From 71c19a86526570bb911bdfb15df32f0250e720c0 Mon Sep 17 00:00:00 2001 From: Ihor Dykhta Date: Sun, 23 Aug 2026 17:31:04 +0300 Subject: [PATCH 01/10] fix: optional collapse for arrow batches Signed-off-by: Ihor Dykhta --- src/duckdb/src/table/duckdb-table.ts | 11 +- src/layers/src/geojson-layer/geojson-layer.ts | 26 +++- src/layers/src/heatmap-layer/heatmap-layer.ts | 5 +- src/layers/src/layer-utils.ts | 129 ++++++++++------ src/layers/src/point-layer/point-layer.ts | 5 +- src/processors/src/data-processor.ts | 3 +- src/utils/src/application-config.ts | 18 ++- src/utils/src/arrow-data-container.ts | 139 ++++++++++++++++-- src/utils/src/filter-utils.ts | 27 ++-- src/utils/src/index.ts | 1 + test/node/utils/data-container-test.js | 40 ++++- test/node/utils/filter-utils-test.js | 48 ++++++ 12 files changed, 359 insertions(+), 93 deletions(-) diff --git a/src/duckdb/src/table/duckdb-table.ts b/src/duckdb/src/table/duckdb-table.ts index f02db4eb23..4d5c38a0ce 100644 --- a/src/duckdb/src/table/duckdb-table.ts +++ b/src/duckdb/src/table/duckdb-table.ts @@ -23,6 +23,7 @@ import { getApplicationConfig, DatabaseAdapter, DatabaseConnection, + compactArrowTable, isArrowTable, isArrowVector } from '@kepler.gl/utils'; @@ -257,11 +258,13 @@ export class KeplerGlDuckDbTable extends KeplerTable { restoreGeoarrowMetadata(arrowResult, geoarrowMetadata); + const compactedResult = compactArrowTable(arrowResult); + fields = useNewFields - ? arrowSchemaToFields(arrowResult, tableDuckDBTypes) - : data.fields ?? arrowSchemaToFields(arrowResult, tableDuckDBTypes); - cols = [...Array(arrowResult.numCols).keys()] - .map(i => arrowResult.getChildAt(i)) + ? arrowSchemaToFields(compactedResult, tableDuckDBTypes) + : data.fields ?? arrowSchemaToFields(compactedResult, tableDuckDBTypes); + cols = [...Array(compactedResult.numCols).keys()] + .map(i => compactedResult.getChildAt(i)) .filter(col => col) as arrow.Vector[]; } catch (error) { console.error('DuckDB table: createTableAndGetArrow', error); diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.ts index eb72f96bfc..5f4e356bb8 100644 --- a/src/layers/src/geojson-layer/geojson-layer.ts +++ b/src/layers/src/geojson-layer/geojson-layer.ts @@ -617,9 +617,20 @@ export default class GeoJsonLayer extends Layer { const geoField = geoFieldAccessor(this.config.columns)(dataContainer); // update the latest batch/chunk of geoarrow data when loading data incrementally - if (geoColumn && geoField && this.dataToFeature.length < dataContainer.numChunks()) { - // for incrementally loading data, we only load and render the latest batch; otherwise, we will load and render all batches - const isIncrementalLoad = dataContainer.numChunks() - this.dataToFeature.length === 1; + const processedRows = this.centroids.length; + if ( + geoColumn && + geoField && + (this.dataToFeature.length < dataContainer.numChunks() || + processedRows < dataContainer.numRows()) + ) { + // Incremental only when a new chunk appeared. Compacted tables stay at 1 + // chunk while row count grows, so those updates reprocess the whole table. + const isIncrementalLoad = + this.dataToFeature.length > 0 && + dataContainer.numChunks() - this.dataToFeature.length === 1 && + processedRows > 0 && + processedRows < dataContainer.numRows(); // TODO: add support for COLUMN_MODE_TABLE in getGeojsonLayerMetaFromArrow const {dataToFeature, bounds, fixedRadius, featureTypes, centroids} = getGeojsonLayerMetaFromArrow({ @@ -628,9 +639,14 @@ export default class GeoJsonLayer extends Layer { geoField, ...(isIncrementalLoad ? {chunkIndex: this.dataToFeature.length} : null) }); - if (centroids) this.centroids = this.centroids.concat(centroids); + if (isIncrementalLoad) { + if (centroids) this.centroids = this.centroids.concat(centroids); + this.dataToFeature = [...this.dataToFeature, ...dataToFeature]; + } else { + this.centroids = centroids || []; + this.dataToFeature = dataToFeature; + } this.updateMeta({bounds, fixedRadius, featureTypes}); - this.dataToFeature = [...this.dataToFeature, ...dataToFeature]; } } else if (this.dataToFeature.length === 0 || this.config.columnMode === COLUMN_MODE_TABLE) { const getFeature = this.getPositionAccessor(dataContainer); diff --git a/src/layers/src/heatmap-layer/heatmap-layer.ts b/src/layers/src/heatmap-layer/heatmap-layer.ts index eabf1289c7..f5685ae740 100644 --- a/src/layers/src/heatmap-layer/heatmap-layer.ts +++ b/src/layers/src/heatmap-layer/heatmap-layer.ts @@ -30,7 +30,7 @@ import {booleanWithin} from '@turf/boolean-within'; import {point as turfPoint} from '@turf/helpers'; import {Feature, Polygon} from 'geojson'; -import {getGeoArrowPointLayerProps, FindDefaultLayerPropsReturnValue} from '../layer-utils'; +import {getGeoArrowPointLayerProps, FindDefaultLayerPropsReturnValue, getGeoArrowPointCoords} from '../layer-utils'; import {getFilterDataFunc} from '../aggregation-layer'; import { parseGeoJsonRawFeature, @@ -88,7 +88,8 @@ export const geoarrowPosAccessor = (dc: DataContainerInterface) => (d: {index: number}): number[] => { const row = dc.valueAt(d.index, geoarrow.fieldIdx); - return [row.get(0), row.get(1)]; + const coords = getGeoArrowPointCoords(row); + return [coords[0], coords[1]]; }; export const geojsonAccessor = diff --git a/src/layers/src/layer-utils.ts b/src/layers/src/layer-utils.ts index 72ab21bca0..e433514400 100644 --- a/src/layers/src/layer-utils.ts +++ b/src/layers/src/layer-utils.ts @@ -31,7 +31,11 @@ import { BinaryPolygonFeature } from '@loaders.gl/schema'; -import {DeckGlGeoTypes, GeojsonDataMaps} from './geojson-layer/geojson-utils'; +import { + DeckGlGeoTypes, + GeojsonDataMaps, + getCentroidFromGeometry +} from './geojson-layer/geojson-utils'; export type FindDefaultLayerProps = { label: string; @@ -87,7 +91,7 @@ function getBinaryGeometriesFromWKBArrow( geoColumn: arrow.Vector, options: {chunkIndex?: number; chunkOffset?: number} ): GeojsonLayerMetaProps { - const dataToFeature: BinaryFeatureCollection[] = []; + const centroids: Array = []; const featureTypes: GeojsonLayerMetaProps['featureTypes'] = { point: false, line: false, @@ -102,8 +106,8 @@ function getBinaryGeometriesFromWKBArrow( let featureIndex = globalFeatureIdOffset; let bounds: [number, number, number, number] = [Infinity, Infinity, -Infinity, -Infinity]; + const geojsonFeatures: Feature[] = []; chunks.forEach(chunk => { - const geojsonFeatures: Feature[] = []; for (let i = 0; i < chunk.length; ++i) { // ignore features without any geometry if (chunk.valueOffsets[i + 1] - chunk.valueOffsets[i] > 0) { @@ -118,54 +122,57 @@ function getBinaryGeometriesFromWKBArrow( properties: {index: featureIndex} }; geojsonFeatures.push(feature); + centroids.push(getCentroidFromGeometry(geometry)); const {type} = geometry; featureTypes.polygon = type === 'Polygon' || type === 'MultiPolygon'; featureTypes.point = type === 'Point' || type === 'MultiPoint'; featureTypes.line = type === 'LineString' || type === 'MultiLineString'; + } else { + centroids.push(null); } featureIndex++; } + }); - const geojsonToBinaryOptions = { - triangulate: true, - fixRingWinding: true - }; - const binaryFeatures = geojsonToBinary(geojsonFeatures, geojsonToBinaryOptions); - - // Need to update globalFeatureIds, to take into account previous batches, - // as geojsonToBinary doesn't have such option. - const featureTypesArr = ['points', 'lines', 'polygons']; - featureTypesArr.forEach(prop => { - const features = binaryFeatures[prop] as - | BinaryPointFeature - | BinaryLineFeature - | BinaryPolygonFeature; - if (features) { - bounds = updateBoundsFromGeoArrowSamples( - features.positions.value as Float64Array, - features.positions.size, - bounds - ); - - const {globalFeatureIds, numericProps} = features; - const {index} = numericProps; - const len = globalFeatureIds.value.length; - for (let i = 0; i < len; ++i) { - globalFeatureIds.value[i] = index.value[i]; - } + // One BinaryFeatureCollection for the whole column. A collection per Arrow + // chunk becomes one GeoJsonLayer (with fill/stroke/point sublayers) each, and + // parquet/DuckDB files easily exceed deck.gl's 255 pickable-layer cap. + const geojsonToBinaryOptions = { + triangulate: true, + fixRingWinding: true + }; + const binaryFeatures = geojsonToBinary(geojsonFeatures, geojsonToBinaryOptions); + + const featureTypesArr = ['points', 'lines', 'polygons']; + featureTypesArr.forEach(prop => { + const features = binaryFeatures[prop] as + | BinaryPointFeature + | BinaryLineFeature + | BinaryPolygonFeature; + if (features) { + bounds = updateBoundsFromGeoArrowSamples( + features.positions.value as Float64Array, + features.positions.size, + bounds + ); + + const {globalFeatureIds, numericProps} = features; + const {index} = numericProps; + const len = globalFeatureIds.value.length; + for (let i = 0; i < len; ++i) { + globalFeatureIds.value[i] = index.value[i]; } - }); - - dataToFeature.push(binaryFeatures); + } }); return { - dataToFeature: dataToFeature, + dataToFeature: [binaryFeatures], featureTypes: featureTypes, bounds, - fixedRadius: false + fixedRadius: false, + centroids }; } @@ -290,6 +297,38 @@ export function getGeoArrowPointFields(fields: Field[]): Field[] { }); } +/** + * Reads lng/lat from a GeoArrow point cell. Arrow may return a FixedSizeList + * (with .get), a typed array, a plain [lng, lat] array, or a {x,y} struct. + */ +export function getGeoArrowPointCoords(row: any): [number, number, number] { + if (row == null) { + return [NaN, NaN, 0]; + } + if (typeof row.get === 'function') { + const x = row.get(0) ?? row.get('x') ?? row.get('lng') ?? row.get('lon'); + const y = row.get(1) ?? row.get('y') ?? row.get('lat'); + if (Number.isFinite(x) && Number.isFinite(y)) { + return [Number(x), Number(y), 0]; + } + } + if (typeof row.toArray === 'function') { + const arr = row.toArray(); + return [Number(arr[0]), Number(arr[1]), 0]; + } + if (Array.isArray(row) || ArrayBuffer.isView(row)) { + return [Number(row[0]), Number(row[1]), 0]; + } + if (typeof row === 'object') { + const x = row.x ?? row.lng ?? row.lon; + const y = row.y ?? row.lat; + if (x !== undefined && y !== undefined) { + return [Number(x), Number(y), 0]; + } + } + return [NaN, NaN, 0]; +} + /** * Builds an arrow vector compatible with ARROW:extension:name geoarrow.point. * @param getPosition Position accessor. @@ -305,7 +344,6 @@ export function createGeoArrowPointVector( // in a correct arrow format, as this approach seems too excessive for just a simple interleaved buffer. const numElements = dataContainer.numRows(); - const table = dataContainer.getTable(); const numCoords = numElements > 0 ? getPosition({index: 0}).length : 2; const precision = 2; @@ -319,21 +357,16 @@ export function createGeoArrowPointVector( const fixedSizeListBuilder = new arrow.FixedSizeListBuilder({type: fixedSizeList}); fixedSizeListBuilder.addChild(floatBuilder); - const assembledBatches: arrow.Data[] = []; + // One record batch, not one per source batch: a batch per parquet/DuckDB + // chunk would create hundreds of pickable ScatterplotLayers and hit deck.gl's + // 255-layer picking cap. const indexData = {index: 0}; - for (let batchIndex = 0; batchIndex < table.batches.length; ++batchIndex) { - const numRowsInBatch = table.batches[batchIndex].numRows; - - for (let i = 0; i < numRowsInBatch; ++i) { - const pos = getPosition(indexData); - fixedSizeListBuilder.append(pos); - - ++indexData.index; - } - assembledBatches.push(fixedSizeListBuilder.flush()); + for (let i = 0; i < numElements; ++i) { + indexData.index = i; + fixedSizeListBuilder.append(getPosition(indexData)); } - return arrow.makeVector(assembledBatches); + return arrow.makeVector(numElements > 0 ? [fixedSizeListBuilder.flush()] : []); } /** diff --git a/src/layers/src/point-layer/point-layer.ts b/src/layers/src/point-layer/point-layer.ts index 5ab058bf9d..5045cca988 100644 --- a/src/layers/src/point-layer/point-layer.ts +++ b/src/layers/src/point-layer/point-layer.ts @@ -41,7 +41,8 @@ import { createGeoArrowPointVector, getFilteredIndex, getNeighbors, - FindDefaultLayerProps + FindDefaultLayerProps, + getGeoArrowPointCoords } from '../layer-utils'; import {getGeojsonPointDataMaps, GeojsonPointDataMaps} from '../geojson-layer/geojson-utils'; import { @@ -131,7 +132,7 @@ export const geoarrowPosAccessor = (dataContainer: DataContainerInterface) => (d: {index: number}) => { const row = dataContainer.valueAt(d.index, geoarrow.fieldIdx); - return [row.get(0), row.get(1), 0]; + return getGeoArrowPointCoords(row); }; export const COLUMN_MODE_POINTS = 'points'; diff --git a/src/processors/src/data-processor.ts b/src/processors/src/data-processor.ts index 2759426b4d..03c6a82172 100644 --- a/src/processors/src/data-processor.ts +++ b/src/processors/src/data-processor.ts @@ -20,6 +20,7 @@ import {ProcessorResult, Field} from '@kepler.gl/types'; import { arrowDataTypeToAnalyzerDataType, arrowDataTypeToFieldType, + compactArrowTable, hasOwnProperty, isPlainObject } from '@kepler.gl/utils'; @@ -626,7 +627,7 @@ export function processArrowBatches(arrowBatches: arrow.RecordBatch[]): Processo if (arrowBatches.length === 0) { return null; } - const arrowTable = castBigIntColumnsToFloat64(new arrow.Table(arrowBatches)); + const arrowTable = compactArrowTable(castBigIntColumnsToFloat64(new arrow.Table(arrowBatches))); const fields = arrowSchemaToFields(arrowTable); const cols = [...Array(arrowTable.numCols).keys()].map(i => arrowTable.getChildAt(i)); diff --git a/src/utils/src/application-config.ts b/src/utils/src/application-config.ts index 0473830989..ec403123bb 100644 --- a/src/utils/src/application-config.ts +++ b/src/utils/src/application-config.ts @@ -218,6 +218,20 @@ export type KeplerApplicationConfig = { /** Controls Redux action logging verbosity in dev mode. See {@link ReduxLogLevel}. */ reduxLogLevel?: ReduxLogLevel; + + /** + * If an Arrow/Parquet table has more record batches than this, they are + * compacted into a single batch. Deck.gl picking supports at most 255 + * pickable layers, so the default is 255. Compacting copies the table, so + * raising this skips that cost for smaller files. Set to `0` to always + * compact; set a very large number to never compact. + * + * @example + * ``` + * initApplicationConfig({maxArrowBatches: 64}); + * ``` + */ + maxArrowBatches?: number; }; const DEFAULT_APPLICATION_CONFIG: Required = { @@ -311,7 +325,9 @@ const DEFAULT_APPLICATION_CONFIG: Required = { customIconUrl: '', - reduxLogLevel: 1 + reduxLogLevel: 1, + + maxArrowBatches: 255 }; const applicationConfig: Required = DEFAULT_APPLICATION_CONFIG; diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.ts index f462b11db3..758c9f8afa 100644 --- a/src/utils/src/arrow-data-container.ts +++ b/src/utils/src/arrow-data-container.ts @@ -7,6 +7,7 @@ import * as arrow from 'apache-arrow'; import {console as globalConsole} from 'global/window'; import {DATA_TYPES as AnalyzerDATA_TYPES} from 'type-analyzer'; +import {getApplicationConfig} from './application-config'; import {DataContainerInterface, RangeOptions} from './data-container-interface'; import {DataRow, SharedRowOptions} from './data-row'; @@ -120,6 +121,98 @@ function* columnIterator(dataContainer: DataContainerInterface, columnIndex: num } } +const DEFAULT_MAX_ARROW_BATCHES = 255; + +/** + * Collapse Arrow record batches into as few chunks as possible. + * + * Deck.gl picking encodes at most 255 pickable leaf layers. Kepler creates one + * Deck layer per Arrow record batch (and GeoJsonLayer adds fill/stroke/point + * sublayers), so parquet/DuckDB tables with hundreds of batches make hover and + * click miss most of the data. Combining chunks keeps picking under that cap + * without changing row values. + * + * apache-arrow JS does not expose Vector.combineChunks in the version Kepler + * uses, so we rebuild each column through a Builder (one chunk per column). + * + * Compaction only runs when `table.batches.length` is greater than + * `maxArrowBatches` from {@link getApplicationConfig} (or the optional override). + */ +export function compactArrowTable(table: arrow.Table, maxArrowBatches?: number): arrow.Table { + if (!table || !Array.isArray(table.batches) || table.batches.length <= 1) { + return table; + } + + const batchLimit = + maxArrowBatches ?? getApplicationConfig().maxArrowBatches ?? DEFAULT_MAX_ARROW_BATCHES; + if (table.batches.length <= batchLimit) { + return table; + } + + try { + const columns: Record = {}; + for (let i = 0; i < table.numCols; i++) { + const field = table.schema.fields[i]; + const column = table.getChildAt(i); + if (!column) { + continue; + } + columns[field.name] = compactArrowVector(column, field.type); + } + + const TableClass = table.constructor as typeof arrow.Table; + let compacted: arrow.Table; + try { + compacted = new TableClass(columns); + } catch { + compacted = new TableClass(table.schema, columns); + } + + if (compacted?.batches?.length && compacted.batches.length < table.batches.length) { + return compacted; + } + } catch { + // Keep the original table if this Arrow build cannot combine chunks. + } + + return table; +} + +function compactArrowVector(column: arrow.Vector, type: arrow.DataType = column.type): arrow.Vector { + if (!column || !column.data || column.data.length <= 1) { + return column; + } + + const maybeCombine = (column as {combineChunks?: () => arrow.Vector}).combineChunks; + if (typeof maybeCombine === 'function') { + try { + const combined = maybeCombine.call(column); + if (combined?.data?.length <= 1) { + return combined; + } + } catch { + // fall through to Builder + } + } + + const builder = arrow.makeBuilder({type, nullValues: [null]}); + const length = column.length; + for (let i = 0; i < length; i++) { + const isValid = typeof (column as {isValid?: (index: number) => boolean}).isValid === 'function' + ? (column as {isValid: (index: number) => boolean}).isValid(i) + : true; + builder.append(isValid ? column.get(i) : null); + } + const finished = builder.finish() as arrow.Vector | {toVector: () => arrow.Vector}; + if (finished instanceof arrow.Vector) { + return finished; + } + if (typeof (finished as {toVector?: () => arrow.Vector}).toVector === 'function') { + return (finished as {toVector: () => arrow.Vector}).toVector(); + } + return column; +} + /** * A data container where all data is stored in raw Arrow table */ @@ -144,14 +237,17 @@ export class ArrowDataContainer implements DataContainerInterface { throw Error("ArrowDataContainer: columns object isn't an array"); } - this._cols = data.cols; - this._numColumns = data.cols.length; - this._numRows = data.cols[0].length; this._fields = data.fields || []; - this._numChunks = data.cols[0].data.length; - // this._colData = data.cols.map(c => c.toArray()); - - this._arrowTable = data.arrowTable || this._createTable(); + this._cols = data.cols; + this._numColumns = this._cols.length; + this._numRows = this._cols[0]?.length ?? 0; + this._numChunks = this._cols[0]?.data?.length ?? 0; + const table = compactArrowTable(data.arrowTable || this._createTable()); + if (table.numCols > 0) { + this._assignTable(table); + } else { + this._arrowTable = table; + } } /** @@ -170,20 +266,33 @@ export class ArrowDataContainer implements DataContainerInterface { return this._arrowTable; } + private _assignTable(table: arrow.Table) { + this._arrowTable = table; + this._cols = Array.from( + {length: table.numCols}, + (_, i) => table.getChildAt(i) as arrow.Vector + ).filter(col => col); + this._numColumns = this._cols.length; + this._numRows = this._cols[0]?.length ?? 0; + this._numChunks = this._cols[0]?.data?.length ?? 0; + } + update(updateData: arrow.Vector[] | arrow.Table) { const isArrow = isArrowTable(updateData); if (isArrow) { - this._cols = Array.from( - {length: updateData.numCols}, - (_, i) => updateData.getChildAt(i) as arrow.Vector - ).filter(col => col); + this._assignTable(compactArrowTable(updateData)); } else { this._cols = updateData; + const table = compactArrowTable(this._createTable()); + if (table.numCols > 0) { + this._assignTable(table); + } else { + this._numColumns = this._cols.length; + this._numRows = this._cols[0]?.length ?? 0; + this._numChunks = this._cols[0]?.data?.length ?? 0; + this._arrowTable = table; + } } - this._numColumns = this._cols?.length ?? 0; - this._numRows = this._cols?.[0]?.length ?? 0; - this._numChunks = this._cols?.[0]?.data?.length ?? 0; - this._arrowTable = isArrow ? updateData : this._createTable(); // cache column data to make valueAt() faster // this._colData = this._cols.map(c => c.toArray()); diff --git a/src/utils/src/filter-utils.ts b/src/utils/src/filter-utils.ts index 954031a1e1..ababc38a30 100644 --- a/src/utils/src/filter-utils.ts +++ b/src/utils/src/filter-utils.ts @@ -376,6 +376,12 @@ export function getFilterProps( } } +function hasFiniteLngLat(pos: unknown): pos is number[] { + return ( + Array.isArray(pos) && pos.length >= 2 && Number.isFinite(pos[0]) && Number.isFinite(pos[1]) + ); +} + export const getPolygonFilterFunctor = (layer, filter, dataContainer) => { const getPosition = layer.getPositionAccessor(dataContainer); @@ -388,22 +394,15 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => { if (!coordinates) return false; if (Array.isArray(coordinates[0])) { return (coordinates as number[][]).some( - coord => - coord.length >= 2 && - coord.every(Number.isFinite) && - isInPolygon(coord, filter.value) + coord => hasFiniteLngLat(coord) && isInPolygon(coord, filter.value) ); } - return ( - coordinates.length >= 2 && - coordinates.every(Number.isFinite) && - isInPolygon(coordinates, filter.value) - ); + return hasFiniteLngLat(coordinates) && isInPolygon(coordinates, filter.value); }; } return data => { const pos = getPosition(data); - return pos.every(Number.isFinite) && isInPolygon(pos, filter.value); + return hasFiniteLngLat(pos) && isInPolygon(pos, filter.value); }; case LAYER_TYPES.grid: case LAYER_TYPES.hexagon: @@ -418,14 +417,14 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => { } return data => { const pos = getPosition(data); - return pos.every(Number.isFinite) && isInPolygon(pos, filter.value); + return hasFiniteLngLat(pos) && isInPolygon(pos, filter.value); }; case LAYER_TYPES.arc: case LAYER_TYPES.line: return data => { const pos = getPosition(data); return ( - pos.every(Number.isFinite) && + hasFiniteLngLat(pos) && [ [pos[0], pos[1]], [pos[3], pos[4]] @@ -446,7 +445,7 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => { return false; } const pos = getCentroid({id}); - return pos.every(Number.isFinite) && isInPolygon(pos, filter.value); + return hasFiniteLngLat(pos) && isInPolygon(pos, filter.value); }; case LAYER_TYPES.geojson: return data => { @@ -461,7 +460,7 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => { } return data => { const pos = getPosition(data); - return pos.every(Number.isFinite) && isInPolygon(pos, filter.value); + return hasFiniteLngLat(pos) && isInPolygon(pos, filter.value); }; default: return () => true; diff --git a/src/utils/src/index.ts b/src/utils/src/index.ts index b38a856548..f730968eeb 100644 --- a/src/utils/src/index.ts +++ b/src/utils/src/index.ts @@ -144,6 +144,7 @@ export * from './map-utils'; export { ArrowDataContainer, + compactArrowTable, arrowDataTypeToAnalyzerDataType, arrowDataTypeToFieldType, isArrowTable, diff --git a/test/node/utils/data-container-test.js b/test/node/utils/data-container-test.js index 200be5c68e..c44478f141 100644 --- a/test/node/utils/data-container-test.js +++ b/test/node/utils/data-container-test.js @@ -3,7 +3,8 @@ import test from 'tape'; -import {createDataContainer, createIndexedDataContainer} from '@kepler.gl/utils'; +import {createDataContainer, createIndexedDataContainer, compactArrowTable, ArrowDataContainer} from '@kepler.gl/utils'; +import * as arrow from 'apache-arrow'; const data = [ [10, 20], // 0 @@ -120,3 +121,40 @@ test('IndexedDataContainer', t => { t.end(); }); + +test('ArrowDataContainer -> compactArrowTable collapses record batches over the limit', t => { + const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]); + const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]); + const combined = batchA.concat(batchB); + + t.ok(combined.batches.length > 1, 'fixture should have multiple record batches'); + + const belowLimit = compactArrowTable(combined, 255); + t.equal( + belowLimit.batches.length, + combined.batches.length, + 'compactArrowTable should leave tables at or under maxArrowBatches unchanged' + ); + + const compacted = compactArrowTable(combined, 1); + t.equal(compacted.numRows, 2, 'compactArrowTable should keep all rows'); + t.equal(compacted.batches.length, 1, 'compactArrowTable should collapse into one batch'); + + const cols = Array.from({length: combined.numCols}, (_, i) => combined.getChildAt(i)).filter( + Boolean + ); + const arrowDc = new ArrowDataContainer({ + cols, + fields: combined.schema.fields.map((field, fieldIdx) => ({ + name: field.name, + fieldIdx + })), + arrowTable: compactArrowTable(combined, 1) + }); + + t.equal(arrowDc.numRows(), 2, 'ArrowDataContainer should keep all rows'); + t.equal(arrowDc.numChunks(), 1, 'ArrowDataContainer should store a compacted table'); + t.deepEqual(arrowDc.valueAt(0, 0), -122.4, 'compacted container should preserve values'); + + t.end(); +}); diff --git a/test/node/utils/filter-utils-test.js b/test/node/utils/filter-utils-test.js index 8ef54b5bb0..f6a1d0a478 100644 --- a/test/node/utils/filter-utils-test.js +++ b/test/node/utils/filter-utils-test.js @@ -804,3 +804,51 @@ test('filterUtils -> getPolygonFilterFunctor -> aggregation layers (grid/hexagon t.end(); }); + +test('filterUtils -> getPolygonFilterFunctor -> point layer ignores non-finite altitude', t => { + const squarePolygon = { + type: 'Feature', + properties: {}, + geometry: { + type: 'Polygon', + coordinates: [ + [ + [-1, -1], + [1, -1], + [1, 1], + [-1, 1], + [-1, -1] + ] + ] + } + }; + + const filter = {value: squarePolygon}; + const layer = { + type: 'point', + config: {columnMode: 'points'}, + getPositionAccessor: () => d => d.position, + dataToFeature: [] + }; + + const fn = getPolygonFilterFunctor(layer, filter, null); + + t.equal(fn({position: [0.5, 0.5, 0]}), true, 'finite altitude should keep a point inside'); + t.equal( + fn({position: [0.5, 0.5, null]}), + true, + 'null altitude should not exclude a valid lng/lat' + ); + t.equal( + fn({position: [0.5, 0.5, undefined]}), + true, + 'undefined altitude should not exclude a valid lng/lat' + ); + t.equal( + fn({position: [10, 10, null]}), + false, + 'null altitude should not include a point outside' + ); + + t.end(); +}); From a504b54b40a7844d2fed3460f711ed8b59a35fa6 Mon Sep 17 00:00:00 2001 From: Ihor Dykhta Date: Sun, 23 Aug 2026 18:29:53 +0300 Subject: [PATCH 02/10] fixes for batches collapse Signed-off-by: Ihor Dykhta --- src/utils/src/application-config.ts | 2 +- src/utils/src/arrow-data-container.ts | 82 +++++++++++++++++++++++--- test/node/utils/data-container-test.js | 15 +++++ 3 files changed, 89 insertions(+), 10 deletions(-) diff --git a/src/utils/src/application-config.ts b/src/utils/src/application-config.ts index ec403123bb..c9439a32b5 100644 --- a/src/utils/src/application-config.ts +++ b/src/utils/src/application-config.ts @@ -327,7 +327,7 @@ const DEFAULT_APPLICATION_CONFIG: Required = { reduxLogLevel: 1, - maxArrowBatches: 255 + maxArrowBatches: 1 }; const applicationConfig: Required = DEFAULT_APPLICATION_CONFIG; diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.ts index 758c9f8afa..d685b5b6bd 100644 --- a/src/utils/src/arrow-data-container.ts +++ b/src/utils/src/arrow-data-container.ts @@ -139,12 +139,14 @@ const DEFAULT_MAX_ARROW_BATCHES = 255; * `maxArrowBatches` from {@link getApplicationConfig} (or the optional override). */ export function compactArrowTable(table: arrow.Table, maxArrowBatches?: number): arrow.Table { + const batchCount = table?.batches?.length ?? 0; + const batchLimit = + maxArrowBatches ?? getApplicationConfig().maxArrowBatches ?? DEFAULT_MAX_ARROW_BATCHES; + if (!table || !Array.isArray(table.batches) || table.batches.length <= 1) { return table; } - const batchLimit = - maxArrowBatches ?? getApplicationConfig().maxArrowBatches ?? DEFAULT_MAX_ARROW_BATCHES; if (table.batches.length <= batchLimit) { return table; } @@ -160,24 +162,86 @@ export function compactArrowTable(table: arrow.Table, maxArrowBatches?: number): columns[field.name] = compactArrowVector(column, field.type); } - const TableClass = table.constructor as typeof arrow.Table; - let compacted: arrow.Table; - try { - compacted = new TableClass(columns); - } catch { - compacted = new TableClass(table.schema, columns); - } + // Always build a kepler apache-arrow Table. Using `table.constructor` can + // mix DuckDB's bundled Arrow Table with rebuilt vectors and drop schema + // metadata (`geo`, ARROW:extension:name) that Kepler needs to create layers. + const compacted = tableFromCompactedColumns(table, columns); if (compacted?.batches?.length && compacted.batches.length < table.batches.length) { + console.log('[kepler.gl] Arrow batches', { + count: batchCount, + maxArrowBatches: batchLimit, + collapsed: true, + after: compacted.batches.length, + hasGeoMetadata: hasArrowGeoMetadata(compacted.schema) + }); return compacted; } } catch { // Keep the original table if this Arrow build cannot combine chunks. } + console.log('[kepler.gl] Arrow batches', { + count: batchCount, + maxArrowBatches: batchLimit, + collapsed: false + }); return table; } +function hasArrowGeoMetadata(schema?: arrow.Schema | null): boolean { + if (!schema) { + return false; + } + if (schema.metadata?.get?.('geo')) { + return true; + } + return (schema.fields || []).some(field => + Boolean(field.metadata?.get?.('ARROW:extension:name')?.startsWith?.('geoarrow')) + ); +} + +function copyArrowSchemaMetadata(from?: arrow.Schema | null, to?: arrow.Schema | null): void { + if (!from || !to) { + return; + } + + const fromMeta = from.metadata as Map | undefined; + const toMeta = to.metadata as Map | undefined; + if (fromMeta && toMeta && typeof fromMeta.forEach === 'function' && typeof toMeta.set === 'function') { + fromMeta.forEach((value, key) => { + toMeta.set(key, value); + }); + } + + const fromByName = new Map((from.fields || []).map(field => [field.name, field])); + (to.fields || []).forEach(toField => { + const fromField = fromByName.get(toField.name); + const src = fromField?.metadata as Map | undefined; + const dst = toField?.metadata as Map | undefined; + if (!src || !dst || typeof src.forEach !== 'function' || typeof dst.set !== 'function') { + return; + } + src.forEach((value, key) => { + dst.set(key, value); + }); + }); +} + +function tableFromCompactedColumns( + source: arrow.Table, + columns: Record +): arrow.Table { + let compacted: arrow.Table; + try { + compacted = new arrow.Table(source.schema, columns); + } catch { + compacted = new arrow.Table(columns); + } + copyArrowSchemaMetadata(source.schema, compacted.schema); + return compacted; +} + function compactArrowVector(column: arrow.Vector, type: arrow.DataType = column.type): arrow.Vector { if (!column || !column.data || column.data.length <= 1) { return column; diff --git a/test/node/utils/data-container-test.js b/test/node/utils/data-container-test.js index c44478f141..e7d92addc3 100644 --- a/test/node/utils/data-container-test.js +++ b/test/node/utils/data-container-test.js @@ -136,9 +136,24 @@ test('ArrowDataContainer -> compactArrowTable collapses record batches over the 'compactArrowTable should leave tables at or under maxArrowBatches unchanged' ); + combined.schema.metadata.set( + 'geo', + JSON.stringify({columns: {lng: {encoding: 'WKB'}}}) + ); + combined.schema.fields[0].metadata.set('ARROW:extension:name', 'geoarrow.wkb'); + const compacted = compactArrowTable(combined, 1); t.equal(compacted.numRows, 2, 'compactArrowTable should keep all rows'); t.equal(compacted.batches.length, 1, 'compactArrowTable should collapse into one batch'); + t.ok( + compacted.schema.metadata.get('geo'), + 'compactArrowTable should keep GeoParquet schema metadata used to create layers' + ); + t.equal( + compacted.schema.fields[0].metadata.get('ARROW:extension:name'), + 'geoarrow.wkb', + 'compactArrowTable should keep geoarrow field metadata' + ); const cols = Array.from({length: combined.numCols}, (_, i) => combined.getChildAt(i)).filter( Boolean From e42b5b84c106d632139505f06eea3b159f4dd24b Mon Sep 17 00:00:00 2001 From: Ihor Dykhta Date: Sun, 23 Aug 2026 18:54:21 +0300 Subject: [PATCH 03/10] fix follow ups Signed-off-by: Ihor Dykhta --- src/layers/src/geojson-layer/geojson-layer.ts | 24 ++++++++++++------ src/utils/src/arrow-data-container.ts | 25 ------------------- 2 files changed, 16 insertions(+), 33 deletions(-) diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.ts index 5f4e356bb8..d51490c42f 100644 --- a/src/layers/src/geojson-layer/geojson-layer.ts +++ b/src/layers/src/geojson-layer/geojson-layer.ts @@ -618,19 +618,27 @@ export default class GeoJsonLayer extends Layer { // update the latest batch/chunk of geoarrow data when loading data incrementally const processedRows = this.centroids.length; - if ( - geoColumn && - geoField && - (this.dataToFeature.length < dataContainer.numChunks() || - processedRows < dataContainer.numRows()) - ) { + const numRows = dataContainer.numRows(); + const numChunks = dataContainer.numChunks(); + // WKB (and other whole-column collections) keep dataToFeature.length at 1 + // even when numChunks() is large. Row count is the progress signal then; + // comparing length to numChunks would re-parse WKB on every meta update. + const processedWholeColumn = + this.dataToFeature.length > 0 && processedRows > 0 && processedRows >= numRows; + const needsUpdate = + this.dataToFeature.length === 0 || + (!processedWholeColumn && this.dataToFeature.length < numChunks) || + (processedRows > 0 && processedRows < numRows); + + if (geoColumn && geoField && needsUpdate) { // Incremental only when a new chunk appeared. Compacted tables stay at 1 // chunk while row count grows, so those updates reprocess the whole table. const isIncrementalLoad = this.dataToFeature.length > 0 && - dataContainer.numChunks() - this.dataToFeature.length === 1 && + !processedWholeColumn && + numChunks - this.dataToFeature.length === 1 && processedRows > 0 && - processedRows < dataContainer.numRows(); + processedRows < numRows; // TODO: add support for COLUMN_MODE_TABLE in getGeojsonLayerMetaFromArrow const {dataToFeature, bounds, fixedRadius, featureTypes, centroids} = getGeojsonLayerMetaFromArrow({ diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.ts index d685b5b6bd..2398084d5c 100644 --- a/src/utils/src/arrow-data-container.ts +++ b/src/utils/src/arrow-data-container.ts @@ -139,7 +139,6 @@ const DEFAULT_MAX_ARROW_BATCHES = 255; * `maxArrowBatches` from {@link getApplicationConfig} (or the optional override). */ export function compactArrowTable(table: arrow.Table, maxArrowBatches?: number): arrow.Table { - const batchCount = table?.batches?.length ?? 0; const batchLimit = maxArrowBatches ?? getApplicationConfig().maxArrowBatches ?? DEFAULT_MAX_ARROW_BATCHES; @@ -168,39 +167,15 @@ export function compactArrowTable(table: arrow.Table, maxArrowBatches?: number): const compacted = tableFromCompactedColumns(table, columns); if (compacted?.batches?.length && compacted.batches.length < table.batches.length) { - console.log('[kepler.gl] Arrow batches', { - count: batchCount, - maxArrowBatches: batchLimit, - collapsed: true, - after: compacted.batches.length, - hasGeoMetadata: hasArrowGeoMetadata(compacted.schema) - }); return compacted; } } catch { // Keep the original table if this Arrow build cannot combine chunks. } - console.log('[kepler.gl] Arrow batches', { - count: batchCount, - maxArrowBatches: batchLimit, - collapsed: false - }); return table; } -function hasArrowGeoMetadata(schema?: arrow.Schema | null): boolean { - if (!schema) { - return false; - } - if (schema.metadata?.get?.('geo')) { - return true; - } - return (schema.fields || []).some(field => - Boolean(field.metadata?.get?.('ARROW:extension:name')?.startsWith?.('geoarrow')) - ); -} - function copyArrowSchemaMetadata(from?: arrow.Schema | null, to?: arrow.Schema | null): void { if (!from || !to) { return; From b136fdd1c659942b033051747028d50d90e6c60b Mon Sep 17 00:00:00 2001 From: Ihor Dykhta Date: Sun, 23 Aug 2026 20:09:14 +0300 Subject: [PATCH 04/10] follow up Signed-off-by: Ihor Dykhta --- src/layers/src/layer-utils.ts | 1 - src/utils/src/arrow-data-container.ts | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/layers/src/layer-utils.ts b/src/layers/src/layer-utils.ts index e433514400..ee074093e6 100644 --- a/src/layers/src/layer-utils.ts +++ b/src/layers/src/layer-utils.ts @@ -24,7 +24,6 @@ import {convertGeoArrowGeometryToGeoJSON} from '@loaders.gl/gis'; import {WKBLoader} from '@loaders.gl/wkt'; import {geojsonToBinary} from '@loaders.gl/gis'; import { - BinaryFeatureCollection, Geometry, BinaryPointFeature, BinaryLineFeature, diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.ts index 2398084d5c..089b7645fa 100644 --- a/src/utils/src/arrow-data-container.ts +++ b/src/utils/src/arrow-data-container.ts @@ -282,10 +282,9 @@ export class ArrowDataContainer implements DataContainerInterface { this._numRows = this._cols[0]?.length ?? 0; this._numChunks = this._cols[0]?.data?.length ?? 0; const table = compactArrowTable(data.arrowTable || this._createTable()); + this._arrowTable = table; if (table.numCols > 0) { this._assignTable(table); - } else { - this._arrowTable = table; } } From 8ab7b009cb3071cf750119ea2874988fbad3f807 Mon Sep 17 00:00:00 2001 From: Ihor Dykhta Date: Sun, 23 Aug 2026 20:39:07 +0300 Subject: [PATCH 05/10] follow ups Signed-off-by: Ihor Dykhta --- src/utils/src/application-config.ts | 2 +- src/utils/src/filter-utils.ts | 7 ++++++- test/node/utils/filter-utils-test.js | 5 +++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/utils/src/application-config.ts b/src/utils/src/application-config.ts index c9439a32b5..ad531b4f38 100644 --- a/src/utils/src/application-config.ts +++ b/src/utils/src/application-config.ts @@ -231,7 +231,7 @@ export type KeplerApplicationConfig = { * initApplicationConfig({maxArrowBatches: 64}); * ``` */ - maxArrowBatches?: number; + maxArrowBatches?: number; }; const DEFAULT_APPLICATION_CONFIG: Required = { diff --git a/src/utils/src/filter-utils.ts b/src/utils/src/filter-utils.ts index ababc38a30..944fec26e3 100644 --- a/src/utils/src/filter-utils.ts +++ b/src/utils/src/filter-utils.ts @@ -377,8 +377,13 @@ export function getFilterProps( } function hasFiniteLngLat(pos: unknown): pos is number[] { + const values = pos as {length?: number; 0?: unknown; 1?: unknown} | null; return ( - Array.isArray(pos) && pos.length >= 2 && Number.isFinite(pos[0]) && Number.isFinite(pos[1]) + Boolean(values) && + (Array.isArray(pos) || ArrayBuffer.isView(pos)) && + (values as {length: number}).length >= 2 && + Number.isFinite(Number(values[0])) && + Number.isFinite(Number(values[1])) ); } diff --git a/test/node/utils/filter-utils-test.js b/test/node/utils/filter-utils-test.js index f6a1d0a478..752454eeee 100644 --- a/test/node/utils/filter-utils-test.js +++ b/test/node/utils/filter-utils-test.js @@ -849,6 +849,11 @@ test('filterUtils -> getPolygonFilterFunctor -> point layer ignores non-finite a false, 'null altitude should not include a point outside' ); + t.equal( + fn({position: new Float64Array([0.5, 0.5, 0])}), + true, + 'typed-array lng/lat should be treated as a valid position' + ); t.end(); }); From 6cf1777640fbca95e539c1cbc8aa40e932618ddf Mon Sep 17 00:00:00 2001 From: Ihor Dykhta Date: Sun, 23 Aug 2026 20:51:08 +0300 Subject: [PATCH 06/10] lint Signed-off-by: Ihor Dykhta --- src/utils/src/filter-utils.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/utils/src/filter-utils.ts b/src/utils/src/filter-utils.ts index 944fec26e3..4c2a4a3cba 100644 --- a/src/utils/src/filter-utils.ts +++ b/src/utils/src/filter-utils.ts @@ -377,14 +377,11 @@ export function getFilterProps( } function hasFiniteLngLat(pos: unknown): pos is number[] { - const values = pos as {length?: number; 0?: unknown; 1?: unknown} | null; - return ( - Boolean(values) && - (Array.isArray(pos) || ArrayBuffer.isView(pos)) && - (values as {length: number}).length >= 2 && - Number.isFinite(Number(values[0])) && - Number.isFinite(Number(values[1])) - ); + if (pos == null || !(Array.isArray(pos) || ArrayBuffer.isView(pos))) { + return false; + } + const values = pos as ArrayLike; + return values.length >= 2 && Number.isFinite(Number(values[0])) && Number.isFinite(Number(values[1])); } export const getPolygonFilterFunctor = (layer, filter, dataContainer) => { From 4e978afc4d5b0a086029c11dc3dc0e679244b45c Mon Sep 17 00:00:00 2001 From: Ihor Dykhta Date: Sun, 23 Aug 2026 22:09:17 +0300 Subject: [PATCH 07/10] fix tests Signed-off-by: Ihor Dykhta --- src/utils/src/filter-utils.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utils/src/filter-utils.ts b/src/utils/src/filter-utils.ts index 4c2a4a3cba..2738d06c23 100644 --- a/src/utils/src/filter-utils.ts +++ b/src/utils/src/filter-utils.ts @@ -831,7 +831,9 @@ export function isInRange(val: any, domain: number[]): boolean { * @return {boolean} */ export function isInPolygon(point: number[], polygon: any): boolean { - return booleanWithin(turfPoint(point), polygon); + // turfPoint requires a plain array; Arrow accessors can return typed arrays + const lngLat = Array.isArray(point) ? point : [Number(point[0]), Number(point[1])]; + return booleanWithin(turfPoint(lngLat), polygon); } export function getTimeWidgetTitleFormatter(domain: [number, number]): string | null { if (!isValidTimeDomain(domain)) { From 206c7fcf1420c31ee6d0e6396c5383c93ed775fb Mon Sep 17 00:00:00 2001 From: Ihor Dykhta Date: Tue, 25 Aug 2026 14:16:40 +0300 Subject: [PATCH 08/10] follow up Signed-off-by: Ihor Dykhta --- src/processors/src/data-processor.ts | 23 ++- src/processors/src/file-handler.ts | 10 +- src/reducers/src/vis-state-updaters.ts | 7 +- src/utils/src/application-config.ts | 10 +- src/utils/src/arrow-data-container.ts | 56 +++++- test/node/processors/file-handler-test.js | 59 +++++- test/node/utils/data-container-test.js | 229 ++++++++++++++++++++++ 7 files changed, 379 insertions(+), 15 deletions(-) diff --git a/src/processors/src/data-processor.ts b/src/processors/src/data-processor.ts index 03c6a82172..17b9c011f3 100644 --- a/src/processors/src/data-processor.ts +++ b/src/processors/src/data-processor.ts @@ -617,17 +617,34 @@ function castBigIntColumnsToFloat64(arrowTable: arrow.Table): arrow.Table { return new arrow.Table(newColumns); } +export type ProcessArrowBatchesOptions = { + /** + * When false, skip compactArrowTable. Progressive Arrow loading uses this so + * each incoming batch does not copy the accumulated table. Defaults to true. + * The final processFileContent call leaves this unset so the completed table + * is compacted once. + */ + compact?: boolean; +}; + /** * Parse arrow batches returned from parseInBatches() * - * @param arrowTable the arrow table to parse + * @param arrowBatches the arrow record batches to parse + * @param options optional processing flags * @returns dataset containing `fields` and `rows` or null */ -export function processArrowBatches(arrowBatches: arrow.RecordBatch[]): ProcessorResult | null { +export function processArrowBatches( + arrowBatches: arrow.RecordBatch[], + options?: ProcessArrowBatchesOptions +): ProcessorResult | null { if (arrowBatches.length === 0) { return null; } - const arrowTable = compactArrowTable(castBigIntColumnsToFloat64(new arrow.Table(arrowBatches))); + const arrowTable = + options?.compact === false + ? castBigIntColumnsToFloat64(new arrow.Table(arrowBatches)) + : compactArrowTable(castBigIntColumnsToFloat64(new arrow.Table(arrowBatches))); const fields = arrowSchemaToFields(arrowTable); const cols = [...Array(arrowTable.numCols).keys()].map(i => arrowTable.getChildAt(i)); diff --git a/src/processors/src/file-handler.ts b/src/processors/src/file-handler.ts index 2e229c93bb..8d090cb264 100644 --- a/src/processors/src/file-handler.ts +++ b/src/processors/src/file-handler.ts @@ -70,6 +70,11 @@ export type ProcessFileDataContent = { sourceUrl?: string; /** User-selected or inferred remote file format (csv, geojson, parquet, …). */ keplerFormat?: string; + /** + * When true, skip Arrow record-batch compaction. Set on intermediate + * progressive Arrow loads so each batch does not copy the growing table. + */ + skipArrowCompact?: boolean; }; export {isArrowTable}; @@ -279,7 +284,10 @@ export async function processFileData({ // eslint-disable-next-line no-useless-catch let result; try { - result = await processor(data); + result = + format === DATASET_FORMATS.arrow && content.skipArrowCompact + ? await processArrowBatches(data as any, {compact: false}) + : await processor(data); } catch (error) { throw new Error(`Can not process uploaded file, ${getError(error as Error)}`); } diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.ts index d037d91863..cfa4e5ca69 100644 --- a/src/reducers/src/vis-state-updaters.ts +++ b/src/reducers/src/vis-state-updaters.ts @@ -3517,7 +3517,12 @@ export const nextFileBatchUpdater = ( fileName.endsWith('arrow') && accumulated?.data?.length > 0 ? [ - PROCESS_FILE_DATA({content: accumulated, fileCache: []}).bimap( + // Skip compaction while batches are still arriving. The completed + // file is compacted once in processFileContent. + PROCESS_FILE_DATA({ + content: {...accumulated, skipArrowCompact: true}, + fileCache: [] + }).bimap( result => loadFilesSuccess(result), err => loadFilesErr(fileName, err) ) diff --git a/src/utils/src/application-config.ts b/src/utils/src/application-config.ts index 1aff2cfe31..d5cd5b05ed 100644 --- a/src/utils/src/application-config.ts +++ b/src/utils/src/application-config.ts @@ -227,10 +227,12 @@ export type KeplerApplicationConfig = { /** * If an Arrow/Parquet table has more record batches than this, they are - * compacted into a single batch. Deck.gl picking supports at most 255 - * pickable layers, so the default is 255. Compacting copies the table, so - * raising this skips that cost for smaller files. Set to `0` to always - * compact; set a very large number to never compact. + * compacted into a single batch. The default is `1`, so any table with more + * than one record batch is compacted. Deck.gl picking supports at most 255 + * pickable leaf layers, and GeoJSON adds fill/stroke/point sublayers per + * batch, so a higher cap still overruns picking. Compacting copies the table. + * Progressive Arrow loading skips compaction on intermediate batches and + * compacts the completed file once. Set a very large number to never compact. * * @example * ``` diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.ts index 089b7645fa..2ae3a460f6 100644 --- a/src/utils/src/arrow-data-container.ts +++ b/src/utils/src/arrow-data-container.ts @@ -121,7 +121,7 @@ function* columnIterator(dataContainer: DataContainerInterface, columnIndex: num } } -const DEFAULT_MAX_ARROW_BATCHES = 255; +const DEFAULT_MAX_ARROW_BATCHES = 1; /** * Collapse Arrow record batches into as few chunks as possible. @@ -136,7 +136,10 @@ const DEFAULT_MAX_ARROW_BATCHES = 255; * uses, so we rebuild each column through a Builder (one chunk per column). * * Compaction only runs when `table.batches.length` is greater than - * `maxArrowBatches` from {@link getApplicationConfig} (or the optional override). + * `maxArrowBatches` from {@link getApplicationConfig} (default `1`) or the + * optional override. Progressive Arrow loading does not compact on each + * incoming batch; {@link ArrowDataContainer.update} assigns new chunks as-is + * and the completed file is compacted once. */ export function compactArrowTable(table: arrow.Table, maxArrowBatches?: number): arrow.Table { const batchLimit = @@ -217,6 +220,43 @@ function tableFromCompactedColumns( return compacted; } +/** + * Convert an Arrow cell from Vector.get() into a value makeBuilder.append accepts. + * Nested GeoArrow (FixedSizeList / List) cells are Vectors, not arrays. + */ +function arrowCellToBuilderValue(value: unknown): unknown { + if (value == null) { + return null; + } + if (ArrayBuffer.isView(value)) { + return Array.from(value as any); + } + + const nested = value as { + toArray?: () => ArrayLike; + numChildren?: number; + get?: (index: number) => unknown; + length?: number; + isValid?: (index: number) => boolean; + }; + + // Primitive numeric child of a FixedSizeList (lng/lat). + if (typeof nested.toArray === 'function' && nested.numChildren === 0) { + return Array.from(nested.toArray()); + } + + if (typeof nested.get === 'function' && typeof nested.length === 'number') { + const out: unknown[] = []; + for (let i = 0; i < nested.length; i++) { + const valid = typeof nested.isValid === 'function' ? nested.isValid(i) : true; + out.push(valid ? arrowCellToBuilderValue(nested.get(i)) : null); + } + return out; + } + + return value; +} + function compactArrowVector(column: arrow.Vector, type: arrow.DataType = column.type): arrow.Vector { if (!column || !column.data || column.data.length <= 1) { return column; @@ -240,7 +280,10 @@ function compactArrowVector(column: arrow.Vector, type: arrow.DataType = column. const isValid = typeof (column as {isValid?: (index: number) => boolean}).isValid === 'function' ? (column as {isValid: (index: number) => boolean}).isValid(i) : true; - builder.append(isValid ? column.get(i) : null); + // FixedSizeList/List .get() returns nested Vectors. Appending those + // silently writes NaN (points) or throws (lines/polygons). Plain JS + // arrays are what makeBuilder expects. + builder.append(isValid ? arrowCellToBuilderValue(column.get(i)) : null); } const finished = builder.finish() as arrow.Vector | {toVector: () => arrow.Vector}; if (finished instanceof arrow.Vector) { @@ -316,12 +359,15 @@ export class ArrowDataContainer implements DataContainerInterface { } update(updateData: arrow.Vector[] | arrow.Table) { + // Incremental Arrow loads append batches here. Compacting on every update + // would copy the growing table (Builder over every cell). Skip collapse; + // processArrowBatches compact once when the file is fully loaded. const isArrow = isArrowTable(updateData); if (isArrow) { - this._assignTable(compactArrowTable(updateData)); + this._assignTable(updateData); } else { this._cols = updateData; - const table = compactArrowTable(this._createTable()); + const table = this._createTable(); if (table.numCols > 0) { this._assignTable(table); } else { diff --git a/test/node/processors/file-handler-test.js b/test/node/processors/file-handler-test.js index fdb6cd782a..b190b92fbb 100644 --- a/test/node/processors/file-handler-test.js +++ b/test/node/processors/file-handler-test.js @@ -2,7 +2,8 @@ // Copyright contributors to the kepler.gl project import test from 'tape'; -import {isKeplerGlMap, makeProgressIterator, filesToDataPayload, processFileData} from '@kepler.gl/processors'; +import {isKeplerGlMap, makeProgressIterator, filesToDataPayload, processFileData, processArrowBatches} from '@kepler.gl/processors'; +import * as arrow from 'apache-arrow'; import {parsedFields, parsedRows} from 'test/fixtures/row-object'; import { savedStateV1InteractionCoordinate as keplerglMap, @@ -277,3 +278,59 @@ test('#file-handler -> processFileData remote ids do not collide on filename', a t.end(); }); + +test('#file-handler -> processArrowBatches skip compact for incremental loads', t => { + const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]); + const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]); + const combined = batchA.concat(batchB); + + t.ok(combined.batches.length > 1, 'fixture should have multiple record batches'); + + const skipped = processArrowBatches(combined.batches, {compact: false}); + t.equal( + skipped.cols[0].data.length, + combined.batches.length, + 'compact:false should leave record batches unchanged' + ); + + const compacted = processArrowBatches(combined.batches); + t.equal(compacted.cols[0].data.length, 1, 'default processArrowBatches should compact'); + t.equal(compacted.cols[0].length, 2, 'compacted table should keep all rows'); + + t.end(); +}); + +test('#file-handler -> processFileData skipArrowCompact', async t => { + const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]); + const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]); + const combined = batchA.concat(batchB); + + const incremental = await processFileData({ + content: { + fileName: 'points.arrow', + data: combined.batches, + skipArrowCompact: true + }, + fileCache: [] + }); + t.equal( + incremental[0].data.cols[0].data.length, + combined.batches.length, + 'progressive processFileData should not compact Arrow batches' + ); + + const finished = await processFileData({ + content: { + fileName: 'points.arrow', + data: combined.batches + }, + fileCache: [] + }); + t.equal( + finished[0].data.cols[0].data.length, + 1, + 'final processFileData should compact Arrow batches' + ); + + t.end(); +}); diff --git a/test/node/utils/data-container-test.js b/test/node/utils/data-container-test.js index e7d92addc3..5a9ea9d743 100644 --- a/test/node/utils/data-container-test.js +++ b/test/node/utils/data-container-test.js @@ -4,6 +4,7 @@ import test from 'tape'; import {createDataContainer, createIndexedDataContainer, compactArrowTable, ArrowDataContainer} from '@kepler.gl/utils'; +import {GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY} from '@kepler.gl/constants'; import * as arrow from 'apache-arrow'; const data = [ @@ -173,3 +174,231 @@ test('ArrowDataContainer -> compactArrowTable collapses record batches over the t.end(); }); + +test('ArrowDataContainer -> update does not collapse record batches', t => { + const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]); + const colsA = Array.from({length: batchA.numCols}, (_, i) => batchA.getChildAt(i)).filter(Boolean); + const dc = new ArrowDataContainer({ + cols: colsA, + fields: batchA.schema.fields.map((field, fieldIdx) => ({ + name: field.name, + fieldIdx + })), + arrowTable: batchA + }); + + const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]); + const combined = batchA.concat(batchB); + dc.update(combined); + + t.equal(dc.numRows(), 2, 'update should keep all rows'); + t.equal( + dc.numChunks(), + combined.batches.length, + 'update should not compact batches during incremental loading' + ); + t.deepEqual(dc.valueAt(1, 0), -122.5, 'update should preserve newly appended values'); + + t.end(); +}); + +function finishArrowVector(builder) { + const finished = builder.finish(); + if (finished instanceof arrow.Vector) { + return finished; + } + return finished.toVector(); +} + +function tableFromColumnValues(name, type, values, metadata) { + const builder = arrow.makeBuilder({type, nullValues: [null]}); + values.forEach(value => builder.append(value)); + const vector = finishArrowVector(builder); + const field = new arrow.Field(name, type, true, metadata); + return new arrow.Table(new arrow.Schema([field]), {[name]: vector}); +} + +function concatColumnTables(name, type, leftValues, rightValues, metadata) { + return tableFromColumnValues(name, type, leftValues, metadata).concat( + tableFromColumnValues(name, type, rightValues, metadata) + ); +} + +function jsArrowCell(value) { + if (value == null) { + return null; + } + if (ArrayBuffer.isView(value)) { + return Array.from(value); + } + if (typeof value.toArray === 'function' && value.numChildren === 0) { + return Array.from(value.toArray()); + } + if (typeof value.get === 'function' && typeof value.length === 'number') { + const out = []; + for (let i = 0; i < value.length; i++) { + const valid = typeof value.isValid === 'function' ? value.isValid(i) : true; + out.push(valid ? jsArrowCell(value.get(i)) : null); + } + return out; + } + return value; +} + +function columnCells(table) { + const column = table.getChildAt(0); + return Array.from({length: column.length}, (_, i) => { + if (typeof column.isValid === 'function' && !column.isValid(i)) { + return null; + } + return jsArrowCell(column.get(i)); + }); +} + +test('compactArrowTable -> concat does not merge chunks; Vector has no combineChunks', t => { + const batchA = arrow.tableFromJSON([{lng: -122.4}]); + const batchB = arrow.tableFromJSON([{lng: -122.5}]); + const combined = batchA.concat(batchB); + const column = combined.getChildAt(0); + + t.equal( + typeof column.combineChunks, + 'undefined', + 'apache-arrow JS Vector has concat, not combineChunks' + ); + t.ok(column.data.length > 1, 'Table.concat keeps one chunk per record batch'); + t.equal(typeof column.concat, 'function', 'Vector.concat exists but does not collapse chunks'); + + t.end(); +}); + +test('compactArrowTable -> dictionary, WKB, and nested GeoArrow keep values', t => { + const geoMeta = metadata => { + const map = new Map(); + map.set(GEOARROW_METADATA_KEY, metadata); + return map; + }; + + const dictType = new arrow.Dictionary(new arrow.Utf8(), new arrow.Int8()); + const dictTable = concatColumnTables('city', dictType, ['sf', null], ['nyc']); + const dictCompacted = compactArrowTable(dictTable, 1); + t.equal(dictCompacted.batches.length, 1, 'dictionary column should collapse to one batch'); + t.deepEqual( + columnCells(dictCompacted), + ['sf', null, 'nyc'], + 'dictionary values and nulls should survive the Builder rebuild' + ); + + const wkbPoint = (lng, lat) => { + const bytes = new Uint8Array(21); + const view = new DataView(bytes.buffer); + view.setUint8(0, 1); + view.setUint32(1, 1, true); + view.setFloat64(5, lng, true); + view.setFloat64(13, lat, true); + return bytes; + }; + const wkbTable = concatColumnTables( + 'geometry', + new arrow.Binary(), + [wkbPoint(-122.4, 37.8), null], + [wkbPoint(-122.5, 37.9)], + geoMeta(GEOARROW_EXTENSIONS.WKB) + ); + wkbTable.schema.metadata.set( + 'geo', + JSON.stringify({columns: {geometry: {encoding: 'WKB'}}}) + ); + const wkbCompacted = compactArrowTable(wkbTable, 1); + t.equal(wkbCompacted.batches.length, 1, 'WKB binary column should collapse to one batch'); + t.deepEqual( + columnCells(wkbCompacted), + columnCells(wkbTable), + 'WKB bytes and nulls should survive the Builder rebuild' + ); + t.equal( + wkbCompacted.schema.fields[0].metadata.get(GEOARROW_METADATA_KEY), + GEOARROW_EXTENSIONS.WKB, + 'geoarrow.wkb field metadata should be copied' + ); + t.ok(wkbCompacted.schema.metadata.get('geo'), 'GeoParquet schema metadata should be copied'); + + const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false)); + const pointTable = concatColumnTables( + 'geom', + pointType, + [[-122.4, 37.8], null], + [[-122.5, 37.9]], + geoMeta(GEOARROW_EXTENSIONS.POINT) + ); + const pointCompacted = compactArrowTable(pointTable, 1); + t.equal(pointCompacted.batches.length, 1, 'geoarrow.point should collapse to one batch'); + t.deepEqual( + columnCells(pointCompacted), + [[-122.4, 37.8], null, [-122.5, 37.9]], + 'geoarrow.point coordinates must not become NaN after Builder rebuild' + ); + + const lineType = new arrow.List(new arrow.Field('vertices', pointType, false)); + const lineTable = concatColumnTables( + 'geom', + lineType, + [[[-122.4, 37.8], [-122.5, 37.9]]], + [[[0, 0], [1, 1]]], + geoMeta(GEOARROW_EXTENSIONS.LINESTRING) + ); + const lineCompacted = compactArrowTable(lineTable, 1); + t.equal(lineCompacted.batches.length, 1, 'geoarrow.linestring should collapse to one batch'); + t.deepEqual( + columnCells(lineCompacted), + columnCells(lineTable), + 'nested geoarrow.linestring coordinates should survive the Builder rebuild' + ); + + const polygonType = new arrow.List(new arrow.Field('rings', lineType, false)); + const ring = [ + [0, 0], + [1, 0], + [1, 1], + [0, 0] + ]; + const polygonTable = concatColumnTables( + 'geom', + polygonType, + [[ring]], + [[ring]], + geoMeta(GEOARROW_EXTENSIONS.POLYGON) + ); + const polygonCompacted = compactArrowTable(polygonTable, 1); + t.equal(polygonCompacted.batches.length, 1, 'geoarrow.polygon should collapse to one batch'); + t.deepEqual( + columnCells(polygonCompacted), + columnCells(polygonTable), + 'nested geoarrow.polygon coordinates should survive the Builder rebuild' + ); + + t.end(); +}); + +test('compactArrowTable -> Builder failure keeps the original table', t => { + const batchA = arrow.tableFromJSON([{lng: -122.4}]); + const batchB = arrow.tableFromJSON([{lng: -122.5}]); + const combined = batchA.concat(batchB); + const exploding = { + batches: combined.batches, + numCols: combined.numCols, + schema: combined.schema, + getChildAt() { + throw new Error('cannot compact'); + } + }; + + const result = compactArrowTable(exploding, 1); + t.equal( + result, + exploding, + 'compactArrowTable should return the original table when the Builder path throws' + ); + + t.end(); +}); From d556813e5ca5ae6cdc6893c9965a9c258c84f84b Mon Sep 17 00:00:00 2001 From: Ihor Dykhta Date: Tue, 25 Aug 2026 18:08:25 +0300 Subject: [PATCH 09/10] follow up again Signed-off-by: Ihor Dykhta --- src/layers/src/geojson-layer/geojson-layer.ts | 7 ++- src/utils/src/arrow-data-container.ts | 11 +++- src/utils/src/filter-utils.ts | 10 ++- test/node/utils/data-container-test.js | 30 +++++++++ test/node/utils/filter-utils-test.js | 61 +++++++++++++++++++ 5 files changed, 114 insertions(+), 5 deletions(-) diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.ts index d51490c42f..c5f9c47058 100644 --- a/src/layers/src/geojson-layer/geojson-layer.ts +++ b/src/layers/src/geojson-layer/geojson-layer.ts @@ -625,10 +625,15 @@ export default class GeoJsonLayer extends Layer { // comparing length to numChunks would re-parse WKB on every meta update. const processedWholeColumn = this.dataToFeature.length > 0 && processedRows > 0 && processedRows >= numRows; + // Progressive loads keep one dataToFeature entry per chunk, then compact + // the finished table to one chunk. Row count already matches, so without + // this check we would skip the rebuild and keep excess Deck layers. + const compactedAfterProgressiveLoad = this.dataToFeature.length > numChunks; const needsUpdate = this.dataToFeature.length === 0 || (!processedWholeColumn && this.dataToFeature.length < numChunks) || - (processedRows > 0 && processedRows < numRows); + (processedRows > 0 && processedRows < numRows) || + compactedAfterProgressiveLoad; if (geoColumn && geoField && needsUpdate) { // Incremental only when a new chunk appeared. Compacted tables stay at 1 diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.ts index 2ae3a460f6..db439bc1d4 100644 --- a/src/utils/src/arrow-data-container.ts +++ b/src/utils/src/arrow-data-container.ts @@ -155,13 +155,18 @@ export function compactArrowTable(table: arrow.Table, maxArrowBatches?: number): try { const columns: Record = {}; + const seenNames = new Map(); for (let i = 0; i < table.numCols; i++) { const field = table.schema.fields[i]; const column = table.getChildAt(i); if (!column) { continue; } - columns[field.name] = compactArrowVector(column, field.type); + // Arrow schemas may repeat field names; object keys would drop earlier columns. + const seen = seenNames.get(field.name) ?? 0; + seenNames.set(field.name, seen + 1); + const key = seen === 0 ? field.name : `${field.name}_${seen}`; + columns[key] = compactArrowVector(column, field.type); } // Always build a kepler apache-arrow Table. Using `table.constructor` can @@ -193,8 +198,8 @@ function copyArrowSchemaMetadata(from?: arrow.Schema | null, to?: arrow.Schema | } const fromByName = new Map((from.fields || []).map(field => [field.name, field])); - (to.fields || []).forEach(toField => { - const fromField = fromByName.get(toField.name); + (to.fields || []).forEach((toField, index) => { + const fromField = from.fields?.[index] || fromByName.get(toField.name); const src = fromField?.metadata as Map | undefined; const dst = toField?.metadata as Map | undefined; if (!src || !dst || typeof src.forEach !== 'function' || typeof dst.set !== 'function') { diff --git a/src/utils/src/filter-utils.ts b/src/utils/src/filter-utils.ts index 2738d06c23..a3ab46edec 100644 --- a/src/utils/src/filter-utils.ts +++ b/src/utils/src/filter-utils.ts @@ -381,7 +381,14 @@ function hasFiniteLngLat(pos: unknown): pos is number[] { return false; } const values = pos as ArrayLike; - return values.length >= 2 && Number.isFinite(Number(values[0])) && Number.isFinite(Number(values[1])); + // Do not coerce with Number(): Number(null/''/false) is 0 and would pass. + return ( + values.length >= 2 && + typeof values[0] === 'number' && + Number.isFinite(values[0]) && + typeof values[1] === 'number' && + Number.isFinite(values[1]) + ); } export const getPolygonFilterFunctor = (layer, filter, dataContainer) => { @@ -427,6 +434,7 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => { const pos = getPosition(data); return ( hasFiniteLngLat(pos) && + hasFiniteLngLat([pos[3], pos[4]]) && [ [pos[0], pos[1]], [pos[3], pos[4]] diff --git a/test/node/utils/data-container-test.js b/test/node/utils/data-container-test.js index 5a9ea9d743..5b532f5d92 100644 --- a/test/node/utils/data-container-test.js +++ b/test/node/utils/data-container-test.js @@ -380,6 +380,36 @@ test('compactArrowTable -> dictionary, WKB, and nested GeoArrow keep values', t t.end(); }); +test('compactArrowTable -> duplicate field names keep both columns', t => { + const duplicateNameTable = (floatValues, intValues) => { + const schema = new arrow.Schema([ + new arrow.Field('x', new arrow.Float64()), + new arrow.Field('x', new arrow.Int32()) + ]); + const floats = arrow.vectorFromArray(floatValues, new arrow.Float64()); + const ints = arrow.vectorFromArray(intValues, new arrow.Int32()); + const data = arrow.makeData({ + type: new arrow.Struct(schema.fields), + children: [floats.data[0], ints.data[0]], + length: floatValues.length + }); + return new arrow.Table(new arrow.RecordBatch(schema, data)); + }; + + const combined = duplicateNameTable([1.5], [1]).concat(duplicateNameTable([2.5], [2])); + t.equal(combined.numCols, 2, 'fixture should expose both duplicate-named columns'); + t.ok(combined.batches.length > 1, 'fixture should have multiple record batches'); + + const compacted = compactArrowTable(combined, 1); + t.equal(compacted.numCols, 2, 'compaction should not drop a duplicate-named column'); + t.equal(compacted.numRows, 2, 'compaction should keep all rows'); + t.equal(compacted.batches.length, 1, 'duplicate-named columns should still collapse to one batch'); + t.deepEqual(Array.from(compacted.getChildAt(0).toArray()), [1.5, 2.5]); + t.deepEqual(Array.from(compacted.getChildAt(1).toArray()), [1, 2]); + + t.end(); +}); + test('compactArrowTable -> Builder failure keeps the original table', t => { const batchA = arrow.tableFromJSON([{lng: -122.4}]); const batchB = arrow.tableFromJSON([{lng: -122.5}]); diff --git a/test/node/utils/filter-utils-test.js b/test/node/utils/filter-utils-test.js index 752454eeee..41078920f6 100644 --- a/test/node/utils/filter-utils-test.js +++ b/test/node/utils/filter-utils-test.js @@ -854,6 +854,67 @@ test('filterUtils -> getPolygonFilterFunctor -> point layer ignores non-finite a true, 'typed-array lng/lat should be treated as a valid position' ); + t.equal( + fn({position: [null, 0.5, 0]}), + false, + 'null longitude should not coerce to 0 and pass the polygon filter' + ); + t.equal( + fn({position: ['', 0.5, 0]}), + false, + 'empty-string longitude should not coerce to 0 and pass the polygon filter' + ); + + t.end(); +}); + +test('filterUtils -> getPolygonFilterFunctor -> arc layer validates both endpoints', t => { + const squarePolygon = { + type: 'Feature', + properties: {}, + geometry: { + type: 'Polygon', + coordinates: [ + [ + [-1, -1], + [1, -1], + [1, 1], + [-1, 1], + [-1, -1] + ] + ] + } + }; + + const fn = getPolygonFilterFunctor( + { + type: 'arc', + getPositionAccessor: () => d => d.position + }, + {value: squarePolygon}, + null + ); + + t.equal( + fn({position: [0.5, 0.5, 0, 0.2, 0.2, 0]}), + true, + 'arc with both endpoints inside should pass' + ); + t.equal( + fn({position: [0.5, 0.5, null, 0.2, 0.2, undefined]}), + true, + 'null altitude should not exclude a valid arc' + ); + t.equal( + fn({position: [0.5, 0.5, 0, null, 0.2, 0]}), + false, + 'missing destination longitude should not reach turfPoint' + ); + t.equal( + fn({position: [0.5, 0.5, 0, 10, 10, 0]}), + false, + 'arc with destination outside should fail' + ); t.end(); }); From afb277d3b2c9a021c8c5f1498e322a763ec4ea05 Mon Sep 17 00:00:00 2001 From: Ihor Dykhta Date: Tue, 25 Aug 2026 18:23:15 +0300 Subject: [PATCH 10/10] fix tests Signed-off-by: Ihor Dykhta --- src/utils/src/arrow-data-container.ts | 2 +- test/node/utils/data-container-test.js | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.ts index db439bc1d4..698bf0e18a 100644 --- a/src/utils/src/arrow-data-container.ts +++ b/src/utils/src/arrow-data-container.ts @@ -166,7 +166,7 @@ export function compactArrowTable(table: arrow.Table, maxArrowBatches?: number): const seen = seenNames.get(field.name) ?? 0; seenNames.set(field.name, seen + 1); const key = seen === 0 ? field.name : `${field.name}_${seen}`; - columns[key] = compactArrowVector(column, field.type); + columns[key] = compactArrowVector(column, column.type || field.type); } // Always build a kepler apache-arrow Table. Using `table.constructor` can diff --git a/test/node/utils/data-container-test.js b/test/node/utils/data-container-test.js index 5b532f5d92..7095ef48e9 100644 --- a/test/node/utils/data-container-test.js +++ b/test/node/utils/data-container-test.js @@ -404,8 +404,16 @@ test('compactArrowTable -> duplicate field names keep both columns', t => { t.equal(compacted.numCols, 2, 'compaction should not drop a duplicate-named column'); t.equal(compacted.numRows, 2, 'compaction should keep all rows'); t.equal(compacted.batches.length, 1, 'duplicate-named columns should still collapse to one batch'); - t.deepEqual(Array.from(compacted.getChildAt(0).toArray()), [1.5, 2.5]); - t.deepEqual(Array.from(compacted.getChildAt(1).toArray()), [1, 2]); + t.deepEqual( + [compacted.getChildAt(0).get(0), compacted.getChildAt(0).get(1)], + [1.5, 2.5], + 'first duplicate-named column should keep its float values' + ); + t.deepEqual( + [compacted.getChildAt(1).get(0), compacted.getChildAt(1).get(1)], + [1, 2], + 'second duplicate-named column should keep its int values' + ); t.end(); });