diff --git a/src/common-utils/src/data-type.ts b/src/common-utils/src/data-type.ts index 4f94f36ffc..b9387d0854 100644 --- a/src/common-utils/src/data-type.ts +++ b/src/common-utils/src/data-type.ts @@ -6,7 +6,7 @@ import {ArrowTableInterface, ApacheVectorInterface, RowData, Field} from '@keple import {ALL_FIELD_TYPES} from '@kepler.gl/constants'; import {console as globalConsole} from 'global/window'; import {range} from 'd3-array'; -import {isHexWkb, notNullorUndefined} from './data'; +import {isHexWkb, isGeoJsonGeometryString, notNullorUndefined} from './data'; import {h3IsValid} from './h3-utils'; const H3_ANALYZER_TYPE = 'H3'; @@ -267,6 +267,14 @@ export function getFieldsFromData(data: RowData, fieldOrder: string[]): Field[] type = data.some(d => isHexWkb(d[name])) ? AnalyzerDATA_TYPES.GEOMETRY : type; } + // GeoJSON geometry/Feature JSON is classified as OBJECT (or STRING). + // Detect it so a `geometry` column exported from parquet/geoarrow re-imports. + if (type === AnalyzerDATA_TYPES.OBJECT || type === AnalyzerDATA_TYPES.STRING) { + type = data.some(d => isGeoJsonGeometryString(d[name])) + ? AnalyzerDATA_TYPES.GEOMETRY + : type; + } + return { name, id: name, diff --git a/src/common-utils/src/data.ts b/src/common-utils/src/data.ts index cade710c51..68f2b1917f 100644 --- a/src/common-utils/src/data.ts +++ b/src/common-utils/src/data.ts @@ -30,6 +30,44 @@ export function isHexWkb(str: string | null): boolean { return /^[0-9a-fA-F]+$/.test(str.slice(2)); } +const GEOJSON_GEOMETRY_TYPES = new Set([ + 'Point', + 'MultiPoint', + 'LineString', + 'MultiLineString', + 'Polygon', + 'MultiPolygon', + 'GeometryCollection', + 'Feature', + 'FeatureCollection' +]); + +/** + * Check if a string is a GeoJSON geometry or Feature (as stored in CSV). + * type-analyzer classifies `{...}` JSON as OBJECT, so a `geometry` column of + * GeoJSON strings would otherwise not be treated as geometry on re-import. + */ +export function isGeoJsonGeometryString(value: unknown): boolean { + if (typeof value !== 'string') { + return false; + } + const trimmed = value.trim(); + if (trimmed.charAt(0) !== '{') { + return false; + } + try { + const parsed = JSON.parse(trimmed); + return Boolean( + parsed && + typeof parsed === 'object' && + typeof parsed.type === 'string' && + GEOJSON_GEOMETRY_TYPES.has(parsed.type) + ); + } catch { + return false; + } +} + /** * Converts non-arrays to arrays. Leaves arrays alone. Converts * undefined values to empty arrays ([] instead of [undefined]). diff --git a/src/processors/src/data-processor.ts b/src/processors/src/data-processor.ts index 2759426b4d..fe62d2239f 100644 --- a/src/processors/src/data-processor.ts +++ b/src/processors/src/data-processor.ts @@ -501,7 +501,7 @@ export function arrowSchemaToFields( const keplerFields = getFieldsFromData(sample, headerRow); const geoArrowMetadata = getGeoArrowMetadataFromSchema(table); - return table.schema.fields.map((field: arrow.Field, fieldIndex: number) => { + const fields = table.schema.fields.map((field: arrow.Field, fieldIndex: number) => { let type = arrowDataTypeToFieldType(field.type); let analyzerType = arrowDataTypeToAnalyzerDataType(field.type); let format = ''; @@ -576,6 +576,57 @@ export function arrowSchemaToFields( metadata: field.metadata }; }); + + logAddedGeometryFields(fields); + return fields; +} + +function getArrowExtensionName(field: { + metadata?: Map | Record; +}): string | undefined { + const metadata = field.metadata; + if (!metadata) { + return undefined; + } + if (typeof (metadata as Map).get === 'function') { + return (metadata as Map).get(GEOARROW_METADATA_KEY); + } + return (metadata as Record)[GEOARROW_METADATA_KEY]; +} + +/** + * Debug helper: log how each geometry column was classified when Arrow/parquet is added. + * kind is one of: geoarrow (native), wkb, geojson-string, simple-arrow. + */ +function logAddedGeometryFields(fields: Field[]): void { + const summary = fields.map(field => { + const encoding = getArrowExtensionName(field); + let kind = 'simple-arrow'; + if (encoding === GEOARROW_EXTENSIONS.WKB || encoding === 'geoarrow.wkb') { + kind = 'wkb'; + } else if (typeof encoding === 'string' && encoding.startsWith('geoarrow')) { + kind = 'geoarrow'; + } else if (field.type === ALL_FIELD_TYPES.geoarrow) { + kind = encoding ? 'geoarrow' : 'geoarrow-unknown'; + } else if (field.type === ALL_FIELD_TYPES.geojson) { + kind = 'geojson-string'; + } + + return { + name: field.name, + keplerType: field.type, + analyzerType: (field as any).analyzerType, + encoding: encoding || null, + kind + }; + }); + + const geometryFields = summary.filter(f => f.kind !== 'simple-arrow'); + // eslint-disable-next-line no-console + console.log('[kepler.gl:geometry] added arrow fields', { + geometry: geometryFields.length ? geometryFields : 'none (simple arrow / no geometry column)', + all: summary + }); } const CAST_BIGINTS = false; diff --git a/src/utils/src/data-utils.ts b/src/utils/src/data-utils.ts index b2d7136788..7f7f6df170 100644 --- a/src/utils/src/data-utils.ts +++ b/src/utils/src/data-utils.ts @@ -5,10 +5,11 @@ import assert from 'assert'; import {format as d3Format} from 'd3-format'; import moment from 'moment-timezone'; -import {convertGeoArrowGeometryToGeoJSON} from '@loaders.gl/gis'; +import {convertGeoArrowGeometryToGeoJSON, convertGeometryToWKT} from '@loaders.gl/gis'; import { ALL_FIELD_TYPES, + GEOARROW_METADATA_KEY, TOOLTIP_FORMATS, TOOLTIP_FORMAT_TYPES, TOOLTIP_KEY, @@ -19,7 +20,6 @@ import {Field, Millisecond, ProtoDatasetField} from '@kepler.gl/types'; import {snapToMarks} from './plot'; import {isPlainObject} from './utils'; -import {isArrowVector} from './arrow-data-container'; export type FieldFormatter = (value: any, field?: ProtoDatasetField) => string; @@ -299,6 +299,80 @@ export function uint8ArrayToHex(data: Uint8Array): string { .join(''); } +function isByteArray(data: unknown): data is ArrayBufferView { + return ArrayBuffer.isView(data) && (data as ArrayBufferView).BYTES_PER_ELEMENT === 1; +} + +function getGeoArrowEncoding(field?: ProtoDatasetField): string | undefined { + const metadata = field?.metadata as Map | Record | undefined; + if (!metadata) { + return undefined; + } + if (typeof (metadata as Map).get === 'function') { + return (metadata as Map).get(GEOARROW_METADATA_KEY); + } + return (metadata as Record)[GEOARROW_METADATA_KEY]; +} + +function serializeGeoJsonGeometry(geometry: {type?: string} | null): string | null { + if (!geometry || typeof geometry !== 'object' || !geometry.type) { + return null; + } + try { + return convertGeometryToWKT(geometry); + } catch { + try { + return JSON.stringify(geometry); + } catch { + return null; + } + } +} + +/** + * Serialize a geoarrow cell so CSV / JSON export can be re-imported. + * Native geoarrow values (points, polygons, …) become WKT (or GeoJSON); + * WKB bytes become hex WKB. Always returns a string. + */ +export function formatGeoArrowValue(data: any, field?: ProtoDatasetField): string { + const encoding = getGeoArrowEncoding(field); + + if (encoding) { + try { + const geometry = convertGeoArrowGeometryToGeoJSON(data, encoding); + const serialized = serializeGeoJsonGeometry(geometry); + if (serialized) { + return serialized; + } + } catch { + // fall through to binary / string fallbacks + } + } + + if (isByteArray(data)) { + const bytes = + data instanceof Uint8Array + ? data + : new Uint8Array(data.buffer, data.byteOffset, data.byteLength); + return uint8ArrayToHex(bytes); + } + + if (typeof data === 'string') { + return data; + } + + try { + const json = JSON.stringify(data); + if (json && json !== '{}' && json !== '[]') { + return json; + } + } catch { + // ignore + } + + return data == null ? '' : String(data); +} + export const FIELD_DISPLAY_FORMAT: { [key: string]: FieldFormatter; } = { @@ -316,22 +390,7 @@ export const FIELD_DISPLAY_FORMAT: { : Array.isArray(d) ? `[${String(d)}]` : '', - [ALL_FIELD_TYPES.geoarrow]: (data, field) => { - if (isArrowVector(data)) { - try { - const encoding = field?.metadata?.get('ARROW:extension:name'); - if (encoding) { - const geometry = convertGeoArrowGeometryToGeoJSON(data, encoding); - return JSON.stringify(geometry); - } - } catch (error) { - // ignore for now - } - } else if (data instanceof Uint8Array) { - return uint8ArrayToHex(data); - } - return data; - }, + [ALL_FIELD_TYPES.geoarrow]: formatGeoArrowValue, [ALL_FIELD_TYPES.object]: (value: any) => { try { return JSON.stringify(value); diff --git a/test/node/utils/data-processor-test.js b/test/node/utils/data-processor-test.js index 476a9b8219..c26763a527 100644 --- a/test/node/utils/data-processor-test.js +++ b/test/node/utils/data-processor-test.js @@ -45,7 +45,7 @@ import { import {formatCsv} from '@kepler.gl/reducers'; -import {ALL_FIELD_TYPES} from '@kepler.gl/constants'; +import {ALL_FIELD_TYPES, GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY} from '@kepler.gl/constants'; import {cmpFields} from '../../helpers/comparison-utils'; test('Processor -> getFieldsFromData', t => { @@ -134,6 +134,21 @@ test('Processor -> getFieldsFromData', t => { t.end(); }); +test('Processor -> getFieldsFromData geojson in geometry column', t => { + const data = [ + {id: '1', geometry: '{"type":"Point","coordinates":[-122.4,37.8]}'}, + {id: '2', geometry: '{"type":"Point","coordinates":[-73.9,40.7]}'} + ]; + const fields = getFieldsFromData(data, ['id', 'geometry']); + t.equal(fields[0].type, 'string', 'id should be string'); + t.equal( + fields[1].type, + 'geojson', + 'GeoJSON JSON in a column named geometry should be detected as geojson' + ); + t.end(); +}); + test('Processor -> processCsvData', t => { t.throws(() => processCsvData(''), 'should throw if csv is empty'); @@ -1009,6 +1024,46 @@ test('Processor -> formatCsv', t => { t.end(); }); +test('Processor -> formatCsv geoarrow geometry roundtrip', t => { + // POINT(1 2) little-endian WKB + const wkb = Uint8Array.from(Buffer.from('0101000000000000000000f03f0000000000000040', 'hex')); + const fields = [ + { + name: 'geometry', + displayName: 'geometry', + type: ALL_FIELD_TYPES.geoarrow, + analyzerType: 'GEOMETRY', + metadata: new Map([[GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.WKB]]) + }, + { + name: 'id', + displayName: 'id', + type: ALL_FIELD_TYPES.integer, + analyzerType: 'INT' + } + ]; + const rows = [[wkb, 1]]; + const dataContainer = createDataContainer(rows, {fields}); + const csv = formatCsv(dataContainer, fields); + + t.equal(typeof csv, 'string', 'should produce a csv string'); + t.ok(csv.startsWith('geometry,id'), 'csv should include geometry header'); + + const reimported = processCsvData(csv); + t.equal( + reimported.fields[0].type, + 'geojson', + 're-imported geometry column should be detected as geojson' + ); + t.ok(reimported.rows[0][0], 're-imported geometry value should be present'); + t.notEqual( + String(reimported.rows[0][0]), + '[object Object]', + 're-imported geometry should not be a broken object string' + ); + t.end(); +}); + test('Processor -> analyzerTypeToFieldType', t => { Object.keys(DATA_TYPES).forEach(atype => { const spy = sinon.spy(Console, 'warn'); diff --git a/test/node/utils/data-utils-test.js b/test/node/utils/data-utils-test.js index 339b792f1d..c9631abc37 100644 --- a/test/node/utils/data-utils-test.js +++ b/test/node/utils/data-utils-test.js @@ -13,10 +13,12 @@ import { getFormatter, defaultFormatter, formatNumber, - roundToFour + roundToFour, + parseFieldValue, + formatGeoArrowValue } from '@kepler.gl/utils'; import {processLayerBounds} from '@kepler.gl/reducers'; -import {ALL_FIELD_TYPES} from '@kepler.gl/constants'; +import {ALL_FIELD_TYPES, GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY} from '@kepler.gl/constants'; test('dataUtils -> clamp', t => { t.equal(clamp([0, 1], 2), 1, 'should clamp 2 to 1 for [0,1]'); @@ -279,3 +281,28 @@ test('dataUtils -> validateBounds', t => { t.end(); }); + +test('dataUtils -> formatGeoArrowValue', t => { + const wkbField = { + name: 'geometry', + type: ALL_FIELD_TYPES.geoarrow, + metadata: new Map([[GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.WKB]]) + }; + const wkb = Uint8Array.from(Buffer.from('0101000000000000000000f03f0000000000000040', 'hex')); + const formatted = formatGeoArrowValue(wkb, wkbField); + t.equal(typeof formatted, 'string', 'geoarrow formatter should always return a string'); + t.ok( + formatted.startsWith('POINT') || formatted.startsWith('01') || formatted.startsWith('{'), + 'WKB cell should export as WKT, hex WKB, or GeoJSON' + ); + + const parsed = parseFieldValue(wkb, ALL_FIELD_TYPES.geoarrow, wkbField); + t.equal(parsed, formatted, 'parseFieldValue should use the geoarrow formatter'); + + const rawObject = formatGeoArrowValue( + {not: 'geometry'}, + {name: 'g', type: ALL_FIELD_TYPES.geoarrow} + ); + t.equal(typeof rawObject, 'string', 'non-geometry objects should still stringify'); + t.end(); +}); diff --git a/test/node/utils/dataset-utils-test.js b/test/node/utils/dataset-utils-test.js index fe44bb82a2..d331c32bad 100644 --- a/test/node/utils/dataset-utils-test.js +++ b/test/node/utils/dataset-utils-test.js @@ -3,7 +3,7 @@ import test from 'tape'; import {findDefaultColorField, createNewDataEntry} from '@kepler.gl/utils'; -import {isHexWkb} from '@kepler.gl/common-utils'; +import {isHexWkb, isGeoJsonGeometryString} from '@kepler.gl/common-utils'; import {processCsvData} from '@kepler.gl/processors'; import csvData from 'test/fixtures/test-layer-data'; @@ -88,4 +88,23 @@ test('datasetUtils.isHexWkb', t => { const validEWktNDR = '0020000001000013ff0000000000400000000000000040'; t.ok(isHexWkb(validEWktNDR), 'A valid hex ewkb in NDR should be valid'); + t.end(); +}); + +test('datasetUtils.isGeoJsonGeometryString', t => { + t.notOk(isGeoJsonGeometryString(''), 'empty string is not GeoJSON geometry'); + t.notOk(isGeoJsonGeometryString(null), 'null is not GeoJSON geometry'); + t.notOk(isGeoJsonGeometryString('{"name":"not geometry"}'), 'plain JSON object is not geometry'); + t.notOk(isGeoJsonGeometryString('POINT (1 2)'), 'WKT is not GeoJSON geometry JSON'); + t.ok( + isGeoJsonGeometryString('{"type":"Point","coordinates":[-122.4,37.8]}'), + 'GeoJSON Point string should be detected' + ); + t.ok( + isGeoJsonGeometryString( + '{"type":"Feature","properties":{},"geometry":{"type":"Point","coordinates":[0,0]}}' + ), + 'GeoJSON Feature string should be detected' + ); + t.end(); });