From 0bb77f4e31dd160c59a5e939670e82a570eb9ad7 Mon Sep 17 00:00:00 2001 From: Ib Green Date: Tue, 5 Nov 2024 21:13:50 -0500 Subject: [PATCH] feat: Arrow loaders for Pointcloud / Mesh formats (#3158) --- .../arrow/src/{ => exports}/arrow-format.ts | 0 modules/arrow/src/index.ts | 2 +- modules/csv/src/csv-format.ts | 15 ++++ modules/csv/src/csv-loader.ts | 10 +-- modules/csv/src/csv-writer.ts | 7 +- modules/draco/src/draco-arrow-loader.ts | 25 ++++++ modules/draco/src/index.ts | 1 + modules/draco/test/draco-arrow-loader.spec.ts | 82 +++++++++++++++++++ modules/draco/test/index.ts | 1 + modules/flatgeobuf/src/flatgeobuf-format.ts | 15 ++++ modules/flatgeobuf/src/flatgeobuf-loader.ts | 10 +-- modules/flatgeobuf/src/floatgeobuf-source.ts | 7 +- modules/flatgeobuf/src/index.ts | 1 + modules/geopackage/src/geopackage-format.ts | 14 ++++ modules/geopackage/src/geopackage-loader.ts | 9 +- modules/geopackage/src/index.ts | 1 + modules/las/src/las-arrow-loader.ts | 4 +- modules/las/src/lib/parse-las.ts | 4 +- modules/obj/src/lib/parse-obj.ts | 2 +- modules/obj/src/obj-arrow-loader.ts | 26 ++++++ modules/pcd/src/pcd-arrow-loader.ts | 4 +- modules/pcd/test/pcd-arrow-loader.spec.ts | 2 +- modules/ply/src/ply-arrow-loader.ts | 25 ++++++ modules/schema-utils/src/index.ts | 3 +- .../arrow-fixed-size-list-utils.ts | 24 +++++- ...rrow-table.ts => convert-mesh-to-table.ts} | 45 +++++++++- .../schema-utils/src/lib/mesh/convert-mesh.ts | 47 ----------- .../src/lib/mesh/convert-table-to-mesh.ts | 49 +++++++++++ .../helpers/geometry-converter.ts | 4 +- 29 files changed, 344 insertions(+), 95 deletions(-) rename modules/arrow/src/{ => exports}/arrow-format.ts (100%) create mode 100644 modules/csv/src/csv-format.ts create mode 100644 modules/draco/src/draco-arrow-loader.ts create mode 100644 modules/draco/test/draco-arrow-loader.spec.ts create mode 100644 modules/flatgeobuf/src/flatgeobuf-format.ts create mode 100644 modules/geopackage/src/geopackage-format.ts create mode 100644 modules/obj/src/obj-arrow-loader.ts create mode 100644 modules/ply/src/ply-arrow-loader.ts rename modules/schema-utils/src/lib/mesh/{convert-mesh-to-arrow-table.ts => convert-mesh-to-table.ts} (50%) delete mode 100644 modules/schema-utils/src/lib/mesh/convert-mesh.ts create mode 100644 modules/schema-utils/src/lib/mesh/convert-table-to-mesh.ts diff --git a/modules/arrow/src/arrow-format.ts b/modules/arrow/src/exports/arrow-format.ts similarity index 100% rename from modules/arrow/src/arrow-format.ts rename to modules/arrow/src/exports/arrow-format.ts diff --git a/modules/arrow/src/index.ts b/modules/arrow/src/index.ts index 53a32806c8..ffd6f02892 100644 --- a/modules/arrow/src/index.ts +++ b/modules/arrow/src/index.ts @@ -7,7 +7,7 @@ export {VECTOR_TYPES} from './lib/types'; // Arrow loader / Writer -export {ArrowFormat} from './arrow-format'; +export {ArrowFormat} from './exports/arrow-format'; export type {ArrowLoaderOptions} from './exports/arrow-loader'; export {ArrowWorkerLoader} from './exports/arrow-loader'; diff --git a/modules/csv/src/csv-format.ts b/modules/csv/src/csv-format.ts new file mode 100644 index 0000000000..9050c6d3f0 --- /dev/null +++ b/modules/csv/src/csv-format.ts @@ -0,0 +1,15 @@ +// loaders.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import type {Format} from '@loaders.gl/loader-utils'; + +/** Comma-Separated Values */ +export const CSVFormat = { + id: 'csv', + module: 'csv', + name: 'CSV', + extensions: ['csv', 'tsv', 'dsv'], + mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'], + category: 'table' +} as const satisfies Format; diff --git a/modules/csv/src/csv-loader.ts b/modules/csv/src/csv-loader.ts index 7329b740c0..47ea5dae02 100644 --- a/modules/csv/src/csv-loader.ts +++ b/modules/csv/src/csv-loader.ts @@ -15,6 +15,7 @@ import { } from '@loaders.gl/schema-utils'; import Papa from './papaparse/papaparse'; import AsyncIteratorStreamer from './papaparse/async-iterator-streamer'; +import {CSVFormat} from './csv-format'; // __VERSION__ is injected by babel-plugin-version-inline // @ts-ignore TS2304: Cannot find name '__VERSION__'. @@ -47,16 +48,11 @@ export type CSVLoaderOptions = LoaderOptions & { }; export const CSVLoader = { + ...CSVFormat, + dataType: null as unknown as ObjectRowTable | ArrayRowTable, batchType: null as unknown as TableBatch, - - id: 'csv', - module: 'csv', - name: 'CSV', version: VERSION, - extensions: ['csv', 'tsv', 'dsv'], - mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'], - category: 'table', parse: async (arrayBuffer: ArrayBuffer, options?: CSVLoaderOptions) => parseCSV(new TextDecoder().decode(arrayBuffer), options), parseText: (text: string, options?: CSVLoaderOptions) => parseCSV(text, options), diff --git a/modules/csv/src/csv-writer.ts b/modules/csv/src/csv-writer.ts index 8a9b17ffef..5cef59a927 100644 --- a/modules/csv/src/csv-writer.ts +++ b/modules/csv/src/csv-writer.ts @@ -6,6 +6,7 @@ import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils'; import type {Table, TableBatch} from '@loaders.gl/schema'; import {encodeTableAsCSV} from './lib/encoders/encode-csv'; +import {CSVFormat} from './csv-format'; export type CSVWriterOptions = WriterOptions & { csv?: { @@ -14,12 +15,8 @@ export type CSVWriterOptions = WriterOptions & { }; export const CSVWriter = { - id: 'csv', + ...CSVFormat, version: 'latest', - module: 'csv', - name: 'CSV', - extensions: ['csv'], - mimeTypes: ['text/csv'], options: { csv: { useDisplayNames: false diff --git a/modules/draco/src/draco-arrow-loader.ts b/modules/draco/src/draco-arrow-loader.ts new file mode 100644 index 0000000000..b07225f859 --- /dev/null +++ b/modules/draco/src/draco-arrow-loader.ts @@ -0,0 +1,25 @@ +// loaders.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import type {ArrowTable} from '@loaders.gl/schema'; +import type {LoaderWithParser} from '@loaders.gl/loader-utils'; +import type {DracoLoaderOptions} from './draco-loader'; +import {DracoLoader} from './draco-loader'; +import {convertMeshToTable} from '@loaders.gl/schema-utils'; + +/** + * Loader for Draco3D compressed geometries + */ +export const DracoArrowLoader = { + ...DracoLoader, + dataType: null as unknown as ArrowTable, + worker: false, + parse +} as const satisfies LoaderWithParser; + +async function parse(arrayBuffer: ArrayBuffer, options?: DracoLoaderOptions): Promise { + const mesh = await DracoLoader.parse(arrayBuffer, options); + const arrowTable = convertMeshToTable(mesh, 'arrow-table'); + return arrowTable; +} diff --git a/modules/draco/src/index.ts b/modules/draco/src/index.ts index d143cfa66b..56529ed5e5 100644 --- a/modules/draco/src/index.ts +++ b/modules/draco/src/index.ts @@ -18,3 +18,4 @@ export {DracoWriterWorker, DracoWriter} from './draco-writer'; export type {DracoLoaderOptions} from './draco-loader'; export {DracoWorkerLoader, DracoLoader} from './draco-loader'; +export {DracoArrowLoader} from './draco-arrow-loader'; diff --git a/modules/draco/test/draco-arrow-loader.spec.ts b/modules/draco/test/draco-arrow-loader.spec.ts new file mode 100644 index 0000000000..2323574cea --- /dev/null +++ b/modules/draco/test/draco-arrow-loader.spec.ts @@ -0,0 +1,82 @@ +/* eslint-disable max-len */ +import test from 'tape-promise/tape'; +import {validateLoader} from 'test/common/conformance'; + +import {DracoArrowLoader} from '@loaders.gl/draco'; +import {setLoaderOptions, load} from '@loaders.gl/core'; +import draco3d from 'draco3d'; + +const BUNNY_DRC_URL = '@loaders.gl/draco/test/data/bunny.drc'; +const CESIUM_TILE_URL = '@loaders.gl/draco/test/data/cesium-tile.drc'; + +setLoaderOptions({ + _workerType: 'test' +}); + +test('DracoArrowLoader#loader conformance', (t) => { + validateLoader(t, DracoArrowLoader, 'DracoArrowLoader'); + t.end(); +}); + +test('DracoArrowLoader#parse(mainthread)', async (t) => { + const table = await load(BUNNY_DRC_URL, DracoArrowLoader, {worker: false}); + // validateMeshCategoryData(t, data); + const {data} = table; + t.equal(data.numRows, 104502 / 3, 'number of rows is correct'); + const positions = data.getChild('POSITION')!; + t.ok(positions, 'POSITION attribute was found'); + t.ok(data.schema, 'Has arrow-like schema'); + t.end(); +}); + +test('DracoArrowLoader#draco3d npm package', async (t) => { + const table = await load(BUNNY_DRC_URL, DracoArrowLoader, { + worker: false, + modules: { + draco3d + } + }); + const {data} = table; + // validateMeshCategoryData(t, data); + t.ok(data.getChild('POSITION'), 'POSITION attribute was found'); + t.end(); +}); + +test('DracoArrowLoader#parse custom attributes(mainthread)', async (t) => { + let table = await load(CESIUM_TILE_URL, DracoArrowLoader, { + worker: false + }); + const {data} = table; + t.equal( + data.getChild('CUSTOM_ATTRIBUTE_2')?.data[0].length, + 173210, + 'Custom (Intensity) attribute was found' + ); + t.equal( + data.getChild('CUSTOM_ATTRIBUTE_3')?.data[0].length, + 173210, + 'Custom (Classification) attribute was found' + ); + + table = await load(CESIUM_TILE_URL, DracoArrowLoader, { + worker: false, + draco: { + extraAttributes: { + Intensity: 2, + Classification: 3 + } + } + }); + t.equal( + table.data.getChild('Intensity')?.data[0].length, + 173210, + 'Intensity attribute was found' + ); + t.equal( + table.data.getChild('Classification')?.data[0].length, + 173210, + 'Classification attribute was found' + ); + + t.end(); +}); diff --git a/modules/draco/test/index.ts b/modules/draco/test/index.ts index 71db1baac7..4190af6ed2 100644 --- a/modules/draco/test/index.ts +++ b/modules/draco/test/index.ts @@ -5,5 +5,6 @@ import './lib/utils/get-draco-schema.spec'; import './draco-loader.spec'; +import './draco-arrow-loader.spec'; import './draco-writer.spec'; import './draco-compression-ratio.spec'; diff --git a/modules/flatgeobuf/src/flatgeobuf-format.ts b/modules/flatgeobuf/src/flatgeobuf-format.ts new file mode 100644 index 0000000000..4712eda29b --- /dev/null +++ b/modules/flatgeobuf/src/flatgeobuf-format.ts @@ -0,0 +1,15 @@ +// loaders.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import type {Format} from '@loaders.gl/loader-utils'; + +/** flatgeobuf format */ +export const FlatGeobufFormat = { + id: 'flatgeobuf', + name: 'FlatGeobuf', + module: 'flatgeobuf', + extensions: ['fgb'], + mimeTypes: ['application/octet-stream'], + category: 'geometry' +} as const satisfies Format; diff --git a/modules/flatgeobuf/src/flatgeobuf-loader.ts b/modules/flatgeobuf/src/flatgeobuf-loader.ts index ca5f1c16da..fafe078901 100644 --- a/modules/flatgeobuf/src/flatgeobuf-loader.ts +++ b/modules/flatgeobuf/src/flatgeobuf-loader.ts @@ -9,6 +9,7 @@ import { parseFlatGeobufInBatches, ParseFlatGeobufOptions } from './lib/parse-flatgeobuf'; +import {FlatGeobufFormat} from './flatgeobuf-format'; // __VERSION__ is injected by babel-plugin-version-inline // @ts-ignore TS2304: Cannot find name '__VERSION__'. @@ -32,17 +33,12 @@ export type FlatGeobufLoaderOptions = LoaderOptions & { /** Load flatgeobuf on a worker */ export const FlatGeobufWorkerLoader = { + ...FlatGeobufFormat, + dataType: null as any, batchType: null as any, - - id: 'flatgeobuf', - name: 'FlatGeobuf', - module: 'flatgeobuf', version: VERSION, worker: true, - extensions: ['fgb'], - mimeTypes: ['application/octet-stream'], - category: 'geometry', tests: [new Uint8Array(FGB_MAGIC_NUMBER).buffer], options: { flatgeobuf: { diff --git a/modules/flatgeobuf/src/floatgeobuf-source.ts b/modules/flatgeobuf/src/floatgeobuf-source.ts index af031b7a3c..4a9ced9cb4 100644 --- a/modules/flatgeobuf/src/floatgeobuf-source.ts +++ b/modules/flatgeobuf/src/floatgeobuf-source.ts @@ -11,6 +11,7 @@ import type { import {Source, DataSource, VectorSource} from '@loaders.gl/loader-utils'; import {FlatGeobufLoader} from './flatgeobuf-loader'; +import {FlatGeobufFormat} from './flatgeobuf-format'; export type FlatGeobuSourceOptions = DataSourceOptions & { flatgeobuf?: {}; @@ -22,12 +23,8 @@ export type FlatGeobuSourceOptions = DataSourceOptions & { * @ndeprecated This is a WIP, not fully implemented */ export const FlatGeobufSource = { - name: 'FlatGeobuf', - id: 'flatgeobuf', - module: 'wms', + ...FlatGeobufFormat, version: '0.0.0', - extensions: [], - mimeTypes: [], type: 'flatgeobuf-server', fromUrl: true, fromBlob: false, // TODO check if supported by library? diff --git a/modules/flatgeobuf/src/index.ts b/modules/flatgeobuf/src/index.ts index dc39082b2e..f6817cc4b9 100644 --- a/modules/flatgeobuf/src/index.ts +++ b/modules/flatgeobuf/src/index.ts @@ -2,5 +2,6 @@ // SPDX-License-Identifier: MIT // Copyright (c) vis.gl contributors +export {FlatGeobufFormat} from './flatgeobuf-format'; export type {FlatGeobufLoaderOptions} from './flatgeobuf-loader'; export {FlatGeobufLoader, FlatGeobufWorkerLoader} from './flatgeobuf-loader'; diff --git a/modules/geopackage/src/geopackage-format.ts b/modules/geopackage/src/geopackage-format.ts new file mode 100644 index 0000000000..96c2298e32 --- /dev/null +++ b/modules/geopackage/src/geopackage-format.ts @@ -0,0 +1,14 @@ +// loaders.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import type {Format} from '@loaders.gl/loader-utils'; + +export const GeoPackageFormat = { + id: 'geopackage', + name: 'GeoPackage', + module: 'geopackage', + extensions: ['gpkg'], + mimeTypes: ['application/geopackage+sqlite3'], + category: 'geometry' +} as const satisfies Format; diff --git a/modules/geopackage/src/geopackage-loader.ts b/modules/geopackage/src/geopackage-loader.ts index b488114b5e..0723c89b3c 100644 --- a/modules/geopackage/src/geopackage-loader.ts +++ b/modules/geopackage/src/geopackage-loader.ts @@ -5,6 +5,7 @@ import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils'; import {Tables, GeoJSONTable} from '@loaders.gl/schema'; import {parseGeoPackage, DEFAULT_SQLJS_CDN} from './lib/parse-geopackage'; +import {GeoPackageFormat} from './geopackage-format'; // __VERSION__ is injected by babel-plugin-version-inline // @ts-ignore TS2304: Cannot find name '__VERSION__'. @@ -30,16 +31,12 @@ export type GeoPackageLoaderOptions = LoaderOptions & { }; export const GeoPackageLoader = { + ...GeoPackageFormat, + dataType: null as unknown as GeoJSONTable | Tables, batchType: null as never, - id: 'geopackage', - name: 'GeoPackage', - module: 'geopackage', version: VERSION, - extensions: ['gpkg'], - mimeTypes: ['application/geopackage+sqlite3'], - category: 'geometry', parse: parseGeoPackage, options: { geopackage: { diff --git a/modules/geopackage/src/index.ts b/modules/geopackage/src/index.ts index a5ef4d4cb2..686f8bdd0c 100644 --- a/modules/geopackage/src/index.ts +++ b/modules/geopackage/src/index.ts @@ -2,5 +2,6 @@ // SPDX-License-Identifier: MIT // Copyright (c) vis.gl contributors +export {GeoPackageFormat} from './geopackage-format'; export type {GeoPackageLoaderOptions} from './geopackage-loader'; export {GeoPackageLoader} from './geopackage-loader'; diff --git a/modules/las/src/las-arrow-loader.ts b/modules/las/src/las-arrow-loader.ts index 9f0192fc28..cf69d16214 100644 --- a/modules/las/src/las-arrow-loader.ts +++ b/modules/las/src/las-arrow-loader.ts @@ -6,7 +6,7 @@ import type {LoaderWithParser} from '@loaders.gl/loader-utils'; import type {ArrowTable} from '@loaders.gl/schema'; import {LASLoaderOptions, LASWorkerLoader} from './las-loader'; -import {convertMesh} from '@loaders.gl/schema-utils'; +import {convertMeshToTable} from '@loaders.gl/schema-utils'; import {parseLAS} from './lib/parse-las'; /** @@ -19,7 +19,7 @@ export const LASArrowLoader = { worker: false, parse: async (arrayBuffer: ArrayBuffer) => { const mesh = parseLAS(arrayBuffer); - const arrowTable = convertMesh(mesh, 'arrow-table'); + const arrowTable = convertMeshToTable(mesh, 'arrow-table'); return arrowTable; } } as const satisfies LoaderWithParser; diff --git a/modules/las/src/lib/parse-las.ts b/modules/las/src/lib/parse-las.ts index 6b197b24f8..e5376c1f6e 100644 --- a/modules/las/src/lib/parse-las.ts +++ b/modules/las/src/lib/parse-las.ts @@ -6,7 +6,7 @@ // import type {ArrowTable, ColumnarTable} from '@loaders.gl/schema'; import type {LASLoaderOptions} from '../las-loader'; import type {LASMesh, LASHeader} from './las-types'; -import {getMeshBoundingBox /* , convertMesh */} from '@loaders.gl/schema-utils'; +import {getMeshBoundingBox /* , convertMeshToTable */} from '@loaders.gl/schema-utils'; import {LASFile} from './laslaz-decoder'; import {getLASSchema} from './get-las-schema'; @@ -28,7 +28,7 @@ export function parseLAS(arrayBuffer: ArrayBuffer, options?: LASLoaderOptions): return parseLASMesh(arrayBuffer, options); // This code breaks pointcloud example on the website // const mesh = parseLASMesh(arrayBuffer, options); - // return convertMesh(mesh, options?.las?.shape || 'mesh') as LASMesh | ArrowTable | ColumnarTable; + // return convertMeshToTable(mesh, options?.las?.shape || 'mesh') as LASMesh | ArrowTable | ColumnarTable; } /** diff --git a/modules/obj/src/lib/parse-obj.ts b/modules/obj/src/lib/parse-obj.ts index a39f872e04..18133c01a7 100644 --- a/modules/obj/src/lib/parse-obj.ts +++ b/modules/obj/src/lib/parse-obj.ts @@ -3,7 +3,7 @@ import {getMeshBoundingBox} from '@loaders.gl/schema-utils'; import {parseOBJMeshes} from './parse-obj-meshes'; import {getOBJSchema} from './get-obj-schema'; -export function parseOBJ(text, options): Mesh { +export function parseOBJ(text: string, options?): Mesh { const {meshes} = parseOBJMeshes(text); // @ts-expect-error diff --git a/modules/obj/src/obj-arrow-loader.ts b/modules/obj/src/obj-arrow-loader.ts new file mode 100644 index 0000000000..bd6852bf95 --- /dev/null +++ b/modules/obj/src/obj-arrow-loader.ts @@ -0,0 +1,26 @@ +// loaders.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import type {LoaderWithParser} from '@loaders.gl/loader-utils'; +import type {ArrowTable} from '@loaders.gl/schema'; + +import {OBJLoaderOptions, OBJWorkerLoader} from './obj-loader'; +import {convertMeshToTable} from '@loaders.gl/schema-utils'; +import {parseOBJ} from './lib/parse-obj'; + +/** + * Worker loader for OBJ - Point Cloud Data + */ +export const OBJArrowLoader = { + ...OBJWorkerLoader, + dataType: null as unknown as ArrowTable, + batchType: null as never, + worker: false, + parse: async (arrayBuffer: ArrayBuffer) => { + const text = new TextDecoder().decode(arrayBuffer); + const mesh = parseOBJ(text); + const arrowTable = convertMeshToTable(mesh, 'arrow-table'); + return arrowTable; + } +} as const satisfies LoaderWithParser; diff --git a/modules/pcd/src/pcd-arrow-loader.ts b/modules/pcd/src/pcd-arrow-loader.ts index ed2254e19a..5bbf482da4 100644 --- a/modules/pcd/src/pcd-arrow-loader.ts +++ b/modules/pcd/src/pcd-arrow-loader.ts @@ -6,7 +6,7 @@ import type {LoaderWithParser} from '@loaders.gl/loader-utils'; import type {ArrowTable} from '@loaders.gl/schema'; import {PCDLoaderOptions, PCDWorkerLoader} from './pcd-loader'; -import {convertMesh} from '@loaders.gl/schema-utils'; +import {convertMeshToTable} from '@loaders.gl/schema-utils'; import {parsePCD} from './lib/parse-pcd'; /** @@ -19,7 +19,7 @@ export const PCDArrowLoader = { worker: false, parse: async (arrayBuffer: ArrayBuffer) => { const mesh = parsePCD(arrayBuffer); - const arrowTable = convertMesh(mesh, 'arrow-table'); + const arrowTable = convertMeshToTable(mesh, 'arrow-table'); return arrowTable; } } as const satisfies LoaderWithParser; diff --git a/modules/pcd/test/pcd-arrow-loader.spec.ts b/modules/pcd/test/pcd-arrow-loader.spec.ts index 91ad2391a8..b4954b63b4 100644 --- a/modules/pcd/test/pcd-arrow-loader.spec.ts +++ b/modules/pcd/test/pcd-arrow-loader.spec.ts @@ -21,7 +21,7 @@ test('PCDArrowLoader#loader conformance', (t) => { t.end(); }); -test.skip('PCDArrowLoader#parse(text)', async (t) => { +test('PCDArrowLoader#parse(text)', async (t) => { const arrowTable = await parse(fetchFile(PCD_ASCII_URL), PCDArrowLoader); // TODO - validate arrow mesh category data? diff --git a/modules/ply/src/ply-arrow-loader.ts b/modules/ply/src/ply-arrow-loader.ts new file mode 100644 index 0000000000..0bf7cb4f15 --- /dev/null +++ b/modules/ply/src/ply-arrow-loader.ts @@ -0,0 +1,25 @@ +// loaders.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import type {LoaderWithParser} from '@loaders.gl/loader-utils'; +import type {ArrowTable} from '@loaders.gl/schema'; + +import {PLYLoaderOptions, PLYWorkerLoader} from './ply-loader'; +import {convertMeshToTable} from '@loaders.gl/schema-utils'; +import {parsePLY} from './lib/parse-ply'; + +/** + * Worker loader for PLY - + */ +export const PLYArrowLoader = { + ...PLYWorkerLoader, + dataType: null as unknown as ArrowTable, + batchType: null as never, + worker: false, + parse: async (arrayBuffer: ArrayBuffer) => { + const mesh = parsePLY(arrayBuffer); + const arrowTable = convertMeshToTable(mesh, 'arrow-table'); + return arrowTable; + } +} as const satisfies LoaderWithParser; diff --git a/modules/schema-utils/src/index.ts b/modules/schema-utils/src/index.ts index de7ea564a1..738da7c839 100644 --- a/modules/schema-utils/src/index.ts +++ b/modules/schema-utils/src/index.ts @@ -77,7 +77,8 @@ export {ArrowLikeTable} from './lib/table/arrow-api/arrow-like-table'; // MESH CATEGORY export {getMeshSize, getMeshBoundingBox} from './lib/mesh/mesh-utils'; -export {convertMesh} from './lib/mesh/convert-mesh'; +export {convertMeshToTable} from './lib/mesh/convert-mesh-to-table'; +export {convertTableToMesh} from './lib/mesh/convert-table-to-mesh'; export { deduceMeshSchema, deduceMeshField, diff --git a/modules/schema-utils/src/lib/arrow-utils/arrow-fixed-size-list-utils.ts b/modules/schema-utils/src/lib/arrow-utils/arrow-fixed-size-list-utils.ts index e47c36284e..f85dc87323 100644 --- a/modules/schema-utils/src/lib/arrow-utils/arrow-fixed-size-list-utils.ts +++ b/modules/schema-utils/src/lib/arrow-utils/arrow-fixed-size-list-utils.ts @@ -30,9 +30,27 @@ export function getFixedSizeListData( stride: number ): arrow.Data { const listType = getFixedSizeListType(typedArray, stride); - const data = new arrow.Data(listType, 0, typedArray.length / stride, 0, [ - typedArray - ]); + const nestedType = listType.children[0].type; + const buffers: Partial> = { + // valueOffsets: undefined, + [arrow.BufferType.DATA]: typedArray // values + // nullBitmap: undefined, + // typeIds: undefined + }; + + // Note: The contiguous array of data is held by the nested "primitive type" column + const nestedData = new arrow.Data(nestedType, 0, typedArray.length, 0, buffers); + + // Wrapped in a FixedSizeList column that provides a "strided" view of the data + const data = new arrow.Data( + listType, + 0, + typedArray.length / stride, + 0, + undefined, + [nestedData] + ); + return data; } diff --git a/modules/schema-utils/src/lib/mesh/convert-mesh-to-arrow-table.ts b/modules/schema-utils/src/lib/mesh/convert-mesh-to-table.ts similarity index 50% rename from modules/schema-utils/src/lib/mesh/convert-mesh-to-arrow-table.ts rename to modules/schema-utils/src/lib/mesh/convert-mesh-to-table.ts index 0270a7574a..d62120d286 100644 --- a/modules/schema-utils/src/lib/mesh/convert-mesh-to-arrow-table.ts +++ b/modules/schema-utils/src/lib/mesh/convert-mesh-to-table.ts @@ -2,11 +2,49 @@ // SPDX-License-Identifier: MIT // Copyright (c) vis.gl contributors +import type {Mesh, ArrowTable, ColumnarTable} from '@loaders.gl/schema'; import * as arrow from 'apache-arrow'; import {getFixedSizeListData} from '../arrow-utils/arrow-fixed-size-list-utils'; -import type {Mesh, ArrowTable} from '@loaders.gl/schema'; import {deserializeArrowSchema} from '../schema/convert-arrow-schema'; -// import {makeMeshAttributeMetadata} from './deduce-mesh-schema'; + +export function convertMeshToTable(mesh: Mesh, shape: 'columnar-table'): ColumnarTable; +export function convertMeshToTable(mesh: Mesh, shape: 'arrow-table'): ArrowTable; + +/** + * Convert a mesh to a specific shape + */ +export function convertMeshToTable( + mesh: Mesh, + shape: 'columnar-table' | 'arrow-table' +): Mesh | ColumnarTable | ArrowTable { + switch (shape) { + case 'columnar-table': + return convertMeshToColumnarTable(mesh); + case 'arrow-table': + return convertMeshToArrowTable(mesh); + default: + throw new Error(shape); + } +} + +/** + * Convert a loaders.gl Mesh to a Columnar Table + * @param mesh + * @returns + */ +export function convertMeshToColumnarTable(mesh: Mesh): ColumnarTable { + const columns = {}; + + for (const [columnName, attribute] of Object.entries(mesh.attributes)) { + columns[columnName] = attribute.value; + } + + return { + shape: 'columnar-table', + schema: mesh.schema, + data: columns + }; +} /** * * Convert a loaders.gl Mesh to an Apache Arrow Table @@ -30,9 +68,10 @@ export function convertMeshToArrowTable(mesh: Mesh, batchSize?: number): ArrowTa } const structField = new arrow.Struct(arrowSchema.fields); + const length = arrowDatas[0].length; const structData = new arrow.Data(structField, 0, length, 0, undefined, arrowDatas); const recordBatch = new arrow.RecordBatch(arrowSchema, structData); - const table = new arrow.Table(arrowSchema, recordBatch); + const table = new arrow.Table([recordBatch]); return {shape: 'arrow-table', schema, data: table}; } diff --git a/modules/schema-utils/src/lib/mesh/convert-mesh.ts b/modules/schema-utils/src/lib/mesh/convert-mesh.ts deleted file mode 100644 index c3529fd86e..0000000000 --- a/modules/schema-utils/src/lib/mesh/convert-mesh.ts +++ /dev/null @@ -1,47 +0,0 @@ -// loaders.gl -// SPDX-License-Identifier: MIT -// Copyright (c) vis.gl contributors - -import type {Mesh, ColumnarTable, ArrowTable} from '@loaders.gl/schema'; -import {convertMeshToArrowTable} from './convert-mesh-to-arrow-table'; - -type TargetShape = 'mesh' | 'columnar-table' | 'arrow-table'; - -export function convertMesh(mesh: Mesh, shape: 'mesh'): Mesh; -export function convertMesh(mesh: Mesh, shape: 'columnar-table'): ColumnarTable; -export function convertMesh(mesh: Mesh, shape: 'arrow-table'): ArrowTable; - -/** - * Convert a mesh to a specific shape - */ -export function convertMesh(mesh: Mesh, shape: TargetShape): Mesh | ColumnarTable | ArrowTable { - switch (shape || 'mesh') { - case 'mesh': - return mesh; - case 'columnar-table': - return convertMeshToColumnarTable(mesh); - case 'arrow-table': - return convertMeshToArrowTable(mesh); - default: - throw new Error(`Unsupported shape ${shape}`); - } -} - -/** - * Convert a loaders.gl Mesh to a Columnar Table - * @param mesh - * @returns - */ -export function convertMeshToColumnarTable(mesh: Mesh): ColumnarTable { - const columns = {}; - - for (const [columnName, attribute] of Object.entries(mesh.attributes)) { - columns[columnName] = attribute.value; - } - - return { - shape: 'columnar-table', - schema: mesh.schema, - data: columns - }; -} diff --git a/modules/schema-utils/src/lib/mesh/convert-table-to-mesh.ts b/modules/schema-utils/src/lib/mesh/convert-table-to-mesh.ts new file mode 100644 index 0000000000..ee9d4ab36f --- /dev/null +++ b/modules/schema-utils/src/lib/mesh/convert-table-to-mesh.ts @@ -0,0 +1,49 @@ +// loaders.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import type {Mesh, ColumnarTable, ArrowTable, Schema} from '@loaders.gl/schema'; +import {getFixedSizeListSize} from '../arrow-utils/arrow-fixed-size-list-utils'; +import {serializeArrowSchema} from '../schema/convert-arrow-schema'; +// import {makeMeshAttributeMetadata} from './deduce-mesh-schema'; + +/** + * Convert a mesh to a specific shape + */ +export function convertTableToMesh(table: ColumnarTable | ArrowTable): Mesh { + switch (table.shape) { + // case 'columnar-table': + // return convertColumnarTableToMesh(table); + case 'arrow-table': + return convertArrowTableToMesh(table); + default: + throw new Error(table.shape); + } +} + +export function convertArrowTableToMesh(table: ArrowTable): Mesh { + const arrowTable = table.data; + + const schema = serializeArrowSchema(arrowTable.schema); + const fields = schema.fields; + + const attributes: Mesh['attributes'] = {}; + for (const field of fields) { + const {name} = field; + const attributeData = arrowTable.getChild(name)!; + const size = getFixedSizeListSize(attributeData); + const typedArray = attributeData?.toArray(); + attributes[name] = {value: typedArray, size}; + } + + fixMetadata(schema); + const topology = schema.metadata.topology as any; + + return {schema, attributes, topology, mode: 0}; +} + +function fixMetadata(schema: Schema) { + schema.metadata ||= {}; + schema.metadata.topology ||= 'point-list'; + schema.metadata.mode ||= '0'; +} diff --git a/modules/tile-converter/src/i3s-converter/helpers/geometry-converter.ts b/modules/tile-converter/src/i3s-converter/helpers/geometry-converter.ts index 9ee11b00b5..e5485533b9 100644 --- a/modules/tile-converter/src/i3s-converter/helpers/geometry-converter.ts +++ b/modules/tile-converter/src/i3s-converter/helpers/geometry-converter.ts @@ -524,7 +524,7 @@ function convertNode({ const mesh = node.mesh; if (mesh) { - convertMesh({ + convertMeshToTable({ mesh, images, cartographicOrigin, @@ -563,7 +563,7 @@ function convertNode({ * @param matrix - transformation matrix - cumulative transformation matrix formed from all parent node matrices * @param featureTexture - feature texture key */ -function convertMesh({ +function convertMeshToTable({ mesh, images, cartographicOrigin,