feat: Arrow loaders for Pointcloud / Mesh formats (#3158)

modules/arrow/src/index.ts

@@ -7,7 +7,7 @@ export {VECTOR_TYPES} from './lib/types';
 
 // Arrow loader / Writer
 
-export {ArrowFormat} from './arrow-format';
+export {ArrowFormat} from './exports/arrow-format';
 
 export type {ArrowLoaderOptions} from './exports/arrow-loader';
 export {ArrowWorkerLoader} from './exports/arrow-loader';

modules/csv/src/csv-format.ts

@@ -0,0 +1,15 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Format} from '@loaders.gl/loader-utils';
+
+/** Comma-Separated Values */
+export const CSVFormat = {
+  id: 'csv',
+  module: 'csv',
+  name: 'CSV',
+  extensions: ['csv', 'tsv', 'dsv'],
+  mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],
+  category: 'table'
+} as const satisfies Format;

modules/csv/src/csv-loader.ts

@@ -15,6 +15,7 @@ import {
 } from '@loaders.gl/schema-utils';
 import Papa from './papaparse/papaparse';
 import AsyncIteratorStreamer from './papaparse/async-iterator-streamer';
+import {CSVFormat} from './csv-format';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
@@ -47,16 +48,11 @@ export type CSVLoaderOptions = LoaderOptions & {
 };
 
 export const CSVLoader = {
+  ...CSVFormat,
+
   dataType: null as unknown as ObjectRowTable | ArrayRowTable,
   batchType: null as unknown as TableBatch,
-
-  id: 'csv',
-  module: 'csv',
-  name: 'CSV',
   version: VERSION,
-  extensions: ['csv', 'tsv', 'dsv'],
-  mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],
-  category: 'table',
   parse: async (arrayBuffer: ArrayBuffer, options?: CSVLoaderOptions) =>
     parseCSV(new TextDecoder().decode(arrayBuffer), options),
   parseText: (text: string, options?: CSVLoaderOptions) => parseCSV(text, options),

modules/csv/src/csv-writer.ts

@@ -6,6 +6,7 @@
 import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';
 import type {Table, TableBatch} from '@loaders.gl/schema';
 import {encodeTableAsCSV} from './lib/encoders/encode-csv';
+import {CSVFormat} from './csv-format';
 
 export type CSVWriterOptions = WriterOptions & {
   csv?: {
@@ -14,12 +15,8 @@ export type CSVWriterOptions = WriterOptions & {
 };
 
 export const CSVWriter = {
-  id: 'csv',
+  ...CSVFormat,
   version: 'latest',
-  module: 'csv',
-  name: 'CSV',
-  extensions: ['csv'],
-  mimeTypes: ['text/csv'],
   options: {
     csv: {
       useDisplayNames: false

modules/draco/src/draco-arrow-loader.ts

@@ -0,0 +1,25 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ArrowTable} from '@loaders.gl/schema';
+import type {LoaderWithParser} from '@loaders.gl/loader-utils';
+import type {DracoLoaderOptions} from './draco-loader';
+import {DracoLoader} from './draco-loader';
+import {convertMeshToTable} from '@loaders.gl/schema-utils';
+
+/**
+ * Loader for Draco3D compressed geometries
+ */
+export const DracoArrowLoader = {
+  ...DracoLoader,
+  dataType: null as unknown as ArrowTable,
+  worker: false,
+  parse
+} as const satisfies LoaderWithParser<ArrowTable, never, DracoLoaderOptions>;
+
+async function parse(arrayBuffer: ArrayBuffer, options?: DracoLoaderOptions): Promise<ArrowTable> {
+  const mesh = await DracoLoader.parse(arrayBuffer, options);
+  const arrowTable = convertMeshToTable(mesh, 'arrow-table');
+  return arrowTable;
+}

modules/draco/src/index.ts

@@ -18,3 +18,4 @@ export {DracoWriterWorker, DracoWriter} from './draco-writer';
 
 export type {DracoLoaderOptions} from './draco-loader';
 export {DracoWorkerLoader, DracoLoader} from './draco-loader';
+export {DracoArrowLoader} from './draco-arrow-loader';

modules/draco/test/draco-arrow-loader.spec.ts

@@ -0,0 +1,82 @@
+/* eslint-disable max-len */
+import test from 'tape-promise/tape';
+import {validateLoader} from 'test/common/conformance';
+
+import {DracoArrowLoader} from '@loaders.gl/draco';
+import {setLoaderOptions, load} from '@loaders.gl/core';
+import draco3d from 'draco3d';
+
+const BUNNY_DRC_URL = '@loaders.gl/draco/test/data/bunny.drc';
+const CESIUM_TILE_URL = '@loaders.gl/draco/test/data/cesium-tile.drc';
+
+setLoaderOptions({
+  _workerType: 'test'
+});
+
+test('DracoArrowLoader#loader conformance', (t) => {
+  validateLoader(t, DracoArrowLoader, 'DracoArrowLoader');
+  t.end();
+});
+
+test('DracoArrowLoader#parse(mainthread)', async (t) => {
+  const table = await load(BUNNY_DRC_URL, DracoArrowLoader, {worker: false});
+  // validateMeshCategoryData(t, data);
+  const {data} = table;
+  t.equal(data.numRows, 104502 / 3, 'number of rows is correct');
+  const positions = data.getChild('POSITION')!;
+  t.ok(positions, 'POSITION attribute was found');
+  t.ok(data.schema, 'Has arrow-like schema');
+  t.end();
+});
+
+test('DracoArrowLoader#draco3d npm package', async (t) => {
+  const table = await load(BUNNY_DRC_URL, DracoArrowLoader, {
+    worker: false,
+    modules: {
+      draco3d
+    }
+  });
+  const {data} = table;
+  // validateMeshCategoryData(t, data);
+  t.ok(data.getChild('POSITION'), 'POSITION attribute was found');
+  t.end();
+});
+
+test('DracoArrowLoader#parse custom attributes(mainthread)', async (t) => {
+  let table = await load(CESIUM_TILE_URL, DracoArrowLoader, {
+    worker: false
+  });
+  const {data} = table;
+  t.equal(
+    data.getChild('CUSTOM_ATTRIBUTE_2')?.data[0].length,
+    173210,
+    'Custom (Intensity) attribute was found'
+  );
+  t.equal(
+    data.getChild('CUSTOM_ATTRIBUTE_3')?.data[0].length,
+    173210,
+    'Custom (Classification) attribute was found'
+  );
+
+  table = await load(CESIUM_TILE_URL, DracoArrowLoader, {
+    worker: false,
+    draco: {
+      extraAttributes: {
+        Intensity: 2,
+        Classification: 3
+      }
+    }
+  });
+  t.equal(
+    table.data.getChild('Intensity')?.data[0].length,
+    173210,
+    'Intensity attribute was found'
+  );
+  t.equal(
+    table.data.getChild('Classification')?.data[0].length,
+    173210,
+    'Classification attribute was found'
+  );
+
+  t.end();
+});

modules/draco/test/index.ts

@@ -5,5 +5,6 @@
 import './lib/utils/get-draco-schema.spec';
 
 import './draco-loader.spec';
+import './draco-arrow-loader.spec';
 import './draco-writer.spec';
 import './draco-compression-ratio.spec';

modules/flatgeobuf/src/flatgeobuf-format.ts

@@ -0,0 +1,15 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Format} from '@loaders.gl/loader-utils';
+
+/** flatgeobuf format */
+export const FlatGeobufFormat = {
+  id: 'flatgeobuf',
+  name: 'FlatGeobuf',
+  module: 'flatgeobuf',
+  extensions: ['fgb'],
+  mimeTypes: ['application/octet-stream'],
+  category: 'geometry'
+} as const satisfies Format;

modules/flatgeobuf/src/flatgeobuf-loader.ts

@@ -9,6 +9,7 @@ import {
   parseFlatGeobufInBatches,
   ParseFlatGeobufOptions
 } from './lib/parse-flatgeobuf';
+import {FlatGeobufFormat} from './flatgeobuf-format';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
@@ -32,17 +33,12 @@ export type FlatGeobufLoaderOptions = LoaderOptions & {
 
 /** Load flatgeobuf on a worker */
 export const FlatGeobufWorkerLoader = {
+  ...FlatGeobufFormat,
+
   dataType: null as any,
   batchType: null as any,
-
-  id: 'flatgeobuf',
-  name: 'FlatGeobuf',
-  module: 'flatgeobuf',
   version: VERSION,
   worker: true,
-  extensions: ['fgb'],
-  mimeTypes: ['application/octet-stream'],
-  category: 'geometry',
   tests: [new Uint8Array(FGB_MAGIC_NUMBER).buffer],
   options: {
     flatgeobuf: {

modules/flatgeobuf/src/floatgeobuf-source.ts

@@ -11,6 +11,7 @@ import type {
 import {Source, DataSource, VectorSource} from '@loaders.gl/loader-utils';
 
 import {FlatGeobufLoader} from './flatgeobuf-loader';
+import {FlatGeobufFormat} from './flatgeobuf-format';
 
 export type FlatGeobuSourceOptions = DataSourceOptions & {
   flatgeobuf?: {};
@@ -22,12 +23,8 @@ export type FlatGeobuSourceOptions = DataSourceOptions & {
  * @ndeprecated This is a WIP, not fully implemented
  */
 export const FlatGeobufSource = {
-  name: 'FlatGeobuf',
-  id: 'flatgeobuf',
-  module: 'wms',
+  ...FlatGeobufFormat,
   version: '0.0.0',
-  extensions: [],
-  mimeTypes: [],
   type: 'flatgeobuf-server',
   fromUrl: true,
   fromBlob: false, // TODO check if supported by library?

modules/flatgeobuf/src/index.ts

@@ -2,5 +2,6 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
+export {FlatGeobufFormat} from './flatgeobuf-format';
 export type {FlatGeobufLoaderOptions} from './flatgeobuf-loader';
 export {FlatGeobufLoader, FlatGeobufWorkerLoader} from './flatgeobuf-loader';

modules/geopackage/src/geopackage-format.ts

@@ -0,0 +1,14 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Format} from '@loaders.gl/loader-utils';
+
+export const GeoPackageFormat = {
+  id: 'geopackage',
+  name: 'GeoPackage',
+  module: 'geopackage',
+  extensions: ['gpkg'],
+  mimeTypes: ['application/geopackage+sqlite3'],
+  category: 'geometry'
+} as const satisfies Format;

modules/geopackage/src/geopackage-loader.ts

@@ -5,6 +5,7 @@
 import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
 import {Tables, GeoJSONTable} from '@loaders.gl/schema';
 import {parseGeoPackage, DEFAULT_SQLJS_CDN} from './lib/parse-geopackage';
+import {GeoPackageFormat} from './geopackage-format';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
@@ -30,16 +31,12 @@ export type GeoPackageLoaderOptions = LoaderOptions & {
 };
 
 export const GeoPackageLoader = {
+  ...GeoPackageFormat,
+
   dataType: null as unknown as GeoJSONTable | Tables<GeoJSONTable>,
   batchType: null as never,
 
-  id: 'geopackage',
-  name: 'GeoPackage',
-  module: 'geopackage',
   version: VERSION,
-  extensions: ['gpkg'],
-  mimeTypes: ['application/geopackage+sqlite3'],
-  category: 'geometry',
   parse: parseGeoPackage,
   options: {
     geopackage: {

modules/geopackage/src/index.ts

@@ -2,5 +2,6 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
+export {GeoPackageFormat} from './geopackage-format';
 export type {GeoPackageLoaderOptions} from './geopackage-loader';
 export {GeoPackageLoader} from './geopackage-loader';

modules/las/src/las-arrow-loader.ts

@@ -6,7 +6,7 @@ import type {LoaderWithParser} from '@loaders.gl/loader-utils';
 import type {ArrowTable} from '@loaders.gl/schema';
 
 import {LASLoaderOptions, LASWorkerLoader} from './las-loader';
-import {convertMesh} from '@loaders.gl/schema-utils';
+import {convertMeshToTable} from '@loaders.gl/schema-utils';
 import {parseLAS} from './lib/parse-las';
 
 /**
@@ -19,7 +19,7 @@ export const LASArrowLoader = {
   worker: false,
   parse: async (arrayBuffer: ArrayBuffer) => {
     const mesh = parseLAS(arrayBuffer);
-    const arrowTable = convertMesh(mesh, 'arrow-table');
+    const arrowTable = convertMeshToTable(mesh, 'arrow-table');
     return arrowTable;
   }
 } as const satisfies LoaderWithParser<ArrowTable, never, LASLoaderOptions>;

modules/las/src/lib/parse-las.ts

@@ -6,7 +6,7 @@
 // import type {ArrowTable, ColumnarTable} from '@loaders.gl/schema';
 import type {LASLoaderOptions} from '../las-loader';
 import type {LASMesh, LASHeader} from './las-types';
-import {getMeshBoundingBox /* , convertMesh */} from '@loaders.gl/schema-utils';
+import {getMeshBoundingBox /* , convertMeshToTable */} from '@loaders.gl/schema-utils';
 import {LASFile} from './laslaz-decoder';
 import {getLASSchema} from './get-las-schema';
 
@@ -28,7 +28,7 @@ export function parseLAS(arrayBuffer: ArrayBuffer, options?: LASLoaderOptions):
   return parseLASMesh(arrayBuffer, options);
   // This code breaks pointcloud example on the website
   // const mesh = parseLASMesh(arrayBuffer, options);
-  // return convertMesh(mesh, options?.las?.shape || 'mesh') as LASMesh | ArrowTable | ColumnarTable;
+  // return convertMeshToTable(mesh, options?.las?.shape || 'mesh') as LASMesh | ArrowTable | ColumnarTable;
 }
 
 /**

modules/obj/src/lib/parse-obj.ts

@@ -3,7 +3,7 @@ import {getMeshBoundingBox} from '@loaders.gl/schema-utils';
 import {parseOBJMeshes} from './parse-obj-meshes';
 import {getOBJSchema} from './get-obj-schema';
 
-export function parseOBJ(text, options): Mesh {
+export function parseOBJ(text: string, options?): Mesh {
   const {meshes} = parseOBJMeshes(text);
 
   // @ts-expect-error

modules/obj/src/obj-arrow-loader.ts

@@ -0,0 +1,26 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {LoaderWithParser} from '@loaders.gl/loader-utils';
+import type {ArrowTable} from '@loaders.gl/schema';
+
+import {OBJLoaderOptions, OBJWorkerLoader} from './obj-loader';
+import {convertMeshToTable} from '@loaders.gl/schema-utils';
+import {parseOBJ} from './lib/parse-obj';
+
+/**
+ * Worker loader for OBJ - Point Cloud Data
+ */
+export const OBJArrowLoader = {
+  ...OBJWorkerLoader,
+  dataType: null as unknown as ArrowTable,
+  batchType: null as never,
+  worker: false,
+  parse: async (arrayBuffer: ArrayBuffer) => {
+    const text = new TextDecoder().decode(arrayBuffer);
+    const mesh = parseOBJ(text);
+    const arrowTable = convertMeshToTable(mesh, 'arrow-table');
+    return arrowTable;
+  }
+} as const satisfies LoaderWithParser<ArrowTable, never, OBJLoaderOptions>;