feat(mvt): Add MVTSource (#2674)

modules/images/src/lib/category-api/binary-image-api.ts

@@ -1,11 +1,6 @@
 // Attributions
 // * Based on binary-gltf-utils under MIT license: Copyright (c) 2016-17 Karl Cheng
 
-// TODO: make these functions work for Node.js buffers?
-// Quarantine references to Buffer to prevent bundler from adding big polyfills
-// import {bufferToArrayBuffer} from '../node/buffer-to-array-buffer';
-// TODO - this should be handled in @loaders.gl/polyfills
-
 import {getISOBMFFMediaType} from './parse-isobmff-binary';
 
 /** MIME type, width and height extracted from binary compressed image data */

modules/loader-utils/src/index.ts

@@ -144,7 +144,8 @@ export {ImageSource} from './lib/sources/image-source';
 export type {
   TileSourceProps,
   TileSourceMetadata,
-  GetTileParameters
+  GetTileParameters,
+  TileLoadParameters
 } from './lib/sources/tile-source';
 export type {TileSource} from './lib/sources/tile-source';
 

modules/mvt/package.json

@@ -33,6 +33,7 @@
   },
   "dependencies": {
     "@loaders.gl/gis": "4.0.0-beta.2",
+    "@loaders.gl/images": "4.0.0-beta.2",
     "@loaders.gl/loader-utils": "4.0.0-beta.2",
     "@loaders.gl/schema": "4.0.0-beta.2",
     "@math.gl/polygon": "^3.5.1",

modules/mvt/src/index.ts

@@ -7,6 +7,8 @@ export type {TileJSON} from './lib/parse-tilejson';
 export type {TileJSONLoaderOptions} from './tilejson-loader';
 export {TileJSONLoader} from './tilejson-loader';
 
+export {MVTSource} from './mvt-source';
+
 // GeoJSONTiler
 
 export type {GeoJSONTilerOptions} from './lib/geojson-tiler/geojson-tiler';

modules/mvt/src/mvt-source.ts

@@ -0,0 +1,110 @@
+// loaders.gl, MIT license
+
+import type {GetTileParameters, ImageType, DataSourceProps} from '@loaders.gl/loader-utils';
+import type {ImageTileSource, VectorTileSource} from '@loaders.gl/loader-utils';
+import {DataSource, resolvePath} from '@loaders.gl/loader-utils';
+import {ImageLoader} from '@loaders.gl/images';
+import {MVTLoader, MVTLoaderOptions, TileJSONLoader, TileJSON} from '@loaders.gl/mvt';
+
+import {TileLoadParameters} from '@loaders.gl/loader-utils';
+
+export type MVTSourceProps = DataSourceProps & {
+  url: string;
+  attributions?: string[];
+};
+
+/**
+ * A PMTiles data source
+ * @note Can be either a raster or vector tile source depending on the contents of the PMTiles file.
+ */
+export class MVTSource extends DataSource implements ImageTileSource, VectorTileSource {
+  props: MVTSourceProps;
+  url: string;
+  schema: 'tms' | 'xyz' = 'tms';
+  metadata: Promise<TileJSON | null>;
+
+  constructor(props: MVTSourceProps) {
+    super(props);
+    this.props = props;
+    this.url = resolvePath(props.url);
+    this.getTileData = this.getTileData.bind(this);
+    this.metadata = this.getMetadata();
+  }
+
+  // @ts-ignore - Metadata type misalignment
+  async getMetadata(): Promise<TileJSON | null> {
+    const metadataUrl = this.getMetadataUrl();
+    const response = await this.fetch(metadataUrl);
+    if (!response.ok) {
+      return null;
+    }
+    const tileJSON = await response.text();
+    const metadata = TileJSONLoader.parseTextSync?.(JSON.stringify(tileJSON)) || null;
+    // metadata.attributions = [...this.props.attributions, ...(metadata.attributions || [])];
+    return metadata;
+  }
+
+  async getTile(tileParams: GetTileParameters): Promise<ArrayBuffer | null> {
+    const {x, y, zoom: z} = tileParams;
+    const tileUrl = this.getTileURL(x, y, z);
+    const response = await this.fetch(tileUrl);
+    if (!response.ok) {
+      return null;
+    }
+    const arrayBuffer = await response.arrayBuffer();
+    return arrayBuffer;
+  }
+
+  // Tile Source interface implementation: deck.gl compatible API
+  // TODO - currently only handles image tiles, not vector tiles
+
+  async getTileData(tileParams: TileLoadParameters): Promise<unknown | null> {
+    const {x, y, z} = tileParams.index;
+    const metadata = await this.metadata;
+    // @ts-expect-error
+    switch (metadata.mimeType || 'application/vnd.mapbox-vector-tile') {
+      case 'application/vnd.mapbox-vector-tile':
+        return await this.getVectorTile({x, y, zoom: z, layers: []});
+      default:
+        return await this.getImageTile({x, y, zoom: z, layers: []});
+    }
+  }
+
+  // ImageTileSource interface implementation
+
+  async getImageTile(tileParams: GetTileParameters): Promise<ImageType | null> {
+    const arrayBuffer = await this.getTile(tileParams);
+    return arrayBuffer ? await ImageLoader.parse(arrayBuffer, this.loadOptions) : null;
+  }
+
+  // VectorTileSource interface implementation
+
+  async getVectorTile(tileParams: GetTileParameters): Promise<unknown | null> {
+    const arrayBuffer = await this.getTile(tileParams);
+    const loadOptions: MVTLoaderOptions = {
+      shape: 'geojson-table',
+      mvt: {
+        coordinates: 'wgs84',
+        tileIndex: {x: tileParams.x, y: tileParams.y, z: tileParams.zoom},
+        ...(this.loadOptions as MVTLoaderOptions)?.mvt
+      },
+      ...this.loadOptions
+    };
+
+    return arrayBuffer ? await MVTLoader.parse(arrayBuffer, loadOptions) : null;
+  }
+
+  getMetadataUrl(): string {
+    return `${this.url}/tilejson.json`;
+  }
+
+  getTileURL(x: number, y: number, z: number) {
+    switch (this.schema) {
+      case 'xyz':
+        return `${this.url}/${x}/${y}/${z}`;
+      case 'tms':
+      default:
+        return `${this.url}/${z}/${x}/${y}`;
+    }
+  }
+}

modules/mvt/test/data/tilesets.ts

@@ -0,0 +1 @@
+export const TILESETS: string[] = [];

modules/mvt/test/index.ts

@@ -4,5 +4,7 @@ import './mvt-loader.spec';
 
 import './tilejson-loader.spec';
 
+import './mvt-source.spec';
+
 // geojson-vt
 import './lib/geojson-tiler';

modules/mvt/test/mvt-source.spec.ts

@@ -0,0 +1,238 @@
+// loaders.gl, MIT license
+
+import test from 'tape-promise/tape';
+import {isBrowser} from '@loaders.gl/core';
+
+import {TILESETS} from './data/tilesets';
+import {MVTSource} from '@loaders.gl/mvt';
+
+test('MVTSource#urls', async (t) => {
+  if (!isBrowser) {
+    t.comment('MVTSource currently only supported in browser');
+    t.end();
+    return;
+  }
+  for (const tilesetUrl of TILESETS) {
+    const source = new MVTSource({url: tilesetUrl});
+    t.ok(source);
+    const metadata = await source.getMetadata();
+    t.ok(metadata);
+    // console.error(JSON.stringify(metadata.tileJSON, null, 2));
+  }
+  t.end();
+});
+
+test('MVTSource#Blobs', async (t) => {
+  if (!isBrowser) {
+    t.comment('MVTSource currently only supported in browser');
+    t.end();
+    return;
+  }
+  for (const tilesetUrl of TILESETS) {
+    const source = new MVTSource({url: tilesetUrl});
+    t.ok(source);
+    const metadata = await source.getMetadata();
+    t.ok(metadata);
+    // console.error(JSON.stringify(metadata.tileJSON, null, 2));
+  }
+  t.end();
+});
+
+// TBA - TILE LOADING TESTS
+
+/*
+import test from 'tape-promise/tape';
+import {validateLoader} from 'test/common/conformance';
+
+import {load} from '@loaders.gl/core';
+import {PMTilesLoader} from '@loaders.gl/pmtiles';
+
+import {PMTILESETS} from './data/tilesets';
+
+test('PMTilesLoader#loader conformance', (t) => {
+  validateLoader(t, PMTilesLoader, 'PMTilesLoader');
+  t.end();
+});
+
+test.skip('PMTilesLoader#load', async (t) => {
+  for (const tilesetUrl of PMTILESETS) {
+    const metadata = await load(tilesetUrl, PMTilesLoader);
+    t.ok(metadata);
+  }
+  t.end();
+});
+
+/*
+// echo '{"type":"Polygon","coordinates":[[[0,0],[0,1],[1,1],[1,0],[0,0]]]}' | ./tippecanoe -zg -o test_fixture_1.pmtiles
+test('cache getHeader', async (t) => {
+  const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1');
+  const cache = new SharedPromiseCache();
+  const header = await cache.getHeader(source);
+  t.strictEqual(header.rootDirectoryOffset, 127);
+  t.strictEqual(header.rootDirectoryLength, 25);
+  t.strictEqual(header.jsonMetadataOffset, 152);
+  t.strictEqual(header.jsonMetadataLength, 247);
+  t.strictEqual(header.leafDirectoryOffset, 0);
+  t.strictEqual(header.leafDirectoryLength, 0);
+  t.strictEqual(header.tileDataOffset, 399);
+  t.strictEqual(header.tileDataLength, 69);
+  t.strictEqual(header.numAddressedTiles, 1);
+  t.strictEqual(header.numTileEntries, 1);
+  t.strictEqual(header.numTileContents, 1);
+  t.strictEqual(header.clustered, false);
+  t.strictEqual(header.internalCompression, 2);
+  t.strictEqual(header.tileCompression, 2);
+  t.strictEqual(header.tileType, 1);
+  t.strictEqual(header.minZoom, 0);
+  t.strictEqual(header.maxZoom, 0);
+  t.strictEqual(header.minLon, 0);
+  t.strictEqual(header.minLat, 0);
+  // t.strictEqual(header.maxLon,1); // TODO fix me
+  t.strictEqual(header.maxLat, 1);
+});
+
+test('cache check against empty', async (t) => {
+  const source = new TestFileSource('@loaders.gl/pmtiles/test/data/empty.pmtiles', '1');
+  const cache = new SharedPromiseCache();
+  t.rejects(async () => {
+    await cache.getHeader(source);
+  });
+});
+
+test('cache check magic number', async (t) => {
+  const source = new TestFileSource('@loaders.gl/pmtiles/test/data/invalid.pmtiles', '1');
+  const cache = new SharedPromiseCache();
+  t.rejects(async () => {
+    await cache.getHeader(source);
+  });
+});
+
+test('cache check future spec version', async (t) => {
+  const source = new TestFileSource('@loaders.gl/pmtiles/test/data/invalid_v4.pmtiles', '1');
+  const cache = new SharedPromiseCache();
+  t.rejects(async () => {
+    await cache.getHeader(source);
+  });
+});
+
+test('cache getDirectory', async (t) => {
+  const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1');
+
+  let cache = new SharedPromiseCache(6400, false);
+  let header = await cache.getHeader(source);
+  t.strictEqual(cache.cache.size, 1);
+
+  cache = new SharedPromiseCache(6400, true);
+  header = await cache.getHeader(source);
+
+  // prepopulates the root directory
+  t.strictEqual(cache.cache.size, 2);
+
+  const directory = await cache.getDirectory(
+    source,
+    header.rootDirectoryOffset,
+    header.rootDirectoryLength,
+    header
+  );
+  t.strictEqual(directory.length, 1);
+  t.strictEqual(directory[0].tileId, 0);
+  t.strictEqual(directory[0].offset, 0);
+  t.strictEqual(directory[0].length, 69);
+  t.strictEqual(directory[0].runLength, 1);
+
+  for (const v of cache.cache.values()) {
+    t.ok(v.lastUsed > 0);
+  }
+});
+
+test('multiple sources in a single cache', async (t) => {
+  const cache = new SharedPromiseCache();
+  const source1 = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1');
+  const source2 = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '2');
+  await cache.getHeader(source1);
+  t.strictEqual(cache.cache.size, 2);
+  await cache.getHeader(source2);
+  t.strictEqual(cache.cache.size, 4);
+});
+
+test('etags are part of key', async (t) => {
+  const cache = new SharedPromiseCache(6400, false);
+  const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1');
+  source.etag = 'etag_1';
+  let header = await cache.getHeader(source);
+  t.strictEqual(header.etag, 'etag_1');
+
+  source.etag = 'etag_2';
+
+  t.rejects(async () => {
+    await cache.getDirectory(
+      source,
+      header.rootDirectoryOffset,
+      header.rootDirectoryLength,
+      header
+    );
+  });
+
+  cache.invalidate(source, 'etag_2');
+  header = await cache.getHeader(source);
+  t.ok(
+    await cache.getDirectory(source, header.rootDirectoryOffset, header.rootDirectoryLength, header)
+  );
+});
+
+test.skip('soft failure on etag weirdness', async (t) => {
+  const cache = new SharedPromiseCache(6400, false);
+  const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1');
+  source.etag = 'etag_1';
+  let header = await cache.getHeader(source);
+  t.strictEqual(header.etag, 'etag_1');
+
+  source.etag = 'etag_2';
+
+  t.rejects(async () => {
+    await cache.getDirectory(
+      source,
+      header.rootDirectoryOffset,
+      header.rootDirectoryLength,
+      header
+    );
+  });
+
+  source.etag = 'etag_1';
+  cache.invalidate(source, 'etag_2');
+
+  header = await cache.getHeader(source);
+  t.strictEqual(header.etag, undefined);
+});
+
+test('cache pruning by byte size', async (t) => {
+  const cache = new SharedPromiseCache(2, false);
+  cache.cache.set('0', {lastUsed: 0, data: Promise.resolve([])});
+  cache.cache.set('1', {lastUsed: 1, data: Promise.resolve([])});
+  cache.cache.set('2', {lastUsed: 2, data: Promise.resolve([])});
+  cache.prune();
+  t.strictEqual(cache.cache.size, 2);
+  t.ok(cache.cache.get('2'));
+  t.ok(cache.cache.get('1'));
+  t.ok(!cache.cache.get('0'));
+});
+
+test('pmtiles get metadata', async (t) => {
+  const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1');
+  const p = new PMTiles(source);
+  const metadata = await p.getMetadata();
+  t.ok(metadata.name);
+});
+
+// echo '{"type":"Polygon","coordinates":[[[0,0],[0,1],[1,0],[0,0]]]}' | ./tippecanoe -zg -o test_fixture_2.pmtiles
+test('pmtiles handle retries', async (t) => {
+  const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1');
+  source.etag = '1';
+  const p = new PMTiles(source);
+  const metadata = await p.getMetadata();
+  t.ok(metadata.name);
+  source.etag = '2';
+  source.replaceData('@loaders.gl/pmtiles/test/data/test_fixture_2.pmtiles');
+  t.ok(await p.getZxy(0, 0, 0));
+});
+*/