diff --git a/modules/images/src/lib/category-api/binary-image-api.ts b/modules/images/src/lib/category-api/binary-image-api.ts index 43c9bb111f..b02cb0e9f5 100644 --- a/modules/images/src/lib/category-api/binary-image-api.ts +++ b/modules/images/src/lib/category-api/binary-image-api.ts @@ -1,11 +1,6 @@ // Attributions // * Based on binary-gltf-utils under MIT license: Copyright (c) 2016-17 Karl Cheng -// TODO: make these functions work for Node.js buffers? -// Quarantine references to Buffer to prevent bundler from adding big polyfills -// import {bufferToArrayBuffer} from '../node/buffer-to-array-buffer'; -// TODO - this should be handled in @loaders.gl/polyfills - import {getISOBMFFMediaType} from './parse-isobmff-binary'; /** MIME type, width and height extracted from binary compressed image data */ diff --git a/modules/loader-utils/src/index.ts b/modules/loader-utils/src/index.ts index 8d985927bf..f28a9ff27b 100644 --- a/modules/loader-utils/src/index.ts +++ b/modules/loader-utils/src/index.ts @@ -144,7 +144,8 @@ export {ImageSource} from './lib/sources/image-source'; export type { TileSourceProps, TileSourceMetadata, - GetTileParameters + GetTileParameters, + TileLoadParameters } from './lib/sources/tile-source'; export type {TileSource} from './lib/sources/tile-source'; diff --git a/modules/mvt/package.json b/modules/mvt/package.json index 5d7579a4b8..1a9a990456 100644 --- a/modules/mvt/package.json +++ b/modules/mvt/package.json @@ -33,6 +33,7 @@ }, "dependencies": { "@loaders.gl/gis": "4.0.0-beta.2", + "@loaders.gl/images": "4.0.0-beta.2", "@loaders.gl/loader-utils": "4.0.0-beta.2", "@loaders.gl/schema": "4.0.0-beta.2", "@math.gl/polygon": "^3.5.1", diff --git a/modules/mvt/src/index.ts b/modules/mvt/src/index.ts index 5f70baa44e..0d7e3e6bc9 100644 --- a/modules/mvt/src/index.ts +++ b/modules/mvt/src/index.ts @@ -7,6 +7,8 @@ export type {TileJSON} from './lib/parse-tilejson'; export type {TileJSONLoaderOptions} from './tilejson-loader'; export {TileJSONLoader} from './tilejson-loader'; +export {MVTSource} from './mvt-source'; + // GeoJSONTiler export type {GeoJSONTilerOptions} from './lib/geojson-tiler/geojson-tiler'; diff --git a/modules/mvt/src/mvt-source.ts b/modules/mvt/src/mvt-source.ts new file mode 100644 index 0000000000..8eb68987ee --- /dev/null +++ b/modules/mvt/src/mvt-source.ts @@ -0,0 +1,110 @@ +// loaders.gl, MIT license + +import type {GetTileParameters, ImageType, DataSourceProps} from '@loaders.gl/loader-utils'; +import type {ImageTileSource, VectorTileSource} from '@loaders.gl/loader-utils'; +import {DataSource, resolvePath} from '@loaders.gl/loader-utils'; +import {ImageLoader} from '@loaders.gl/images'; +import {MVTLoader, MVTLoaderOptions, TileJSONLoader, TileJSON} from '@loaders.gl/mvt'; + +import {TileLoadParameters} from '@loaders.gl/loader-utils'; + +export type MVTSourceProps = DataSourceProps & { + url: string; + attributions?: string[]; +}; + +/** + * A PMTiles data source + * @note Can be either a raster or vector tile source depending on the contents of the PMTiles file. + */ +export class MVTSource extends DataSource implements ImageTileSource, VectorTileSource { + props: MVTSourceProps; + url: string; + schema: 'tms' | 'xyz' = 'tms'; + metadata: Promise; + + constructor(props: MVTSourceProps) { + super(props); + this.props = props; + this.url = resolvePath(props.url); + this.getTileData = this.getTileData.bind(this); + this.metadata = this.getMetadata(); + } + + // @ts-ignore - Metadata type misalignment + async getMetadata(): Promise { + const metadataUrl = this.getMetadataUrl(); + const response = await this.fetch(metadataUrl); + if (!response.ok) { + return null; + } + const tileJSON = await response.text(); + const metadata = TileJSONLoader.parseTextSync?.(JSON.stringify(tileJSON)) || null; + // metadata.attributions = [...this.props.attributions, ...(metadata.attributions || [])]; + return metadata; + } + + async getTile(tileParams: GetTileParameters): Promise { + const {x, y, zoom: z} = tileParams; + const tileUrl = this.getTileURL(x, y, z); + const response = await this.fetch(tileUrl); + if (!response.ok) { + return null; + } + const arrayBuffer = await response.arrayBuffer(); + return arrayBuffer; + } + + // Tile Source interface implementation: deck.gl compatible API + // TODO - currently only handles image tiles, not vector tiles + + async getTileData(tileParams: TileLoadParameters): Promise { + const {x, y, z} = tileParams.index; + const metadata = await this.metadata; + // @ts-expect-error + switch (metadata.mimeType || 'application/vnd.mapbox-vector-tile') { + case 'application/vnd.mapbox-vector-tile': + return await this.getVectorTile({x, y, zoom: z, layers: []}); + default: + return await this.getImageTile({x, y, zoom: z, layers: []}); + } + } + + // ImageTileSource interface implementation + + async getImageTile(tileParams: GetTileParameters): Promise { + const arrayBuffer = await this.getTile(tileParams); + return arrayBuffer ? await ImageLoader.parse(arrayBuffer, this.loadOptions) : null; + } + + // VectorTileSource interface implementation + + async getVectorTile(tileParams: GetTileParameters): Promise { + const arrayBuffer = await this.getTile(tileParams); + const loadOptions: MVTLoaderOptions = { + shape: 'geojson-table', + mvt: { + coordinates: 'wgs84', + tileIndex: {x: tileParams.x, y: tileParams.y, z: tileParams.zoom}, + ...(this.loadOptions as MVTLoaderOptions)?.mvt + }, + ...this.loadOptions + }; + + return arrayBuffer ? await MVTLoader.parse(arrayBuffer, loadOptions) : null; + } + + getMetadataUrl(): string { + return `${this.url}/tilejson.json`; + } + + getTileURL(x: number, y: number, z: number) { + switch (this.schema) { + case 'xyz': + return `${this.url}/${x}/${y}/${z}`; + case 'tms': + default: + return `${this.url}/${z}/${x}/${y}`; + } + } +} diff --git a/modules/mvt/test/data/tilesets.ts b/modules/mvt/test/data/tilesets.ts new file mode 100644 index 0000000000..b83e80aa3c --- /dev/null +++ b/modules/mvt/test/data/tilesets.ts @@ -0,0 +1 @@ +export const TILESETS: string[] = []; diff --git a/modules/mvt/test/index.ts b/modules/mvt/test/index.ts index 361df49aa2..435065d08e 100644 --- a/modules/mvt/test/index.ts +++ b/modules/mvt/test/index.ts @@ -4,5 +4,7 @@ import './mvt-loader.spec'; import './tilejson-loader.spec'; +import './mvt-source.spec'; + // geojson-vt import './lib/geojson-tiler'; diff --git a/modules/mvt/test/mvt-source.spec.ts b/modules/mvt/test/mvt-source.spec.ts new file mode 100644 index 0000000000..f16744972f --- /dev/null +++ b/modules/mvt/test/mvt-source.spec.ts @@ -0,0 +1,238 @@ +// loaders.gl, MIT license + +import test from 'tape-promise/tape'; +import {isBrowser} from '@loaders.gl/core'; + +import {TILESETS} from './data/tilesets'; +import {MVTSource} from '@loaders.gl/mvt'; + +test('MVTSource#urls', async (t) => { + if (!isBrowser) { + t.comment('MVTSource currently only supported in browser'); + t.end(); + return; + } + for (const tilesetUrl of TILESETS) { + const source = new MVTSource({url: tilesetUrl}); + t.ok(source); + const metadata = await source.getMetadata(); + t.ok(metadata); + // console.error(JSON.stringify(metadata.tileJSON, null, 2)); + } + t.end(); +}); + +test('MVTSource#Blobs', async (t) => { + if (!isBrowser) { + t.comment('MVTSource currently only supported in browser'); + t.end(); + return; + } + for (const tilesetUrl of TILESETS) { + const source = new MVTSource({url: tilesetUrl}); + t.ok(source); + const metadata = await source.getMetadata(); + t.ok(metadata); + // console.error(JSON.stringify(metadata.tileJSON, null, 2)); + } + t.end(); +}); + +// TBA - TILE LOADING TESTS + +/* +import test from 'tape-promise/tape'; +import {validateLoader} from 'test/common/conformance'; + +import {load} from '@loaders.gl/core'; +import {PMTilesLoader} from '@loaders.gl/pmtiles'; + +import {PMTILESETS} from './data/tilesets'; + +test('PMTilesLoader#loader conformance', (t) => { + validateLoader(t, PMTilesLoader, 'PMTilesLoader'); + t.end(); +}); + +test.skip('PMTilesLoader#load', async (t) => { + for (const tilesetUrl of PMTILESETS) { + const metadata = await load(tilesetUrl, PMTilesLoader); + t.ok(metadata); + } + t.end(); +}); + +/* +// echo '{"type":"Polygon","coordinates":[[[0,0],[0,1],[1,1],[1,0],[0,0]]]}' | ./tippecanoe -zg -o test_fixture_1.pmtiles +test('cache getHeader', async (t) => { + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1'); + const cache = new SharedPromiseCache(); + const header = await cache.getHeader(source); + t.strictEqual(header.rootDirectoryOffset, 127); + t.strictEqual(header.rootDirectoryLength, 25); + t.strictEqual(header.jsonMetadataOffset, 152); + t.strictEqual(header.jsonMetadataLength, 247); + t.strictEqual(header.leafDirectoryOffset, 0); + t.strictEqual(header.leafDirectoryLength, 0); + t.strictEqual(header.tileDataOffset, 399); + t.strictEqual(header.tileDataLength, 69); + t.strictEqual(header.numAddressedTiles, 1); + t.strictEqual(header.numTileEntries, 1); + t.strictEqual(header.numTileContents, 1); + t.strictEqual(header.clustered, false); + t.strictEqual(header.internalCompression, 2); + t.strictEqual(header.tileCompression, 2); + t.strictEqual(header.tileType, 1); + t.strictEqual(header.minZoom, 0); + t.strictEqual(header.maxZoom, 0); + t.strictEqual(header.minLon, 0); + t.strictEqual(header.minLat, 0); + // t.strictEqual(header.maxLon,1); // TODO fix me + t.strictEqual(header.maxLat, 1); +}); + +test('cache check against empty', async (t) => { + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/empty.pmtiles', '1'); + const cache = new SharedPromiseCache(); + t.rejects(async () => { + await cache.getHeader(source); + }); +}); + +test('cache check magic number', async (t) => { + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/invalid.pmtiles', '1'); + const cache = new SharedPromiseCache(); + t.rejects(async () => { + await cache.getHeader(source); + }); +}); + +test('cache check future spec version', async (t) => { + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/invalid_v4.pmtiles', '1'); + const cache = new SharedPromiseCache(); + t.rejects(async () => { + await cache.getHeader(source); + }); +}); + +test('cache getDirectory', async (t) => { + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1'); + + let cache = new SharedPromiseCache(6400, false); + let header = await cache.getHeader(source); + t.strictEqual(cache.cache.size, 1); + + cache = new SharedPromiseCache(6400, true); + header = await cache.getHeader(source); + + // prepopulates the root directory + t.strictEqual(cache.cache.size, 2); + + const directory = await cache.getDirectory( + source, + header.rootDirectoryOffset, + header.rootDirectoryLength, + header + ); + t.strictEqual(directory.length, 1); + t.strictEqual(directory[0].tileId, 0); + t.strictEqual(directory[0].offset, 0); + t.strictEqual(directory[0].length, 69); + t.strictEqual(directory[0].runLength, 1); + + for (const v of cache.cache.values()) { + t.ok(v.lastUsed > 0); + } +}); + +test('multiple sources in a single cache', async (t) => { + const cache = new SharedPromiseCache(); + const source1 = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1'); + const source2 = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '2'); + await cache.getHeader(source1); + t.strictEqual(cache.cache.size, 2); + await cache.getHeader(source2); + t.strictEqual(cache.cache.size, 4); +}); + +test('etags are part of key', async (t) => { + const cache = new SharedPromiseCache(6400, false); + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1'); + source.etag = 'etag_1'; + let header = await cache.getHeader(source); + t.strictEqual(header.etag, 'etag_1'); + + source.etag = 'etag_2'; + + t.rejects(async () => { + await cache.getDirectory( + source, + header.rootDirectoryOffset, + header.rootDirectoryLength, + header + ); + }); + + cache.invalidate(source, 'etag_2'); + header = await cache.getHeader(source); + t.ok( + await cache.getDirectory(source, header.rootDirectoryOffset, header.rootDirectoryLength, header) + ); +}); + +test.skip('soft failure on etag weirdness', async (t) => { + const cache = new SharedPromiseCache(6400, false); + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1'); + source.etag = 'etag_1'; + let header = await cache.getHeader(source); + t.strictEqual(header.etag, 'etag_1'); + + source.etag = 'etag_2'; + + t.rejects(async () => { + await cache.getDirectory( + source, + header.rootDirectoryOffset, + header.rootDirectoryLength, + header + ); + }); + + source.etag = 'etag_1'; + cache.invalidate(source, 'etag_2'); + + header = await cache.getHeader(source); + t.strictEqual(header.etag, undefined); +}); + +test('cache pruning by byte size', async (t) => { + const cache = new SharedPromiseCache(2, false); + cache.cache.set('0', {lastUsed: 0, data: Promise.resolve([])}); + cache.cache.set('1', {lastUsed: 1, data: Promise.resolve([])}); + cache.cache.set('2', {lastUsed: 2, data: Promise.resolve([])}); + cache.prune(); + t.strictEqual(cache.cache.size, 2); + t.ok(cache.cache.get('2')); + t.ok(cache.cache.get('1')); + t.ok(!cache.cache.get('0')); +}); + +test('pmtiles get metadata', async (t) => { + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1'); + const p = new PMTiles(source); + const metadata = await p.getMetadata(); + t.ok(metadata.name); +}); + +// echo '{"type":"Polygon","coordinates":[[[0,0],[0,1],[1,0],[0,0]]]}' | ./tippecanoe -zg -o test_fixture_2.pmtiles +test('pmtiles handle retries', async (t) => { + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1'); + source.etag = '1'; + const p = new PMTiles(source); + const metadata = await p.getMetadata(); + t.ok(metadata.name); + source.etag = '2'; + source.replaceData('@loaders.gl/pmtiles/test/data/test_fixture_2.pmtiles'); + t.ok(await p.getZxy(0, 0, 0)); +}); +*/ diff --git a/modules/mvt/tsconfig.json b/modules/mvt/tsconfig.json index a1f485e8d3..46b940c801 100644 --- a/modules/mvt/tsconfig.json +++ b/modules/mvt/tsconfig.json @@ -8,6 +8,7 @@ "outDir": "dist" }, "references": [ + {"path": "../images"}, {"path": "../loader-utils"}, {"path": "../gis"} ] diff --git a/modules/pmtiles/src/pmtiles-source.ts b/modules/pmtiles/src/pmtiles-source.ts index d0172c7585..52aa793bc3 100644 --- a/modules/pmtiles/src/pmtiles-source.ts +++ b/modules/pmtiles/src/pmtiles-source.ts @@ -1,6 +1,7 @@ // loaders.gl, MIT license -import type {GetTileParameters, ImageType, DataSourceProps} from '@loaders.gl/loader-utils'; +import type {TileLoadParameters, GetTileParameters} from '@loaders.gl/loader-utils'; +import type {ImageType, DataSourceProps} from '@loaders.gl/loader-utils'; import type {ImageTileSource, VectorTileSource} from '@loaders.gl/loader-utils'; import {DataSource, resolvePath} from '@loaders.gl/loader-utils'; import {ImageLoader} from '@loaders.gl/images'; @@ -10,7 +11,6 @@ import {PMTiles, Source, RangeResponse} from 'pmtiles'; import type {PMTilesMetadata} from './lib/parse-pmtiles'; import {parsePMTilesHeader} from './lib/parse-pmtiles'; -import {TileLoadParameters} from 'modules/loader-utils/src/lib/sources/tile-source'; const VERSION = '1.0.0';