diff --git a/dev-docs/RFCs/v3.4/data-source-rfc.md b/dev-docs/RFCs/v3.4/data-source-rfc.md new file mode 100644 index 0000000000..52e8ba707a --- /dev/null +++ b/dev-docs/RFCs/v3.4/data-source-rfc.md @@ -0,0 +1,41 @@ +# Data Sources API + +Build a data source API that can encompass services such +- loaded data +- URLS +- tile service +- WMS +- Incremental fetch with range requests etc. +- programmatic data generation +- ... + +### Related + +- deck.gl has a semi-internal data source API. +- + + + +## Main problems + +### Refresh / Dirty state handling. + +How does the application (typically deck.gl)) know when to redraw? + +```typescript +DataSource.setNeedsRefresh(); +DataSource.getNeedsRefresh(clear: boolean = true); +``` + +## Updates + +`DataSource.setProps()` + +Typing is a bit messy when overriding child class definitions. + +## Declarative usage + +Fully declarative usage requires a lifecycle management system, which seems too heavy. + + + diff --git a/docs/modules/pmtiles/README.md b/docs/modules/pmtiles/README.md new file mode 100644 index 0000000000..489f7ea2ff --- /dev/null +++ b/docs/modules/pmtiles/README.md @@ -0,0 +1,5 @@ +# @loaders.gl/pmtiles + +> The pmtiles loaders are still under development and are not yet considered ready for use. + +Support for loading and traversing [pmtiles](http://potree.org/) format point clouds. diff --git a/docs/modules/pmtiles/api-reference/pmtiles-loader.md b/docs/modules/pmtiles/api-reference/pmtiles-loader.md new file mode 100644 index 0000000000..7c0d5f2579 --- /dev/null +++ b/docs/modules/pmtiles/api-reference/pmtiles-loader.md @@ -0,0 +1,27 @@ +# PMTilesLoader + +The `PMTilesLoader` parses header/metadata from a pmtiles archive + +| Loader | Characteristic | +| --------------------- | --------------------------------------------- | +| File Extension | `.pmtiles` | +| File Type | Binary/Text | +| File Format | [PLY](http://paulbourke.net/dataformats/ply/) | +| Data Format | [Mesh](/docs/specifications/category-mesh) | +| Decoder Type | Synchronous | +| Worker Thread Support | Yes | +| Streaming Support | No | + +## Usage + +```typescript +import {PMTilesLoader} from '@loaders.gl/ply'; +import {load} from '@loaders.gl/core'; + +const data = await load(url, PMTilesLoader, options); +``` + +## Options + +| Option | Type | Default | Description | +| ------ | ---- | ------- | ----------- | diff --git a/docs/modules/pmtiles/api-reference/pmtiles-source.md b/docs/modules/pmtiles/api-reference/pmtiles-source.md new file mode 100644 index 0000000000..dc6ab52a72 --- /dev/null +++ b/docs/modules/pmtiles/api-reference/pmtiles-source.md @@ -0,0 +1,27 @@ +# PMTilesSource + +The `PMTilesSource` parses simple meshes in the Polygon File Format or the Stanford Triangle Format. + +| Loader | Characteristic | +| --------------------- | --------------------------------------------- | +| File Extension | `.ply` | +| File Type | Binary/Text | +| File Format | [PLY](http://paulbourke.net/dataformats/ply/) | +| Data Format | [Mesh](/docs/specifications/category-mesh) | +| Decoder Type | Synchronous | +| Worker Thread Support | Yes | +| Streaming Support | No | + +## Usage + +```typescript +import {PMTilesSource} from '@loaders.gl/ply'; +import {load} from '@loaders.gl/core'; + +const data = await load(url, PMTilesSource, options); +``` + +## Options + +| Option | Type | Default | Description | +| ------ | ---- | ------- | ----------- | diff --git a/docs/modules/pmtiles/formats/pmtiles.md b/docs/modules/pmtiles/formats/pmtiles.md new file mode 100644 index 0000000000..b0ab32770c --- /dev/null +++ b/docs/modules/pmtiles/formats/pmtiles.md @@ -0,0 +1,161 @@ +# PMTiles + +PMTiles is a single-file archive format for tiled data. A PMTiles archive can be hosted on a commodity storage platform such as Amazon S3. + +- https://github.com/protomaps/PMTiles + + +## Versions + +## Version 3 + +- File Structure +97% smaller overhead - Spec version 2 would always issue a 512 kilobyte initial request; version 3 reduces this to 16 kilobytes. What remains the same is that nearly any map tile can be retrieved in at most two additional requests. +- Unlimited metadata - version 2 had a hard cap on the amount of JSON metadata of about 300 kilobytes; version 3 removes this limit. This is essential for tools like tippecanoe to store detailed column statistics. Essential archive information, such as tile type and compression methods, are stored in a binary header separate from application metadata. +- Hilbert tile IDs - tiles internally are addressed by a single 64-bit Hilbert tile ID instead of Z/X/Y. See the blog post on Tile IDs for details. +- Archive ordering - An optional clustered mode enforces that tile contents are laid out in Tile ID order. +- Compressed directories and metadata - Directories used to fetch offsets of tile data consume about 10% the space of those in version 2. See the blog post on compressed directories for details. +- JavaScript +Compression - The TypeScript pmtiles library now includes a decompressor - fflate - to allow reading compressed vector tile archives directly in the browser. This reduces the size and latency of vector tiles by as much as 70%. +- Tile Cancellation - All JavaScript plugins now support tile cancellation, meaning quick zooming across many levels will interrupt the loading of tiles that are never shown. This has a significant effect on the perceived user experience, as tiles at the end of a animation will appear earlier. +- ETag support - clients can detect when files change on static storage by reading the ETag HTTP header. This means that PMTiles-based map applications can update datasets in place at low frequency without running into caching problems. + +## Version 3 Specification + +### File structure + +A PMTiles archive is a single-file archive of square tiles with five main sections: + +1. A fixed-size, 127-byte **Header** starting with `PMTiles` and then the spec version - currently `3` - that contains offsets to the next sections. +2. A root **Directory**, described below. The Header and Root combined must be less than 16,384 bytes. +3. JSON metadata. +4. Optionally, a section of **Leaf Directories**, encoded the same way as the root. +5. The tile data. + +### Entries + +A Directory is a list of `Entries`, in ascending order by `TileId`: + + Entry = (TileId uint64, Offset uint64, Length uint32, RunLength uint32) + +* `TileId` starts at 0 and corresponds to a cumulative position on the series of square Hilbert curves starting at z=0. +* `Offset` is the position of the tile in the file relative to the start of the data section. +* `Length` is the size of the tile in bytes. +* `RunLength` is how many times this tile is repeated: the `TileId=5,RunLength=2` means that tile is present at IDs 5 and 6. +* If `RunLength=0`, the offset/length points to a Leaf Directory where `TileId` is the first entry. + +### Directory Serialization + +Entries are stored in memory as integers, but serialized to disk using these compression steps: + +1. A little-endian varint indicating the # of entries +2. Delta encoding of `TileId` +3. Zeroing of `Offset`: + * `0` if it is equal to the `Offset` + `Length` of the previous entry + * `Offset+1` otherwise +4. Varint encoding of all numbers +5. Columnar ordering: all `TileId`s, all `RunLength`s, all `Length`s, then all `Offset`s +6. Finally, general purpose compression as described by the `Header`'s `InternalCompression` field + +##3 Directory Hierarchy + +* The number of entries in the root directory and leaf directories is up to the implementation. +* However, the compressed size of the header plus root directory is required in v3 to be under **16,384 bytes**. This is to allow latency-optimized clients to prefetch the root directory and guarantee it is complete. A sophisticated writer might need several attempts to optimize this. +* Root size, leaf sizes and depth should be configurable by the user to optimize for different trade-offs: cost, bandwidth, latency. + +### Header Design + +*Certain fields belonging to metadata in v2 are promoted to fixed-size header fields. This allows a map container to be initialized to the desired extent or center without blocking on the JSON metadata, and allows proxies to return well-defined HTTP headers.* + +The `Header` is 127 bytes, with little-endian integer values: + +| offset | description | width | +| ------ | ----------------------------------------------------------------------------------------- | ----- | +| 0 | magic number `PMTiles` | 7 | +| 7 | spec version, currently `3` | 1 | +| 8 | offset of root directory | 8 | +| 16 | length of root directory | 8 | +| 24 | offset of JSON metadata, possibly compressed by `InternalCompression` | 8 | +| 32 | length of JSON metadata | 8 | +| 40 | offset of leaf directories | 8 | +| 48 | length of leaf directories | 8 | +| 56 | offset of tile data | 8 | +| 64 | length of tile data | 8 | +| 72 | # of addressed tiles, 0 if unknown | 8 | +| 80 | # of tile entries, 0 if unknown | 8 | +| 88 | # of tile contents, 0 if unknown | 8 | +| 96 | boolean clustered flag, `0x1` if true | 1 | +| 97 | `InternalCompression` enum (0 = Unknown, 1 = None, 2 = Gzip, 3 = Brotli, 4 = Zstd) | 1 | +| 98 | `TileCompression` enum | 1 | +| 99 | tile type enum (0 = Unknown/Other, 1 = MVT (PBF Vector Tile), 2 = PNG, 3 = JPEG, 4 = WEBP | 1 | +| 100 | min zoom | 1 | +| 101 | max zoom | 1 | +| 102 | min longitude (signed 32-bit integer: longitude * 10,000,000) | 4 | +| 106 | min latitude | 4 | +| 110 | max longitude | 4 | +| 114 | max latitude | 4 | +| 118 | center zoom | 1 | +| 119 | center longitude | 4 | +| 123 | center latitude | 4 | + +### Notes + +* **# of addressed tiles**: the total number of tiles before run-length encoding, i.e. `Sum(RunLength)` over all entries. +* **# of tile entries**: the total number of entries across all directories where `RunLength > 0`. +* **# # of tile contents**: the number of referenced blobs in the tile section, or the unique # of offsets. If the archive is completely deduplicated, this is equal to the # of unique tile contents. If there is no deduplication, this is equal to the number of tile entries above. +* **boolean clustered flag**: if true, blobs in the data section are ordered by Hilbert `TileId`. When writing with deduplication, this means that offsets are either contiguous with the previous offset+length, or refer to a lesser offset. +* **compression enum**: Mandatory, tells the client how to decompress contents as well as provide correct `Content-Encoding` headers to browsers. +* **tile type**: A hint as to the tile contents. Clients and proxies may use this to: + * Automatically determine a visualization method + * provide a conventional MIME type `Content-Type` HTTP header + * Enforce a canonical extension e.g. `.mvt`, `png`, `jpeg`, `.webp` to prevent duplication in caches + +### Organization + +In most cases, the archive should be in the order `Header`, Root Directory, JSON Metadata, Leaf Directories, Tile Data. It is possible to relocate sections other than `Header` arbitrarily, but no current writers/readers take advantage of this. A future design may allow for reverse-ordered archives to enable single-pass writing. + + +## Version 2 + +*Note: this is deprecated in favor of spec version 3.* + +PMTiles is a binary serialization format designed for two main access patterns: over the network, via HTTP 1.1 Byte Serving (`Range:` requests), or via memory-mapped files on disk. **All integer values are little-endian.** + +A PMTiles archive is composed of: +* a fixed-size 512,000 byte header section +* Followed by any number of tiles in arbitrary format +* Optionally followed by any number of *leaf directories* + +### Header +* The header begins with a 2-byte magic number, "PM" +* Followed by 2 bytes, the PMTiles specification version (currently 2). +* Followed by 4 bytes, the length of metadata (M bytes) +* Followed by 2 bytes, the number of entries in the *root directory* (N entries) +* Followed by M bytes of metadata, which **must be a JSON string with bounds, minzoom and maxzoom properties (new in v2)** +* Followed by N * 17 bytes, the root directory. + +### Directory structure + +A directory is a contiguous sequence of 17 byte entries. A directory can have at most 21,845 entries. **A directory must be sorted by Z, X and then Y order (new in v2).** + +An entry consists of: +* 1 byte: the zoom level (Z) of the entry, with the top bit set to 1 instead of 0 to indicate the offset/length points to a leaf directory and not a tile. +* 3 bytes: the X (column) of the entry. +* 3 bytes: the Y (row) of the entry. +* 6 bytes: the offset of where the tile begins in the archive. +* 4 bytes: the length of the tile, in bytes. + +**All leaf directory entries follow non-leaf entries. All leaf directories in a single directory must have the same Z value. (new in v2).** + +### Notes +* A full directory of 21,845 entries holds exactly a complete pyramid with 8 levels, or 1+4+16+64+256+1024+4096+16384. +* A PMTiles archive with less than 21,845 tiles should have a root directory and no leaf directories. +* Multiple tile entries can point to the same offset; this is useful for de-duplicating certain tiles, such as an empty "ocean" tile. +* Analogously, multiple leaf directory entries can point to the same offset; this can avoid inefficiently-packed small leaf directories. +* The tentative media type for PMTiles archives is `application/vnd.pmtiles`. + +### Implementation suggestions + +* PMTiles is designed to make implementing a writer simple. Reserve 512KB, then write all tiles, recording their entry information; then write all leaf directories; finally, rewind to 0 and write the header. +* The order of tile data in the archive is unspecified; an optimized implementation should arrange tiles on a 2D space-filling curve. +* PMTiles readers should cache directory entries by byte offset, not by Z/X/Y. This means that deduplicated leaf directories result in cache hits. \ No newline at end of file diff --git a/examples/website/geospatial/package.json b/examples/website/geospatial/package.json index 4eef7b48fa..8c1235211d 100644 --- a/examples/website/geospatial/package.json +++ b/examples/website/geospatial/package.json @@ -16,6 +16,7 @@ "@loaders.gl/core": "4.0.0-alpha.23", "@loaders.gl/flatgeobuf": "4.0.0-alpha.23", "@loaders.gl/geopackage": "4.0.0-alpha.23", + "@loaders.gl/parquet": "4.0.0-alpha.23", "mapbox-gl": "npm:empty-npm-package@^1.0.0", "maplibre-gl": "^2.4.0", "react": "^18.0.0", diff --git a/examples/website/tiles/app.tsx b/examples/website/tiles/app.tsx new file mode 100644 index 0000000000..e2807052b0 --- /dev/null +++ b/examples/website/tiles/app.tsx @@ -0,0 +1,131 @@ +// loaders.gl, MIT license + +import React from 'react'; +import {createRoot} from 'react-dom/client'; + +import DeckGL from '@deck.gl/react/typed'; +import {MapView} from '@deck.gl/core/typed'; +import {TileLayer} from '@deck.gl/geo-layers/typed'; +import {BitmapLayer, PathLayer} from '@deck.gl/layers/typed'; +import {PMTilesImageSource} from '@loaders.gl/pmtiles'; + +const INITIAL_VIEW_STATE = { + latitude: 47.65, + longitude: 7, + zoom: 4.5, + maxZoom: 20, + maxPitch: 89, + bearing: 0 +}; + +const COPYRIGHT_LICENSE_STYLE = { + position: 'absolute', + right: 0, + bottom: 0, + backgroundColor: 'hsla(0,0%,100%,.5)', + padding: '0 5px', + font: '12px/20px Helvetica Neue,Arial,Helvetica,sans-serif' +}; + +const LINK_STYLE = { + textDecoration: 'none', + color: 'rgba(0,0,0,.75)', + cursor: 'grab' +}; + +/* global window */ +const devicePixelRatio = (typeof window !== 'undefined' && window.devicePixelRatio) || 1; + +function getTooltip({tile}) { + if (tile) { + const {x, y, z} = tile.index; + return `tile: x: ${x}, y: ${y}, z: ${z}`; + } + return null; +} + +// const vectorTileSource = new PMTilesVectorSource({ +// url: "https://r2-public.protomaps.com/protomaps-sample-datasets/nz-buildings-v3.pmtiles", +// attributions: ["© Land Information New Zealand"], +// }); + +const rasterTileSource = new PMTilesImageSource({ + url:"https://r2-public.protomaps.com/protomaps-sample-datasets/terrarium_z9.pmtiles" + // attributions:["https://github.com/tilezen/joerd/blob/master/docs/attribution.md"], + // tileSize: [512,512] +}); + +export default function App({showBorder = false, onTilesLoad = null}) { + const tileLayer = new TileLayer({ + // https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Tile_servers + data: [ + 'https://a.tile.openstreetmap.org/{z}/{x}/{y}.png', + 'https://b.tile.openstreetmap.org/{z}/{x}/{y}.png', + 'https://c.tile.openstreetmap.org/{z}/{x}/{y}.png' + ], + getTileData: rasterTileSource.getTileData, + // Since these OSM tiles support HTTP/2, we can make many concurrent requests + // and we aren't limited by the browser to a certain number per domain. + maxRequests: 20, + + pickable: true, + onViewportLoad: onTilesLoad, + autoHighlight: showBorder, + highlightColor: [60, 60, 60, 40], + // https://wiki.openstreetmap.org/wiki/Zoom_levels + minZoom: 0, + maxZoom: 19, + tileSize: 256, + zoomOffset: devicePixelRatio === 1 ? -1 : 0, + renderSubLayers: props => { + const { + bbox: {west, south, east, north} + } = props.tile; + + return [ + new BitmapLayer(props, { + data: null, + image: props.data, + bounds: [west, south, east, north] + }), + // showBorder && + // new PathLayer({ + // id: `${props.id}-border`, + // data: [ + // [ + // [west, north], + // [west, south], + // [east, south], + // [east, north], + // [west, north] + // ] + // ], + // getPath: d => d, + // getColor: [255, 0, 0], + // widthMinPixels: 4 + // }) + ]; + } + }); + + return ( + +
+ {'© '} + + OpenStreetMap contributors + +
+ + ); +} + +export function renderToDOM(container) { + createRoot(container).render(); +} diff --git a/examples/website/tiles/index.html b/examples/website/tiles/index.html new file mode 100644 index 0000000000..b09a17fedd --- /dev/null +++ b/examples/website/tiles/index.html @@ -0,0 +1,27 @@ + + + + + Tiled Loaders + + + + +
+ + + diff --git a/examples/website/tiles/package.json b/examples/website/tiles/package.json new file mode 100644 index 0000000000..f5d6a3529d --- /dev/null +++ b/examples/website/tiles/package.json @@ -0,0 +1,31 @@ +{ + "private": true, + "name": "geospatial-loaders-example", + "description": "Minimal example of using loaders.gl with vite.", + "version": "0.0.0", + "license": "MIT", + "scripts": { + "start": "vite", + "build": "tsc && vite build", + "serve": "vite preview" + }, + "dependencies": { + "@deck.gl/core": "^8.9.28", + "@deck.gl/layers": "^8.9.28", + "@deck.gl/geo-layers": "^8.9.28", + "@deck.gl/mesh-layers": "^8.9.28", + "@deck.gl/extensions": "8.9.28", + "@deck.gl/react": "^8.9.28", + "@loaders.gl/core": "4.0.0-alpha.26", + "mapbox-gl": "npm:empty-npm-package@^1.0.0", + "maplibre-gl": "^2.4.0", + "react": "^18.0.0", + "react-dom": "^18.0.0", + "react-map-gl": "^7.0.0", + "styled-components": "^4.2.0" + }, + "devDependencies": { + "typescript": "^5.0.4", + "vite": "^4.4.9" + } +} diff --git a/examples/website/geospatial/tsconfig.json b/examples/website/tiles/tsconfig.json similarity index 94% rename from examples/website/geospatial/tsconfig.json rename to examples/website/tiles/tsconfig.json index 49a2fb115e..f614f74e8a 100644 --- a/examples/website/geospatial/tsconfig.json +++ b/examples/website/tiles/tsconfig.json @@ -16,5 +16,5 @@ "noEmit": true, "jsx": "react-jsx" }, - "include": ["./app.js"] + "include": ["."] } diff --git a/examples/website/geospatial/vite.config.ts b/examples/website/tiles/vite.config.ts similarity index 100% rename from examples/website/geospatial/vite.config.ts rename to examples/website/tiles/vite.config.ts diff --git a/modules/core/src/lib/filesystems/read-array-buffer.ts b/modules/core/src/lib/filesystems/read-array-buffer.ts index 6ac17582cd..0cdc1cd4a6 100644 --- a/modules/core/src/lib/filesystems/read-array-buffer.ts +++ b/modules/core/src/lib/filesystems/read-array-buffer.ts @@ -1,5 +1,11 @@ // Random-Access read +/** + * Read a slice of a Blob or File, without loading the entire file into memory + * The trick when reading File objects is to read successive "slices" of the File + * Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields + * @param file to read + */ export async function readArrayBuffer( file: Blob | ArrayBuffer | any, start: number, @@ -11,21 +17,3 @@ export async function readArrayBuffer( } return await file.read(start, start + length); } - -/** - * Read a slice of a Blob or File, without loading the entire file into memory - * The trick when reading File objects is to read successive "slices" of the File - * Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields - * Actually reading from file happens in `readAsArrayBuffer` - * @param blob to read - export async function readBlob(blob: Blob): Promise { - return await new Promise((resolve, reject) => { - const fileReader = new FileReader(); - fileReader.onload = (event: ProgressEvent) => - resolve(event?.target?.result as ArrayBuffer); - // TODO - reject with a proper Error - fileReader.onerror = (error: ProgressEvent) => reject(error); - fileReader.readAsArrayBuffer(blob); - }); -} -*/ diff --git a/modules/loader-utils/src/index.ts b/modules/loader-utils/src/index.ts index ac914d5aa8..7828c7527b 100644 --- a/modules/loader-utils/src/index.ts +++ b/modules/loader-utils/src/index.ts @@ -118,7 +118,8 @@ export {fs}; import * as stream from './lib/node/stream'; export {stream}; -// EXPERIMENTAL +// EXPERIMENTAL: FILE SYSTEMS + export type {FileSystem, RandomAccessReadFileSystem} from './lib/filesystems/filesystem'; export {NodeFileSystem as _NodeFileSystem} from './lib/filesystems/node-filesystem'; @@ -134,3 +135,22 @@ export {makeReadableFile} from './lib/filesystems/readable-file'; export type {WritableFile} from './lib/filesystems/writable-file'; export {makeWritableFile} from './lib/filesystems/writable-file'; + +// EXPERIMENTAL: DATA SOURCES +export type {DataSourceProps} from './lib/sources/data-source'; +export {DataSource} from './lib/sources/data-source'; + +export type {ImageType} from './lib/sources/image-source'; +export type {ImageSourceProps, ImageSourceMetadata} from './lib/sources/image-source'; +export type {GetImageParameters} from './lib/sources/image-source'; +export {ImageSource} from './lib/sources/image-source'; + +export type { + TileSourceProps, + TileSourceMetadata, + GetTileParameters +} from './lib/sources/tile-source'; +export {TileSource} from './lib/sources/tile-source'; + +export type {ImageTileSourceProps} from './lib/sources/image-tile-source'; +export {ImageTileSource} from './lib/sources/image-tile-source'; diff --git a/modules/wms/src/lib/sources/data-source.ts b/modules/loader-utils/src/lib/sources/data-source.ts similarity index 97% rename from modules/wms/src/lib/sources/data-source.ts rename to modules/loader-utils/src/lib/sources/data-source.ts index 7283d2e3ea..ddc2b52c94 100644 --- a/modules/wms/src/lib/sources/data-source.ts +++ b/modules/loader-utils/src/lib/sources/data-source.ts @@ -2,6 +2,7 @@ import type {LoaderOptions} from '@loaders.gl/loader-utils'; +/** Common properties for all data sources */ export type DataSourceProps = { /** LoaderOptions provide an option to override `fetch`. Will also be passed to any sub loaders */ loadOptions?: LoaderOptions; diff --git a/modules/wms/src/lib/sources/image-source.ts b/modules/loader-utils/src/lib/sources/image-source.ts similarity index 88% rename from modules/wms/src/lib/sources/image-source.ts rename to modules/loader-utils/src/lib/sources/image-source.ts index 92505715db..4f8bf7b7d6 100644 --- a/modules/wms/src/lib/sources/image-source.ts +++ b/modules/loader-utils/src/lib/sources/image-source.ts @@ -1,9 +1,19 @@ // loaders.gl, MIT license -import type {ImageType} from '@loaders.gl/images'; import type {DataSourceProps} from './data-source'; import {DataSource} from './data-source'; +/** data images @note duplicates definition in images/schema to avoid circular deps */ +export type ImageDataType = { + data: Uint8Array; + width: number; + height: number; + compressed?: boolean; +}; + +/** Supported Image Types @note duplicates definition in images/schema to avoid circular deps */ +export type ImageType = ImageBitmap | ImageDataType | HTMLImageElement; + /** * Normalized capabilities of an Image service * @example diff --git a/modules/loader-utils/src/lib/sources/image-tile-source.ts b/modules/loader-utils/src/lib/sources/image-tile-source.ts new file mode 100644 index 0000000000..5d9f6133c2 --- /dev/null +++ b/modules/loader-utils/src/lib/sources/image-tile-source.ts @@ -0,0 +1,22 @@ +// loaders.gl, MIT license + +import type {ImageType} from './image-source'; +import {TileSource, TileSourceProps} from './tile-source'; +import {GetTileParameters, TileLoadParameters} from './tile-source'; + +export type ImageTileSourceProps = TileSourceProps; + +/** + * MapTileSource - data sources that allow data to be queried by (geospatial) tile + * @note If geospatial, bounding box is expected to be in web mercator coordinates + */ +export abstract class ImageTileSource< + PropsT extends ImageTileSourceProps = ImageTileSourceProps +> extends TileSource { + constructor(props: PropsT) { + super(props); + this.getTileData = this.getTileData.bind(this); + } + abstract getTile(parameters: GetTileParameters): Promise; + abstract getTileData(props: TileLoadParameters): Promise; +} diff --git a/modules/wms/src/lib/sources/tile-source.ts b/modules/loader-utils/src/lib/sources/tile-source.ts similarity index 56% rename from modules/wms/src/lib/sources/tile-source.ts rename to modules/loader-utils/src/lib/sources/tile-source.ts index e7ec113662..82e21a0c4a 100644 --- a/modules/wms/src/lib/sources/tile-source.ts +++ b/modules/loader-utils/src/lib/sources/tile-source.ts @@ -1,12 +1,13 @@ // loaders.gl, MIT license -import type {ImageType} from '@loaders.gl/images'; +// import type {ImageType} from '@loaders.gl/images'; import {DataSource, DataSourceProps} from './data-source'; /** * Normalized capabilities of an Image service + * Sources are expected to normalize the response to capabilities * @example - * The WMSService will normalize the response to the WMS `GetCapabilities` data structure extracted from WMS XML response + * A WMS service would normalize the response to the WMS `GetCapabilities` data structure extracted from WMS XML response * into an TileSourceMetadata. */ export type TileSourceMetadata = { @@ -23,6 +24,9 @@ export type TileSourceMetadata = { }; }; +/** + * Description of one data layer in the image source + */ export type TileSourceLayer = { name: string; title?: string; @@ -31,6 +35,9 @@ export type TileSourceLayer = { layers: TileSourceLayer[]; }; +/** + * Generic parameters for requesting an image from an image source + */ export type GetTileParameters = { /** Layers to render */ layers: string | string[]; @@ -48,7 +55,25 @@ export type GetTileParameters = { format?: 'image/png'; }; -type TileSourceProps = DataSourceProps; +export type TileLoadParameters = { + index: {x: number; y: number; z: number}; + id: string; + bbox: TileBoundingBox; + zoom?: number; + url?: string | null; + signal?: AbortSignal; + userData?: Record; +}; + +/** deck.gl compatible bounding box */ +export type TileBoundingBox = NonGeoBoundingBox | GeoBoundingBox; +export type GeoBoundingBox = {west: number; north: number; east: number; south: number}; +export type NonGeoBoundingBox = {left: number; top: number; right: number; bottom: number}; + +/** + * Props for a TileSource + */ +export type TileSourceProps = DataSourceProps; /** * MapTileSource - data sources that allow data to be queried by (geospatial) extents @@ -57,5 +82,8 @@ type TileSourceProps = DataSourceProps; */ export abstract class TileSource extends DataSource { abstract getMetadata(): Promise; - abstract getTile(parameters: GetTileParameters): Promise; + /** Flat parameters */ + abstract getTile(parameters: GetTileParameters): Promise; + /** deck.gl compatible method */ + abstract getTileData(parameters: TileLoadParameters): Promise; } diff --git a/modules/wms/src/lib/sources/utils/utils.ts b/modules/loader-utils/src/lib/sources/utils/utils.ts similarity index 100% rename from modules/wms/src/lib/sources/utils/utils.ts rename to modules/loader-utils/src/lib/sources/utils/utils.ts diff --git a/modules/loader-utils/test/lib/file-provider/file-handle-file.spec.js b/modules/loader-utils/test/lib/file-provider/file-handle-file.spec.js index d76da23ab5..fe497aea08 100644 --- a/modules/loader-utils/test/lib/file-provider/file-handle-file.spec.js +++ b/modules/loader-utils/test/lib/file-provider/file-handle-file.spec.js @@ -9,7 +9,7 @@ const SLPKUrl = 'modules/i3s/test/data/DA12_subset.slpk'; test('FileHandleFile#slice', async (t) => { if (!isBrowser) { const provider = await FileHandleFile.from(SLPKUrl); - t.equals(Buffer.from(await provider.slice(0n, 4n)).compare(signature), 0); + t.equals(Buffer.from(await provider.slice(0n, 4n)).compare(Buffer.from(signature)), 0); } t.end(); }); diff --git a/modules/pmtiles/README.md b/modules/pmtiles/README.md new file mode 100644 index 0000000000..b4ce9f83aa --- /dev/null +++ b/modules/pmtiles/README.md @@ -0,0 +1,7 @@ +# @loaders.gl/pmtiles + +[loaders.gl](https://loaders.gl/docs) is a collection of framework-independent 3D and geospatial parsers and encoders. + +This module contains loaders for the pmtiles format. + +For documentation please visit the [website](https://loaders.gl). diff --git a/modules/pmtiles/package.json b/modules/pmtiles/package.json new file mode 100644 index 0000000000..92dbf57f81 --- /dev/null +++ b/modules/pmtiles/package.json @@ -0,0 +1,40 @@ +{ + "name": "@loaders.gl/pmtiles", + "version": "3.4.0-alpha.3", + "description": "Framework-independent loader for the pmtiles format", + "license": "MIT", + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "https://github.com/visgl/loaders.gl" + }, + "keywords": [ + "webgl", + "loader", + "3d", + "mesh", + "point cloud", + "PCD" + ], + "types": "dist/index.d.ts", + "main": "dist/es5/index.js", + "module": "dist/esm/index.js", + "sideEffects": false, + "files": [ + "src", + "dist", + "README.md" + ], + "scripts": { + "pre-build": "npm run build-bundle", + "build-bundle": "esbuild src/bundle.ts --bundle --outfile=dist/dist.min.js" + }, + "dependencies": { + "@loaders.gl/loader-utils": "3.4.0-alpha.3", + "@loaders.gl/schema": "3.4.0-alpha.3", + "pmtiles": "^2.7.2" + }, + "gitHead": "c95a4ff72512668a93d9041ce8636bac09333fd5" +} diff --git a/modules/pmtiles/src/bundle.ts b/modules/pmtiles/src/bundle.ts new file mode 100644 index 0000000000..0db0c48b55 --- /dev/null +++ b/modules/pmtiles/src/bundle.ts @@ -0,0 +1,4 @@ +// @ts-nocheck +const moduleExports = require('./index'); +globalThis.loaders = globalThis.loaders || {}; +module.exports = Object.assign(globalThis.loaders, moduleExports); diff --git a/modules/pmtiles/src/index.ts b/modules/pmtiles/src/index.ts new file mode 100644 index 0000000000..576669128e --- /dev/null +++ b/modules/pmtiles/src/index.ts @@ -0,0 +1,7 @@ +// loaders.gl, MIT license + +export type {PMTilesImageSourceProps} from './pmtiles-source'; +export {PMTilesImageSource} from './pmtiles-source'; + +// EXPERIMENTAL +export {PMTilesLoader} from './pmtiles-loader'; diff --git a/modules/pmtiles/src/lib/parse-pmtiles.ts b/modules/pmtiles/src/lib/parse-pmtiles.ts new file mode 100644 index 0000000000..f15b662462 --- /dev/null +++ b/modules/pmtiles/src/lib/parse-pmtiles.ts @@ -0,0 +1,87 @@ +import {Source, PMTiles, Header, TileType} from 'pmtiles'; +// import {Source, RangeResponse, PMTiles, Header, Compression, TileType} from 'pmtiles'; + +// export enum Compression { +// Unknown = 0, +// None = 1, +// Gzip = 2, +// Brotli = 3, +// Zstd = 4, +// } + +// export enum TileType { +// Unknown = 0, +// Mvt = 1, +// Png = 2, +// Jpeg = 3, +// Webp = 4, +// } + +export interface PMTilesMetadata { + format: 'pmtiles'; + /** PMTiles format specific header */ + formatHeader?: Header; + version: number; + tileType: TileType; + minZoom: number; + maxZoom: number; + minLon: number; + minLat: number; + maxLon: number; + maxLat: number; + centerZoom: number; + centerLon: number; + centerLat: number; + etag?: string; + // Addition + metadata: unknown; +} + +export type ParsePMTilesOptions = { + tileZxy?: [number, number, number]; +}; + +export async function loadPMTilesHeader(source: Source): Promise { + const pmTiles = new PMTiles(source); + const header = await pmTiles.getHeader(); + const metadata = await pmTiles.getMetadata(); + return parsePMTilesMetadata(header, metadata); +} + +export async function loadPMTile( + source: Source, + options: ParsePMTilesOptions +): Promise { + const pmTiles = new PMTiles(source); + if (!options.tileZxy) { + throw new Error('tile zxy missing'); + } + const [z, x, y] = options.tileZxy; + const tile = await pmTiles.getZxy(z, x, y); + return tile?.data; +} + +export async function parsePMTilesMetadata( + header: Header, + metadata: unknown +): Promise { + return { + // The assumption is that this is a tileJSON style metadata generated by e.g. tippecanone + metadata, + format: 'pmtiles', + version: header.specVersion, + tileType: header.tileType, + minZoom: header.minZoom, + maxZoom: header.maxZoom, + minLon: header.minLon, + minLat: header.minLat, + maxLon: header.maxLon, + maxLat: header.maxLat, + centerZoom: header.centerZoom, + centerLon: header.centerLon, + centerLat: header.centerLat, + etag: header.etag, + + formatHeader: header + }; +} diff --git a/modules/pmtiles/src/lib/sources.ts b/modules/pmtiles/src/lib/sources.ts new file mode 100644 index 0000000000..ab45a1f30d --- /dev/null +++ b/modules/pmtiles/src/lib/sources.ts @@ -0,0 +1,150 @@ +/* +import {fetchFile} from '@loaders.gl/core'; + +import {Source as PMTilesSource, RangeResponse} from 'pmtiles'; + +/** @note "source" here is a PMTiles library type, referring to * +export function makeSource(data: string | Blob, fetch?) { + if (typeof data === 'string') { + return new FetchSource(data, fetch); + } + if (data instanceof Blob) { + const url = ''; + return new BlobSource(data, url); + } +} + +export class BlobSource implements PMTilesSource { + blob: Blob; + key: string; + + constructor(blob: Blob, key: string) { + this.blob = blob; + this.key = key; + } + + // TODO - how is this used? + getKey() { + return this.blob.url || ''; + } + + async getBytes(offset: number, length: number, signal?: AbortSignal): Promise { + const data = await this.blob.arrayBuffer(); + return { + data + // etag: response.headers.get('ETag') || undefined, + // cacheControl: response.headers.get('Cache-Control') || undefined, + // expires: response.headers.get('Expires') || undefined + }; + } +} + +export class FetchSource implements PMTilesSource { + url: string; + fetch; + + constructor(url: string, fetch = fetchFile) { + this.url = url; + this.fetch = fetch; + } + + // TODO - how is this used? + getKey() { + return this.url; + } + + async getBytes(offset: number, length: number, signal?: AbortSignal): Promise { + let controller; + if (!signal) { + // ToDO why is it so important to abort in case 200? + // TODO check this works or assert 206 + controller = new AbortController(); + signal = controller.signal; + } + + let response = await fetch(this.url, { + signal, + headers: {Range: `bytes=${offset}-${offset + length - 1}`} + }); + + switch (response.status) { + case 206: // Partial Content success + // This is the expected success code for a range request + break; + + case 200: + // some well-behaved backends, e.g. DigitalOcean CDN, respond with 200 instead of 206 + // but we also need to detect no support for Byte Serving which is returning the whole file + const content_length = response.headers.get('Content-Length'); + if (!content_length || Number(content_length) > length) { + if (controller) { + controller.abort(); + } + throw Error( + 'content-length header missing or exceeding request. Server must support HTTP Byte Serving.' + ); + } + + case 416: // "Range Not Satisfiable" + // some HTTP servers don't accept ranges beyond the end of the resource. + // Retry with the exact length + // TODO: can return 416 with offset > 0 if content changed, which will have a blank etag. + // See https://github.com/protomaps/PMTiles/issues/90 + if (offset === 0) { + const content_range = response.headers.get('Content-Range'); + if (!content_range || !content_range.startsWith('bytes *')) { + throw Error('Missing content-length on 416 response'); + } + const actual_length = Number(content_range.substr(8)); + response = await fetch(this.url, { + signal, + headers: {Range: `bytes=0-${actual_length - 1}`} + }); + } + break; + + default: + if (response.status >= 300) { + throw Error(`Bad response code: ${response.status}`); + } + } + + const data = await response.arrayBuffer(); + return { + data, + etag: response.headers.get('ETag') || undefined, + cacheControl: response.headers.get('Cache-Control') || undefined, + expires: response.headers.get('Expires') || undefined + }; + } +} + +/* +class TestNodeFileSource implements Source { + buffer: ArrayBuffer; + path: string; + key: string; + etag?: string; + + constructor(path: string, key: string) { + this.path = path; + this.buffer = fs.readFileSync(path); + this.key = key; + } + + getKey() { + return this.key; + } + + replaceData(path: string) { + this.path = path; + this.buffer = fs.readFileSync(path); + } + + async getBytes(offset: number, length: number): Promise { + const slice = new Uint8Array(this.buffer.slice(offset, offset + length)) + .buffer; + return { data: slice, etag: this.etag }; + } +} +*/ diff --git a/modules/pmtiles/src/pmtiles-loader.ts b/modules/pmtiles/src/pmtiles-loader.ts new file mode 100644 index 0000000000..c46dc2991f --- /dev/null +++ b/modules/pmtiles/src/pmtiles-loader.ts @@ -0,0 +1,23 @@ +import type {Loader} from '@loaders.gl/loader-utils'; + +// __VERSION__ is injected by babel-plugin-version-inline +// @ts-ignore TS2304: Cannot find name '__VERSION__'. +const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest'; + +/** + * Worker loader for PCD - Point Cloud Data + */ +export const PMTilesLoader = { + name: 'PMTiles', + id: 'pmtiles', + module: 'pmtiles', + version: VERSION, + worker: true, + extensions: ['pmtilse'], + mimeTypes: ['application/octet-stream'], + options: { + pmtiles: {} + } +}; + +export const _typecheckPCDLoader: Loader = PMTilesLoader; diff --git a/modules/pmtiles/src/pmtiles-source.ts b/modules/pmtiles/src/pmtiles-source.ts new file mode 100644 index 0000000000..64add62d34 --- /dev/null +++ b/modules/pmtiles/src/pmtiles-source.ts @@ -0,0 +1,42 @@ +import {GetTileParameters, ImageType, ImageTileSourceProps} from '@loaders.gl/loader-utils'; +import {ImageTileSource} from '@loaders.gl/loader-utils'; +import {ImageLoader} from '@loaders.gl/images'; + +import {PMTiles} from 'pmtiles'; +// import {PMTiles, DecompressFunc} from 'pmtiles'; +import {parsePMTilesMetadata} from './lib/parse-pmtiles'; +import {TileLoadParameters} from 'modules/loader-utils/src/lib/sources/tile-source'; + +export type PMTilesImageSourceProps = ImageTileSourceProps & { + url: string; +}; + +export class PMTilesImageSource extends ImageTileSource { + pmtiles: PMTiles; + + constructor(props: PMTilesImageSourceProps) { + super(props); + this.pmtiles = new PMTiles(props.url); + } + + async getMetadata(): Promise { + const header = await this.pmtiles.getHeader(); + const metadata = await this.pmtiles.getMetadata(); + return parsePMTilesMetadata(header, metadata); + } + + async getTile(tileParams: GetTileParameters): Promise { + const {x, y, zoom: z} = tileParams; + const rangeResponse = await this.pmtiles.getZxy(z, x, y); + const arrayBuffer = rangeResponse?.data; + if (!arrayBuffer) { + return null; + } + return ImageLoader.parse(arrayBuffer, this.loadOptions); + } + + async getTileData(tileParams: TileLoadParameters): Promise { + const {x, y, z} = tileParams.index; + return await this.getTile({x, y, zoom: z, layers: []}); + } +} diff --git a/modules/pmtiles/test/data/README.md b/modules/pmtiles/test/data/README.md new file mode 100644 index 0000000000..316f59bc65 --- /dev/null +++ b/modules/pmtiles/test/data/README.md @@ -0,0 +1,5 @@ +# Attributions for Sample Files + +## pmtiles + +https://github.com/protomaps/PMTiles/tree/main/js/test/data under BSD-3 license \ No newline at end of file diff --git a/modules/pmtiles/test/data/empty.pmtiles b/modules/pmtiles/test/data/empty.pmtiles new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/pmtiles/test/data/invalid.pmtiles b/modules/pmtiles/test/data/invalid.pmtiles new file mode 100644 index 0000000000..f2b720ba30 --- /dev/null +++ b/modules/pmtiles/test/data/invalid.pmtiles @@ -0,0 +1 @@ +This is an invalid tile archive, a test case to make sure that the code throws an error, but it needs to be the minimum size to pass the first test diff --git a/modules/pmtiles/test/data/invalid_v4.pmtiles b/modules/pmtiles/test/data/invalid_v4.pmtiles new file mode 100644 index 0000000000..1871cb2759 Binary files /dev/null and b/modules/pmtiles/test/data/invalid_v4.pmtiles differ diff --git a/modules/pmtiles/test/data/test_fixture_1.pmtiles b/modules/pmtiles/test/data/test_fixture_1.pmtiles new file mode 100644 index 0000000000..c86db1f27b Binary files /dev/null and b/modules/pmtiles/test/data/test_fixture_1.pmtiles differ diff --git a/modules/pmtiles/test/data/test_fixture_2.pmtiles b/modules/pmtiles/test/data/test_fixture_2.pmtiles new file mode 100644 index 0000000000..cb19dd5f11 Binary files /dev/null and b/modules/pmtiles/test/data/test_fixture_2.pmtiles differ diff --git a/modules/pmtiles/test/data/v3/stamen_toner(raster)CC-BY+ODbL_z3.pmtiles b/modules/pmtiles/test/data/v3/stamen_toner(raster)CC-BY+ODbL_z3.pmtiles new file mode 100644 index 0000000000..8d170ca89e Binary files /dev/null and b/modules/pmtiles/test/data/v3/stamen_toner(raster)CC-BY+ODbL_z3.pmtiles differ diff --git a/modules/pmtiles/test/index.ts b/modules/pmtiles/test/index.ts new file mode 100644 index 0000000000..527e907dd8 --- /dev/null +++ b/modules/pmtiles/test/index.ts @@ -0,0 +1,2 @@ +import './pmtiles-loader.spec'; +import './pmtiles-source.spec'; diff --git a/modules/pmtiles/test/pmtiles-loader.spec.ts b/modules/pmtiles/test/pmtiles-loader.spec.ts new file mode 100644 index 0000000000..1ee4b790f0 --- /dev/null +++ b/modules/pmtiles/test/pmtiles-loader.spec.ts @@ -0,0 +1,186 @@ +/* eslint-disable max-len */ +import test from 'tape-promise/tape'; +import {validateLoader} from 'test/common/conformance'; + +import {PMTilesLoader} from '@loaders.gl/pmtiles'; +// import {PMTiles, SharedPromiseCache} from 'pmtiles'; + +test('PMTilesLoader#loader conformance', (t) => { + validateLoader(t, PMTilesLoader, 'PMTilesLoader'); + t.end(); +}); + +/* +// echo '{"type":"Polygon","coordinates":[[[0,0],[0,1],[1,1],[1,0],[0,0]]]}' | ./tippecanoe -zg -o test_fixture_1.pmtiles +test('cache getHeader', async (t) => { + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1'); + const cache = new SharedPromiseCache(); + const header = await cache.getHeader(source); + t.strictEqual(header.rootDirectoryOffset, 127); + t.strictEqual(header.rootDirectoryLength, 25); + t.strictEqual(header.jsonMetadataOffset, 152); + t.strictEqual(header.jsonMetadataLength, 247); + t.strictEqual(header.leafDirectoryOffset, 0); + t.strictEqual(header.leafDirectoryLength, 0); + t.strictEqual(header.tileDataOffset, 399); + t.strictEqual(header.tileDataLength, 69); + t.strictEqual(header.numAddressedTiles, 1); + t.strictEqual(header.numTileEntries, 1); + t.strictEqual(header.numTileContents, 1); + t.strictEqual(header.clustered, false); + t.strictEqual(header.internalCompression, 2); + t.strictEqual(header.tileCompression, 2); + t.strictEqual(header.tileType, 1); + t.strictEqual(header.minZoom, 0); + t.strictEqual(header.maxZoom, 0); + t.strictEqual(header.minLon, 0); + t.strictEqual(header.minLat, 0); + // t.strictEqual(header.maxLon,1); // TODO fix me + t.strictEqual(header.maxLat, 1); +}); + +test('cache check against empty', async (t) => { + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/empty.pmtiles', '1'); + const cache = new SharedPromiseCache(); + t.rejects(async () => { + await cache.getHeader(source); + }); +}); + +test('cache check magic number', async (t) => { + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/invalid.pmtiles', '1'); + const cache = new SharedPromiseCache(); + t.rejects(async () => { + await cache.getHeader(source); + }); +}); + +test('cache check future spec version', async (t) => { + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/invalid_v4.pmtiles', '1'); + const cache = new SharedPromiseCache(); + t.rejects(async () => { + await cache.getHeader(source); + }); +}); + +test('cache getDirectory', async (t) => { + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1'); + + let cache = new SharedPromiseCache(6400, false); + let header = await cache.getHeader(source); + t.strictEqual(cache.cache.size, 1); + + cache = new SharedPromiseCache(6400, true); + header = await cache.getHeader(source); + + // prepopulates the root directory + t.strictEqual(cache.cache.size, 2); + + const directory = await cache.getDirectory( + source, + header.rootDirectoryOffset, + header.rootDirectoryLength, + header + ); + t.strictEqual(directory.length, 1); + t.strictEqual(directory[0].tileId, 0); + t.strictEqual(directory[0].offset, 0); + t.strictEqual(directory[0].length, 69); + t.strictEqual(directory[0].runLength, 1); + + for (const v of cache.cache.values()) { + t.ok(v.lastUsed > 0); + } +}); + +test('multiple sources in a single cache', async (t) => { + const cache = new SharedPromiseCache(); + const source1 = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1'); + const source2 = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '2'); + await cache.getHeader(source1); + t.strictEqual(cache.cache.size, 2); + await cache.getHeader(source2); + t.strictEqual(cache.cache.size, 4); +}); + +test('etags are part of key', async (t) => { + const cache = new SharedPromiseCache(6400, false); + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1'); + source.etag = 'etag_1'; + let header = await cache.getHeader(source); + t.strictEqual(header.etag, 'etag_1'); + + source.etag = 'etag_2'; + + t.rejects(async () => { + await cache.getDirectory( + source, + header.rootDirectoryOffset, + header.rootDirectoryLength, + header + ); + }); + + cache.invalidate(source, 'etag_2'); + header = await cache.getHeader(source); + t.ok( + await cache.getDirectory(source, header.rootDirectoryOffset, header.rootDirectoryLength, header) + ); +}); + +test.skip('soft failure on etag weirdness', async (t) => { + const cache = new SharedPromiseCache(6400, false); + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1'); + source.etag = 'etag_1'; + let header = await cache.getHeader(source); + t.strictEqual(header.etag, 'etag_1'); + + source.etag = 'etag_2'; + + t.rejects(async () => { + await cache.getDirectory( + source, + header.rootDirectoryOffset, + header.rootDirectoryLength, + header + ); + }); + + source.etag = 'etag_1'; + cache.invalidate(source, 'etag_2'); + + header = await cache.getHeader(source); + t.strictEqual(header.etag, undefined); +}); + +test('cache pruning by byte size', async (t) => { + const cache = new SharedPromiseCache(2, false); + cache.cache.set('0', {lastUsed: 0, data: Promise.resolve([])}); + cache.cache.set('1', {lastUsed: 1, data: Promise.resolve([])}); + cache.cache.set('2', {lastUsed: 2, data: Promise.resolve([])}); + cache.prune(); + t.strictEqual(cache.cache.size, 2); + t.ok(cache.cache.get('2')); + t.ok(cache.cache.get('1')); + t.ok(!cache.cache.get('0')); +}); + +test('pmtiles get metadata', async (t) => { + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1'); + const p = new PMTiles(source); + const metadata = await p.getMetadata(); + t.ok(metadata.name); +}); + +// echo '{"type":"Polygon","coordinates":[[[0,0],[0,1],[1,0],[0,0]]]}' | ./tippecanoe -zg -o test_fixture_2.pmtiles +test('pmtiles handle retries', async (t) => { + const source = new TestFileSource('@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles', '1'); + source.etag = '1'; + const p = new PMTiles(source); + const metadata = await p.getMetadata(); + t.ok(metadata.name); + source.etag = '2'; + source.replaceData('@loaders.gl/pmtiles/test/data/test_fixture_2.pmtiles'); + t.ok(await p.getZxy(0, 0, 0)); +}); +*/ diff --git a/modules/pmtiles/test/pmtiles-source.spec.ts b/modules/pmtiles/test/pmtiles-source.spec.ts new file mode 100644 index 0000000000..563bc314b5 --- /dev/null +++ b/modules/pmtiles/test/pmtiles-source.spec.ts @@ -0,0 +1,12 @@ +/* eslint-disable max-len */ +import test from 'tape-promise/tape'; +import {PMTilesImageSource} from '@loaders.gl/pmtiles'; + +test('cache getHeader', async (t) => { + const source = new PMTilesImageSource({ + url: '@loaders.gl/pmtiles/test/data/test_fixture_1.pmtiles' + }); // , '1'); + t.ok(source); +}); + +// TBA diff --git a/modules/pmtiles/tsconfig.json b/modules/pmtiles/tsconfig.json new file mode 100644 index 0000000000..4e11b7c942 --- /dev/null +++ b/modules/pmtiles/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.module.json", + "include": ["src/**/*"], + "exclude": ["node_modules"], + "compilerOptions": { + "composite": true, + "rootDir": "src", + "outDir": "dist" + }, + "references": [ + {"path": "../loader-utils"}, + {"path": "../schema"} + ] +} diff --git a/modules/schema/src/types/category-image.ts b/modules/schema/src/types/category-image.ts index ee327abc25..8f8acab8f7 100644 --- a/modules/schema/src/types/category-image.ts +++ b/modules/schema/src/types/category-image.ts @@ -11,7 +11,7 @@ export type ImageDataType = { /** * Supported Image Types */ -export type ImageType = ImageBitmap | typeof Image | ImageDataType; +export type ImageType = ImageBitmap | ImageDataType | HTMLImageElement; /** * Image type string used to control or determine the type of images returned from ImageLoader diff --git a/modules/wms/src/index.ts b/modules/wms/src/index.ts index 0bfa30359d..ccfeaffab8 100644 --- a/modules/wms/src/index.ts +++ b/modules/wms/src/index.ts @@ -54,8 +54,6 @@ export {GMLLoader as _GMLLoader} from './gml-loader'; // EXPERIMENTAL: DATA SOURCES export type {ImageType} from '@loaders.gl/images'; -export type {ImageSourceProps, ImageSourceMetadata} from './lib/sources/image-source'; -export {ImageSource} from './lib/sources/image-source'; export type {ImageServiceType} from './lib/create-image-source'; export {createImageSource} from './lib/create-image-source'; @@ -82,3 +80,6 @@ export {ArcGISImageServer as _ArcGISImageServer} from './lib/services/arcgis/arc /** @deprecated Use WMSCapabilitiesLoaderOptions */ export type {WMSCapabilitiesLoaderOptions as WMSLoaderOptions} from './wms-capabilities-loader'; + +// TODO - restore once deck.gl has been udpated +export {ImageSource} from '@loaders.gl/loader-utils'; diff --git a/modules/wms/src/lib/create-image-source.ts b/modules/wms/src/lib/create-image-source.ts index b8bd402c72..12fd2abc15 100644 --- a/modules/wms/src/lib/create-image-source.ts +++ b/modules/wms/src/lib/create-image-source.ts @@ -1,6 +1,6 @@ // loaders.gl, MIT license -import {ImageSource} from './sources/image-source'; +import {ImageSource} from '@loaders.gl/loader-utils'; import {ImageService, ImageServiceProps} from './services/generic/image-service'; import type {WMSServiceProps} from './services/ogc/wms-service'; import {WMSService} from './services/ogc/wms-service'; diff --git a/modules/wms/src/lib/services/arcgis/arcgis-image-service.ts b/modules/wms/src/lib/services/arcgis/arcgis-image-service.ts index ac63f9109c..7a4d090f19 100644 --- a/modules/wms/src/lib/services/arcgis/arcgis-image-service.ts +++ b/modules/wms/src/lib/services/arcgis/arcgis-image-service.ts @@ -1,9 +1,9 @@ // loaders.gl, MIT license import {ImageType} from '@loaders.gl/images'; -import type {ImageSourceMetadata, GetImageParameters} from '../../sources/image-source'; -import type {ImageSourceProps} from '../../sources/image-source'; -import {ImageSource} from '../../sources/image-source'; +import type {ImageSourceMetadata, GetImageParameters} from '@loaders.gl/loader-utils'; +import type {ImageSourceProps} from '@loaders.gl/loader-utils'; +import {ImageSource} from '@loaders.gl/loader-utils'; export type ArcGISImageServerProps = ImageSourceProps & { url: string; diff --git a/modules/wms/src/lib/services/generic/image-service.ts b/modules/wms/src/lib/services/generic/image-service.ts index 9d26be6b08..ac9d023e3d 100644 --- a/modules/wms/src/lib/services/generic/image-service.ts +++ b/modules/wms/src/lib/services/generic/image-service.ts @@ -4,8 +4,8 @@ import {LoaderOptions} from '@loaders.gl/loader-utils'; import type {ImageType} from '@loaders.gl/images'; import {ImageLoader} from '@loaders.gl/images'; -import type {ImageSourceMetadata, GetImageParameters} from '../../sources/image-source'; -import {ImageSource} from '../../sources/image-source'; +import type {ImageSourceMetadata, GetImageParameters} from '@loaders.gl/loader-utils'; +import {ImageSource} from '@loaders.gl/loader-utils'; /** Template URL string should contain `${width}` etc which will be substituted. */ export type ImageServiceProps = { diff --git a/modules/wms/src/lib/services/ogc/csw-service.ts b/modules/wms/src/lib/services/ogc/csw-service.ts index d4c98bba83..7826b71a1a 100644 --- a/modules/wms/src/lib/services/ogc/csw-service.ts +++ b/modules/wms/src/lib/services/ogc/csw-service.ts @@ -2,8 +2,8 @@ /* eslint-disable camelcase */ -import type {DataSourceProps} from '../../sources/data-source'; -import {DataSource} from '../../sources/data-source'; +import type {DataSourceProps} from '@loaders.gl/loader-utils'; +import {DataSource} from '@loaders.gl/loader-utils'; import type {CSWCapabilities} from '../../../csw-capabilities-loader'; import {CSWCapabilitiesLoader} from '../../../csw-capabilities-loader'; diff --git a/modules/wms/src/lib/services/ogc/wms-service.ts b/modules/wms/src/lib/services/ogc/wms-service.ts index ed3e5de3bd..e50ac59b56 100644 --- a/modules/wms/src/lib/services/ogc/wms-service.ts +++ b/modules/wms/src/lib/services/ogc/wms-service.ts @@ -5,9 +5,9 @@ import type {ImageType} from '@loaders.gl/images'; import {ImageLoader} from '@loaders.gl/images'; import {mergeLoaderOptions} from '@loaders.gl/loader-utils'; -import type {ImageSourceMetadata, GetImageParameters} from '../../sources/image-source'; -import type {ImageSourceProps} from '../../sources/image-source'; -import {ImageSource} from '../../sources/image-source'; +import type {ImageSourceMetadata, GetImageParameters} from '@loaders.gl/loader-utils'; +import type {ImageSourceProps} from '@loaders.gl/loader-utils'; +import {ImageSource} from '@loaders.gl/loader-utils'; import type {WMSCapabilities} from '../../../wms-capabilities-loader'; import type {WMSFeatureInfo} from '../../../wip/wms-feature-info-loader'; diff --git a/modules/zip/src/parse-zip/end-of-central-directory.ts b/modules/zip/src/parse-zip/end-of-central-directory.ts index 808bcfec77..ff40a359dd 100644 --- a/modules/zip/src/parse-zip/end-of-central-directory.ts +++ b/modules/zip/src/parse-zip/end-of-central-directory.ts @@ -13,8 +13,8 @@ export type ZipEoCDRecord = { }; const eoCDSignature: ZipSignature = [0x50, 0x4b, 0x05, 0x06]; -const zip64EoCDLocatorSignature = Buffer.from([0x50, 0x4b, 0x06, 0x07]); -const zip64EoCDSignature = Buffer.from([0x50, 0x4b, 0x06, 0x06]); +const zip64EoCDLocatorSignature = [0x50, 0x4b, 0x06, 0x07]; +const zip64EoCDSignature = [0x50, 0x4b, 0x06, 0x06]; // offsets accroding to https://en.wikipedia.org/wiki/ZIP_(file_format) const CD_RECORDS_NUMBER_OFFSET = 8n; @@ -42,7 +42,7 @@ export const parseEoCDRecord = async (fileProvider: FileProvider): Promise => { - if (Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare(signature) !== 0) { + if ( + Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare( + Buffer.from(signature) + ) !== 0 + ) { return null; } diff --git a/test/aliases.js b/test/aliases.js index e0eaba7f38..3c8fb05b55 100644 --- a/test/aliases.js +++ b/test/aliases.js @@ -32,6 +32,7 @@ function makeAliases(basename = __dirname) { '@loaders.gl/parquet/test': resolve(basename, '../modules/parquet/test'), '@loaders.gl/pcd/test': resolve(basename, '../modules/pcd/test'), '@loaders.gl/ply/test': resolve(basename, '../modules/ply/test'), + '@loaders.gl/pmtiles/test': resolve(basename, '../modules/pmtiles/test'), '@loaders.gl/polyfills/test': resolve(basename, '../modules/polyfills/test'), '@loaders.gl/potree/test': resolve(basename, '../modules/potree/test'), '@loaders.gl/shapefile/test': resolve(basename, '../modules/shapefile/test'), diff --git a/test/modules.js b/test/modules.js index 63eaec83a3..2e45cc8b1f 100644 --- a/test/modules.js +++ b/test/modules.js @@ -50,11 +50,15 @@ import '@loaders.gl/tiles/test'; import '@loaders.gl/geopackage/test'; import '@loaders.gl/gis/test'; import '@loaders.gl/kml/test'; -import '@loaders.gl/mvt/test'; import '@loaders.gl/shapefile/test'; import '@loaders.gl/wkt/test'; import '@loaders.gl/wms/test'; +import '@loaders.gl/mvt/test'; + +// Range request archive style formats +import '@loaders.gl/pmtiles/test'; + // Table Formats import '@loaders.gl/schema/test'; import '@loaders.gl/arrow/test'; diff --git a/tsconfig.json b/tsconfig.json index 0f9883540b..8ab303f6f2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -78,6 +78,8 @@ "@loaders.gl/parquet/test": ["modules/parquet/test"], "@loaders.gl/pcd": ["modules/pcd/src"], "@loaders.gl/pcd/test": ["modules/pcd/test"], + "@loaders.gl/pmtiles": ["modules/pmtiles/src"], + "@loaders.gl/pmtiles/test": ["modules/pmtiles/test"], "@loaders.gl/ply": ["modules/ply/src"], "@loaders.gl/ply/test": ["modules/ply/test"], "@loaders.gl/potree": ["modules/potree/src"], diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 951dd514ee..e37458f0b4 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -78,6 +78,7 @@ const config = { '@loaders.gl/parquet': resolve('../modules/parquet/src'), '@loaders.gl/pcd': resolve('../modules/pcd/src'), '@loaders.gl/ply': resolve('../modules/ply/src'), + '@loaders.gl/pmtiles': resolve('../modules/pmtiles/src'), '@loaders.gl/polyfills': resolve('../modules/polyfills/src'), '@loaders.gl/potree': resolve('../modules/potree/src'), '@loaders.gl/schema': resolve('../modules/schema/src'), diff --git a/website/src/examples-sidebar.js b/website/src/examples-sidebar.js index 2b744815f6..15bd50d16b 100644 --- a/website/src/examples-sidebar.js +++ b/website/src/examples-sidebar.js @@ -20,6 +20,7 @@ label: 'Geospatial Loaders', items: [ 'geospatial', + 'pmtiles', 'wms' ], }, diff --git a/website/src/examples/pmtiles.mdx b/website/src/examples/pmtiles.mdx new file mode 100644 index 0000000000..7ef312e666 --- /dev/null +++ b/website/src/examples/pmtiles.mdx @@ -0,0 +1,7 @@ +# PMTiles + +import Demo from 'examples/website/tiles/app'; + +
+ +
diff --git a/website/static/images/examples/pmtiles.jpg b/website/static/images/examples/pmtiles.jpg new file mode 100644 index 0000000000..516accdf6f Binary files /dev/null and b/website/static/images/examples/pmtiles.jpg differ diff --git a/yarn.lock b/yarn.lock index c3422c95fa..75cd70c9da 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1961,6 +1961,15 @@ dependencies: "@loaders.gl/loader-utils" "3.4.14" +"@loaders.gl/loader-utils@3.4.0-alpha.3": + version "3.4.0-alpha.3" + resolved "https://registry.yarnpkg.com/@loaders.gl/loader-utils/-/loader-utils-3.4.0-alpha.3.tgz#a4a6e5acc1075988b3b88fb6a889f8cd71c39fc4" + integrity sha512-/7si5ZbeuB2DX7kuZ2Egir/0OAL4qbIqYOs90vj+jarEbEflsedete/TPrjsdVwKLFT7RfDQPohTqMVEy9hdEg== + dependencies: + "@babel/runtime" "^7.3.1" + "@loaders.gl/worker-utils" "3.4.0-alpha.3" + "@probe.gl/stats" "^4.0.1" + "@loaders.gl/loader-utils@3.4.14": version "3.4.14" resolved "https://registry.yarnpkg.com/@loaders.gl/loader-utils/-/loader-utils-3.4.14.tgz#d94decc279fd2304b8762c87d8d9626058d91f21" @@ -1970,6 +1979,20 @@ "@loaders.gl/worker-utils" "3.4.14" "@probe.gl/stats" "^4.0.1" +"@loaders.gl/schema@3.4.0-alpha.3": + version "3.4.0-alpha.3" + resolved "https://registry.yarnpkg.com/@loaders.gl/schema/-/schema-3.4.0-alpha.3.tgz#c97fa3c553e0fbddb6ff6497cc59e78f9781181d" + integrity sha512-syMpg7GHGS1o82PH884AhV9RualUfXs7Hul0B3y/T3jBv2Z60nzxpsRlQcqJ9r1vJE0q8ZimQGpAsccQcSdMRA== + dependencies: + "@types/geojson" "^7946.0.7" + +"@loaders.gl/worker-utils@3.4.0-alpha.3": + version "3.4.0-alpha.3" + resolved "https://registry.yarnpkg.com/@loaders.gl/worker-utils/-/worker-utils-3.4.0-alpha.3.tgz#15deba3310b1bbe2933b2816b08c26e5fe211888" + integrity sha512-Ifxvm6g8v5KdJk5mwN5xG2lYNRozB9svgRcnJQyoyf3NJd1K+Dq4SRatAHxzGzbmpTa6dm8TLgK8cYF/78++BA== + dependencies: + "@babel/runtime" "^7.3.1" + "@loaders.gl/worker-utils@3.4.14": version "3.4.14" resolved "https://registry.yarnpkg.com/@loaders.gl/worker-utils/-/worker-utils-3.4.14.tgz#5391a416a3d60e03b9edcedb285af44312d40d2e" @@ -6264,6 +6287,11 @@ fflate@0.7.4: resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.7.4.tgz#61587e5d958fdabb5a9368a302c25363f4f69f50" integrity sha512-5u2V/CDW15QM1XbbgS+0DfPxVB+jUKhWEKuuFuHncbk3tEEqzmoXL+2KyOFuKGqOnmdIy0/davWF1CkuwtibCw== +fflate@^0.8.0: + version "0.8.1" + resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.8.1.tgz#1ed92270674d2ad3c73f077cd0acf26486dae6c9" + integrity sha512-/exOvEuc+/iaUm105QIiOt4LpBdMTWsXxqR0HDF35vx3fmaKzw7354gTilCh5rkzEt8WYyG//ku3h3nRmd7CHQ== + figgy-pudding@^3.4.1, figgy-pudding@^3.5.1: version "3.5.2" resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.2.tgz#b4eee8148abb01dcf1d1ac34367d59e12fa61d6e" @@ -10185,6 +10213,13 @@ pkg-dir@^4.1.0: dependencies: find-up "^4.0.0" +pmtiles@^2.7.2: + version "2.10.0" + resolved "https://registry.yarnpkg.com/pmtiles/-/pmtiles-2.10.0.tgz#894e2954723924add7dd9637c89c0329ffe62501" + integrity sha512-X+s6JyperpcAkKwv55MKx72ckOUB0ZjcfK4929iM0SS0MkLydEi2FSW1E8YTE1E2XaZ2TVk/MIUrbsZuXV7K2g== + dependencies: + fflate "^0.8.0" + pngjs-nozlib@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/pngjs-nozlib/-/pngjs-nozlib-1.0.0.tgz#9e64d602cfe9cce4d9d5997d0687429a73f0b7d7"