diff --git a/modules/wkt/src/hex-wkb-loader.ts b/modules/wkt/src/hex-wkb-loader.ts new file mode 100644 index 0000000000..1654fbdbc9 --- /dev/null +++ b/modules/wkt/src/hex-wkb-loader.ts @@ -0,0 +1,61 @@ +// loaders.gl, MIT license + +import type {LoaderWithParser} from '@loaders.gl/loader-utils'; +import {BinaryGeometry} from '@loaders.gl/schema'; + +import type {WKBLoaderOptions} from './wkb-loader'; +import {WKBLoader} from './wkb-loader'; +import {VERSION} from './lib/utils/version'; +import {decodeHex} from './lib/utils/hex-transcoder'; + +export type HexWKBLoaderOptions = WKBLoaderOptions; + +/** + * Worker loader for Hex-encoded WKB (Well-Known Binary) + */ +export const HexWKBLoader: LoaderWithParser = { + name: 'Hexadecimal WKB', + id: 'wkb', + module: 'wkt', + version: VERSION, + worker: true, + category: 'geometry', + extensions: ['wkb'], + mimeTypes: [], + options: WKBLoader.options, + text: true, + testText: isHexWKB, + // TODO - encoding here seems wasteful - extend hex transcoder? + parse: async (arrayBuffer: ArrayBuffer) => parseHexWKB(new TextDecoder().decode(arrayBuffer)), + parseTextSync: parseHexWKB +}; + +function parseHexWKB(text: string, options?: HexWKBLoaderOptions): BinaryGeometry { + const uint8Array = decodeHex(text); + const binaryGeometry = WKBLoader.parseSync?.(uint8Array.buffer, options); + // @ts-expect-error + return binaryGeometry; +} + +/** + * Check if string is a valid Well-known binary (WKB) in HEX format + * https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry + * + * @param str input string + * @returns true if string is a valid WKB in HEX format + */ +export function isHexWKB(string: string | null): boolean { + if (!string) { + return false; + } + // check if the length of the string is even and is at least 10 characters long + if (string.length < 10 || string.length % 2 !== 0) { + return false; + } + // check if first two characters are 00 or 01 + if (!string.startsWith('00') && !string.startsWith('01')) { + return false; + } + // check if the rest of the string is a valid hex + return /^[0-9a-fA-F]+$/.test(string.slice(2)); +} diff --git a/modules/wkt/src/index.ts b/modules/wkt/src/index.ts index 118a7adb5b..03fc550b55 100644 --- a/modules/wkt/src/index.ts +++ b/modules/wkt/src/index.ts @@ -1,4 +1,11 @@ -export {WKBLoader, WKBWorkerLoader} from './wkb-loader'; +// luma.gl, MIT license export {WKTLoader, WKTWorkerLoader} from './wkt-loader'; export {WKTWriter} from './wkt-writer'; + +export {WKBLoader, WKBWorkerLoader} from './wkb-loader'; export {WKBWriter} from './wkb-writer'; + +export {HexWKBLoader} from './hex-wkb-loader'; + +// EXPERIMENTAL EXPORTS +export {encodeHex, decodeHex} from './lib/utils/hex-transcoder'; diff --git a/modules/wkt/src/lib/parse-wkb.ts b/modules/wkt/src/lib/parse-wkb.ts index 32b0ec8676..6c2171b6b9 100644 --- a/modules/wkt/src/lib/parse-wkb.ts +++ b/modules/wkt/src/lib/parse-wkb.ts @@ -3,8 +3,11 @@ import type { BinaryGeometry, BinaryPointGeometry, BinaryLineGeometry, - BinaryPolygonGeometry + BinaryPolygonGeometry, + Geometry } from '@loaders.gl/schema'; +import {binaryToGeometry} from '@loaders.gl/gis'; +import type {WKBLoaderOptions} from '../wkb-loader'; const NUM_DIMENSIONS = { 0: 2, // 2D @@ -13,7 +16,23 @@ const NUM_DIMENSIONS = { 3: 4 // 4D (ZM) }; -export default function parseWKB(arrayBuffer: ArrayBuffer): BinaryGeometry { +export function parseWKB( + arrayBuffer: ArrayBuffer, + options?: WKBLoaderOptions +): BinaryGeometry | Geometry { + const binaryGeometry = parseWKBToBinary(arrayBuffer, options); + const shape = options?.wkb?.shape || 'binary-geometry'; + switch (shape) { + case 'binary-geometry': + return binaryGeometry; + case 'geometry': + return binaryToGeometry(binaryGeometry); + default: + throw new Error(shape); + } +} + +function parseWKBToBinary(arrayBuffer: ArrayBuffer, options?: WKBLoaderOptions): BinaryGeometry { const view = new DataView(arrayBuffer); let offset = 0; diff --git a/modules/wkt/src/lib/utils/hex-transcoder.ts b/modules/wkt/src/lib/utils/hex-transcoder.ts new file mode 100644 index 0000000000..5eeb82eca1 --- /dev/null +++ b/modules/wkt/src/lib/utils/hex-transcoder.ts @@ -0,0 +1,50 @@ +// Forked from https://github.com/jessetane/hex-transcoder under MIT license + +const alphabet = '0123456789abcdef'; +const encodeLookup: string[] = []; +const decodeLookup: number[] = []; + +for (let i = 0; i < 256; i++) { + encodeLookup[i] = alphabet[(i >> 4) & 0xf] + alphabet[i & 0xf]; + if (i < 16) { + if (i < 10) { + decodeLookup[0x30 + i] = i; + } else { + decodeLookup[0x61 - 10 + i] = i; + } + } +} + +/** + * Encode a Uint8Array to a hex string + * + * @param array Bytes to encode to string + * @return hex string + */ +export function encodeHex(array: Uint8Array): string { + const length = array.length; + let string = ''; + let i = 0; + while (i < length) { + string += encodeLookup[array[i++]]; + } + return string; +} + +/** + * Decodes a hex string to a Uint8Array + * + * @param string hex string to decode to Uint8Array + * @return Uint8Array + */ +export function decodeHex(string: string): Uint8Array { + const sizeof = string.length >> 1; + const length = sizeof << 1; + const array = new Uint8Array(sizeof); + let n = 0; + let i = 0; + while (i < length) { + array[n++] = (decodeLookup[string.charCodeAt(i++)] << 4) | decodeLookup[string.charCodeAt(i++)]; + } + return array; +} diff --git a/modules/wkt/src/wkb-loader.ts b/modules/wkt/src/wkb-loader.ts index 75ffaca396..1ce20544ce 100644 --- a/modules/wkt/src/wkb-loader.ts +++ b/modules/wkt/src/wkb-loader.ts @@ -1,11 +1,18 @@ -import type {Loader, LoaderWithParser} from '@loaders.gl/loader-utils'; +import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils'; import {VERSION} from './lib/utils/version'; -import parseWKB from './lib/parse-wkb'; +import {parseWKB} from './lib/parse-wkb'; +import {BinaryGeometry} from '@loaders.gl/schema'; + +export type WKBLoaderOptions = LoaderOptions & { + wkb?: { + shape: 'binary-geometry' | 'geometry'; + }; +}; /** * Worker loader for WKB (Well-Known Binary) */ -export const WKBWorkerLoader = { +export const WKBWorkerLoader: Loader = { name: 'WKB', id: 'wkb', module: 'wkt', @@ -15,18 +22,17 @@ export const WKBWorkerLoader = { extensions: ['wkb'], mimeTypes: [], options: { - wkb: {} + wkb: { + shape: 'binary-geometry' + } } }; /** * Loader for WKB (Well-Known Binary) */ -export const WKBLoader = { +export const WKBLoader: LoaderWithParser = { ...WKBWorkerLoader, parse: async (arrayBuffer: ArrayBuffer) => parseWKB(arrayBuffer), parseSync: parseWKB }; - -export const _typecheckWKBWorkerLoader: Loader = WKBWorkerLoader; -export const _typecheckWKBLoader: LoaderWithParser = WKBLoader; diff --git a/modules/wkt/test/hex-wkb-loader.spec.ts b/modules/wkt/test/hex-wkb-loader.spec.ts new file mode 100644 index 0000000000..079aff6a56 --- /dev/null +++ b/modules/wkt/test/hex-wkb-loader.spec.ts @@ -0,0 +1,55 @@ +import test from 'tape-promise/tape'; +import {fetchFile, parseSync} from '@loaders.gl/core'; +import {HexWKBLoader} from '@loaders.gl/wkt'; +import {parseTestCases} from './utils/parse-test-cases'; + +const WKB_2D_TEST_CASES = '@loaders.gl/wkt/test/data/wkb-testdata2d.json'; +const WKB_Z_TEST_CASES = '@loaders.gl/wkt/test/data/wkb-testdataZ.json'; + +test('HexWKBLoader#2D', async (t) => { + const response = await fetchFile(WKB_2D_TEST_CASES); + const TEST_CASES = parseTestCases(await response.json()); + + // TODO parseWKB outputs TypedArrays; testCase contains regular arrays + for (const testCase of Object.values(TEST_CASES)) { + // Little endian + if (testCase.wkbHex && testCase.binary) { + t.deepEqual(parseSync(testCase.wkbHex, HexWKBLoader), testCase.binary); + } + + // Big endian + if (testCase.wkbHexXdr && testCase.binary) { + t.deepEqual(parseSync(testCase.wkbHexXdr, HexWKBLoader), testCase.binary); + } + } + + t.end(); +}); + +test('HexWKBLoader#Z', async (t) => { + const response = await fetchFile(WKB_Z_TEST_CASES); + const TEST_CASES = parseTestCases(await response.json()); + + // TODO parseWKB outputs TypedArrays; testCase contains regular arrays + for (const testCase of Object.values(TEST_CASES)) { + // Little endian + if (testCase.wkbHex && testCase.binary) { + t.deepEqual( + parseSync(testCase.wkbHex, HexWKBLoader), + testCase.binary, + testCase.wkbHex.slice(0, 60) + ); + } + + // Big endian + if (testCase.wkbHexXdr && testCase.binary) { + t.deepEqual( + parseSync(testCase.wkbHexXdr, HexWKBLoader), + testCase.binary, + testCase.wkbHexXdr.slice(0, 60) + ); + } + } + + t.end(); +}); diff --git a/modules/wkt/test/index.js b/modules/wkt/test/index.js.disabled similarity index 100% rename from modules/wkt/test/index.js rename to modules/wkt/test/index.js.disabled diff --git a/modules/wkt/test/index.ts b/modules/wkt/test/index.ts new file mode 100644 index 0000000000..4e1e198962 --- /dev/null +++ b/modules/wkt/test/index.ts @@ -0,0 +1,13 @@ +// loaders.gl, MIT license + +import './lib/utils/hex-transcoder.spec'; + +import './wkb-loader.spec'; +import './wkb-writer.spec'; + +import './hex-wkb-loader.spec'; + +import './wkt-loader.spec'; +import './wkt-writer.spec'; + +import './wkt-crs-loader.spec'; diff --git a/modules/wkt/test/lib/utils/hex-transcoder.spec.ts b/modules/wkt/test/lib/utils/hex-transcoder.spec.ts new file mode 100644 index 0000000000..225f9914d0 --- /dev/null +++ b/modules/wkt/test/lib/utils/hex-transcoder.spec.ts @@ -0,0 +1,36 @@ +// loaders.gl, MIT license + +import test from 'tape-promise/tape'; +import {HexWKBLoader} from '@loaders.gl/wkt'; + +const isHexWKB = HexWKBLoader.testText!; + +test('datasetUtils.isHexWKB', (t) => { + t.notOk(isHexWKB(''), 'empty string is not a valid hex wkb'); + + // @ts-ignore null is not a string + t.notOk(isHexWKB(null), 'null is not a valid hex wkb'); + + const countyFIPS = '06075'; + t.notOk(isHexWKB(countyFIPS), 'FIPS code should not be a valid hex wkb'); + + const h3Code = '8a2a1072b59ffff'; + t.notOk(isHexWKB(h3Code), 'H3 code should not be a valid hex wkb'); + + const randomHexStr = '8a2a1072b59ffff'; + t.notOk(isHexWKB(randomHexStr), 'A random hex string should not be a valid hex wkb'); + + const validWkt = '0101000000000000000000f03f0000000000000040'; + t.ok(isHexWKB(validWkt), 'A valid hex wkb should be valid'); + + const validEWkt = '0101000020e6100000000000000000f03f0000000000000040'; + t.ok(isHexWKB(validEWkt), 'A valid hex ewkb should be valid'); + + const validWktNDR = '00000000013ff0000000000000400000000000000040'; + t.ok(isHexWKB(validWktNDR), 'A valid hex wkb in NDR should be valid'); + + const validEWktNDR = '0020000001000013ff0000000000400000000000000040'; + t.ok(isHexWKB(validEWktNDR), 'A valid hex ewkb in NDR should be valid'); + + t.end(); +}); diff --git a/modules/wkt/test/wkb/parse-wkb.spec.ts b/modules/wkt/test/wkb/parse-wkb.spec.ts index 7005d5cbff..87cafc7d78 100644 --- a/modules/wkt/test/wkb/parse-wkb.spec.ts +++ b/modules/wkt/test/wkb/parse-wkb.spec.ts @@ -41,6 +41,10 @@ test('parseWKB Z', async (t) => { if (testCase.wkbXdr && testCase.binary) { t.deepEqual(parseWKB(testCase.wkbXdr), testCase.binary); } + + // if (testCase.wkbXdr && testCase.binary && testCase.geoJSON) { + // t.deepEqual(parseSync(testCase.wkbXdr, WKBLoader, {wkb: {shape: 'geometry'}}), testCase.geoJSON); + // } } t.end(); diff --git a/modules/wkt/test/wkb/utils.ts b/modules/wkt/test/wkb/utils.ts index efff562393..cedeac0394 100644 --- a/modules/wkt/test/wkb/utils.ts +++ b/modules/wkt/test/wkb/utils.ts @@ -36,6 +36,9 @@ interface ParsedTestCase { /** Geometry in WKT */ wkt: string; + /** Geometry in WKB, stored in hex */ + wkbHex: string; + /** Geometry in WKB */ wkb: ArrayBuffer; @@ -45,6 +48,9 @@ interface ParsedTestCase { /** Geometry in WKB XDR (big endian) */ wkbXdr: ArrayBuffer; + /** Geometry in WKB, stored in hex */ + wkbHexXdr: string; + /** Geometry in EWKB XDR (big endian) */ ewkbXdr: ArrayBuffer; @@ -106,6 +112,8 @@ export function parseTestCases( const parsedTestCase: ParsedTestCase = { wkt, geoJSON, + wkbHex: wkb, + wkbHexXdr: wkbXdr, wkb: hexStringToArrayBuffer(wkb), ewkb: hexStringToArrayBuffer(ewkb), twkb: hexStringToArrayBuffer(twkb), diff --git a/modules/wkt/tsconfig.json b/modules/wkt/tsconfig.json index 4e11b7c942..f6a2a8083f 100644 --- a/modules/wkt/tsconfig.json +++ b/modules/wkt/tsconfig.json @@ -9,6 +9,7 @@ }, "references": [ {"path": "../loader-utils"}, + {"path": "../gis"}, {"path": "../schema"} ] }