feat(wkt): hex wkb loader

modules/wkt/src/hex-wkb-loader.ts

@@ -0,0 +1,61 @@
+// loaders.gl, MIT license
+
+import type {LoaderWithParser} from '@loaders.gl/loader-utils';
+import {BinaryGeometry} from '@loaders.gl/schema';
+
+import type {WKBLoaderOptions} from './wkb-loader';
+import {WKBLoader} from './wkb-loader';
+import {VERSION} from './lib/utils/version';
+import {decodeHex} from './lib/utils/hex-transcoder';
+
+export type HexWKBLoaderOptions = WKBLoaderOptions;
+
+/**
+ * Worker loader for Hex-encoded WKB (Well-Known Binary)
+ */
+export const HexWKBLoader: LoaderWithParser<BinaryGeometry, never, HexWKBLoaderOptions> = {
+  name: 'Hexadecimal WKB',
+  id: 'wkb',
+  module: 'wkt',
+  version: VERSION,
+  worker: true,
+  category: 'geometry',
+  extensions: ['wkb'],
+  mimeTypes: [],
+  options: WKBLoader.options,
+  text: true,
+  testText: isHexWKB,
+  // TODO - encoding here seems wasteful - extend hex transcoder?
+  parse: async (arrayBuffer: ArrayBuffer) => parseHexWKB(new TextDecoder().decode(arrayBuffer)),
+  parseTextSync: parseHexWKB
+};
+
+function parseHexWKB(text: string, options?: HexWKBLoaderOptions): BinaryGeometry {
+  const uint8Array = decodeHex(text);
+  const binaryGeometry = WKBLoader.parseSync?.(uint8Array.buffer, options);
+  // @ts-expect-error
+  return binaryGeometry;
+}
+
+/**
+ * Check if string is a valid Well-known binary (WKB) in HEX format
+ * https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry
+ *
+ * @param str input string
+ * @returns true if string is a valid WKB in HEX format
+ */
+export function isHexWKB(string: string | null): boolean {
+  if (!string) {
+    return false;
+  }
+  // check if the length of the string is even and is at least 10 characters long
+  if (string.length < 10 || string.length % 2 !== 0) {
+    return false;
+  }
+  // check if first two characters are 00 or 01
+  if (!string.startsWith('00') && !string.startsWith('01')) {
+    return false;
+  }
+  // check if the rest of the string is a valid hex
+  return /^[0-9a-fA-F]+$/.test(string.slice(2));
+}

modules/wkt/src/index.ts

@@ -1,4 +1,11 @@
-export {WKBLoader, WKBWorkerLoader} from './wkb-loader';
+// luma.gl, MIT license
 export {WKTLoader, WKTWorkerLoader} from './wkt-loader';
 export {WKTWriter} from './wkt-writer';
+
+export {WKBLoader, WKBWorkerLoader} from './wkb-loader';
 export {WKBWriter} from './wkb-writer';
+
+export {HexWKBLoader} from './hex-wkb-loader';
+
+// EXPERIMENTAL EXPORTS
+export {encodeHex, decodeHex} from './lib/utils/hex-transcoder';

modules/wkt/src/lib/parse-wkb.ts

@@ -3,8 +3,11 @@ import type {
   BinaryGeometry,
   BinaryPointGeometry,
   BinaryLineGeometry,
-  BinaryPolygonGeometry
+  BinaryPolygonGeometry,
+  Geometry
 } from '@loaders.gl/schema';
+import {binaryToGeometry} from '@loaders.gl/gis';
+import type {WKBLoaderOptions} from '../wkb-loader';
 
 const NUM_DIMENSIONS = {
   0: 2, // 2D
@@ -13,7 +16,23 @@ const NUM_DIMENSIONS = {
   3: 4 // 4D (ZM)
 };
 
-export default function parseWKB(arrayBuffer: ArrayBuffer): BinaryGeometry {
+export function parseWKB(
+  arrayBuffer: ArrayBuffer,
+  options?: WKBLoaderOptions
+): BinaryGeometry | Geometry {
+  const binaryGeometry = parseWKBToBinary(arrayBuffer, options);
+  const shape = options?.wkb?.shape || 'binary-geometry';
+  switch (shape) {
+    case 'binary-geometry':
+      return binaryGeometry;
+    case 'geometry':
+      return binaryToGeometry(binaryGeometry);
+    default:
+      throw new Error(shape);
+  }
+}
+
+function parseWKBToBinary(arrayBuffer: ArrayBuffer, options?: WKBLoaderOptions): BinaryGeometry {
   const view = new DataView(arrayBuffer);
   let offset = 0;
 

modules/wkt/src/lib/utils/hex-transcoder.ts

@@ -0,0 +1,50 @@
+// Forked from https://github.com/jessetane/hex-transcoder under MIT license
+
+const alphabet = '0123456789abcdef';
+const encodeLookup: string[] = [];
+const decodeLookup: number[] = [];
+
+for (let i = 0; i < 256; i++) {
+  encodeLookup[i] = alphabet[(i >> 4) & 0xf] + alphabet[i & 0xf];
+  if (i < 16) {
+    if (i < 10) {
+      decodeLookup[0x30 + i] = i;
+    } else {
+      decodeLookup[0x61 - 10 + i] = i;
+    }
+  }
+}
+
+/**
+ * Encode a Uint8Array to a hex string
+ *
+ * @param  array Bytes to encode to string
+ * @return hex string
+ */
+export function encodeHex(array: Uint8Array): string {
+  const length = array.length;
+  let string = '';
+  let i = 0;
+  while (i < length) {
+    string += encodeLookup[array[i++]];
+  }
+  return string;
+}
+
+/**
+ * Decodes a hex string to a Uint8Array
+ *
+ * @param string hex string to decode to Uint8Array
+ * @return Uint8Array
+ */
+export function decodeHex(string: string): Uint8Array {
+  const sizeof = string.length >> 1;
+  const length = sizeof << 1;
+  const array = new Uint8Array(sizeof);
+  let n = 0;
+  let i = 0;
+  while (i < length) {
+    array[n++] = (decodeLookup[string.charCodeAt(i++)] << 4) | decodeLookup[string.charCodeAt(i++)];
+  }
+  return array;
+}

modules/wkt/src/wkb-loader.ts

@@ -1,11 +1,18 @@
-import type {Loader, LoaderWithParser} from '@loaders.gl/loader-utils';
+import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
 import {VERSION} from './lib/utils/version';
-import parseWKB from './lib/parse-wkb';
+import {parseWKB} from './lib/parse-wkb';
+import {BinaryGeometry} from '@loaders.gl/schema';
+
+export type WKBLoaderOptions = LoaderOptions & {
+  wkb?: {
+    shape: 'binary-geometry' | 'geometry';
+  };
+};
 
 /**
  * Worker loader for WKB (Well-Known Binary)
  */
-export const WKBWorkerLoader = {
+export const WKBWorkerLoader: Loader<BinaryGeometry, never, WKBLoaderOptions> = {
   name: 'WKB',
   id: 'wkb',
   module: 'wkt',
@@ -15,18 +22,17 @@ export const WKBWorkerLoader = {
   extensions: ['wkb'],
   mimeTypes: [],
   options: {
-    wkb: {}
+    wkb: {
+      shape: 'binary-geometry'
+    }
   }
 };
 
 /**
  * Loader for WKB (Well-Known Binary)
  */
-export const WKBLoader = {
+export const WKBLoader: LoaderWithParser<BinaryGeometry, never, WKBLoaderOptions> = {
   ...WKBWorkerLoader,
   parse: async (arrayBuffer: ArrayBuffer) => parseWKB(arrayBuffer),
   parseSync: parseWKB
 };
-
-export const _typecheckWKBWorkerLoader: Loader = WKBWorkerLoader;
-export const _typecheckWKBLoader: LoaderWithParser = WKBLoader;

modules/wkt/test/hex-wkb-loader.spec.ts

@@ -0,0 +1,55 @@
+import test from 'tape-promise/tape';
+import {fetchFile, parseSync} from '@loaders.gl/core';
+import {HexWKBLoader} from '@loaders.gl/wkt';
+import {parseTestCases} from './utils/parse-test-cases';
+
+const WKB_2D_TEST_CASES = '@loaders.gl/wkt/test/data/wkb-testdata2d.json';
+const WKB_Z_TEST_CASES = '@loaders.gl/wkt/test/data/wkb-testdataZ.json';
+
+test('HexWKBLoader#2D', async (t) => {
+  const response = await fetchFile(WKB_2D_TEST_CASES);
+  const TEST_CASES = parseTestCases(await response.json());
+
+  // TODO parseWKB outputs TypedArrays; testCase contains regular arrays
+  for (const testCase of Object.values(TEST_CASES)) {
+    // Little endian
+    if (testCase.wkbHex && testCase.binary) {
+      t.deepEqual(parseSync(testCase.wkbHex, HexWKBLoader), testCase.binary);
+    }
+
+    // Big endian
+    if (testCase.wkbHexXdr && testCase.binary) {
+      t.deepEqual(parseSync(testCase.wkbHexXdr, HexWKBLoader), testCase.binary);
+    }
+  }
+
+  t.end();
+});
+
+test('HexWKBLoader#Z', async (t) => {
+  const response = await fetchFile(WKB_Z_TEST_CASES);
+  const TEST_CASES = parseTestCases(await response.json());
+
+  // TODO parseWKB outputs TypedArrays; testCase contains regular arrays
+  for (const testCase of Object.values(TEST_CASES)) {
+    // Little endian
+    if (testCase.wkbHex && testCase.binary) {
+      t.deepEqual(
+        parseSync(testCase.wkbHex, HexWKBLoader),
+        testCase.binary,
+        testCase.wkbHex.slice(0, 60)
+      );
+    }
+
+    // Big endian
+    if (testCase.wkbHexXdr && testCase.binary) {
+      t.deepEqual(
+        parseSync(testCase.wkbHexXdr, HexWKBLoader),
+        testCase.binary,
+        testCase.wkbHexXdr.slice(0, 60)
+      );
+    }
+  }
+
+  t.end();
+});

modules/wkt/test/index.ts

@@ -0,0 +1,13 @@
+// loaders.gl, MIT license
+
+import './lib/utils/hex-transcoder.spec';
+
+import './wkb-loader.spec';
+import './wkb-writer.spec';
+
+import './hex-wkb-loader.spec';
+
+import './wkt-loader.spec';
+import './wkt-writer.spec';
+
+import './wkt-crs-loader.spec';

modules/wkt/test/lib/utils/hex-transcoder.spec.ts

@@ -0,0 +1,36 @@
+// loaders.gl, MIT license
+
+import test from 'tape-promise/tape';
+import {HexWKBLoader} from '@loaders.gl/wkt';
+
+const isHexWKB = HexWKBLoader.testText!;
+
+test('datasetUtils.isHexWKB', (t) => {
+  t.notOk(isHexWKB(''), 'empty string is not a valid hex wkb');
+
+  // @ts-ignore null is not a string
+  t.notOk(isHexWKB(null), 'null is not a valid hex wkb');
+
+  const countyFIPS = '06075';
+  t.notOk(isHexWKB(countyFIPS), 'FIPS code should not be a valid hex wkb');
+
+  const h3Code = '8a2a1072b59ffff';
+  t.notOk(isHexWKB(h3Code), 'H3 code should not be a valid hex wkb');
+
+  const randomHexStr = '8a2a1072b59ffff';
+  t.notOk(isHexWKB(randomHexStr), 'A random hex string should not be a valid hex wkb');
+
+  const validWkt = '0101000000000000000000f03f0000000000000040';
+  t.ok(isHexWKB(validWkt), 'A valid hex wkb should be valid');
+
+  const validEWkt = '0101000020e6100000000000000000f03f0000000000000040';
+  t.ok(isHexWKB(validEWkt), 'A valid hex ewkb should be valid');
+
+  const validWktNDR = '00000000013ff0000000000000400000000000000040';
+  t.ok(isHexWKB(validWktNDR), 'A valid hex wkb in NDR should be valid');
+
+  const validEWktNDR = '0020000001000013ff0000000000400000000000000040';
+  t.ok(isHexWKB(validEWktNDR), 'A valid hex ewkb in NDR should be valid');
+
+  t.end();
+});

modules/wkt/test/wkb/parse-wkb.spec.ts

@@ -41,6 +41,10 @@ test('parseWKB Z', async (t) => {
     if (testCase.wkbXdr && testCase.binary) {
       t.deepEqual(parseWKB(testCase.wkbXdr), testCase.binary);
     }
+
+    // if (testCase.wkbXdr && testCase.binary && testCase.geoJSON) {
+    //   t.deepEqual(parseSync(testCase.wkbXdr, WKBLoader, {wkb: {shape: 'geometry'}}), testCase.geoJSON);
+    // }
   }
 
   t.end();

modules/wkt/test/wkb/utils.ts

@@ -36,6 +36,9 @@ interface ParsedTestCase {
   /** Geometry in WKT */
   wkt: string;
 
+  /** Geometry in WKB, stored in hex */
+  wkbHex: string;
+
   /** Geometry in WKB */
   wkb: ArrayBuffer;
 
@@ -45,6 +48,9 @@ interface ParsedTestCase {
   /** Geometry in WKB XDR (big endian) */
   wkbXdr: ArrayBuffer;
 
+  /** Geometry in WKB, stored in hex */
+  wkbHexXdr: string;
+
   /** Geometry in EWKB XDR (big endian) */
   ewkbXdr: ArrayBuffer;
 
@@ -106,6 +112,8 @@ export function parseTestCases(
     const parsedTestCase: ParsedTestCase = {
       wkt,
       geoJSON,
+      wkbHex: wkb,
+      wkbHexXdr: wkbXdr,
       wkb: hexStringToArrayBuffer(wkb),
       ewkb: hexStringToArrayBuffer(ewkb),
       twkb: hexStringToArrayBuffer(twkb),

modules/wkt/tsconfig.json

@@ -9,6 +9,7 @@
   },
   "references": [
     {"path": "../loader-utils"},
+    {"path": "../gis"},
     {"path": "../schema"}
   ]
 }