chore: Remove Buffer usage

docs/upgrade-guide.md

@@ -58,7 +58,7 @@ and loaders.gl v4.0 aligns with this practice.
 
 **@loaders.gl/crypto**
 
-- All hashes now require an encoding parameter. To get previous behavior, just specify `'base64'`.
+- All hashes now require an encoding parameter. To get previous behavior, just specify `.hash...(..., 'base64')`.
 
 **@loaders.gl/arrow**
 

modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-archive.ts

@@ -62,6 +62,9 @@ export class Tiles3DArchive {
     const arrayBuffer = new TextEncoder().encode(path).buffer;
     const nameHash = await new MD5Hash().hash(arrayBuffer, 'hex');
     const byteOffset = this.hashTable[nameHash];
+    if (byteOffset === undefined) {
+      return null;
+    }
 
     const localFileHeader = await parseZipLocalFileHeader(byteOffset, this.fileProvider);
     if (!localFileHeader) {

modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts

@@ -45,18 +45,18 @@ const PATH_DESCRIPTIONS: {test: RegExp; extensions: string[]}[] = [
 export class SLPKArchive {
   /** A DataView representation of the archive */
   private slpkArchive: FileProvider;
-  // Maps hex-encoded md5 hashes to bigint offsets into the archive
-  private hashedOffsetMap: Record<string, bigint>;
+  // Maps hex-encoded md5 filename hashes to bigint offsets into the archive
+  private hashTable: Record<string, bigint>;
   /** Array of hashes and offsets into archive */
   // hashToOffsetMap: Record<string, number>;
 
   protected _textEncoder = new TextEncoder();
   protected _textDecoder = new TextDecoder();
   protected _md5Hash = new MD5Hash();
 
-  constructor(slpkArchive: FileProvider, hashedOffsetMap: Record<string, bigint>) {
+  constructor(slpkArchive: FileProvider, hashTable: Record<string, bigint>) {
     this.slpkArchive = slpkArchive;
-    this.hashedOffsetMap = hashedOffsetMap;
+    this.hashTable = hashTable;
   }
 
   /**
@@ -65,7 +65,7 @@ export class SLPKArchive {
    * @param mode - currently only raw mode supported
    * @returns buffer with ready to use file
    */
-  async getFile(path: string, mode: 'http' | 'raw' = 'raw'): Promise<Buffer> {
+  async getFile(path: string, mode: 'http' | 'raw' = 'raw'): Promise<ArrayBuffer> {
     if (mode === 'http') {
       const extensions = PATH_DESCRIPTIONS.find((val) => val.test.test(path))?.extensions;
       if (extensions) {
@@ -77,18 +77,18 @@ export class SLPKArchive {
           }
         }
         if (data) {
-          return Buffer.from(data);
+          return data;
         }
       }
     }
     if (mode === 'raw') {
       const decompressedFile = await this.getDataByPath(`${path}.gz`);
       if (decompressedFile) {
-        return Buffer.from(decompressedFile);
+        return decompressedFile;
       }
       const fileWithoutCompression = await this.getFileBytes(path);
       if (fileWithoutCompression) {
-        return Buffer.from(fileWithoutCompression);
+        return fileWithoutCompression;
       }
     }
 
@@ -116,7 +116,7 @@ export class SLPKArchive {
       const decompressedData = await compression.decompress(data);
       return decompressedData;
     }
-    return Buffer.from(data);
+    return data;
   }
 
   /**
@@ -128,7 +128,7 @@ export class SLPKArchive {
     const binaryPath = this._textEncoder.encode(path);
     const nameHash = await this._md5Hash.hash(binaryPath.buffer, 'hex');
 
-    const offset = this.hashedOffsetMap[nameHash];
+    const offset = this.hashTable[nameHash];
     if (offset === undefined) {
       return undefined;
     }

modules/tile-converter/src/i3s-server/controllers/slpk-controller.ts

@@ -20,15 +20,15 @@ export async function loadArchive(fullLayerPath: string): Promise<void> {
  * @param url - I3S HTTP URL
  * @returns  - file content
  */
-export async function getFileByUrl(url: string): Promise<Buffer | null> {
+export async function getFileByUrl(url: string): Promise<ArrayBuffer | null> {
   const trimmedPath = /^\/?(.*)\/?$/.exec(url);
-  let uncompressedFile: Buffer | null = null;
   if (trimmedPath) {
     try {
-      uncompressedFile = await slpkArchive.getFile(trimmedPath[1], 'http');
+      const uncompressedFile = await slpkArchive.getFile(trimmedPath[1], 'http');
+      return uncompressedFile;
     } catch {
       // TODO - log error?
     }
   }
-  return uncompressedFile;
+  return null;
 }

modules/zip/src/hash-file-utility.ts

@@ -28,29 +28,6 @@ function bufferToHex(buffer: ArrayBuffer, start: number, length: number): string
     .join('');
 }
 
-/**
- * generates hash info from central directory
- * @param fileProvider - provider of the archive
- * @returns ready to use hash info
- */
-export async function getHashMapFromZipArchive(
-  fileProvider: FileProvider
-): Promise<Record<string, bigint>> {
-  const zipCDIterator = makeZipCDHeaderIterator(fileProvider);
-  const md5Hash = new MD5Hash();
-  const textEncoder = new TextEncoder();
-
-  const hashMap: Record<string, bigint> = {};
-  for await (const cdHeader of zipCDIterator) {
-    const filename = cdHeader.fileName.split('\\').join('/').toLocaleLowerCase();
-    const arrayBuffer = textEncoder.encode(filename).buffer;
-    const md5 = await md5Hash.hash(arrayBuffer, 'hex');
-    hashMap[md5] = cdHeader.localHeaderOffset;
-  }
-  return hashMap;
-}
-
-//
 /**
  * generates hash info from zip files "central directory"
  * @param fileProvider - provider of the archive

modules/zip/src/parse-zip/cd-file-header.ts

@@ -1,4 +1,4 @@
-import {FileProvider} from '@loaders.gl/loader-utils';
+import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils';
 import {parseEoCDRecord} from './end-of-central-directory';
 import {ZipSignature} from './search-from-the-end';
 
@@ -31,7 +31,7 @@ const CD_EXTRA_FIELD_LENGTH_OFFSET = 30n;
 const CD_LOCAL_HEADER_OFFSET_OFFSET = 42n;
 const CD_FILE_NAME_OFFSET = 46n;
 
-export const signature: ZipSignature = [0x50, 0x4b, 0x01, 0x02];
+export const signature: ZipSignature = new Uint8Array([0x50, 0x4b, 0x01, 0x02]);
 
 /**
  * Parses central directory file header of zip file
@@ -41,48 +41,39 @@ export const signature: ZipSignature = [0x50, 0x4b, 0x01, 0x02];
  */
 export const parseZipCDFileHeader = async (
   headerOffset: bigint,
-  buffer: FileProvider
+  file: FileProvider
 ): Promise<ZipCDFileHeader | null> => {
-  if (
-    Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare(
-      Buffer.from(signature)
-    ) !== 0
-  ) {
+  const magicBytes = await file.slice(headerOffset, headerOffset + 4n);
+  if (!compareArrayBuffers(magicBytes, signature.buffer)) {
     return null;
   }
 
-  let compressedSize = BigInt(await buffer.getUint32(headerOffset + CD_COMPRESSED_SIZE_OFFSET));
-
-  let uncompressedSize = BigInt(await buffer.getUint32(headerOffset + CD_UNCOMPRESSED_SIZE_OFFSET));
-
-  const extraFieldLength = await buffer.getUint16(headerOffset + CD_EXTRA_FIELD_LENGTH_OFFSET);
-
-  const fileNameLength = await buffer.getUint16(headerOffset + CD_FILE_NAME_LENGTH_OFFSET);
-
-  const fileName = new TextDecoder().decode(
-    await buffer.slice(
-      headerOffset + CD_FILE_NAME_OFFSET,
-      headerOffset + CD_FILE_NAME_OFFSET + BigInt(fileNameLength)
-    )
+  let compressedSize = BigInt(await file.getUint32(headerOffset + CD_COMPRESSED_SIZE_OFFSET));
+  let uncompressedSize = BigInt(await file.getUint32(headerOffset + CD_UNCOMPRESSED_SIZE_OFFSET));
+  const extraFieldLength = await file.getUint16(headerOffset + CD_EXTRA_FIELD_LENGTH_OFFSET);
+  const fileNameLength = await file.getUint16(headerOffset + CD_FILE_NAME_LENGTH_OFFSET);
+  const filenameBytes = await file.slice(
+    headerOffset + CD_FILE_NAME_OFFSET,
+    headerOffset + CD_FILE_NAME_OFFSET + BigInt(fileNameLength)
   );
+  const fileName = new TextDecoder().decode(filenameBytes);
 
   const extraOffset = headerOffset + CD_FILE_NAME_OFFSET + BigInt(fileNameLength);
-
-  const oldFormatOffset = await buffer.getUint32(headerOffset + CD_LOCAL_HEADER_OFFSET_OFFSET);
+  const oldFormatOffset = await file.getUint32(headerOffset + CD_LOCAL_HEADER_OFFSET_OFFSET);
 
   let fileDataOffset = BigInt(oldFormatOffset);
   let offsetInZip64Data = 4n;
   // looking for info that might be also be in zip64 extra field
   if (uncompressedSize === BigInt(0xffffffff)) {
-    uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
+    uncompressedSize = await file.getBigUint64(extraOffset + offsetInZip64Data);
     offsetInZip64Data += 8n;
   }
   if (compressedSize === BigInt(0xffffffff)) {
-    compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
+    compressedSize = await file.getBigUint64(extraOffset + offsetInZip64Data);
     offsetInZip64Data += 8n;
   }
   if (fileDataOffset === BigInt(0xffffffff)) {
-    fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64
+    fileDataOffset = await file.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64
   }
   const localHeaderOffset = fileDataOffset;
 

modules/zip/src/parse-zip/end-of-central-directory.ts

@@ -1,4 +1,4 @@
-import {FileProvider} from '@loaders.gl/loader-utils';
+import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils';
 import {ZipSignature, searchFromTheEnd} from './search-from-the-end';
 
 /**
@@ -12,9 +12,9 @@ export type ZipEoCDRecord = {
   cdRecordsNumber: bigint;
 };
 
-const eoCDSignature: ZipSignature = [0x50, 0x4b, 0x05, 0x06];
-const zip64EoCDLocatorSignature = [0x50, 0x4b, 0x06, 0x07];
-const zip64EoCDSignature = [0x50, 0x4b, 0x06, 0x06];
+const eoCDSignature: ZipSignature = new Uint8Array([0x50, 0x4b, 0x05, 0x06]);
+const zip64EoCDLocatorSignature = new Uint8Array([0x50, 0x4b, 0x06, 0x07]);
+const zip64EoCDSignature = new Uint8Array([0x50, 0x4b, 0x06, 0x06]);
 
 // offsets accroding to https://en.wikipedia.org/wiki/ZIP_(file_format)
 const CD_RECORDS_NUMBER_OFFSET = 8n;
@@ -25,43 +25,33 @@ const ZIP64_CD_START_OFFSET_OFFSET = 48n;
 
 /**
  * Parses end of central directory record of zip file
- * @param fileProvider - FileProvider instance
+ * @param file - FileProvider instance
  * @returns Info from the header
  */
-export const parseEoCDRecord = async (fileProvider: FileProvider): Promise<ZipEoCDRecord> => {
-  const zipEoCDOffset = await searchFromTheEnd(fileProvider, eoCDSignature);
+export const parseEoCDRecord = async (file: FileProvider): Promise<ZipEoCDRecord> => {
+  const zipEoCDOffset = await searchFromTheEnd(file, eoCDSignature);
 
-  let cdRecordsNumber = BigInt(
-    await fileProvider.getUint16(zipEoCDOffset + CD_RECORDS_NUMBER_OFFSET)
-  );
-  let cdStartOffset = BigInt(await fileProvider.getUint32(zipEoCDOffset + CD_START_OFFSET_OFFSET));
+  let cdRecordsNumber = BigInt(await file.getUint16(zipEoCDOffset + CD_RECORDS_NUMBER_OFFSET));
+  let cdStartOffset = BigInt(await file.getUint32(zipEoCDOffset + CD_START_OFFSET_OFFSET));
 
   if (cdStartOffset === BigInt(0xffffffff) || cdRecordsNumber === BigInt(0xffffffff)) {
     const zip64EoCDLocatorOffset = zipEoCDOffset - 20n;
 
-    if (
-      Buffer.from(
-        await fileProvider.slice(zip64EoCDLocatorOffset, zip64EoCDLocatorOffset + 4n)
-      ).compare(Buffer.from(zip64EoCDLocatorSignature)) !== 0
-    ) {
+    const magicBytes = await file.slice(zip64EoCDLocatorOffset, zip64EoCDLocatorOffset + 4n);
+    if (!compareArrayBuffers(magicBytes, zip64EoCDLocatorSignature)) {
       throw new Error('zip64 EoCD locator not found');
     }
-    const zip64EoCDOffset = await fileProvider.getBigUint64(
+    const zip64EoCDOffset = await file.getBigUint64(
       zip64EoCDLocatorOffset + ZIP64_EOCD_START_OFFSET_OFFSET
     );
 
-    if (
-      Buffer.from(await fileProvider.slice(zip64EoCDOffset, zip64EoCDOffset + 4n)).compare(
-        Buffer.from(zip64EoCDSignature)
-      ) !== 0
-    ) {
+    const endOfCDMagicBytes = await file.slice(zip64EoCDOffset, zip64EoCDOffset + 4n);
+    if (!compareArrayBuffers(endOfCDMagicBytes, zip64EoCDSignature.buffer)) {
       throw new Error('zip64 EoCD not found');
     }
 
-    cdRecordsNumber = await fileProvider.getBigUint64(
-      zip64EoCDOffset + ZIP64_CD_RECORDS_NUMBER_OFFSET
-    );
-    cdStartOffset = await fileProvider.getBigUint64(zip64EoCDOffset + ZIP64_CD_START_OFFSET_OFFSET);
+    cdRecordsNumber = await file.getBigUint64(zip64EoCDOffset + ZIP64_CD_RECORDS_NUMBER_OFFSET);
+    cdStartOffset = await file.getBigUint64(zip64EoCDOffset + ZIP64_CD_START_OFFSET_OFFSET);
   }
 
   return {

modules/zip/src/parse-zip/local-file-header.ts

@@ -1,4 +1,5 @@
-import {FileProvider} from '@loaders.gl/loader-utils';
+import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils';
+import {ZipSignature} from './search-from-the-end';
 
 /**
  * zip local file header info
@@ -27,7 +28,7 @@ const FILE_NAME_LENGTH_OFFSET = 26n;
 const EXTRA_FIELD_LENGTH_OFFSET = 28n;
 const FILE_NAME_OFFSET = 30n;
 
-export const signature = [0x50, 0x4b, 0x03, 0x04];
+export const signature: ZipSignature = new Uint8Array([0x50, 0x4b, 0x03, 0x04]);
 
 /**
  * Parses local file header of zip file
@@ -39,11 +40,8 @@ export const parseZipLocalFileHeader = async (
   headerOffset: bigint,
   buffer: FileProvider
 ): Promise<ZipLocalFileHeader | null> => {
-  if (
-    Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare(
-      Buffer.from(signature)
-    ) !== 0
-  ) {
+  const magicBytes = await buffer.slice(headerOffset, headerOffset + 4n);
+  if (!compareArrayBuffers(magicBytes, signature)) {
     return null;
   }
 

modules/zip/src/parse-zip/search-from-the-end.ts

@@ -1,7 +1,7 @@
 import {FileProvider} from '@loaders.gl/loader-utils';
 
 /** Description of zip signature type */
-export type ZipSignature = [number, number, number, number];
+export type ZipSignature = Uint8Array;
 
 /**
  * looking for the last occurrence of the provided

modules/zip/test/zip-utils/search-from-the-end.spec.ts

@@ -8,7 +8,7 @@ test('SLPKLoader#searchFromTheEnd', async (t) => {
   t.equals(
     await searchFromTheEnd(
       new DataViewFile(new DataView(DATA_ARRAY.buffer)),
-      [0x50, 0x4b, 0x03, 0x04]
+      new Uint8Array([0x50, 0x4b, 0x03, 0x04])
     ),
     0n
   );