diff --git a/docs/upgrade-guide.md b/docs/upgrade-guide.md index d70753c64e..c746b5f7fc 100644 --- a/docs/upgrade-guide.md +++ b/docs/upgrade-guide.md @@ -58,7 +58,7 @@ and loaders.gl v4.0 aligns with this practice. **@loaders.gl/crypto** -- All hashes now require an encoding parameter. To get previous behavior, just specify `'base64'`. +- All hashes now require an encoding parameter. To get previous behavior, just specify `.hash...(..., 'base64')`. **@loaders.gl/arrow** diff --git a/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-archive.ts b/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-archive.ts index 6b94ceaa7e..681b9b7bba 100644 --- a/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-archive.ts +++ b/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-archive.ts @@ -62,6 +62,9 @@ export class Tiles3DArchive { const arrayBuffer = new TextEncoder().encode(path).buffer; const nameHash = await new MD5Hash().hash(arrayBuffer, 'hex'); const byteOffset = this.hashTable[nameHash]; + if (byteOffset === undefined) { + return null; + } const localFileHeader = await parseZipLocalFileHeader(byteOffset, this.fileProvider); if (!localFileHeader) { diff --git a/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts b/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts index cce96c0943..da6b63aca2 100644 --- a/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts +++ b/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts @@ -45,8 +45,8 @@ const PATH_DESCRIPTIONS: {test: RegExp; extensions: string[]}[] = [ export class SLPKArchive { /** A DataView representation of the archive */ private slpkArchive: FileProvider; - // Maps hex-encoded md5 hashes to bigint offsets into the archive - private hashedOffsetMap: Record; + // Maps hex-encoded md5 filename hashes to bigint offsets into the archive + private hashTable: Record; /** Array of hashes and offsets into archive */ // hashToOffsetMap: Record; @@ -54,9 +54,9 @@ export class SLPKArchive { protected _textDecoder = new TextDecoder(); protected _md5Hash = new MD5Hash(); - constructor(slpkArchive: FileProvider, hashedOffsetMap: Record) { + constructor(slpkArchive: FileProvider, hashTable: Record) { this.slpkArchive = slpkArchive; - this.hashedOffsetMap = hashedOffsetMap; + this.hashTable = hashTable; } /** @@ -65,7 +65,7 @@ export class SLPKArchive { * @param mode - currently only raw mode supported * @returns buffer with ready to use file */ - async getFile(path: string, mode: 'http' | 'raw' = 'raw'): Promise { + async getFile(path: string, mode: 'http' | 'raw' = 'raw'): Promise { if (mode === 'http') { const extensions = PATH_DESCRIPTIONS.find((val) => val.test.test(path))?.extensions; if (extensions) { @@ -77,18 +77,18 @@ export class SLPKArchive { } } if (data) { - return Buffer.from(data); + return data; } } } if (mode === 'raw') { const decompressedFile = await this.getDataByPath(`${path}.gz`); if (decompressedFile) { - return Buffer.from(decompressedFile); + return decompressedFile; } const fileWithoutCompression = await this.getFileBytes(path); if (fileWithoutCompression) { - return Buffer.from(fileWithoutCompression); + return fileWithoutCompression; } } @@ -116,7 +116,7 @@ export class SLPKArchive { const decompressedData = await compression.decompress(data); return decompressedData; } - return Buffer.from(data); + return data; } /** @@ -128,7 +128,7 @@ export class SLPKArchive { const binaryPath = this._textEncoder.encode(path); const nameHash = await this._md5Hash.hash(binaryPath.buffer, 'hex'); - const offset = this.hashedOffsetMap[nameHash]; + const offset = this.hashTable[nameHash]; if (offset === undefined) { return undefined; } diff --git a/modules/tile-converter/src/i3s-server/controllers/slpk-controller.ts b/modules/tile-converter/src/i3s-server/controllers/slpk-controller.ts index 7cf2f3abd4..302d28f8bc 100644 --- a/modules/tile-converter/src/i3s-server/controllers/slpk-controller.ts +++ b/modules/tile-converter/src/i3s-server/controllers/slpk-controller.ts @@ -20,15 +20,15 @@ export async function loadArchive(fullLayerPath: string): Promise { * @param url - I3S HTTP URL * @returns - file content */ -export async function getFileByUrl(url: string): Promise { +export async function getFileByUrl(url: string): Promise { const trimmedPath = /^\/?(.*)\/?$/.exec(url); - let uncompressedFile: Buffer | null = null; if (trimmedPath) { try { - uncompressedFile = await slpkArchive.getFile(trimmedPath[1], 'http'); + const uncompressedFile = await slpkArchive.getFile(trimmedPath[1], 'http'); + return uncompressedFile; } catch { // TODO - log error? } } - return uncompressedFile; + return null; } diff --git a/modules/zip/src/hash-file-utility.ts b/modules/zip/src/hash-file-utility.ts index 3ac2ccf05e..4970c82248 100644 --- a/modules/zip/src/hash-file-utility.ts +++ b/modules/zip/src/hash-file-utility.ts @@ -28,29 +28,6 @@ function bufferToHex(buffer: ArrayBuffer, start: number, length: number): string .join(''); } -/** - * generates hash info from central directory - * @param fileProvider - provider of the archive - * @returns ready to use hash info - */ -export async function getHashMapFromZipArchive( - fileProvider: FileProvider -): Promise> { - const zipCDIterator = makeZipCDHeaderIterator(fileProvider); - const md5Hash = new MD5Hash(); - const textEncoder = new TextEncoder(); - - const hashMap: Record = {}; - for await (const cdHeader of zipCDIterator) { - const filename = cdHeader.fileName.split('\\').join('/').toLocaleLowerCase(); - const arrayBuffer = textEncoder.encode(filename).buffer; - const md5 = await md5Hash.hash(arrayBuffer, 'hex'); - hashMap[md5] = cdHeader.localHeaderOffset; - } - return hashMap; -} - -// /** * generates hash info from zip files "central directory" * @param fileProvider - provider of the archive diff --git a/modules/zip/src/parse-zip/cd-file-header.ts b/modules/zip/src/parse-zip/cd-file-header.ts index aa0d2518f1..85dafde0a9 100644 --- a/modules/zip/src/parse-zip/cd-file-header.ts +++ b/modules/zip/src/parse-zip/cd-file-header.ts @@ -1,4 +1,4 @@ -import {FileProvider} from '@loaders.gl/loader-utils'; +import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils'; import {parseEoCDRecord} from './end-of-central-directory'; import {ZipSignature} from './search-from-the-end'; @@ -31,7 +31,7 @@ const CD_EXTRA_FIELD_LENGTH_OFFSET = 30n; const CD_LOCAL_HEADER_OFFSET_OFFSET = 42n; const CD_FILE_NAME_OFFSET = 46n; -export const signature: ZipSignature = [0x50, 0x4b, 0x01, 0x02]; +export const signature: ZipSignature = new Uint8Array([0x50, 0x4b, 0x01, 0x02]); /** * Parses central directory file header of zip file @@ -41,48 +41,39 @@ export const signature: ZipSignature = [0x50, 0x4b, 0x01, 0x02]; */ export const parseZipCDFileHeader = async ( headerOffset: bigint, - buffer: FileProvider + file: FileProvider ): Promise => { - if ( - Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare( - Buffer.from(signature) - ) !== 0 - ) { + const magicBytes = await file.slice(headerOffset, headerOffset + 4n); + if (!compareArrayBuffers(magicBytes, signature.buffer)) { return null; } - let compressedSize = BigInt(await buffer.getUint32(headerOffset + CD_COMPRESSED_SIZE_OFFSET)); - - let uncompressedSize = BigInt(await buffer.getUint32(headerOffset + CD_UNCOMPRESSED_SIZE_OFFSET)); - - const extraFieldLength = await buffer.getUint16(headerOffset + CD_EXTRA_FIELD_LENGTH_OFFSET); - - const fileNameLength = await buffer.getUint16(headerOffset + CD_FILE_NAME_LENGTH_OFFSET); - - const fileName = new TextDecoder().decode( - await buffer.slice( - headerOffset + CD_FILE_NAME_OFFSET, - headerOffset + CD_FILE_NAME_OFFSET + BigInt(fileNameLength) - ) + let compressedSize = BigInt(await file.getUint32(headerOffset + CD_COMPRESSED_SIZE_OFFSET)); + let uncompressedSize = BigInt(await file.getUint32(headerOffset + CD_UNCOMPRESSED_SIZE_OFFSET)); + const extraFieldLength = await file.getUint16(headerOffset + CD_EXTRA_FIELD_LENGTH_OFFSET); + const fileNameLength = await file.getUint16(headerOffset + CD_FILE_NAME_LENGTH_OFFSET); + const filenameBytes = await file.slice( + headerOffset + CD_FILE_NAME_OFFSET, + headerOffset + CD_FILE_NAME_OFFSET + BigInt(fileNameLength) ); + const fileName = new TextDecoder().decode(filenameBytes); const extraOffset = headerOffset + CD_FILE_NAME_OFFSET + BigInt(fileNameLength); - - const oldFormatOffset = await buffer.getUint32(headerOffset + CD_LOCAL_HEADER_OFFSET_OFFSET); + const oldFormatOffset = await file.getUint32(headerOffset + CD_LOCAL_HEADER_OFFSET_OFFSET); let fileDataOffset = BigInt(oldFormatOffset); let offsetInZip64Data = 4n; // looking for info that might be also be in zip64 extra field if (uncompressedSize === BigInt(0xffffffff)) { - uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data); + uncompressedSize = await file.getBigUint64(extraOffset + offsetInZip64Data); offsetInZip64Data += 8n; } if (compressedSize === BigInt(0xffffffff)) { - compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data); + compressedSize = await file.getBigUint64(extraOffset + offsetInZip64Data); offsetInZip64Data += 8n; } if (fileDataOffset === BigInt(0xffffffff)) { - fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64 + fileDataOffset = await file.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64 } const localHeaderOffset = fileDataOffset; diff --git a/modules/zip/src/parse-zip/end-of-central-directory.ts b/modules/zip/src/parse-zip/end-of-central-directory.ts index ff40a359dd..ddb5e7f331 100644 --- a/modules/zip/src/parse-zip/end-of-central-directory.ts +++ b/modules/zip/src/parse-zip/end-of-central-directory.ts @@ -1,4 +1,4 @@ -import {FileProvider} from '@loaders.gl/loader-utils'; +import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils'; import {ZipSignature, searchFromTheEnd} from './search-from-the-end'; /** @@ -12,9 +12,9 @@ export type ZipEoCDRecord = { cdRecordsNumber: bigint; }; -const eoCDSignature: ZipSignature = [0x50, 0x4b, 0x05, 0x06]; -const zip64EoCDLocatorSignature = [0x50, 0x4b, 0x06, 0x07]; -const zip64EoCDSignature = [0x50, 0x4b, 0x06, 0x06]; +const eoCDSignature: ZipSignature = new Uint8Array([0x50, 0x4b, 0x05, 0x06]); +const zip64EoCDLocatorSignature = new Uint8Array([0x50, 0x4b, 0x06, 0x07]); +const zip64EoCDSignature = new Uint8Array([0x50, 0x4b, 0x06, 0x06]); // offsets accroding to https://en.wikipedia.org/wiki/ZIP_(file_format) const CD_RECORDS_NUMBER_OFFSET = 8n; @@ -25,43 +25,33 @@ const ZIP64_CD_START_OFFSET_OFFSET = 48n; /** * Parses end of central directory record of zip file - * @param fileProvider - FileProvider instance + * @param file - FileProvider instance * @returns Info from the header */ -export const parseEoCDRecord = async (fileProvider: FileProvider): Promise => { - const zipEoCDOffset = await searchFromTheEnd(fileProvider, eoCDSignature); +export const parseEoCDRecord = async (file: FileProvider): Promise => { + const zipEoCDOffset = await searchFromTheEnd(file, eoCDSignature); - let cdRecordsNumber = BigInt( - await fileProvider.getUint16(zipEoCDOffset + CD_RECORDS_NUMBER_OFFSET) - ); - let cdStartOffset = BigInt(await fileProvider.getUint32(zipEoCDOffset + CD_START_OFFSET_OFFSET)); + let cdRecordsNumber = BigInt(await file.getUint16(zipEoCDOffset + CD_RECORDS_NUMBER_OFFSET)); + let cdStartOffset = BigInt(await file.getUint32(zipEoCDOffset + CD_START_OFFSET_OFFSET)); if (cdStartOffset === BigInt(0xffffffff) || cdRecordsNumber === BigInt(0xffffffff)) { const zip64EoCDLocatorOffset = zipEoCDOffset - 20n; - if ( - Buffer.from( - await fileProvider.slice(zip64EoCDLocatorOffset, zip64EoCDLocatorOffset + 4n) - ).compare(Buffer.from(zip64EoCDLocatorSignature)) !== 0 - ) { + const magicBytes = await file.slice(zip64EoCDLocatorOffset, zip64EoCDLocatorOffset + 4n); + if (!compareArrayBuffers(magicBytes, zip64EoCDLocatorSignature)) { throw new Error('zip64 EoCD locator not found'); } - const zip64EoCDOffset = await fileProvider.getBigUint64( + const zip64EoCDOffset = await file.getBigUint64( zip64EoCDLocatorOffset + ZIP64_EOCD_START_OFFSET_OFFSET ); - if ( - Buffer.from(await fileProvider.slice(zip64EoCDOffset, zip64EoCDOffset + 4n)).compare( - Buffer.from(zip64EoCDSignature) - ) !== 0 - ) { + const endOfCDMagicBytes = await file.slice(zip64EoCDOffset, zip64EoCDOffset + 4n); + if (!compareArrayBuffers(endOfCDMagicBytes, zip64EoCDSignature.buffer)) { throw new Error('zip64 EoCD not found'); } - cdRecordsNumber = await fileProvider.getBigUint64( - zip64EoCDOffset + ZIP64_CD_RECORDS_NUMBER_OFFSET - ); - cdStartOffset = await fileProvider.getBigUint64(zip64EoCDOffset + ZIP64_CD_START_OFFSET_OFFSET); + cdRecordsNumber = await file.getBigUint64(zip64EoCDOffset + ZIP64_CD_RECORDS_NUMBER_OFFSET); + cdStartOffset = await file.getBigUint64(zip64EoCDOffset + ZIP64_CD_START_OFFSET_OFFSET); } return { diff --git a/modules/zip/src/parse-zip/local-file-header.ts b/modules/zip/src/parse-zip/local-file-header.ts index 1cf02a8050..1bfa7a2bc1 100644 --- a/modules/zip/src/parse-zip/local-file-header.ts +++ b/modules/zip/src/parse-zip/local-file-header.ts @@ -1,4 +1,5 @@ -import {FileProvider} from '@loaders.gl/loader-utils'; +import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils'; +import {ZipSignature} from './search-from-the-end'; /** * zip local file header info @@ -27,7 +28,7 @@ const FILE_NAME_LENGTH_OFFSET = 26n; const EXTRA_FIELD_LENGTH_OFFSET = 28n; const FILE_NAME_OFFSET = 30n; -export const signature = [0x50, 0x4b, 0x03, 0x04]; +export const signature: ZipSignature = new Uint8Array([0x50, 0x4b, 0x03, 0x04]); /** * Parses local file header of zip file @@ -39,11 +40,8 @@ export const parseZipLocalFileHeader = async ( headerOffset: bigint, buffer: FileProvider ): Promise => { - if ( - Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare( - Buffer.from(signature) - ) !== 0 - ) { + const magicBytes = await buffer.slice(headerOffset, headerOffset + 4n); + if (!compareArrayBuffers(magicBytes, signature)) { return null; } diff --git a/modules/zip/src/parse-zip/search-from-the-end.ts b/modules/zip/src/parse-zip/search-from-the-end.ts index b832563654..5630ee0f5b 100644 --- a/modules/zip/src/parse-zip/search-from-the-end.ts +++ b/modules/zip/src/parse-zip/search-from-the-end.ts @@ -1,7 +1,7 @@ import {FileProvider} from '@loaders.gl/loader-utils'; /** Description of zip signature type */ -export type ZipSignature = [number, number, number, number]; +export type ZipSignature = Uint8Array; /** * looking for the last occurrence of the provided diff --git a/modules/zip/test/zip-utils/search-from-the-end.spec.ts b/modules/zip/test/zip-utils/search-from-the-end.spec.ts index da6270e37f..1332c536c4 100644 --- a/modules/zip/test/zip-utils/search-from-the-end.spec.ts +++ b/modules/zip/test/zip-utils/search-from-the-end.spec.ts @@ -8,7 +8,7 @@ test('SLPKLoader#searchFromTheEnd', async (t) => { t.equals( await searchFromTheEnd( new DataViewFile(new DataView(DATA_ARRAY.buffer)), - [0x50, 0x4b, 0x03, 0x04] + new Uint8Array([0x50, 0x4b, 0x03, 0x04]) ), 0n );