From 4be14fbfc234bd5e6f1b70e1e6a7723738fdb522 Mon Sep 17 00:00:00 2001 From: Ib Green Date: Fri, 6 Oct 2023 22:23:15 +0200 Subject: [PATCH 1/4] chore(zip): Refactor zip content hash tables (#2500) --- .../3d-tiles-archive-archive.ts | 24 ++- .../3d-tiles-archive-parser.ts | 22 +-- .../tiles-3d-archive-file-system.ts | 19 ++- modules/3d-tiles/src/types.ts | 8 +- modules/i3s/package.json | 4 +- modules/i3s/src/i3s-slpk-loader.ts | 6 +- modules/i3s/src/index.ts | 4 +- .../src/lib/parsers/parse-slpk/parse-slpk.ts | 26 ++-- .../lib/parsers/parse-slpk/slpk-archieve.ts | 36 +++-- modules/i3s/tsconfig.json | 1 + .../i3s-server/controllers/slpk-controller.ts | 18 ++- .../controllers/slpk-controller.spec.ts | 3 +- modules/zip/package.json | 1 + modules/zip/src/filesystems/zip-filesystem.ts | 6 +- modules/zip/src/hash-file-utility.ts | 137 +++++++----------- modules/zip/src/index.ts | 6 +- modules/zip/src/parse-zip/cd-file-header.ts | 4 +- modules/zip/tsconfig.json | 3 +- yarn.lock | 6 +- 19 files changed, 161 insertions(+), 173 deletions(-) diff --git a/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-archive.ts b/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-archive.ts index ef0ad0564b..6b94ceaa7e 100644 --- a/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-archive.ts +++ b/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-archive.ts @@ -1,7 +1,7 @@ -import md5 from 'md5'; import {FileProvider} from '@loaders.gl/loader-utils'; -import {parseZipLocalFileHeader, HashElement, findBin} from '@loaders.gl/zip'; +import {MD5Hash} from '@loaders.gl/crypto'; import {DeflateCompression, NoCompression} from '@loaders.gl/compression'; +import {parseZipLocalFileHeader} from '@loaders.gl/zip'; type CompressionHandler = (compressedFile: ArrayBuffer) => Promise; @@ -22,16 +22,16 @@ export class Tiles3DArchive { /** FileProvider with whe whole file */ private fileProvider: FileProvider; /** hash info */ - private hashArray: HashElement[]; + private hashTable: Record; /** * creates Tiles3DArchive handler * @param fileProvider - FileProvider with the whole file - * @param hashFile - hash info + * @param hashTable - hash info */ - constructor(fileProvider: FileProvider, hashFile: HashElement[]) { + constructor(fileProvider: FileProvider, hashTable: Record) { this.fileProvider = fileProvider; - this.hashArray = hashFile; + this.hashTable = hashTable; } /** @@ -47,7 +47,7 @@ export class Tiles3DArchive { data = await this.getFileBytes(path); } if (!data) { - throw new Error('No such file in the archive'); + throw new Error(`No such file in the archive: ${path}`); } return data; @@ -59,13 +59,11 @@ export class Tiles3DArchive { * @returns buffer with the raw file data */ private async getFileBytes(path: string): Promise { - const nameHash = Buffer.from(md5(path), 'hex'); - const fileInfo = findBin(nameHash, this.hashArray); // implement binary search - if (!fileInfo) { - return null; - } + const arrayBuffer = new TextEncoder().encode(path).buffer; + const nameHash = await new MD5Hash().hash(arrayBuffer, 'hex'); + const byteOffset = this.hashTable[nameHash]; - const localFileHeader = await parseZipLocalFileHeader(fileInfo.offset, this.fileProvider); + const localFileHeader = await parseZipLocalFileHeader(byteOffset, this.fileProvider); if (!localFileHeader) { return null; } diff --git a/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-parser.ts b/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-parser.ts index 08042f16b8..49906d8497 100644 --- a/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-parser.ts +++ b/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-parser.ts @@ -1,9 +1,8 @@ import {FileProvider} from '@loaders.gl/loader-utils'; import { - HashElement, cdSignature as cdHeaderSignature, - generateHashInfo, - parseHashFile, + makeHashTableFromZipHeaders, + parseHashTable, parseZipCDFileHeader, parseZipLocalFileHeader, searchFromTheEnd @@ -24,19 +23,20 @@ export const parse3DTilesArchive = async ( const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider); - let hashData: HashElement[]; + let hashTable: Record; if (cdFileHeader?.fileName !== '@3dtilesIndex1@') { - cb?.('3tz doesnt contain hash file'); - hashData = await generateHashInfo(fileProvider); - cb?.('hash info has been composed according to central directory records'); + hashTable = await makeHashTableFromZipHeaders(fileProvider); + cb?.( + '3tz doesnt contain hash file, hash info has been composed according to zip archive headers' + ); } else { - cb?.('3tz contains hash file'); + // cb?.('3tz contains hash file'); const localFileHeader = await parseZipLocalFileHeader( cdFileHeader.localHeaderOffset, fileProvider ); if (!localFileHeader) { - throw new Error('corrupted 3tz'); + throw new Error('corrupted 3tz zip archive'); } const fileDataOffset = localFileHeader.fileDataOffset; @@ -45,8 +45,8 @@ export const parse3DTilesArchive = async ( fileDataOffset + localFileHeader.compressedSize ); - hashData = parseHashFile(hashFile); + hashTable = parseHashTable(hashFile); } - return new Tiles3DArchive(fileProvider, hashData); + return new Tiles3DArchive(fileProvider, hashTable); }; diff --git a/modules/3d-tiles/src/lib/filesystems/tiles-3d-archive-file-system.ts b/modules/3d-tiles/src/lib/filesystems/tiles-3d-archive-file-system.ts index a6cd670d3b..011dd11198 100644 --- a/modules/3d-tiles/src/lib/filesystems/tiles-3d-archive-file-system.ts +++ b/modules/3d-tiles/src/lib/filesystems/tiles-3d-archive-file-system.ts @@ -4,8 +4,7 @@ import { cdSignature as cdHeaderSignature, searchFromTheEnd, parseZipCDFileHeader, - HashElement, - parseHashFile, + parseHashTable, parseZipLocalFileHeader } from '@loaders.gl/zip'; import {Tiles3DArchive} from '../../3d-tiles-archive/3d-tiles-archive-archive'; @@ -18,7 +17,7 @@ import {Tiles3DArchive} from '../../3d-tiles-archive/3d-tiles-archive-archive'; * @see https://github.com/erikdahlstrom/3tz-specification/blob/master/Specification.md */ export class Tiles3DArchiveFileSystem extends ZipFileSystem { - hashData?: HashElement[] | null; + hashTable?: Record | null; /** * Constructor @@ -40,9 +39,9 @@ export class Tiles3DArchiveFileSystem extends ZipFileSystem { if (!fileProvider) { throw new Error('No data detected in the zip archive'); } - await this.parseHashFile(); - if (this.hashData) { - const archive = new Tiles3DArchive(fileProvider, this.hashData); + await this.parseHashTable(); + if (this.hashTable) { + const archive = new Tiles3DArchive(fileProvider, this.hashTable); const fileData = await archive.getFile(filename); const response = new Response(fileData); @@ -57,8 +56,8 @@ export class Tiles3DArchiveFileSystem extends ZipFileSystem { * to files inside the archive * @returns void */ - private async parseHashFile(): Promise { - if (this.hashData !== undefined) { + private async parseHashTable(): Promise { + if (this.hashTable !== undefined) { return; } @@ -89,9 +88,9 @@ export class Tiles3DArchiveFileSystem extends ZipFileSystem { fileDataOffset + localFileHeader.compressedSize ); - this.hashData = parseHashFile(hashFile); + this.hashTable = parseHashTable(hashFile); } else { - this.hashData = null; + this.hashTable = null; } } } diff --git a/modules/3d-tiles/src/types.ts b/modules/3d-tiles/src/types.ts index daa13970bd..f3705024c7 100644 --- a/modules/3d-tiles/src/types.ts +++ b/modules/3d-tiles/src/types.ts @@ -301,9 +301,9 @@ export type Tiles3DTileContent = { */ export type Subtree = { /** An array of buffers. */ - buffers: Buffer[]; + buffers: GLTFStyleBuffer[]; /** An array of buffer views. */ - bufferViews: BufferView[]; + bufferViews: GLTFStyleBufferView[]; /** The availability of tiles in the subtree. The availability bitstream is a 1D boolean array where tiles are ordered by their level in the subtree and Morton index * within that level. A tile's availability is determined by a single bit, 1 meaning a tile exists at that spatial index, and 0 meaning it does not. * The number of elements in the array is `(N^subtreeLevels - 1)/(N - 1)` where N is 4 for subdivision scheme `QUADTREE` and 8 for `OCTREE`. @@ -356,14 +356,14 @@ export type ExplicitBitstream = Uint8Array; */ export type SubdivisionScheme = 'QUADTREE' | 'OCTREE'; -type Buffer = { +type GLTFStyleBuffer = { name: string; uri?: string; byteLength: number; }; /** Subtree buffer view */ -export type BufferView = { +export type GLTFStyleBufferView = { buffer: number; byteOffset: number; byteLength: number; diff --git a/modules/i3s/package.json b/modules/i3s/package.json index c5adacbfd1..1e8410525b 100644 --- a/modules/i3s/package.json +++ b/modules/i3s/package.json @@ -34,6 +34,7 @@ }, "dependencies": { "@loaders.gl/compression": "4.0.0-beta.2", + "@loaders.gl/crypto": "4.0.0-beta.2", "@loaders.gl/draco": "4.0.0-beta.2", "@loaders.gl/images": "4.0.0-beta.2", "@loaders.gl/loader-utils": "4.0.0-beta.2", @@ -43,8 +44,7 @@ "@luma.gl/constants": "^8.5.4", "@math.gl/core": "^3.5.1", "@math.gl/culling": "^3.5.1", - "@math.gl/geospatial": "^3.5.1", - "md5": "^2.3.0" + "@math.gl/geospatial": "^3.5.1" }, "peerDependencies": { "@loaders.gl/core": "^4.0.0-alpha.8" diff --git a/modules/i3s/src/i3s-slpk-loader.ts b/modules/i3s/src/i3s-slpk-loader.ts index 098553445d..1d8d97bf0a 100644 --- a/modules/i3s/src/i3s-slpk-loader.ts +++ b/modules/i3s/src/i3s-slpk-loader.ts @@ -1,6 +1,6 @@ import type {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils'; import {DataViewFile} from '@loaders.gl/loader-utils'; -import {parseSLPK as parseSLPKFromProvider} from './lib/parsers/parse-slpk/parse-slpk'; +import {parseSLPKArchive as parseSLPKFromProvider} from './lib/parsers/parse-slpk/parse-slpk'; // __VERSION__ is injected by babel-plugin-version-inline // @ts-ignore TS2304: Cannot find name '__VERSION__'. @@ -25,7 +25,7 @@ export const SLPKLoader: LoaderWithParser module: 'i3s', version: VERSION, mimeTypes: ['application/octet-stream'], - parse: parseSLPK, + parse: parseSLPKArchive, extensions: ['slpk'], options: {} }; @@ -37,7 +37,7 @@ export const SLPKLoader: LoaderWithParser * @returns requested file */ -async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) { +async function parseSLPKArchive(data: ArrayBuffer, options: SLPKLoaderOptions = {}) { return (await parseSLPKFromProvider(new DataViewFile(new DataView(data)))).getFile( options.slpk?.path ?? '', options.slpk?.pathMode diff --git a/modules/i3s/src/index.ts b/modules/i3s/src/index.ts index e12cc895c7..ec48965ef3 100644 --- a/modules/i3s/src/index.ts +++ b/modules/i3s/src/index.ts @@ -46,4 +46,6 @@ export {I3SAttributeLoader, loadFeatureAttributes} from './i3s-attribute-loader' export {I3SBuildingSceneLayerLoader} from './i3s-building-scene-layer-loader'; export {I3SNodePageLoader} from './i3s-node-page-loader'; export {ArcGISWebSceneLoader} from './arcgis-webscene-loader'; -export {parseSLPK} from './lib/parsers/parse-slpk/parse-slpk'; + +export type {SLPKArchive} from './lib/parsers/parse-slpk/slpk-archieve'; +export {parseSLPKArchive} from './lib/parsers/parse-slpk/parse-slpk'; diff --git a/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts b/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts index cdeb035f72..51150ec5f8 100644 --- a/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts +++ b/modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts @@ -4,9 +4,8 @@ import { cdSignature as cdHeaderSignature, parseZipLocalFileHeader, searchFromTheEnd, - HashElement, - parseHashFile, - generateHashInfo + parseHashTable, + makeHashTableFromZipHeaders } from '@loaders.gl/zip'; import {SLPKArchive} from './slpk-archieve'; @@ -16,21 +15,22 @@ import {SLPKArchive} from './slpk-archieve'; * @param cb is called with information message during parsing * @returns slpk file handler */ -export const parseSLPK = async ( +export async function parseSLPKArchive( fileProvider: FileProvider, cb?: (msg: string) => void -): Promise => { +): Promise { const hashCDOffset = await searchFromTheEnd(fileProvider, cdHeaderSignature); const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider); - let hashData: HashElement[]; + let hashTable: Record; if (cdFileHeader?.fileName !== '@specialIndexFileHASH128@') { - cb?.('SLPK doesnt contain hash file'); - hashData = await generateHashInfo(fileProvider); - cb?.('hash info has been composed according to central directory records'); + hashTable = await makeHashTableFromZipHeaders(fileProvider); + cb?.( + 'SLPK doesnt contain hash file, hash info has been composed according to zip archive headers' + ); } else { - cb?.('SLPK contains hash file'); + // cb?.('SLPK contains hash file'); const localFileHeader = await parseZipLocalFileHeader( cdFileHeader.localHeaderOffset, fileProvider @@ -45,8 +45,8 @@ export const parseSLPK = async ( fileDataOffset + localFileHeader.compressedSize ); - hashData = parseHashFile(hashFile); + hashTable = parseHashTable(hashFile); } - return new SLPKArchive(fileProvider, hashData); -}; + return new SLPKArchive(fileProvider, hashTable); +} diff --git a/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts b/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts index 087dd30dcc..cce96c0943 100644 --- a/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts +++ b/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts @@ -1,6 +1,6 @@ -import md5 from 'md5'; +import {MD5Hash} from '@loaders.gl/crypto'; import {FileProvider} from '@loaders.gl/loader-utils'; -import {parseZipLocalFileHeader, HashElement, findBin} from '@loaders.gl/zip'; +import {parseZipLocalFileHeader} from '@loaders.gl/zip'; import {GZipCompression} from '@loaders.gl/compression'; /** Description of real paths for different file types */ @@ -43,11 +43,20 @@ const PATH_DESCRIPTIONS: {test: RegExp; extensions: string[]}[] = [ * Class for handling information about slpk file */ export class SLPKArchive { + /** A DataView representation of the archive */ private slpkArchive: FileProvider; - private hashArray: HashElement[]; - constructor(slpkArchive: FileProvider, hashFile: HashElement[]) { + // Maps hex-encoded md5 hashes to bigint offsets into the archive + private hashedOffsetMap: Record; + /** Array of hashes and offsets into archive */ + // hashToOffsetMap: Record; + + protected _textEncoder = new TextEncoder(); + protected _textDecoder = new TextDecoder(); + protected _md5Hash = new MD5Hash(); + + constructor(slpkArchive: FileProvider, hashedOffsetMap: Record) { this.slpkArchive = slpkArchive; - this.hashArray = hashFile; + this.hashedOffsetMap = hashedOffsetMap; } /** @@ -83,7 +92,7 @@ export class SLPKArchive { } } - throw new Error('No such file in the archieve'); + throw new Error(`No such file in the archive: ${path}`); } /** @@ -111,18 +120,20 @@ export class SLPKArchive { } /** - * Trying to get raw file data by adress + * Trying to get raw file data by address * @param path - path inside the archive * @returns buffer with the raw file data */ private async getFileBytes(path: string): Promise { - const nameHash = Buffer.from(md5(path), 'hex'); - const fileInfo = findBin(nameHash, this.hashArray); // implement binary search - if (!fileInfo) { + const binaryPath = this._textEncoder.encode(path); + const nameHash = await this._md5Hash.hash(binaryPath.buffer, 'hex'); + + const offset = this.hashedOffsetMap[nameHash]; + if (offset === undefined) { return undefined; } - const localFileHeader = await parseZipLocalFileHeader(fileInfo.offset, this.slpkArchive); + const localFileHeader = await parseZipLocalFileHeader(offset, this.slpkArchive); if (!localFileHeader) { return undefined; } @@ -134,7 +145,4 @@ export class SLPKArchive { return compressedFile; } - findBin(nameHash: Buffer) { - throw new Error('Method not implemented.'); - } } diff --git a/modules/i3s/tsconfig.json b/modules/i3s/tsconfig.json index 8d900c7e9b..911d637bbd 100644 --- a/modules/i3s/tsconfig.json +++ b/modules/i3s/tsconfig.json @@ -10,6 +10,7 @@ "references": [ {"path": "../core"}, {"path": "../compression"}, + {"path": "../crypto"}, {"path": "../draco"}, {"path": "../gltf"}, {"path": "../images"}, diff --git a/modules/tile-converter/src/i3s-server/controllers/slpk-controller.ts b/modules/tile-converter/src/i3s-server/controllers/slpk-controller.ts index 9e6941cda6..7cf2f3abd4 100644 --- a/modules/tile-converter/src/i3s-server/controllers/slpk-controller.ts +++ b/modules/tile-converter/src/i3s-server/controllers/slpk-controller.ts @@ -1,32 +1,34 @@ import '@loaders.gl/polyfills'; -import {parseSLPK} from '@loaders.gl/i3s'; +import {parseSLPKArchive, SLPKArchive} from '@loaders.gl/i3s'; import {FileHandleFile} from '@loaders.gl/loader-utils'; -let slpkArchive; +let slpkArchive: SLPKArchive; /** * Open SLPK file for reading and load HASH file * @param fullLayerPath - full path to SLPK file */ -export const loadArchive = async (fullLayerPath: string): Promise => { - slpkArchive = await parseSLPK(await FileHandleFile.from(fullLayerPath), (msg) => +export async function loadArchive(fullLayerPath: string): Promise { + slpkArchive = await parseSLPKArchive(await FileHandleFile.from(fullLayerPath), (msg) => console.log(msg) ); console.log('The server is ready to use'); -}; +} /** * Get a file from SLPK * @param url - I3S HTTP URL * @returns - file content */ -export async function getFileByUrl(url: string) { +export async function getFileByUrl(url: string): Promise { const trimmedPath = /^\/?(.*)\/?$/.exec(url); let uncompressedFile: Buffer | null = null; if (trimmedPath) { try { - uncompressedFile = Buffer.from(await slpkArchive.getFile(trimmedPath[1], 'http')); - } catch (e) {} + uncompressedFile = await slpkArchive.getFile(trimmedPath[1], 'http'); + } catch { + // TODO - log error? + } } return uncompressedFile; } diff --git a/modules/tile-converter/test/i3s-server/controllers/slpk-controller.spec.ts b/modules/tile-converter/test/i3s-server/controllers/slpk-controller.spec.ts index 324b7783ac..85f0f3137f 100644 --- a/modules/tile-converter/test/i3s-server/controllers/slpk-controller.spec.ts +++ b/modules/tile-converter/test/i3s-server/controllers/slpk-controller.spec.ts @@ -1,6 +1,6 @@ import test from 'tape-promise/tape'; import {isBrowser} from '@loaders.gl/core'; -import path from 'path'; +import {path} from '@loaders.gl/loader-utils'; import {getFileByUrl, loadArchive} from '../../../src/i3s-server/controllers/slpk-controller'; const URL_PREFIX = ''; @@ -32,7 +32,6 @@ test('tile-converter(i3s-server)#getFileByUrl return files content', async (t) = t.end(); return; } - const FULL_LAYER_PATH = path.join(process.cwd(), SLPK_URL); // eslint-disable-line no-undef await loadArchive(FULL_LAYER_PATH); diff --git a/modules/zip/package.json b/modules/zip/package.json index 6d9f51e7a0..54928414c0 100644 --- a/modules/zip/package.json +++ b/modules/zip/package.json @@ -31,6 +31,7 @@ }, "dependencies": { "@loaders.gl/compression": "4.0.0-beta.2", + "@loaders.gl/crypto": "4.0.0-beta.2", "@loaders.gl/loader-utils": "4.0.0-beta.2", "jszip": "^3.1.5", "md5": "^2.3.0" diff --git a/modules/zip/src/filesystems/zip-filesystem.ts b/modules/zip/src/filesystems/zip-filesystem.ts index c17006b954..1f48d67013 100644 --- a/modules/zip/src/filesystems/zip-filesystem.ts +++ b/modules/zip/src/filesystems/zip-filesystem.ts @@ -1,7 +1,7 @@ import {FileSystem, isBrowser} from '@loaders.gl/loader-utils'; import {FileProvider, isFileProvider} from '@loaders.gl/loader-utils'; import {FileHandleFile} from '@loaders.gl/loader-utils'; -import {ZipCDFileHeader, zipCDFileHeaderGenerator} from '../parse-zip/cd-file-header'; +import {ZipCDFileHeader, makeZipCDHeaderIterator} from '../parse-zip/cd-file-header'; import {parseZipLocalFileHeader} from '../parse-zip/local-file-header'; import {DeflateCompression} from '@loaders.gl/compression'; @@ -63,7 +63,7 @@ export class ZipFileSystem implements FileSystem { throw new Error('No data detected in the zip archive'); } const fileNames: string[] = []; - const zipCDIterator = zipCDFileHeaderGenerator(fileProvider); + const zipCDIterator = makeZipCDHeaderIterator(fileProvider); for await (const cdHeader of zipCDIterator) { fileNames.push(cdHeader.fileName); } @@ -126,7 +126,7 @@ export class ZipFileSystem implements FileSystem { if (!fileProvider) { throw new Error('No data detected in the zip archive'); } - const zipCDIterator = zipCDFileHeaderGenerator(fileProvider); + const zipCDIterator = makeZipCDHeaderIterator(fileProvider); let result: ZipCDFileHeader | null = null; for await (const cdHeader of zipCDIterator) { if (cdHeader.fileName === filename) { diff --git a/modules/zip/src/hash-file-utility.ts b/modules/zip/src/hash-file-utility.ts index 9b1c568986..3ac2ccf05e 100644 --- a/modules/zip/src/hash-file-utility.ts +++ b/modules/zip/src/hash-file-utility.ts @@ -1,101 +1,76 @@ -import md5 from 'md5'; +import {MD5Hash} from '@loaders.gl/crypto'; import {FileProvider} from '@loaders.gl/loader-utils'; -import {zipCDFileHeaderGenerator} from './parse-zip/cd-file-header'; - -/** Element of hash array */ -export type HashElement = { - /** File name hash */ - hash: Buffer; - /** File offset in the archive */ - offset: bigint; -}; +import {makeZipCDHeaderIterator} from './parse-zip/cd-file-header'; /** - * Comparing md5 hashes according to https://github.com/Esri/i3s-spec/blob/master/docs/2.0/slpk_hashtable.pcsl.md step 5 - * @param hash1 hash to compare - * @param hash2 hash to compare - * @returns -1 if hash1 < hash2, 0 of hash1 === hash2, 1 if hash1 > hash2 + * Reads hash file from buffer and returns it in ready-to-use form + * @param arrayBuffer - buffer containing hash file + * @returns Map containing hash and offset */ -export const compareHashes = (hash1: Buffer, hash2: Buffer): number => { - const h1 = new BigUint64Array(hash1.buffer, hash1.byteOffset, 2); - const h2 = new BigUint64Array(hash2.buffer, hash2.byteOffset, 2); +export function parseHashTable(arrayBuffer: ArrayBuffer): Record { + const dataView = new DataView(arrayBuffer); - const diff = h1[0] === h2[0] ? h1[1] - h2[1] : h1[0] - h2[0]; + const hashMap: Record = {}; - if (diff < 0n) { - return -1; - } else if (diff === 0n) { - return 0; + for (let i = 0; i < arrayBuffer.byteLength; i = i + 24) { + const offset = dataView.getBigUint64(i + 16, true); + const hash = bufferToHex(arrayBuffer, i, 16); + hashMap[hash] = offset; } - return 1; -}; -/** - * Reads hash file from buffer and returns it in ready-to-use form - * @param hashFile - bufer containing hash file - * @returns Array containing file info - */ -export const parseHashFile = (hashFile: ArrayBuffer): HashElement[] => { - const hashFileBuffer = Buffer.from(hashFile); - const hashArray: HashElement[] = []; - for (let i = 0; i < hashFileBuffer.buffer.byteLength; i = i + 24) { - const offsetBuffer = new DataView( - hashFileBuffer.buffer.slice( - hashFileBuffer.byteOffset + i + 16, - hashFileBuffer.byteOffset + i + 24 - ) - ); - const offset = offsetBuffer.getBigUint64(offsetBuffer.byteOffset, true); - hashArray.push({ - hash: Buffer.from( - hashFileBuffer.subarray(hashFileBuffer.byteOffset + i, hashFileBuffer.byteOffset + i + 16) - ), - offset - }); - } - return hashArray; -}; + return hashMap; +} + +function bufferToHex(buffer: ArrayBuffer, start: number, length: number): string { + // buffer is an ArrayBuffer + return [...new Uint8Array(buffer, start, length)] + .map((x) => x.toString(16).padStart(2, '0')) + .join(''); +} /** - * Binary search in the hash info - * @param hashToSearch hash that we need to find - * @returns required hash element or undefined if not found + * generates hash info from central directory + * @param fileProvider - provider of the archive + * @returns ready to use hash info */ -export const findBin = ( - hashToSearch: Buffer, - hashArray: HashElement[] -): HashElement | undefined => { - let lowerBorder = 0; - let upperBorder = hashArray.length; +export async function getHashMapFromZipArchive( + fileProvider: FileProvider +): Promise> { + const zipCDIterator = makeZipCDHeaderIterator(fileProvider); + const md5Hash = new MD5Hash(); + const textEncoder = new TextEncoder(); - while (upperBorder - lowerBorder > 1) { - const middle = lowerBorder + Math.floor((upperBorder - lowerBorder) / 2); - const value = compareHashes(hashArray[middle].hash, hashToSearch); - if (value === 0) { - return hashArray[middle]; - } else if (value < 0) { - lowerBorder = middle; - } else { - upperBorder = middle; - } + const hashMap: Record = {}; + for await (const cdHeader of zipCDIterator) { + const filename = cdHeader.fileName.split('\\').join('/').toLocaleLowerCase(); + const arrayBuffer = textEncoder.encode(filename).buffer; + const md5 = await md5Hash.hash(arrayBuffer, 'hex'); + hashMap[md5] = cdHeader.localHeaderOffset; } - return undefined; -}; + return hashMap; +} +// /** - * generates hash info from central directory + * generates hash info from zip files "central directory" * @param fileProvider - provider of the archive * @returns ready to use hash info */ -export const generateHashInfo = async (fileProvider: FileProvider): Promise => { - const zipCDIterator = zipCDFileHeaderGenerator(fileProvider); - const hashInfo: HashElement[] = []; +export async function makeHashTableFromZipHeaders( + fileProvider: FileProvider +): Promise> { + const zipCDIterator = makeZipCDHeaderIterator(fileProvider); + const md5Hash = new MD5Hash(); + const textEncoder = new TextEncoder(); + + const hashTable: Record = {}; + for await (const cdHeader of zipCDIterator) { - hashInfo.push({ - hash: Buffer.from(md5(cdHeader.fileName.split('\\').join('/').toLocaleLowerCase()), 'hex'), - offset: cdHeader.localHeaderOffset - }); + const filename = cdHeader.fileName.split('\\').join('/').toLocaleLowerCase(); + const arrayBuffer = textEncoder.encode(filename).buffer; + const md5 = await md5Hash.hash(arrayBuffer, 'hex'); + hashTable[md5] = cdHeader.localHeaderOffset; } - hashInfo.sort((a, b) => compareHashes(a.hash, b.hash)); - return hashInfo; -}; + + return hashTable; +} diff --git a/modules/zip/src/index.ts b/modules/zip/src/index.ts index 92f943383f..3de69fab5e 100644 --- a/modules/zip/src/index.ts +++ b/modules/zip/src/index.ts @@ -6,7 +6,7 @@ export {TarBuilder} from './tar-builder'; export { parseZipCDFileHeader, - zipCDFileHeaderGenerator, + makeZipCDHeaderIterator, signature as cdSignature } from './parse-zip/cd-file-header'; export { @@ -16,7 +16,7 @@ export { export {parseEoCDRecord} from './parse-zip/end-of-central-directory'; export {searchFromTheEnd} from './parse-zip/search-from-the-end'; -export type {HashElement} from './hash-file-utility'; -export {compareHashes, parseHashFile, findBin, generateHashInfo} from './hash-file-utility'; +// export type {HashElement} from './hash-file-utility'; +export {parseHashTable, makeHashTableFromZipHeaders} from './hash-file-utility'; export {ZipFileSystem} from './filesystems/zip-filesystem'; diff --git a/modules/zip/src/parse-zip/cd-file-header.ts b/modules/zip/src/parse-zip/cd-file-header.ts index d538cb24bf..aa0d2518f1 100644 --- a/modules/zip/src/parse-zip/cd-file-header.ts +++ b/modules/zip/src/parse-zip/cd-file-header.ts @@ -101,7 +101,9 @@ export const parseZipCDFileHeader = async ( * Create iterator over files of zip archive * @param fileProvider - file provider that provider random access to the file */ -export async function* zipCDFileHeaderGenerator(fileProvider: FileProvider) { +export async function* makeZipCDHeaderIterator( + fileProvider: FileProvider +): AsyncIterable { const {cdStartOffset} = await parseEoCDRecord(fileProvider); let cdHeader = await parseZipCDFileHeader(cdStartOffset, fileProvider); while (cdHeader) { diff --git a/modules/zip/tsconfig.json b/modules/zip/tsconfig.json index 4e6af14083..1580d14c14 100644 --- a/modules/zip/tsconfig.json +++ b/modules/zip/tsconfig.json @@ -10,6 +10,7 @@ "references": [ {"path": "../loader-utils"}, {"path": "../core"}, - {"path": "../compression"} + {"path": "../compression"}, + {"path": "../crypto"} ] } diff --git a/yarn.lock b/yarn.lock index eccf837751..da5494b7cb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4141,9 +4141,9 @@ camelcase@^5.0.0, camelcase@^5.3.1: integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg== caniuse-lite@^1.0.30001449: - version "1.0.30001542" - resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001542.tgz" - integrity sha512-UrtAXVcj1mvPBFQ4sKd38daP8dEcXXr5sQe6QNNinaPd0iA/cxg9/l3VrSdL73jgw5sKyuQ6jNgiKO12W3SsVA== + version "1.0.30001546" + resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001546.tgz" + integrity sha512-zvtSJwuQFpewSyRrI3AsftF6rM0X80mZkChIt1spBGEvRglCrjTniXvinc8JKRoqTwXAgvqTImaN9igfSMtUBw== caseless@~0.12.0: version "0.12.0" From 4ffe878050461d751028c77d33dcc56a05c6fdee Mon Sep 17 00:00:00 2001 From: Ib Green Date: Fri, 6 Oct 2023 23:32:40 +0200 Subject: [PATCH 2/4] chore: Remove Buffer usage (#2675) --- docs/upgrade-guide.md | 2 +- .../3d-tiles-archive-archive.ts | 3 ++ .../lib/parsers/parse-slpk/slpk-archieve.ts | 20 ++++----- .../i3s-server/controllers/slpk-controller.ts | 8 ++-- modules/zip/src/hash-file-utility.ts | 23 ---------- modules/zip/src/parse-zip/cd-file-header.ts | 43 ++++++++----------- .../src/parse-zip/end-of-central-directory.ts | 42 +++++++----------- .../zip/src/parse-zip/local-file-header.ts | 12 +++--- .../zip/src/parse-zip/search-from-the-end.ts | 2 +- .../zip-utils/search-from-the-end.spec.ts | 2 +- 10 files changed, 58 insertions(+), 99 deletions(-) diff --git a/docs/upgrade-guide.md b/docs/upgrade-guide.md index d70753c64e..c746b5f7fc 100644 --- a/docs/upgrade-guide.md +++ b/docs/upgrade-guide.md @@ -58,7 +58,7 @@ and loaders.gl v4.0 aligns with this practice. **@loaders.gl/crypto** -- All hashes now require an encoding parameter. To get previous behavior, just specify `'base64'`. +- All hashes now require an encoding parameter. To get previous behavior, just specify `.hash...(..., 'base64')`. **@loaders.gl/arrow** diff --git a/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-archive.ts b/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-archive.ts index 6b94ceaa7e..681b9b7bba 100644 --- a/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-archive.ts +++ b/modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-archive.ts @@ -62,6 +62,9 @@ export class Tiles3DArchive { const arrayBuffer = new TextEncoder().encode(path).buffer; const nameHash = await new MD5Hash().hash(arrayBuffer, 'hex'); const byteOffset = this.hashTable[nameHash]; + if (byteOffset === undefined) { + return null; + } const localFileHeader = await parseZipLocalFileHeader(byteOffset, this.fileProvider); if (!localFileHeader) { diff --git a/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts b/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts index cce96c0943..da6b63aca2 100644 --- a/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts +++ b/modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts @@ -45,8 +45,8 @@ const PATH_DESCRIPTIONS: {test: RegExp; extensions: string[]}[] = [ export class SLPKArchive { /** A DataView representation of the archive */ private slpkArchive: FileProvider; - // Maps hex-encoded md5 hashes to bigint offsets into the archive - private hashedOffsetMap: Record; + // Maps hex-encoded md5 filename hashes to bigint offsets into the archive + private hashTable: Record; /** Array of hashes and offsets into archive */ // hashToOffsetMap: Record; @@ -54,9 +54,9 @@ export class SLPKArchive { protected _textDecoder = new TextDecoder(); protected _md5Hash = new MD5Hash(); - constructor(slpkArchive: FileProvider, hashedOffsetMap: Record) { + constructor(slpkArchive: FileProvider, hashTable: Record) { this.slpkArchive = slpkArchive; - this.hashedOffsetMap = hashedOffsetMap; + this.hashTable = hashTable; } /** @@ -65,7 +65,7 @@ export class SLPKArchive { * @param mode - currently only raw mode supported * @returns buffer with ready to use file */ - async getFile(path: string, mode: 'http' | 'raw' = 'raw'): Promise { + async getFile(path: string, mode: 'http' | 'raw' = 'raw'): Promise { if (mode === 'http') { const extensions = PATH_DESCRIPTIONS.find((val) => val.test.test(path))?.extensions; if (extensions) { @@ -77,18 +77,18 @@ export class SLPKArchive { } } if (data) { - return Buffer.from(data); + return data; } } } if (mode === 'raw') { const decompressedFile = await this.getDataByPath(`${path}.gz`); if (decompressedFile) { - return Buffer.from(decompressedFile); + return decompressedFile; } const fileWithoutCompression = await this.getFileBytes(path); if (fileWithoutCompression) { - return Buffer.from(fileWithoutCompression); + return fileWithoutCompression; } } @@ -116,7 +116,7 @@ export class SLPKArchive { const decompressedData = await compression.decompress(data); return decompressedData; } - return Buffer.from(data); + return data; } /** @@ -128,7 +128,7 @@ export class SLPKArchive { const binaryPath = this._textEncoder.encode(path); const nameHash = await this._md5Hash.hash(binaryPath.buffer, 'hex'); - const offset = this.hashedOffsetMap[nameHash]; + const offset = this.hashTable[nameHash]; if (offset === undefined) { return undefined; } diff --git a/modules/tile-converter/src/i3s-server/controllers/slpk-controller.ts b/modules/tile-converter/src/i3s-server/controllers/slpk-controller.ts index 7cf2f3abd4..302d28f8bc 100644 --- a/modules/tile-converter/src/i3s-server/controllers/slpk-controller.ts +++ b/modules/tile-converter/src/i3s-server/controllers/slpk-controller.ts @@ -20,15 +20,15 @@ export async function loadArchive(fullLayerPath: string): Promise { * @param url - I3S HTTP URL * @returns - file content */ -export async function getFileByUrl(url: string): Promise { +export async function getFileByUrl(url: string): Promise { const trimmedPath = /^\/?(.*)\/?$/.exec(url); - let uncompressedFile: Buffer | null = null; if (trimmedPath) { try { - uncompressedFile = await slpkArchive.getFile(trimmedPath[1], 'http'); + const uncompressedFile = await slpkArchive.getFile(trimmedPath[1], 'http'); + return uncompressedFile; } catch { // TODO - log error? } } - return uncompressedFile; + return null; } diff --git a/modules/zip/src/hash-file-utility.ts b/modules/zip/src/hash-file-utility.ts index 3ac2ccf05e..4970c82248 100644 --- a/modules/zip/src/hash-file-utility.ts +++ b/modules/zip/src/hash-file-utility.ts @@ -28,29 +28,6 @@ function bufferToHex(buffer: ArrayBuffer, start: number, length: number): string .join(''); } -/** - * generates hash info from central directory - * @param fileProvider - provider of the archive - * @returns ready to use hash info - */ -export async function getHashMapFromZipArchive( - fileProvider: FileProvider -): Promise> { - const zipCDIterator = makeZipCDHeaderIterator(fileProvider); - const md5Hash = new MD5Hash(); - const textEncoder = new TextEncoder(); - - const hashMap: Record = {}; - for await (const cdHeader of zipCDIterator) { - const filename = cdHeader.fileName.split('\\').join('/').toLocaleLowerCase(); - const arrayBuffer = textEncoder.encode(filename).buffer; - const md5 = await md5Hash.hash(arrayBuffer, 'hex'); - hashMap[md5] = cdHeader.localHeaderOffset; - } - return hashMap; -} - -// /** * generates hash info from zip files "central directory" * @param fileProvider - provider of the archive diff --git a/modules/zip/src/parse-zip/cd-file-header.ts b/modules/zip/src/parse-zip/cd-file-header.ts index aa0d2518f1..85dafde0a9 100644 --- a/modules/zip/src/parse-zip/cd-file-header.ts +++ b/modules/zip/src/parse-zip/cd-file-header.ts @@ -1,4 +1,4 @@ -import {FileProvider} from '@loaders.gl/loader-utils'; +import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils'; import {parseEoCDRecord} from './end-of-central-directory'; import {ZipSignature} from './search-from-the-end'; @@ -31,7 +31,7 @@ const CD_EXTRA_FIELD_LENGTH_OFFSET = 30n; const CD_LOCAL_HEADER_OFFSET_OFFSET = 42n; const CD_FILE_NAME_OFFSET = 46n; -export const signature: ZipSignature = [0x50, 0x4b, 0x01, 0x02]; +export const signature: ZipSignature = new Uint8Array([0x50, 0x4b, 0x01, 0x02]); /** * Parses central directory file header of zip file @@ -41,48 +41,39 @@ export const signature: ZipSignature = [0x50, 0x4b, 0x01, 0x02]; */ export const parseZipCDFileHeader = async ( headerOffset: bigint, - buffer: FileProvider + file: FileProvider ): Promise => { - if ( - Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare( - Buffer.from(signature) - ) !== 0 - ) { + const magicBytes = await file.slice(headerOffset, headerOffset + 4n); + if (!compareArrayBuffers(magicBytes, signature.buffer)) { return null; } - let compressedSize = BigInt(await buffer.getUint32(headerOffset + CD_COMPRESSED_SIZE_OFFSET)); - - let uncompressedSize = BigInt(await buffer.getUint32(headerOffset + CD_UNCOMPRESSED_SIZE_OFFSET)); - - const extraFieldLength = await buffer.getUint16(headerOffset + CD_EXTRA_FIELD_LENGTH_OFFSET); - - const fileNameLength = await buffer.getUint16(headerOffset + CD_FILE_NAME_LENGTH_OFFSET); - - const fileName = new TextDecoder().decode( - await buffer.slice( - headerOffset + CD_FILE_NAME_OFFSET, - headerOffset + CD_FILE_NAME_OFFSET + BigInt(fileNameLength) - ) + let compressedSize = BigInt(await file.getUint32(headerOffset + CD_COMPRESSED_SIZE_OFFSET)); + let uncompressedSize = BigInt(await file.getUint32(headerOffset + CD_UNCOMPRESSED_SIZE_OFFSET)); + const extraFieldLength = await file.getUint16(headerOffset + CD_EXTRA_FIELD_LENGTH_OFFSET); + const fileNameLength = await file.getUint16(headerOffset + CD_FILE_NAME_LENGTH_OFFSET); + const filenameBytes = await file.slice( + headerOffset + CD_FILE_NAME_OFFSET, + headerOffset + CD_FILE_NAME_OFFSET + BigInt(fileNameLength) ); + const fileName = new TextDecoder().decode(filenameBytes); const extraOffset = headerOffset + CD_FILE_NAME_OFFSET + BigInt(fileNameLength); - - const oldFormatOffset = await buffer.getUint32(headerOffset + CD_LOCAL_HEADER_OFFSET_OFFSET); + const oldFormatOffset = await file.getUint32(headerOffset + CD_LOCAL_HEADER_OFFSET_OFFSET); let fileDataOffset = BigInt(oldFormatOffset); let offsetInZip64Data = 4n; // looking for info that might be also be in zip64 extra field if (uncompressedSize === BigInt(0xffffffff)) { - uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data); + uncompressedSize = await file.getBigUint64(extraOffset + offsetInZip64Data); offsetInZip64Data += 8n; } if (compressedSize === BigInt(0xffffffff)) { - compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data); + compressedSize = await file.getBigUint64(extraOffset + offsetInZip64Data); offsetInZip64Data += 8n; } if (fileDataOffset === BigInt(0xffffffff)) { - fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64 + fileDataOffset = await file.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64 } const localHeaderOffset = fileDataOffset; diff --git a/modules/zip/src/parse-zip/end-of-central-directory.ts b/modules/zip/src/parse-zip/end-of-central-directory.ts index ff40a359dd..ddb5e7f331 100644 --- a/modules/zip/src/parse-zip/end-of-central-directory.ts +++ b/modules/zip/src/parse-zip/end-of-central-directory.ts @@ -1,4 +1,4 @@ -import {FileProvider} from '@loaders.gl/loader-utils'; +import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils'; import {ZipSignature, searchFromTheEnd} from './search-from-the-end'; /** @@ -12,9 +12,9 @@ export type ZipEoCDRecord = { cdRecordsNumber: bigint; }; -const eoCDSignature: ZipSignature = [0x50, 0x4b, 0x05, 0x06]; -const zip64EoCDLocatorSignature = [0x50, 0x4b, 0x06, 0x07]; -const zip64EoCDSignature = [0x50, 0x4b, 0x06, 0x06]; +const eoCDSignature: ZipSignature = new Uint8Array([0x50, 0x4b, 0x05, 0x06]); +const zip64EoCDLocatorSignature = new Uint8Array([0x50, 0x4b, 0x06, 0x07]); +const zip64EoCDSignature = new Uint8Array([0x50, 0x4b, 0x06, 0x06]); // offsets accroding to https://en.wikipedia.org/wiki/ZIP_(file_format) const CD_RECORDS_NUMBER_OFFSET = 8n; @@ -25,43 +25,33 @@ const ZIP64_CD_START_OFFSET_OFFSET = 48n; /** * Parses end of central directory record of zip file - * @param fileProvider - FileProvider instance + * @param file - FileProvider instance * @returns Info from the header */ -export const parseEoCDRecord = async (fileProvider: FileProvider): Promise => { - const zipEoCDOffset = await searchFromTheEnd(fileProvider, eoCDSignature); +export const parseEoCDRecord = async (file: FileProvider): Promise => { + const zipEoCDOffset = await searchFromTheEnd(file, eoCDSignature); - let cdRecordsNumber = BigInt( - await fileProvider.getUint16(zipEoCDOffset + CD_RECORDS_NUMBER_OFFSET) - ); - let cdStartOffset = BigInt(await fileProvider.getUint32(zipEoCDOffset + CD_START_OFFSET_OFFSET)); + let cdRecordsNumber = BigInt(await file.getUint16(zipEoCDOffset + CD_RECORDS_NUMBER_OFFSET)); + let cdStartOffset = BigInt(await file.getUint32(zipEoCDOffset + CD_START_OFFSET_OFFSET)); if (cdStartOffset === BigInt(0xffffffff) || cdRecordsNumber === BigInt(0xffffffff)) { const zip64EoCDLocatorOffset = zipEoCDOffset - 20n; - if ( - Buffer.from( - await fileProvider.slice(zip64EoCDLocatorOffset, zip64EoCDLocatorOffset + 4n) - ).compare(Buffer.from(zip64EoCDLocatorSignature)) !== 0 - ) { + const magicBytes = await file.slice(zip64EoCDLocatorOffset, zip64EoCDLocatorOffset + 4n); + if (!compareArrayBuffers(magicBytes, zip64EoCDLocatorSignature)) { throw new Error('zip64 EoCD locator not found'); } - const zip64EoCDOffset = await fileProvider.getBigUint64( + const zip64EoCDOffset = await file.getBigUint64( zip64EoCDLocatorOffset + ZIP64_EOCD_START_OFFSET_OFFSET ); - if ( - Buffer.from(await fileProvider.slice(zip64EoCDOffset, zip64EoCDOffset + 4n)).compare( - Buffer.from(zip64EoCDSignature) - ) !== 0 - ) { + const endOfCDMagicBytes = await file.slice(zip64EoCDOffset, zip64EoCDOffset + 4n); + if (!compareArrayBuffers(endOfCDMagicBytes, zip64EoCDSignature.buffer)) { throw new Error('zip64 EoCD not found'); } - cdRecordsNumber = await fileProvider.getBigUint64( - zip64EoCDOffset + ZIP64_CD_RECORDS_NUMBER_OFFSET - ); - cdStartOffset = await fileProvider.getBigUint64(zip64EoCDOffset + ZIP64_CD_START_OFFSET_OFFSET); + cdRecordsNumber = await file.getBigUint64(zip64EoCDOffset + ZIP64_CD_RECORDS_NUMBER_OFFSET); + cdStartOffset = await file.getBigUint64(zip64EoCDOffset + ZIP64_CD_START_OFFSET_OFFSET); } return { diff --git a/modules/zip/src/parse-zip/local-file-header.ts b/modules/zip/src/parse-zip/local-file-header.ts index 1cf02a8050..1bfa7a2bc1 100644 --- a/modules/zip/src/parse-zip/local-file-header.ts +++ b/modules/zip/src/parse-zip/local-file-header.ts @@ -1,4 +1,5 @@ -import {FileProvider} from '@loaders.gl/loader-utils'; +import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils'; +import {ZipSignature} from './search-from-the-end'; /** * zip local file header info @@ -27,7 +28,7 @@ const FILE_NAME_LENGTH_OFFSET = 26n; const EXTRA_FIELD_LENGTH_OFFSET = 28n; const FILE_NAME_OFFSET = 30n; -export const signature = [0x50, 0x4b, 0x03, 0x04]; +export const signature: ZipSignature = new Uint8Array([0x50, 0x4b, 0x03, 0x04]); /** * Parses local file header of zip file @@ -39,11 +40,8 @@ export const parseZipLocalFileHeader = async ( headerOffset: bigint, buffer: FileProvider ): Promise => { - if ( - Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare( - Buffer.from(signature) - ) !== 0 - ) { + const magicBytes = await buffer.slice(headerOffset, headerOffset + 4n); + if (!compareArrayBuffers(magicBytes, signature)) { return null; } diff --git a/modules/zip/src/parse-zip/search-from-the-end.ts b/modules/zip/src/parse-zip/search-from-the-end.ts index b832563654..5630ee0f5b 100644 --- a/modules/zip/src/parse-zip/search-from-the-end.ts +++ b/modules/zip/src/parse-zip/search-from-the-end.ts @@ -1,7 +1,7 @@ import {FileProvider} from '@loaders.gl/loader-utils'; /** Description of zip signature type */ -export type ZipSignature = [number, number, number, number]; +export type ZipSignature = Uint8Array; /** * looking for the last occurrence of the provided diff --git a/modules/zip/test/zip-utils/search-from-the-end.spec.ts b/modules/zip/test/zip-utils/search-from-the-end.spec.ts index da6270e37f..1332c536c4 100644 --- a/modules/zip/test/zip-utils/search-from-the-end.spec.ts +++ b/modules/zip/test/zip-utils/search-from-the-end.spec.ts @@ -8,7 +8,7 @@ test('SLPKLoader#searchFromTheEnd', async (t) => { t.equals( await searchFromTheEnd( new DataViewFile(new DataView(DATA_ARRAY.buffer)), - [0x50, 0x4b, 0x03, 0x04] + new Uint8Array([0x50, 0x4b, 0x03, 0x04]) ), 0n ); From d60184d28be93bd2ca47c8e1cb7ac49fe235983c Mon Sep 17 00:00:00 2001 From: Ib Green Date: Sun, 8 Oct 2023 10:39:44 +0200 Subject: [PATCH 3/4] feat(loader-utils): Refactor FileSystem to be independent of fs (#2676) --- .eslintrc.js | 2 +- .prettierignore | 4 +- modules/core/src/index.ts | 5 +- modules/core/src/lib/fetch/fetch-file.node.ts | 61 --------- modules/core/src/lib/fetch/fetch-file.ts | 10 +- .../src/lib/filesystems/browser-filesystem.ts | 37 +++--- modules/csv/test/papaparse/papaparse.spec.ts | 1 - modules/loader-utils/src/index.ts | 13 +- .../src/lib/file-provider/data-view-file.ts | 1 - .../loader-utils/src/lib/files/blob-file.ts | 29 +++++ modules/loader-utils/src/lib/files/file.ts | 29 +++++ .../loader-utils/src/lib/files/http-file.ts | 110 ++++++++++++++++ .../src/lib/files/node-file-facade.ts | 39 ++++++ .../src/lib/files}/sources.ts | 0 .../src/lib/filesystems/filesystem.ts | 87 +++---------- .../lib/filesystems/node-filesystem-facade.ts | 63 ++++++++++ .../src/lib/filesystems/node-filesystem.ts | 67 ---------- .../src/lib/filesystems/readable-file.ts | 30 ----- .../src/lib/filesystems/writable-file.ts | 44 ------- .../test/lib/geojson-tiler/get-tile.spec.ts | 1 - .../lib/parsers/parse-parquet-to-columns.ts | 4 +- .../src/lib/parsers/parse-parquet-to-rows.ts | 4 +- .../src/parquetjs/parser/parquet-reader.ts | 15 ++- .../test/parquetjs/integration.spec.ts | 4 +- modules/parquet/test/parquetjs/reader.spec.ts | 4 +- modules/pmtiles/src/pmtiles-source.ts | 39 +++--- modules/polyfills/package.json | 1 + .../polyfills/src/filesystems/fetch-node.ts | 96 ++++++++++++++ .../polyfills/src/filesystems/node-file.ts | 118 ++++++++++++++++++ .../src/filesystems/node-filesystem.ts | 79 +++--------- .../stream-utils.node.ts | 45 ++++++- modules/polyfills/src/index.browser.ts | 5 +- modules/polyfills/src/index.ts | 70 ++++++----- .../polyfills/src/node/fetch/response.node.ts | 2 +- .../test/filesystems/fetch-node.spec.js | 64 ++++++++++ modules/polyfills/test/index.ts | 1 + .../src/lib/node/worker_threads-browser.ts | 1 - 37 files changed, 753 insertions(+), 432 deletions(-) delete mode 100644 modules/core/src/lib/fetch/fetch-file.node.ts create mode 100644 modules/loader-utils/src/lib/files/blob-file.ts create mode 100644 modules/loader-utils/src/lib/files/file.ts create mode 100644 modules/loader-utils/src/lib/files/http-file.ts create mode 100644 modules/loader-utils/src/lib/files/node-file-facade.ts rename modules/{pmtiles/src/lib => loader-utils/src/lib/files}/sources.ts (100%) create mode 100644 modules/loader-utils/src/lib/filesystems/node-filesystem-facade.ts delete mode 100644 modules/loader-utils/src/lib/filesystems/node-filesystem.ts delete mode 100644 modules/loader-utils/src/lib/filesystems/readable-file.ts delete mode 100644 modules/loader-utils/src/lib/filesystems/writable-file.ts create mode 100644 modules/polyfills/src/filesystems/fetch-node.ts create mode 100644 modules/polyfills/src/filesystems/node-file.ts rename modules/polyfills/src/{node/fetch/utils => filesystems}/stream-utils.node.ts (64%) create mode 100644 modules/polyfills/test/filesystems/fetch-node.spec.js diff --git a/.eslintrc.js b/.eslintrc.js index a99ddae4a8..42e7c4a02a 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -58,7 +58,7 @@ const config = deepMerge(defaultConfig, { // '@typescript-eslint/no-floating-promises': ['warn'], // '@typescript-eslint/await-thenable': ['warn'], // '@typescript-eslint/no-misused-promises': ['warn'], - '@typescript-eslint/no-empty-function': ['warn', {allow: ['arrowFunctions']}], + '@typescript-eslint/no-empty-function': 0, // We use function hoisting '@typescript-eslint/no-use-before-define': 0, // We always want explicit typing, e.g `field: string = ''` diff --git a/.prettierignore b/.prettierignore index ec9ac93cff..2c08686670 100644 --- a/.prettierignore +++ b/.prettierignore @@ -9,8 +9,8 @@ public modules/core/src/iterators/make-stream/make-node-stream.ts -modules/loader-utils/src/lib/filesystems/node-filesystem.browser.ts -modules/loader-utils/src/lib/filesystems/node-filesystem.ts +modules/loader-utils/src/lib/files/node-file-facade.ts +modules/loader-utils/src/lib/filesystems/node-filesystem-facade.ts modules/3d-tiles/test/lib/classes/tile-3d-batch-table-hierarchy.spec.ts diff --git a/modules/core/src/index.ts b/modules/core/src/index.ts index 935b4069ef..b25bc5f5b6 100644 --- a/modules/core/src/index.ts +++ b/modules/core/src/index.ts @@ -11,8 +11,11 @@ export type { DataType, SyncDataType, BatchableDataType, + ReadableFile, + WritableFile, + Stat, FileSystem, - RandomAccessReadFileSystem + RandomAccessFileSystem } from '@loaders.gl/loader-utils'; // FILE READING AND WRITING diff --git a/modules/core/src/lib/fetch/fetch-file.node.ts b/modules/core/src/lib/fetch/fetch-file.node.ts deleted file mode 100644 index 8be94ee224..0000000000 --- a/modules/core/src/lib/fetch/fetch-file.node.ts +++ /dev/null @@ -1,61 +0,0 @@ -// loaders.gl, MIT license - -import {fs} from '@loaders.gl/loader-utils'; - -/** - * Enables - * @param url - * @param options - * @returns - */ -export async function fetchFileNode(url: string, options): Promise { - // Support `file://` protocol - const FILE_PROTOCOL_REGEX = /^file:\/\//; - url.replace(FILE_PROTOCOL_REGEX, '/'); - - const noqueryUrl = url.split('?')[0]; - - try { - // Now open the stream - const body = await new Promise((resolve, reject) => { - // @ts-ignore - const stream = fs.createReadStream(noqueryUrl, {encoding: null}); - stream.once('readable', () => resolve(stream)); - stream.on('error', (error) => reject(error)); - }); - - const status = 200; - const statusText = 'OK'; - const headers = getHeadersForFile(noqueryUrl); - // @ts-expect-error - const response = new Response(body, {headers, status, statusText}); - Object.defineProperty(response, 'url', {value: url}); - return response; - } catch (error) { - const errorMessage = (error as Error).message; - const status = 400; - const statusText = errorMessage; - const headers = {}; - const response = new Response(errorMessage, {headers, status, statusText}); - Object.defineProperty(response, 'url', {value: url}); - return response; - } -} - -function getHeadersForFile(noqueryUrl: string): Headers { - const headers = {}; - - // Fix up content length if we can for best progress experience - if (!headers['content-length']) { - const stats = fs.statSync(noqueryUrl); - headers['content-length'] = stats.size; - } - - // Automatically decompress gzipped files with .gz extension - if (noqueryUrl.endsWith('.gz')) { - noqueryUrl = noqueryUrl.slice(0, -3); - headers['content-encoding'] = 'gzip'; - } - - return new Headers(headers); -} diff --git a/modules/core/src/lib/fetch/fetch-file.ts b/modules/core/src/lib/fetch/fetch-file.ts index aee84e108a..c31502c6eb 100644 --- a/modules/core/src/lib/fetch/fetch-file.ts +++ b/modules/core/src/lib/fetch/fetch-file.ts @@ -2,7 +2,6 @@ import {resolvePath} from '@loaders.gl/loader-utils'; import {makeResponse} from '../utils/response-utils'; -import * as node from './fetch-file.node'; export function isNodePath(url: string): boolean { return !isRequestURL(url) && !isDataURL(url); @@ -29,8 +28,13 @@ export async function fetchFile( const url = resolvePath(urlOrData); // Support fetching from local file system - if (isNodePath(url) && node?.fetchFileNode) { - return node.fetchFileNode(url, fetchOptions); + if (isNodePath(url)) { + if (globalThis.loaders?.fetchNode) { + return globalThis.loaders?.fetchNode(url, fetchOptions); + } + // throw new Error( + // 'fetchFile: globalThis.loaders.fetchNode not defined. Install @loaders.gl/polyfills' + // ); } // Call global fetch diff --git a/modules/core/src/lib/filesystems/browser-filesystem.ts b/modules/core/src/lib/filesystems/browser-filesystem.ts index 020c0efce0..f3018ee6e6 100644 --- a/modules/core/src/lib/filesystems/browser-filesystem.ts +++ b/modules/core/src/lib/filesystems/browser-filesystem.ts @@ -1,4 +1,5 @@ -import type {FileSystem} from '@loaders.gl/loader-utils'; +import type {FileSystem, ReadableFile} from '@loaders.gl/loader-utils'; +import {BlobFile} from '@loaders.gl/loader-utils'; type BrowserFileSystemOptions = { fetch?: typeof fetch; @@ -109,18 +110,30 @@ export class BrowserFileSystem implements FileSystem { // implements IRandomAccessFileSystem // RANDOM ACCESS - async open(pathname: string, flags: unknown, mode?: unknown): Promise { - return this.files[pathname]; + async openReadableFile(pathname: string, flags: unknown): Promise { + return new BlobFile(this.files[pathname]); } - /** + // PRIVATE + + // Supports case independent paths, and file usage tracking + _getFile(path: string, used: boolean): File { + // Prefer case match, but fall back to case independent. + const file = this.files[path] || this.lowerCaseFiles[path]; + if (file && used) { + this.usedFiles[path] = true; + } + return file; + } +} +/* * Read a range into a buffer * @todo - handle position memory * @param buffer is the buffer that the data (read from the fd) will be written to. * @param offset is the offset in the buffer to start writing at. * @param length is an integer specifying the number of bytes to read. * @param position is an argument specifying where to begin reading from in the file. If position is null, data will be read from the current file position, and the file position will be updated. If position is an integer, the file position will remain unchanged. - */ + * async read( fd: any, buffer: ArrayBuffer, @@ -140,16 +153,4 @@ export class BrowserFileSystem implements FileSystem { } // fstat(fd: number): Promise; // Stat - - // PRIVATE - - // Supports case independent paths, and file usage tracking - _getFile(path: string, used: boolean): File { - // Prefer case match, but fall back to case independent. - const file = this.files[path] || this.lowerCaseFiles[path]; - if (file && used) { - this.usedFiles[path] = true; - } - return file; - } -} + */ diff --git a/modules/csv/test/papaparse/papaparse.spec.ts b/modules/csv/test/papaparse/papaparse.spec.ts index 6b2d4e89c2..8e67114599 100644 --- a/modules/csv/test/papaparse/papaparse.spec.ts +++ b/modules/csv/test/papaparse/papaparse.spec.ts @@ -488,7 +488,6 @@ const CUSTOM_TESTS = [ Papa.parse(BASE_PATH + 'long-sample.csv', { download: true, chunkSize: 500, - // eslint-disable-next-line @typescript-eslint/no-empty-function beforeFirstChunk(chunk) {}, step(response) { updates++; diff --git a/modules/loader-utils/src/index.ts b/modules/loader-utils/src/index.ts index e03b51b513..3c7fa6f1a5 100644 --- a/modules/loader-utils/src/index.ts +++ b/modules/loader-utils/src/index.ts @@ -120,9 +120,6 @@ export {stream}; // EXPERIMENTAL: FILE SYSTEMS -export type {FileSystem, RandomAccessReadFileSystem} from './lib/filesystems/filesystem'; -export {NodeFileSystem as _NodeFileSystem} from './lib/filesystems/node-filesystem'; - export type {FileProvider} from './lib/file-provider/file-provider'; export {isFileProvider} from './lib/file-provider/file-provider'; @@ -130,11 +127,13 @@ export {FileHandle} from './lib/file-provider/file-handle'; export {FileHandleFile} from './lib/file-provider/file-handle-file'; export {DataViewFile} from './lib/file-provider/data-view-file'; -export type {ReadableFile} from './lib/filesystems/readable-file'; -export {makeReadableFile} from './lib/filesystems/readable-file'; +export type {ReadableFile, WritableFile, Stat} from './lib/files/file'; +export {BlobFile} from './lib/files/blob-file'; +export {HttpFile} from './lib/files/http-file'; +export {NodeFileFacade as NodeFile} from './lib/files/node-file-facade'; -export type {WritableFile} from './lib/filesystems/writable-file'; -export {makeWritableFile} from './lib/filesystems/writable-file'; +export type {FileSystem, RandomAccessFileSystem} from './lib/filesystems/filesystem'; +export {NodeFileSystemFacade as NodeFilesystem} from './lib/filesystems/node-filesystem-facade'; // EXPERIMENTAL: DATA SOURCES export type {Service} from './service-types'; diff --git a/modules/loader-utils/src/lib/file-provider/data-view-file.ts b/modules/loader-utils/src/lib/file-provider/data-view-file.ts index 8d1df1158f..a08e12ec33 100644 --- a/modules/loader-utils/src/lib/file-provider/data-view-file.ts +++ b/modules/loader-utils/src/lib/file-provider/data-view-file.ts @@ -21,7 +21,6 @@ export class DataViewFile implements FileProvider { this.file = file; } - // eslint-disable-next-line @typescript-eslint/no-empty-function async destroy(): Promise {} /** diff --git a/modules/loader-utils/src/lib/files/blob-file.ts b/modules/loader-utils/src/lib/files/blob-file.ts new file mode 100644 index 0000000000..09eb3d050d --- /dev/null +++ b/modules/loader-utils/src/lib/files/blob-file.ts @@ -0,0 +1,29 @@ +// loaders.gl, MIT license + +import {ReadableFile} from './file'; + +export class BlobFile implements ReadableFile { + readonly handle: Blob; + readonly size: number; + readonly url: string; + + constructor(blob: Blob | File | ArrayBuffer) { + this.handle = blob instanceof ArrayBuffer ? new Blob([blob]) : blob; + this.size = blob instanceof ArrayBuffer ? blob.byteLength : blob.size; + this.url = blob instanceof File ? blob.name : ''; + } + + async close() {} + + async stat() { + return { + size: this.handle.size, + isDirectory: false + }; + } + + async read(start: number, length: number): Promise { + const arrayBuffer = await this.handle.slice(start, start + length).arrayBuffer(); + return arrayBuffer; + } +} diff --git a/modules/loader-utils/src/lib/files/file.ts b/modules/loader-utils/src/lib/files/file.ts new file mode 100644 index 0000000000..7c0e31ed41 --- /dev/null +++ b/modules/loader-utils/src/lib/files/file.ts @@ -0,0 +1,29 @@ +export type Stat = { + size: number; + isDirectory: boolean; +}; + +export interface ReadableFile { + /** The underlying file handle (Blob, Node.js file descriptor etc) */ + handle: unknown; + /** Length of file in bytes, if available */ + size: number; + /** Read data */ + read(start?: number, end?: number): Promise; + /** Read data */ + fetchRange?(offset: number, length: number, signal?: AbortSignal): Promise; + /** Get information about file */ + stat?(): Promise; + /** Close the file */ + close(): Promise; +} + +export interface WritableFile { + handle: unknown; + /** Write to file. The number of bytes written will be returned */ + write: (arrayBuffer: ArrayBuffer, offset?: number, length?: number) => Promise; + /** Get information about the file */ + stat?(): Promise; + /** Close the file */ + close(): Promise; +} diff --git a/modules/loader-utils/src/lib/files/http-file.ts b/modules/loader-utils/src/lib/files/http-file.ts new file mode 100644 index 0000000000..433d281c80 --- /dev/null +++ b/modules/loader-utils/src/lib/files/http-file.ts @@ -0,0 +1,110 @@ +// loaders.gl, MIT license + +import {ReadableFile, Stat} from './file'; + +export class HttpFile implements ReadableFile { + readonly handle: string; + readonly size: number = 0; + readonly url: string; + + constructor(url: string) { + this.handle = url; + this.url = url; + } + + async close(): Promise {} + + async stat(): Promise { + const response = await fetch(this.handle, {method: 'HEAD'}); + if (!response.ok) { + throw new Error(`Failed to fetch HEAD ${this.handle}`); + } + return { + size: parseInt(response.headers.get('Content-Length') || '0'), + isDirectory: false + }; + } + + async read(offset: number, length: number): Promise { + const response = await this.fetchRange(offset, length); + const arrayBuffer = await response.arrayBuffer(); + return arrayBuffer; + } + + /** + * + * @param offset + * @param length + * @param signal + * @returns + * @see https://github.com/protomaps/PMTiles + */ + // eslint-disable-next-line complexity + async fetchRange(offset: number, length: number, signal?: AbortSignal): Promise { + let controller: AbortController | undefined; + if (!signal) { + // ToDO why is it so important to abort in case 200? + // TODO check this works or assert 206 + controller = new AbortController(); + signal = controller.signal; + } + + const url = this.handle; + let response = await fetch(url, { + signal, + headers: {Range: `bytes=${offset}-${offset + length - 1}`} + }); + + switch (response.status) { + case 206: // Partial Content success + // This is the expected success code for a range request + break; + + case 200: + // some well-behaved backends, e.g. DigitalOcean CDN, respond with 200 instead of 206 + // but we also need to detect no support for Byte Serving which is returning the whole file + const contentLength = response.headers.get('Content-Length'); + if (!contentLength || Number(contentLength) > length) { + if (controller) { + controller.abort(); + } + throw Error( + 'content-length header missing or exceeding request. Server must support HTTP Byte Serving.' + ); + } + + // @eslint-disable-next-line no-fallthrough + case 416: // "Range Not Satisfiable" + // some HTTP servers don't accept ranges beyond the end of the resource. + // Retry with the exact length + // TODO: can return 416 with offset > 0 if content changed, which will have a blank etag. + // See https://github.com/protomaps/PMTiles/issues/90 + if (offset === 0) { + const contentRange = response.headers.get('Content-Range'); + if (!contentRange || !contentRange.startsWith('bytes *')) { + throw Error('Missing content-length on 416 response'); + } + const actualLength = Number(contentRange.substr(8)); + response = await fetch(this.url, { + signal, + headers: {Range: `bytes=0-${actualLength - 1}`} + }); + } + break; + + default: + if (response.status >= 300) { + throw Error(`Bad response code: ${response.status}`); + } + } + + return response; + // const data = await response.arrayBuffer(); + // return { + // data, + // etag: response.headers.get('ETag') || undefined, + // cacheControl: response.headers.get('Cache-Control') || undefined, + // expires: response.headers.get('Expires') || undefined + // }; + } +} diff --git a/modules/loader-utils/src/lib/files/node-file-facade.ts b/modules/loader-utils/src/lib/files/node-file-facade.ts new file mode 100644 index 0000000000..73c9155c06 --- /dev/null +++ b/modules/loader-utils/src/lib/files/node-file-facade.ts @@ -0,0 +1,39 @@ +// loaders.gl, MIT license + +import {isBrowser} from '../env-utils/globals'; +import {ReadableFile, WritableFile, Stat} from './file'; + +const NOT_IMPLEMENTED = new Error('Not implemented'); + +/** This class is a facade that gets replaced with an actual NodeFile instance */ +export class NodeFileFacade implements ReadableFile, WritableFile { + /** The underlying file handle (Blob, Node.js file descriptor etc) */ + handle: unknown; + /** Length of file in bytes, if available */ + size: number = 0; + + constructor(options) { + // Return the actual implementation instance + if (globalThis.loaders?.NodeFile) { + return new globalThis.loaders.NodeFile(options); + } + if (isBrowser) { + throw new Error('Can\'t instantiate NodeFile in browser.'); + } + throw new Error('Can\'t instantiate NodeFile. Make sure to import @loaders.gl/polyfills first.'); + } + /** Read data */ + async read(start?: number, end?: number): Promise { + throw NOT_IMPLEMENTED; + } + /** Write to file. The number of bytes written will be returned */ + async write(arrayBuffer: ArrayBuffer, offset?: number, length?: number): Promise { + throw NOT_IMPLEMENTED; + } + /** Get information about file */ + async stat(): Promise { + throw NOT_IMPLEMENTED; + } + /** Close the file */ + async close(): Promise {} +} diff --git a/modules/pmtiles/src/lib/sources.ts b/modules/loader-utils/src/lib/files/sources.ts similarity index 100% rename from modules/pmtiles/src/lib/sources.ts rename to modules/loader-utils/src/lib/files/sources.ts diff --git a/modules/loader-utils/src/lib/filesystems/filesystem.ts b/modules/loader-utils/src/lib/filesystems/filesystem.ts index b4bf46c452..fe1f8837fb 100644 --- a/modules/loader-utils/src/lib/filesystems/filesystem.ts +++ b/modules/loader-utils/src/lib/filesystems/filesystem.ts @@ -1,87 +1,38 @@ // loaders.gl, MIT license -export type ReadOptions = {}; - -export type Stat = { - size: number; - isDirectory: () => boolean; -}; +import {ReadableFile, WritableFile} from '../files/file'; /** * A FileSystem interface can encapsulate various file sources, - * a FileList, a ZipFile, a GoogleDrive etc. + * a FileList, a Node.js filesystem, a ZipFile, a GoogleDrive etc. */ export interface FileSystem { - /** - * Return a list of file names - * @param dirname directory name. file system root directory if omitted - */ + /** Return a list of file names in a "directory" */ readdir(dirname?: string, options?: {recursive?: boolean}): Promise; - /** - * Gets information from a local file from the filesystem - * @param filename file name to stat - * @param options currently unused - * @throws if filename is not in local filesystem - */ + /** Gets information from a local file from the filesystem */ stat(filename: string, options?: object): Promise<{size: number}>; - /** - * Fetches a local file from the filesystem (or a URL) - * @param filename - * @param options - */ - fetch(filename: RequestInfo, options?: RequestInit): Promise; -} + /** Removes a file from the file system */ + unlink?(path: string): Promise; -/** - * A random access file system - */ -export interface RandomAccessReadFileSystem extends FileSystem { - open(path: string, flags: unknown, mode?: unknown): Promise; - close(fd: unknown): Promise; - fstat(fd: unknown): Promise; - read(fd: any, options?: ReadOptions): Promise<{bytesRead: number; buffer: Uint8Array}>; - // read( - // fd: any, - // buffer: ArrayBuffer | ArrayBufferView, - // offset?: number, - // length?: number, - // position?: number - // ): Promise<{bytesRead: number; buffer: ArrayBuffer}>; + /** Fetches the full contents of a file from the filesystem (or a URL) */ + fetch(path: string, options?: RequestInit): Promise; } /** - * A FileSystem interface can encapsulate a FileList, a ZipFile, a GoogleDrive etc. - * -export interface IFileSystem { - /** - * Return a list of file names - * @param dirname directory name. file system root directory if omitted - * - readdir(dirname?: string, options?: {recursive?: boolean}): Promise; + * A random access file system, open readable and/or writable files + */ +export interface RandomAccessFileSystem extends FileSystem { + /** Can open readable files */ + readonly readable: boolean; - /** - * Gets information from a local file from the filesystem - * @param filename file name to stat - * @param options currently unused - * @throws if filename is not in local filesystem - * - stat(filename: string, options?: object): Promise<{size: number}>; + /** Can open writable files */ + readonly writable: boolean; - /** - * Fetches a local file from the filesystem (or a URL) - * @param filename - * @param options - * - fetch(filename: string, options?: object): Promise; -} + /** Open a readable file */ + openReadableFile(path: string, flags?: 'a'): Promise; -type ReadOptions = {buffer?: ArrayBuffer; offset?: number; length?: number; position?: number}; -export interface IRandomAccessReadFileSystem extends IFileSystem { - open(path: string, flags: string | number, mode?: any): Promise; - close(fd: any): Promise; - fstat(fd: any): Promise; - read(fd: any, options?: ReadOptions): Promise<{bytesRead: number; buffer: Buffer}>; + /** Open a writable file */ + openWritableFile(path: string, flags?: 'w' | 'wx', mode?: number): Promise; } -*/ diff --git a/modules/loader-utils/src/lib/filesystems/node-filesystem-facade.ts b/modules/loader-utils/src/lib/filesystems/node-filesystem-facade.ts new file mode 100644 index 0000000000..045b88925c --- /dev/null +++ b/modules/loader-utils/src/lib/filesystems/node-filesystem-facade.ts @@ -0,0 +1,63 @@ +// loaders.gl, MIT license + +import {isBrowser} from '../env-utils/globals'; +import {Stat} from '../files/file'; +import {NodeFileFacade as NodeFile} from '../files/node-file-facade'; +import {RandomAccessFileSystem} from './filesystem'; + +const NOT_IMPLEMENTED = new Error('Not implemented'); + +/** + * FileSystem pass-through for Node.js + * Compatible with BrowserFileSystem. + * @note Dummy implementation, not used (constructor returns a real NodeFileSystem instance) + * @param options + */ +export class NodeFileSystemFacade implements RandomAccessFileSystem { + // implements FileSystem + constructor(options: {[key: string]: any}) { + if (globalThis.loaders?.NodeFileSystem) { + return new globalThis.loaders.NodeFileSystem(options); + } + if (isBrowser) { + throw new Error('Can\'t instantiate NodeFileSystem in browser.'); + } + throw new Error( + 'Can\'t instantiate NodeFileSystem. Make sure to import @loaders.gl/polyfills first.' + ); + } + + // DUMMY IMPLEMENTATION, not used (constructor returns a real NodeFileSystem instance) + + // implements RandomAccessReadFileSystem + + readonly readable = true; + readonly writable = true; + + async openReadableFile(path: string, flags): Promise { + throw NOT_IMPLEMENTED; + } + + // implements RandomAccessWriteFileSystem + async openWritableFile(path: string, flags, mode): Promise { + throw NOT_IMPLEMENTED; + } + + // Implements file system + + async readdir(dirname = '.', options?: {}): Promise { + throw NOT_IMPLEMENTED; + } + + async stat(path: string, options?: {}): Promise { + throw NOT_IMPLEMENTED; + } + + async unlink(path: string): Promise { + throw NOT_IMPLEMENTED; + } + + async fetch(path: RequestInfo, options?: RequestInit): Promise { + throw NOT_IMPLEMENTED; + } +} diff --git a/modules/loader-utils/src/lib/filesystems/node-filesystem.ts b/modules/loader-utils/src/lib/filesystems/node-filesystem.ts deleted file mode 100644 index 1ddae25083..0000000000 --- a/modules/loader-utils/src/lib/filesystems/node-filesystem.ts +++ /dev/null @@ -1,67 +0,0 @@ -import * as fs from '../node/fs'; -import {FileSystem, RandomAccessReadFileSystem} from './filesystem'; -// import {fetchFile} from "../fetch/fetch-file" -// import {selectLoader} from "../api/select-loader"; - -type Stat = { - size: number; - isDirectory: () => boolean; - info?: fs.Stats; -}; - -type ReadOptions = { - buffer?: Buffer; - offset?: number; - length?: number; - position?: number; -}; - -/** - * FileSystem pass-through for Node.js - * Compatible with BrowserFileSystem. - * @param options - */ -export class NodeFileSystem implements FileSystem, RandomAccessReadFileSystem { - // implements FileSystem - constructor(options: {[key: string]: any}) { - if (globalThis.loaders?.NodeFileSystem) { - return new globalThis.loaders.NodeFileSystem(options); - } - throw new Error( - 'Can\'t instantiate NodeFileSystem in browser. Make sure to import @loaders.gl/polyfills first.' - ); - } - - async readdir(dirname = '.', options?: {}): Promise { - return []; - } - - async stat(path: string, options?: {}): Promise { - return {size: 0, isDirectory: () => false}; - } - - async fetch(path: string, options: {[key: string]: any}) { - return globalThis.fetch(path, options); - } - - // implements IRandomAccessFileSystem - - async open(path: string, flags: string | number, mode?: any): Promise { - return 0; - } - - // eslint-disable-next-line @typescript-eslint/no-empty-function - async close(fd: number): Promise {} - - async fstat(fd: number): Promise { - return {size: 0, isDirectory: () => false}; - } - - async read( - fd: number, - // @ts-ignore Possibly null - {buffer = null, offset = 0, length = buffer.byteLength, position = null}: ReadOptions - ): Promise<{bytesRead: number; buffer: Uint8Array}> { - return {bytesRead: 0, buffer: new Uint8Array(0)}; - } -} diff --git a/modules/loader-utils/src/lib/filesystems/readable-file.ts b/modules/loader-utils/src/lib/filesystems/readable-file.ts deleted file mode 100644 index 3f755bfc7b..0000000000 --- a/modules/loader-utils/src/lib/filesystems/readable-file.ts +++ /dev/null @@ -1,30 +0,0 @@ -// loaders.gl, MIT license - -export type ReadableFile = { - read: (position: number, length: number) => Promise; - close: () => Promise; - /** Length of file in bytes */ - size: number; -}; - -/** Helper function to create an envelope reader for a binary memory input */ -export function makeReadableFile(data: Blob | ArrayBuffer): ReadableFile { - if (data instanceof ArrayBuffer) { - const arrayBuffer: ArrayBuffer = data; - return { - read: async (start: number, length: number) => Buffer.from(data, start, length), - close: async () => {}, - size: arrayBuffer.byteLength - }; - } - - const blob: Blob = data; - return { - read: async (start: number, length: number) => { - const arrayBuffer = await blob.slice(start, start + length).arrayBuffer(); - return Buffer.from(arrayBuffer); - }, - close: async () => {}, - size: blob.size - }; -} diff --git a/modules/loader-utils/src/lib/filesystems/writable-file.ts b/modules/loader-utils/src/lib/filesystems/writable-file.ts deleted file mode 100644 index 24c5bec5f3..0000000000 --- a/modules/loader-utils/src/lib/filesystems/writable-file.ts +++ /dev/null @@ -1,44 +0,0 @@ -// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.) -import {isBrowser} from '../env-utils/globals'; -import * as fs from '../node/fs'; -import type {Writable} from 'stream'; - -export type WritableFile = { - write: (buf: Buffer) => Promise; - close: () => Promise; -}; - -export interface WriteStreamOptions { - flags?: string; - encoding?: 'utf8'; - fd?: number; - mode?: number; - autoClose?: boolean; - start?: number; -} - -/** Helper function to create an envelope reader for a binary memory input */ -export function makeWritableFile( - pathOrStream: string | Writable, - options?: WriteStreamOptions -): WritableFile { - if (isBrowser) { - return { - write: async () => {}, - close: async () => {} - }; - } - - const outputStream: Writable = - typeof pathOrStream === 'string' ? fs.createWriteStream(pathOrStream, options) : pathOrStream; - return { - write: async (buffer: Buffer) => - new Promise((resolve, reject) => { - outputStream.write(buffer, (err) => (err ? reject(err) : resolve())); - }), - close: () => - new Promise((resolve, reject) => { - (outputStream as any).close((err) => (err ? reject(err) : resolve())); - }) - }; -} diff --git a/modules/mvt/test/lib/geojson-tiler/get-tile.spec.ts b/modules/mvt/test/lib/geojson-tiler/get-tile.spec.ts index b6d57eea34..3e78bbc177 100644 --- a/modules/mvt/test/lib/geojson-tiler/get-tile.spec.ts +++ b/modules/mvt/test/lib/geojson-tiler/get-tile.spec.ts @@ -28,7 +28,6 @@ const square = [ test('GeoJSONVT#getTile#us-states.json', async (t) => { const log = console.log; - // eslint-disable-next-line @typescript-eslint/no-empty-function console.log = function () {}; const geojson = await getJSON('us-states.json'); diff --git a/modules/parquet/src/lib/parsers/parse-parquet-to-columns.ts b/modules/parquet/src/lib/parsers/parse-parquet-to-columns.ts index fd2baa7fb0..e1e6d83f4d 100644 --- a/modules/parquet/src/lib/parsers/parse-parquet-to-columns.ts +++ b/modules/parquet/src/lib/parsers/parse-parquet-to-columns.ts @@ -1,7 +1,7 @@ // loaders.gl, MIT license import {ColumnarTable, ColumnarTableBatch, Schema} from '@loaders.gl/schema'; -import {makeReadableFile} from '@loaders.gl/loader-utils'; +import {BlobFile} from '@loaders.gl/loader-utils'; import type {ParquetLoaderOptions} from '../../parquet-loader'; import {ParquetReader} from '../../parquetjs/parser/parquet-reader'; import {ParquetRowGroup} from '../../parquetjs/schema/declare'; @@ -16,7 +16,7 @@ export async function parseParquetInColumns( ): Promise { installBufferPolyfill(); const blob = new Blob([arrayBuffer]); - const file = makeReadableFile(blob); + const file = new BlobFile(blob); const reader = new ParquetReader(file); for await (const batch of parseParquetFileInColumnarBatches(reader, options)) { diff --git a/modules/parquet/src/lib/parsers/parse-parquet-to-rows.ts b/modules/parquet/src/lib/parsers/parse-parquet-to-rows.ts index d1784298d3..e495c90078 100644 --- a/modules/parquet/src/lib/parsers/parse-parquet-to-rows.ts +++ b/modules/parquet/src/lib/parsers/parse-parquet-to-rows.ts @@ -1,6 +1,6 @@ // import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils'; // import {ColumnarTableBatch} from '@loaders.gl/schema'; -import {makeReadableFile} from '@loaders.gl/loader-utils'; +import {BlobFile} from '@loaders.gl/loader-utils'; import {GeoJSONTable, ObjectRowTable, ObjectRowTableBatch} from '@loaders.gl/schema'; import type {ParquetLoaderOptions} from '../../parquet-loader'; import type {ParquetRow} from '../../parquetjs/schema/declare'; @@ -16,7 +16,7 @@ export async function parseParquet( installBufferPolyfill(); const blob = new Blob([arrayBuffer]); - const file = makeReadableFile(blob); + const file = new BlobFile(blob); const reader = new ParquetReader(file, { preserveBinary: options?.parquet?.preserveBinary }); diff --git a/modules/parquet/src/parquetjs/parser/parquet-reader.ts b/modules/parquet/src/parquetjs/parser/parquet-reader.ts index 01983a1655..558bf4b681 100644 --- a/modules/parquet/src/parquetjs/parser/parquet-reader.ts +++ b/modules/parquet/src/parquetjs/parser/parquet-reader.ts @@ -134,7 +134,8 @@ export class ParquetReader { /** Metadata is stored in the footer */ async readHeader(): Promise { - const buffer = await this.file.read(0, PARQUET_MAGIC.length); + const arrayBuffer = await this.file.read(0, PARQUET_MAGIC.length); + const buffer = Buffer.from(arrayBuffer); const magic = buffer.toString(); switch (magic) { case PARQUET_MAGIC: @@ -149,7 +150,8 @@ export class ParquetReader { /** Metadata is stored in the footer */ async readFooter(): Promise { const trailerLen = PARQUET_MAGIC.length + 4; - const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen); + const arrayBuffer = await this.file.read(this.file.size - trailerLen, trailerLen); + const trailerBuf = Buffer.from(arrayBuffer); const magic = trailerBuf.slice(4).toString(); if (magic !== PARQUET_MAGIC) { @@ -162,7 +164,8 @@ export class ParquetReader { throw new Error(`Invalid metadata size ${metadataOffset}`); } - const metadataBuf = await this.file.read(metadataOffset, metadataSize); + const arrayBuffer2 = await this.file.read(metadataOffset, metadataSize); + const metadataBuf = Buffer.from(arrayBuffer2); // let metadata = new parquet_thrift.FileMetaData(); // parquet_util.decodeThrift(metadata, metadataBuf); @@ -244,7 +247,8 @@ export class ParquetReader { } dictionary = context.dictionary?.length ? context.dictionary : dictionary; - const pagesBuf = await this.file.read(pagesOffset, pagesSize); + const arrayBuffer = await this.file.read(pagesOffset, pagesSize); + const pagesBuf = Buffer.from(arrayBuffer); return await decodeDataPages(pagesBuf, {...context, dictionary}); } @@ -275,7 +279,8 @@ export class ParquetReader { this.file.size - dictionaryPageOffset, this.props.defaultDictionarySize ); - const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize); + const arrayBuffer = await this.file.read(dictionaryPageOffset, dictionarySize); + const pagesBuf = Buffer.from(arrayBuffer); const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length}; const decodedPage = await decodePage(cursor, context); diff --git a/modules/parquet/test/parquetjs/integration.spec.ts b/modules/parquet/test/parquetjs/integration.spec.ts index 057e386d86..e9769b223a 100644 --- a/modules/parquet/test/parquetjs/integration.spec.ts +++ b/modules/parquet/test/parquetjs/integration.spec.ts @@ -1,7 +1,7 @@ /* eslint-disable camelcase */ import test, {Test} from 'tape-promise/tape'; import {isBrowser, fetchFile} from '@loaders.gl/core'; -import {makeReadableFile} from '@loaders.gl/loader-utils'; +import {BlobFile} from '@loaders.gl/loader-utils'; import {ParquetSchema, ParquetReader, ParquetEncoder} from '@loaders.gl/parquet'; const FRUITS_URL = '@loaders.gl/parquet/test/data/fruits.parquet'; @@ -115,7 +115,7 @@ async function writeTestFile(opts) { async function readTestFile(t: Test) { const response = await fetchFile(FRUITS_URL); const arrayBuffer = await response.arrayBuffer(); - const reader = new ParquetReader(makeReadableFile(arrayBuffer)); + const reader = new ParquetReader(new BlobFile(arrayBuffer)); t.equal(reader.getRowCount(), TEST_NUM_ROWS * 4); t.deepEqual(reader.getSchemaMetadata(), {myuid: '420', fnord: 'dronf'}); diff --git a/modules/parquet/test/parquetjs/reader.spec.ts b/modules/parquet/test/parquetjs/reader.spec.ts index 25e197d6bd..2ff793f3a2 100644 --- a/modules/parquet/test/parquetjs/reader.spec.ts +++ b/modules/parquet/test/parquetjs/reader.spec.ts @@ -1,6 +1,6 @@ /* eslint-disable camelcase */ import test from 'tape-promise/tape'; -import {makeReadableFile} from '@loaders.gl/loader-utils'; +import {BlobFile} from '@loaders.gl/loader-utils'; import {ParquetReader} from '@loaders.gl/parquet'; import {fetchFile} from '@loaders.gl/core'; @@ -12,7 +12,7 @@ const FRUITS_URL = '@loaders.gl/parquet/test/data/fruits.parquet'; test('ParquetReader#fruits.parquet', async t => { const response = await fetchFile(FRUITS_URL); const arrayBuffer = await response.arrayBuffer(); - const reader = new ParquetReader(makeReadableFile(arrayBuffer)); + const reader = new ParquetReader(new BlobFile(arrayBuffer)); // t.equal(reader.getRowCount(), TEST_NUM_ROWS * 4, 'rowCount'); const metadata = await reader.getSchemaMetadata(); diff --git a/modules/pmtiles/src/pmtiles-source.ts b/modules/pmtiles/src/pmtiles-source.ts index 53add8d641..d0172c7585 100644 --- a/modules/pmtiles/src/pmtiles-source.ts +++ b/modules/pmtiles/src/pmtiles-source.ts @@ -12,37 +12,36 @@ import type {PMTilesMetadata} from './lib/parse-pmtiles'; import {parsePMTilesHeader} from './lib/parse-pmtiles'; import {TileLoadParameters} from 'modules/loader-utils/src/lib/sources/tile-source'; -// export const PMTilesService: Service = { -// name: 'PMTiles', -// id: 'pmtiles', -// module: 'pmtiles', -// // version: VERSION, -// extensions: ['pmtiles'], -// mimeTypes: ['application/octet-stream'], -// options: { -// pmtiles: {} -// } -// }; +const VERSION = '1.0.0'; + +export type Service = { + name: string; + id: string; + module: string; + version: string; + extensions: string[]; + mimeTypes: string[]; + options: Record; +}; -/** - * WIP - Loader for pmtiles metadata - * @note loads metadata only. To load individual tiles, use PMTilesSource -export const PMTilesLoader: LoaderWithParser = { +export type ServiceWithSource = Service & { + _source?: SourceT; + _sourceProps?: SourcePropsT; + createSource: (props: SourcePropsT) => SourceT; +}; + +export const PMTilesService: ServiceWithSource = { name: 'PMTiles', id: 'pmtiles', module: 'pmtiles', version: VERSION, - worker: true, extensions: ['pmtiles'], mimeTypes: ['application/octet-stream'], options: { pmtiles: {} }, - parse: async (arrayBuffer, options) => { - throw new Error('not implemented'); - } + createSource: (props: PMTilesSourceProps) => new PMTilesSource(props) }; - */ export type PMTilesSourceProps = DataSourceProps & { url: string | Blob; diff --git a/modules/polyfills/package.json b/modules/polyfills/package.json index 5b92a0310b..5e8f0eb969 100644 --- a/modules/polyfills/package.json +++ b/modules/polyfills/package.json @@ -36,6 +36,7 @@ "build-bundle": "esbuild src/bundle.ts --bundle --outfile=dist/dist.min.js" }, "dependencies": { + "@loaders.gl/loader-utils": "4.0.0-beta.1", "@babel/runtime": "^7.3.1", "buffer": "^6.0.3", "get-pixels": "^3.3.2", diff --git a/modules/polyfills/src/filesystems/fetch-node.ts b/modules/polyfills/src/filesystems/fetch-node.ts new file mode 100644 index 0000000000..a114f97af6 --- /dev/null +++ b/modules/polyfills/src/filesystems/fetch-node.ts @@ -0,0 +1,96 @@ +// loaders.gl, MIT license + +import fs from 'fs'; +import {Readable} from 'stream'; +import {resolvePath} from '@loaders.gl/loader-utils'; +import {decompressReadStream} from './stream-utils.node'; + +const isBoolean = (x) => typeof x === 'boolean'; +const isFunction = (x) => typeof x === 'function'; +const isObject = (x) => x !== null && typeof x === 'object'; +const isReadableNodeStream = (x) => + isObject(x) && isFunction(x.read) && isFunction(x.pipe) && isBoolean(x.readable); + +/** + * Enables + * @param url + * @param options + * @returns + */ +// eslint-disable-next-line max-statements +export async function fetchNode(url: string, options?: RequestInit): Promise { + // Support `file://` protocol + const FILE_PROTOCOL_REGEX = /^file:\/\//; + url.replace(FILE_PROTOCOL_REGEX, '/'); + + // Remove any query parameters, as they have no meaning + let noqueryUrl = url.split('?')[0]; + noqueryUrl = resolvePath(noqueryUrl); + + const responseHeaders = new Headers(); + // Automatically decompress gzipped files with .gz extension + if (url.endsWith('.gz')) { + // url = url.slice(0, -3); + responseHeaders['content-encoding'] = 'gzip'; + } + if (url.endsWith('.br')) { + // url = url.slice(0, -3); + responseHeaders['content-encoding'] = 'br'; + } + + try { + // Now open the stream + const body = await new Promise((resolve, reject) => { + // @ts-ignore + const stream = fs.createReadStream(noqueryUrl, {encoding: null}); + stream.once('readable', () => resolve(stream)); + stream.on('error', (error) => reject(error)); + }); + + let bodyStream: Readable = body; + + // Check for content-encoding and create a decompression stream + if (isReadableNodeStream(body)) { + bodyStream = decompressReadStream(body, responseHeaders); + } else if (typeof body === 'string') { + bodyStream = Readable.from([new TextEncoder().encode(body)]); + } else { + bodyStream = Readable.from([body || new ArrayBuffer(0)]); + } + + const status = 200; + const statusText = 'OK'; + const headers = getHeadersForFile(noqueryUrl); + // @ts-expect-error + const response = new Response(bodyStream, {headers, status, statusText}); + Object.defineProperty(response, 'url', {value: url}); + return response; + } catch (error) { + // console.error(error); + const errorMessage = (error as Error).message; + const status = 400; + const statusText = errorMessage; + const headers = {}; + const response = new Response(errorMessage, {headers, status, statusText}); + Object.defineProperty(response, 'url', {value: url}); + return response; + } +} + +function getHeadersForFile(noqueryUrl: string): Headers { + const headers = {}; + + // Fix up content length if we can for best progress experience + if (!headers['content-length']) { + const stats = fs.statSync(noqueryUrl); + headers['content-length'] = stats.size; + } + + // Automatically decompress gzipped files with .gz extension + if (noqueryUrl.endsWith('.gz')) { + noqueryUrl = noqueryUrl.slice(0, -3); + headers['content-encoding'] = 'gzip'; + } + + return new Headers(headers); +} diff --git a/modules/polyfills/src/filesystems/node-file.ts b/modules/polyfills/src/filesystems/node-file.ts new file mode 100644 index 0000000000..353e6bc9a2 --- /dev/null +++ b/modules/polyfills/src/filesystems/node-file.ts @@ -0,0 +1,118 @@ +import {ReadableFile, WritableFile, Stat} from '@loaders.gl/loader-utils'; +import fs from 'fs'; + +export class NodeFile implements ReadableFile, WritableFile { + handle: number; + size: number; + + constructor(path, flags: 'a' | 'w' | 'wx', mode?: number) { + this.handle = fs.openSync(path, flags, mode); + this.size = fs.fstatSync(this.handle).size; + } + + async close(): Promise { + return new Promise((resolve, reject) => { + fs.close(this.handle, (err) => (err ? reject(err) : resolve())); + }); + } + + async stat(): Promise { + return await new Promise((resolve, reject) => + fs.fstat(this.handle, (err, info) => + err ? reject(err) : resolve({size: info.size, isDirectory: info.isDirectory()}) + ) + ); + } + + async read(offset: number, length: number): Promise { + const arrayBuffer = new ArrayBuffer(length); + + let totalBytesRead = 0; + const uint8Array = new Uint8Array(arrayBuffer); + + let position; + // Read in loop until we get required number of bytes + while (length > 0) { + const bytesRead = await readBytes(this.handle, uint8Array, 0, length, offset); + + // Check if end of file reached + if (bytesRead === 0) { + break; + } + + totalBytesRead += bytesRead; + offset += bytesRead; + length -= bytesRead; + + // Advance position unless we are using built-in position advancement + if (position !== undefined) { + position += bytesRead; + } + } + return totalBytesRead < length ? arrayBuffer.slice(0, totalBytesRead) : arrayBuffer; + } + + async write( + arrayBuffer: ArrayBuffer, + offset: number = 0, + length: number = arrayBuffer.byteLength + ): Promise { + return new Promise((resolve, reject) => { + const uint8Array = new Uint8Array(arrayBuffer, offset, length); + fs.write(this.handle, uint8Array, (err, bytesWritten) => + err ? reject(err) : resolve(bytesWritten) + ); + }); + } +} + +async function readBytes( + fd: number, + uint8Array: Uint8Array, + offset: number, + length: number, + position: number | null +): Promise { + return await new Promise((resolve, reject) => + fs.read(fd, uint8Array, offset, length, position, (err, bytesRead) => + err ? reject(err) : resolve(bytesRead) + ) + ); +} + +// TODO - implement streaming write +/* +export interface WriteStreamOptions { + flags?: string; + encoding?: 'utf8'; + fd?: number; + mode?: number; + autoClose?: boolean; + start?: number; +} + +export class NodeStreamWritableFile implements WritableFile { + outputStream: fs.WriteStream | Writable; + + constructor(pathOrStream: string | Writable, options?: WriteStreamOptions) { + this.outputStream = + typeof pathOrStream === 'string' ? fs.createWriteStream(pathOrStream, options) : pathOrStream; + } + + async write(buffer: ArrayBuffer): Promise { + return new Promise((resolve, reject) => { + const uint8Array = new Uint8Array(buffer); + this.outputStream.write(uint8Array, (err) => (err ? reject(err) : resolve())); + }); + } + + async close(): Promise { + if (this.outputStream instanceof fs.WriteStream) { + return new Promise((resolve, reject) => { + const stream = this.outputStream as fs.WriteStream; + stream.close((err) => (err ? reject(err) : resolve())); + }); + } + } +} +*/ diff --git a/modules/polyfills/src/filesystems/node-filesystem.ts b/modules/polyfills/src/filesystems/node-filesystem.ts index 496c469cf2..d4039e5b16 100644 --- a/modules/polyfills/src/filesystems/node-filesystem.ts +++ b/modules/polyfills/src/filesystems/node-filesystem.ts @@ -1,33 +1,24 @@ -import {FileSystem, RandomAccessReadFileSystem} from '@loaders.gl/loader-utils'; -import fs from 'fs'; +// loaders.gl, MIT license + +import {Stat, RandomAccessFileSystem} from '@loaders.gl/loader-utils'; import fsPromise from 'fs/promises'; +import {NodeFile} from './node-file'; +import {fetchNode} from './fetch-node'; // import {fetchFile} from "../fetch/fetch-file" // import {selectLoader} from "../api/select-loader"; -type Stat = { - size: number; - isDirectory: () => boolean; - info?: fs.Stats; -}; - -type ReadOptions = { - buffer?: Buffer; - offset?: number; - length?: number; - position?: number; -}; - /** * FileSystem pass-through for Node.js * Compatible with BrowserFileSystem. * @param options */ -export class NodeFileSystem implements FileSystem, RandomAccessReadFileSystem { +export class NodeFileSystem implements RandomAccessFileSystem { + readable: boolean = true; + writable: boolean = true; + // implements FileSystem - constructor(options: {[key: string]: any}) { - this.fetch = options._fetch; - } + constructor() {} async readdir(dirname = '.', options?: {}): Promise { return await fsPromise.readdir(dirname, options); @@ -35,53 +26,23 @@ export class NodeFileSystem implements FileSystem, RandomAccessReadFileSystem { async stat(path: string, options?: {}): Promise { const info = await fsPromise.stat(path, options); - return {size: Number(info.size), isDirectory: () => false, info}; + return {size: Number(info.size), isDirectory: info.isDirectory()}; } - async fetch(path: string, options: {[key: string]: any}) { - // Falls back to handle https:/http:/data: etc fetches - // eslint-disable-next-line - const fallbackFetch = options.fetch || this.fetch; - return fallbackFetch(path, options); + async unlink(path: string): Promise { + return await fsPromise.unlink(path); } - // implements IRandomAccessFileSystem - async open(path: string, flags: string | number, mode?: any): Promise { - return (await fsPromise.open(path, flags)) as unknown as number; - } - - async close(fd: number): Promise { - fs.close(fd); + async fetch(path: string, options: RequestInit): Promise { + return await fetchNode(path, options); } - async fstat(fd: number): Promise { - return await new Promise((resolve, reject) => - fs.fstat(fd, (err, info) => (err ? reject(err) : resolve(info))) - ); + // implements IRandomAccessFileSystem + async openReadableFile(path: string, flags: 'a' = 'a'): Promise { + return new NodeFile(path, flags); } - async read( - fd: number, - // @ts-ignore Possibly null - {buffer = null, offset = 0, length = buffer.byteLength, position = null}: ReadOptions - ): Promise<{bytesRead: number; buffer: Uint8Array}> { - let totalBytesRead = 0; - // Read in loop until we get required number of bytes - while (totalBytesRead < length) { - const {bytesRead} = await new Promise<{bytesRead: number; buffer: Buffer}>( - // eslint-disable-next-line no-loop-func - (resolve, reject) => - fs.read( - fd, - buffer, - offset + totalBytesRead, - length - totalBytesRead, - position + totalBytesRead, - (err, bytesRead, buffer) => (err ? reject(err) : resolve({bytesRead, buffer})) - ) - ); - totalBytesRead += bytesRead; - } - return {bytesRead: totalBytesRead, buffer}; + async openWritableFile(path: string, flags: 'w' | 'wx' = 'w', mode?: any): Promise { + return new NodeFile(path, flags, mode); } } diff --git a/modules/polyfills/src/node/fetch/utils/stream-utils.node.ts b/modules/polyfills/src/filesystems/stream-utils.node.ts similarity index 64% rename from modules/polyfills/src/node/fetch/utils/stream-utils.node.ts rename to modules/polyfills/src/filesystems/stream-utils.node.ts index 0ea98face6..57e66b19a7 100644 --- a/modules/polyfills/src/node/fetch/utils/stream-utils.node.ts +++ b/modules/polyfills/src/filesystems/stream-utils.node.ts @@ -1,14 +1,16 @@ // loaders.gl, MIT license import zlib from 'zlib'; +import {Readable} from 'stream'; -import {toArrayBuffer} from './decode-data-uri.node'; +const isArrayBuffer = (x) => x && x instanceof ArrayBuffer; +const isBuffer = (x) => x && x instanceof Buffer; /** * */ -export function decompressReadStream(readStream, headers) { - switch (headers.get('content-encoding')) { +export function decompressReadStream(readStream: Readable, headers?: Headers) { + switch (headers?.get('content-encoding')) { case 'br': return readStream.pipe(zlib.createBrotliDecompress()); case 'gzip': @@ -77,3 +79,40 @@ export function concatenateArrayBuffers(sources: (ArrayBuffer | Uint8Array)[]): // We work with ArrayBuffers, discard the typed array wrapper return result.buffer; } + +/** + * @param data + * @todo Duplicate of core + */ +export function toArrayBuffer(data: unknown): ArrayBuffer { + if (isArrayBuffer(data)) { + return data as ArrayBuffer; + } + + // TODO - per docs we should just be able to call buffer.buffer, but there are issues + if (isBuffer(data)) { + // @ts-expect-error + const typedArray = new Uint8Array(data); + return typedArray.buffer; + } + + // Careful - Node Buffers will look like ArrayBuffers (keep after isBuffer) + if (ArrayBuffer.isView(data)) { + return data.buffer; + } + + if (typeof data === 'string') { + const text = data; + const uint8Array = new TextEncoder().encode(text); + return uint8Array.buffer; + } + + // HACK to support Blob polyfill + // @ts-expect-error + if (data && typeof data === 'object' && data._toArrayBuffer) { + // @ts-expect-error + return data._toArrayBuffer(); + } + + throw new Error(`toArrayBuffer(${JSON.stringify(data, null, 2).slice(10)})`); +} diff --git a/modules/polyfills/src/index.browser.ts b/modules/polyfills/src/index.browser.ts index 8f0cfe4d9f..8fc69de1ac 100644 --- a/modules/polyfills/src/index.browser.ts +++ b/modules/polyfills/src/index.browser.ts @@ -1,7 +1,10 @@ // loaders.gl, MIT License -// eslint-disable-next-line @typescript-eslint/no-empty-function export function installFilePolyfills() {} // Dummy export to avoid import errors in browser tests export const NodeFileSystem = null; + +export function fetchNode(path: string, options: RequestInit): Promise { + throw new Error('fetchNode not available in browser'); +} diff --git a/modules/polyfills/src/index.ts b/modules/polyfills/src/index.ts index a7f23ab37d..4fa147284d 100644 --- a/modules/polyfills/src/index.ts +++ b/modules/polyfills/src/index.ts @@ -4,17 +4,28 @@ import {isBrowser} from './utils/is-browser'; import {TextDecoder, TextEncoder} from './lib/encoding'; // Node specific -import * as base64 from './node/buffer/btoa.node'; +import {atob, btoa} from './node/buffer/btoa.node'; import {encodeImageNode} from './node/images/encode-image.node'; import {parseImageNode, NODE_FORMAT_SUPPORT} from './node/images/parse-image.node'; +// FILESYSTEM POLYFILLS +export {NodeFile} from './filesystems/node-file'; +export {NodeFileSystem} from './filesystems/node-filesystem'; +export {fetchNode} from './filesystems/fetch-node'; +import {NodeFile} from './filesystems/node-file'; +import {NodeFileSystem} from './filesystems/node-filesystem'; +import {fetchNode} from './filesystems/fetch-node'; + // export {ReadableStreamPolyfill} from './node/file/readable-stream'; // export {BlobPolyfill} from './node/file/blob'; export {FileReaderPolyfill} from './node/file/file-reader'; export {FilePolyfill} from './node/file/file'; export {installFilePolyfills} from './node/file/install-file-polyfills'; +const {versions} = require('node:process'); +export const nodeVersion = parseInt(versions.node.split('.')[0]); + if (isBrowser) { // eslint-disable-next-line no-console console.error( @@ -22,6 +33,12 @@ if (isBrowser) { ); } +// FILESYSTEM POLYFILLS +globalThis.loaders = globalThis.loaders || {}; +globalThis.loaders.NodeFile = NodeFile; +globalThis.loaders.NodeFileSystem = NodeFileSystem; +globalThis.loaders.fetchNode = fetchNode; + // POLYFILLS: TextEncoder, TextDecoder // - Recent Node versions have these classes but virtually no encodings unless special build. // - Browser: Edge, IE11 do not have these @@ -40,29 +57,22 @@ if (!globalThis.TextDecoder) { // - Node: Yes // - Browser: No -if (!isBrowser && !('atob' in globalThis) && base64.atob) { - globalThis['atob'] = base64.atob; +if (!('atob' in globalThis) && atob) { + globalThis['atob'] = atob; } -if (!isBrowser && !('btoa' in globalThis) && base64.btoa) { - globalThis['btoa'] = base64.btoa; +if (!('btoa' in globalThis) && btoa) { + globalThis['btoa'] = btoa; } -globalThis.loaders = globalThis.loaders || {}; - -// FILESYSTEM POLYFILLS: -export {NodeFileSystem} from './filesystems/node-filesystem'; -import {NodeFileSystem} from './filesystems/node-filesystem'; -globalThis.loaders.NodeFileSystem = NodeFileSystem; - // NODE IMAGE FUNCTIONS: // These are not official polyfills but used by the @loaders.gl/images module if installed // TODO - is there an appropriate Image API we could polyfill using an adapter? -if (!isBrowser && !('_encodeImageNode' in globalThis) && encodeImageNode) { +if (!('_encodeImageNode' in globalThis) && encodeImageNode) { globalThis['_encodeImageNode'] = encodeImageNode; } -if (!isBrowser && !('_parseImageNode' in globalThis) && parseImageNode) { +if (!('_parseImageNode' in globalThis) && parseImageNode) { globalThis['_parseImageNode'] = parseImageNode; globalThis['_imageFormatsNode'] = NODE_FORMAT_SUPPORT; } @@ -74,19 +84,21 @@ if (!isBrowser && !('_parseImageNode' in globalThis) && parseImageNode) { // - IE11: No. This polyfill is node only, install external polyfill import {Headers as HeadersNode} from './node/fetch/headers.node'; import {Response as ResponseNode} from './node/fetch/response.node'; -import {fetchNode as fetchNode} from './node/fetch/fetch.node'; - -if (!isBrowser && !('Headers' in globalThis) && HeadersNode) { - // @ts-expect-error - globalThis.Headers = HeadersNode; -} - -if (!isBrowser && !('Response' in globalThis) && ResponseNode) { - // @ts-expect-error - globalThis.Response = ResponseNode; -} - -if (!isBrowser && !('fetch' in globalThis) && fetchNode) { - // @ts-expect-error - globalThis.fetch = fetchNode; +import {fetchNode as fetchNodePolyfill} from './node/fetch/fetch.node'; + +if (nodeVersion < 18) { + if (!('Headers' in globalThis) && HeadersNode) { + // @ts-expect-error + globalThis.Headers = HeadersNode; + } + + if (!('Response' in globalThis) && ResponseNode) { + // @ts-expect-error + globalThis.Response = ResponseNode; + } + + if (!('fetch' in globalThis) && fetchNodePolyfill) { + // @ts-expect-error + globalThis.fetch = fetchNodePolyfill; + } } diff --git a/modules/polyfills/src/node/fetch/response.node.ts b/modules/polyfills/src/node/fetch/response.node.ts index 1387a7619f..4c0652c544 100644 --- a/modules/polyfills/src/node/fetch/response.node.ts +++ b/modules/polyfills/src/node/fetch/response.node.ts @@ -1,7 +1,7 @@ // loaders.gl, MIT license import {assert} from '../../utils/assert'; -import {decompressReadStream, concatenateReadStream} from './utils/stream-utils.node'; +import {decompressReadStream, concatenateReadStream} from '../../filesystems/stream-utils.node'; import {Headers} from './headers.node'; const isBoolean = (x) => typeof x === 'boolean'; diff --git a/modules/polyfills/test/filesystems/fetch-node.spec.js b/modules/polyfills/test/filesystems/fetch-node.spec.js new file mode 100644 index 0000000000..ceb0649276 --- /dev/null +++ b/modules/polyfills/test/filesystems/fetch-node.spec.js @@ -0,0 +1,64 @@ +import test from 'tape-promise/tape'; +import {isBrowser} from '@loaders.gl/core'; +import '@loaders.gl/polyfills'; +// eslint-disable-next-line import/named +import {fetchNode} from '@loaders.gl/polyfills'; + +const GITHUB_MASTER = 'https://raw.githubusercontent.com/visgl/loaders.gl/master/modules/'; +const PLY_CUBE_ATT_URL = `${GITHUB_MASTER}ply/test/data/cube_att.ply`; +const TEXT_URL = `@loaders.gl/polyfills/test/data/data.txt`; +const TEXT_URL_GZIPPED = `@loaders.gl/polyfills/test/data/data.txt.gz`; + +test('polyfills#fetchNode() (NODE)', async (t) => { + if (!isBrowser) { + const response = await fetchNode(PLY_CUBE_ATT_URL); + t.ok(response.headers, 'fetchNode polyfill successfully returned headers under Node.js'); + const data = await response.arrayBuffer(); + t.ok(data, 'fetchNode polyfill successfully loaded data under Node.js'); + } + t.end(); +}); + +test('polyfills#fetchNode() ignores url query params when loading file (NODE)', async (t) => { + if (!isBrowser) { + const response = await fetchNode(`${PLY_CUBE_ATT_URL}?v=1.2.3`); + const data = await response.text(); + t.ok(response.headers, 'fetchNode polyfill successfully returned headers under Node.js'); + t.ok(data, 'fetchNode polyfill successfully loaded data under Node.js'); + } + t.end(); +}); + +test('polyfills#fetchNode() error handling (NODE)', async (t) => { + if (!isBrowser) { + let response = await fetchNode('non-existent-file'); + t.comment(response.statusText); + t.ok(response.statusText.includes('ENOENT'), 'fetchNode statusText forwards node ENOENT error'); + t.notOk(response.ok, 'fetchNode polyfill fails cleanly on non-existent file'); + t.ok(response.arrayBuffer(), 'Response.arrayBuffer() does not throw'); + + response = await fetchNode('.'); + t.comment(response.statusText); + t.ok(response.statusText.includes('EISDIR'), 'fetchNode statusText forwards node error'); + t.notOk(response.ok, 'fetchNode polyfill fails cleanly on directory'); + t.ok(response.arrayBuffer(), 'Response.arrayBuffer() does not throw'); + } + t.end(); +}); + +test('polyfills#fetchNode() able to decompress .gz extension (NODE)', async (t) => { + if (!isBrowser) { + let response = await fetchNode(TEXT_URL); + t.ok(response.ok, response.statusText); + let data = await response.text(); + t.equal(data, '123456', 'fetchNode polyfill correctly read text file'); + + if (!isBrowser) { + response = await fetchNode(TEXT_URL_GZIPPED); + t.ok(response.ok, response.statusText); + data = await response.text(); + t.equal(data, '123456', 'fetchNode polyfill correctly decompressed gzipped ".gz" file'); + } + } + t.end(); +}); diff --git a/modules/polyfills/test/index.ts b/modules/polyfills/test/index.ts index c1d914093f..334d808269 100644 --- a/modules/polyfills/test/index.ts +++ b/modules/polyfills/test/index.ts @@ -12,3 +12,4 @@ import './file/file-polyfill.spec'; import './promise/all-settled.spec'; import './filesystems/node-filesystem.spec'; +import './filesystems/fetch-node.spec'; diff --git a/modules/worker-utils/src/lib/node/worker_threads-browser.ts b/modules/worker-utils/src/lib/node/worker_threads-browser.ts index 555037f88a..b8c393fcba 100644 --- a/modules/worker-utils/src/lib/node/worker_threads-browser.ts +++ b/modules/worker-utils/src/lib/node/worker_threads-browser.ts @@ -3,7 +3,6 @@ // `import 'worker_threads` doesn't break browser builds. // The replacement is done in package.json browser field export class Worker { - // eslint-disable-next-line @typescript-eslint/no-empty-function terminate() {} } From 84df1e945d09893ec59ce1416897d21c3778d7a6 Mon Sep 17 00:00:00 2001 From: Ib Green Date: Sun, 8 Oct 2023 11:41:13 +0200 Subject: [PATCH 4/4] chore(core): Remove writeSync, save and fs dependencies (#2678) --- .../tiles-3d-archive-file-system.ts | 4 +- modules/core/src/index.ts | 5 +- modules/core/src/lib/api/encode.ts | 6 +- modules/core/src/lib/api/save.ts | 13 --- .../core/src/lib/fetch/read-array-buffer.ts | 8 +- modules/core/src/lib/fetch/read-file.ts | 31 -------- modules/core/src/lib/fetch/write-file.ts | 27 ------- modules/core/test/index.ts | 2 +- ...ile.spec.ts => read-file.spec.ts.disabled} | 0 modules/loader-utils/src/index.ts | 17 ++-- .../src/lib/file-provider/data-view-file.ts | 5 +- .../src/lib/file-provider/file-handle-file.ts | 72 +++++++---------- .../src/lib/file-provider/file-handle.ts | 79 ------------------- .../src/lib/file-provider/file-provider.ts | 1 + .../loader-utils/src/lib/files/blob-file.ts | 3 + modules/loader-utils/src/lib/files/file.ts | 18 +++-- .../loader-utils/src/lib/files/http-file.ts | 18 ++++- .../src/lib/files/node-file-facade.ts | 12 +-- .../src/lib/filesystems/filesystem.ts | 2 +- modules/loader-utils/src/lib/node/fs.ts | 49 ------------ modules/loader-utils/test/index.ts | 4 +- .../file-provider/file-handle-file.spec.js | 10 +-- .../lib/file-provider/file-handle.spec.js | 20 ----- .../parquet/src/parquetjs/utils/file-utils.ts | 3 +- .../polyfills/src/filesystems/node-file.ts | 45 ++++++++--- .../src/filesystems/node-filesystem.ts | 12 ++- .../test/filesystems/node-file.spec.ts | 19 +++++ modules/polyfills/test/index.ts | 2 + .../i3s-server/controllers/slpk-controller.ts | 2 +- .../src/slpk-extractor/slpk-extractor.ts | 2 +- modules/zip/src/filesystems/zip-filesystem.ts | 28 +++---- .../test/filesystems/zip-filesystem.spec.ts | 2 +- yarn.lock | 16 ++++ 33 files changed, 188 insertions(+), 349 deletions(-) delete mode 100644 modules/core/src/lib/api/save.ts delete mode 100644 modules/core/src/lib/fetch/read-file.ts delete mode 100644 modules/core/src/lib/fetch/write-file.ts rename modules/core/test/lib/fetch/{read-file.spec.ts => read-file.spec.ts.disabled} (100%) delete mode 100644 modules/loader-utils/src/lib/file-provider/file-handle.ts delete mode 100644 modules/loader-utils/src/lib/node/fs.ts delete mode 100644 modules/loader-utils/test/lib/file-provider/file-handle.spec.js create mode 100644 modules/polyfills/test/filesystems/node-file.spec.ts diff --git a/modules/3d-tiles/src/lib/filesystems/tiles-3d-archive-file-system.ts b/modules/3d-tiles/src/lib/filesystems/tiles-3d-archive-file-system.ts index 011dd11198..4756f289f2 100644 --- a/modules/3d-tiles/src/lib/filesystems/tiles-3d-archive-file-system.ts +++ b/modules/3d-tiles/src/lib/filesystems/tiles-3d-archive-file-system.ts @@ -35,7 +35,7 @@ export class Tiles3DArchiveFileSystem extends ZipFileSystem { * @returns - Response with file data */ async fetch(filename: string): Promise { - const fileProvider = await this.fileProvider; + const fileProvider = this.fileProvider; if (!fileProvider) { throw new Error('No data detected in the zip archive'); } @@ -61,7 +61,7 @@ export class Tiles3DArchiveFileSystem extends ZipFileSystem { return; } - const fileProvider = await this.fileProvider; + const fileProvider = this.fileProvider; if (!fileProvider) { throw new Error('No data detected in the zip archive'); } diff --git a/modules/core/src/index.ts b/modules/core/src/index.ts index b25bc5f5b6..0461d98bc9 100644 --- a/modules/core/src/index.ts +++ b/modules/core/src/index.ts @@ -22,8 +22,8 @@ export type { export {fetchFile} from './lib/fetch/fetch-file'; export {readArrayBuffer} from './lib/fetch/read-array-buffer'; -export {readFileSync} from './lib/fetch/read-file'; -export {writeFile, writeFileSync} from './lib/fetch/write-file'; +// export {readFileSync} from './lib/fetch/read-file'; +// export {writeFile, writeFileSync} from './lib/fetch/write-file'; // CONFIGURATION export {setLoaderOptions, getLoaderOptions} from './lib/api/loader-options'; @@ -42,7 +42,6 @@ export {loadInBatches} from './lib/api/load-in-batches'; export {encodeTable, encodeTableAsText, encodeTableInBatches} from './lib/api/encode-table'; export {encode, encodeSync, encodeInBatches, encodeURLtoURL} from './lib/api/encode'; export {encodeText, encodeTextSync} from './lib/api/encode'; -export {save, saveSync} from './lib/api/save'; // CORE UTILS SHARED WITH LOADERS (RE-EXPORTED FROM LOADER-UTILS) export {setPathPrefix, getPathPrefix, resolvePath} from '@loaders.gl/loader-utils'; diff --git a/modules/core/src/lib/api/encode.ts b/modules/core/src/lib/api/encode.ts index caf8a400bc..a61a5c0c6f 100644 --- a/modules/core/src/lib/api/encode.ts +++ b/modules/core/src/lib/api/encode.ts @@ -1,8 +1,7 @@ import {Writer, WriterOptions, canEncodeWithWorker} from '@loaders.gl/loader-utils'; +import {concatenateArrayBuffers, resolvePath, NodeFile} from '@loaders.gl/loader-utils'; import {processOnWorker} from '@loaders.gl/worker-utils'; -import {concatenateArrayBuffers, resolvePath} from '@loaders.gl/loader-utils'; import {isBrowser} from '@loaders.gl/loader-utils'; -import {writeFile} from '../fetch/write-file'; import {fetchFile} from '../fetch/fetch-file'; import {getLoaderOptions} from './loader-options'; @@ -51,7 +50,8 @@ export async function encode( if (!isBrowser && writer.encodeURLtoURL) { // TODO - how to generate filenames with correct extensions? const tmpInputFilename = getTemporaryFilename('input'); - await writeFile(tmpInputFilename, data as ArrayBuffer); + const file = new NodeFile(tmpInputFilename, 'w'); + await file.write(data as ArrayBuffer); const tmpOutputFilename = getTemporaryFilename('output'); diff --git a/modules/core/src/lib/api/save.ts b/modules/core/src/lib/api/save.ts deleted file mode 100644 index 6f2a7b81e5..0000000000 --- a/modules/core/src/lib/api/save.ts +++ /dev/null @@ -1,13 +0,0 @@ -import type {Writer, WriterOptions} from '@loaders.gl/loader-utils'; -import {encode, encodeSync} from './encode'; -import {writeFile, writeFileSync} from '../fetch/write-file'; - -export async function save(data: unknown, url: string, writer: Writer, options: WriterOptions) { - const encodedData = await encode(data, writer, options); - return await writeFile(url, encodedData); -} - -export function saveSync(data: unknown, url: string, writer: Writer, options: WriterOptions) { - const encodedData = encodeSync(data, writer, options); - return writeFileSync(url, encodedData); -} diff --git a/modules/core/src/lib/fetch/read-array-buffer.ts b/modules/core/src/lib/fetch/read-array-buffer.ts index 76384c9ecd..918d6f47e5 100644 --- a/modules/core/src/lib/fetch/read-array-buffer.ts +++ b/modules/core/src/lib/fetch/read-array-buffer.ts @@ -1,5 +1,4 @@ -// -import {fs} from '@loaders.gl/loader-utils'; +// loaders.gl, MIT license /** * Reads a chunk from a random access file @@ -9,13 +8,10 @@ import {fs} from '@loaders.gl/loader-utils'; * @returns */ export async function readArrayBuffer( - file: Blob | ArrayBuffer | string | number, + file: Blob | ArrayBuffer | string, start: number, length: number ): Promise { - if (typeof file === 'number') { - return await fs._readToArrayBuffer(file, start, length); - } // TODO - we can do better for ArrayBuffer and string if (!(file instanceof Blob)) { file = new Blob([file]); diff --git a/modules/core/src/lib/fetch/read-file.ts b/modules/core/src/lib/fetch/read-file.ts deleted file mode 100644 index 6830c304dd..0000000000 --- a/modules/core/src/lib/fetch/read-file.ts +++ /dev/null @@ -1,31 +0,0 @@ -// File read -import {isBrowser, resolvePath, fs, toArrayBuffer} from '@loaders.gl/loader-utils'; -import {assert} from '@loaders.gl/loader-utils'; - -// TODO - this is not tested -// const isDataURL = (url) => url.startsWith('data:'); - -/** - * In a few cases (data URIs, node.js) "files" can be read synchronously - */ -export function readFileSync(url: string, options: object = {}) { - url = resolvePath(url); - - // Only support this if we can also support sync data URL decoding in browser - // if (isDataURL(url)) { - // return decodeDataUri(url); - // } - - if (!isBrowser) { - const buffer = fs.readFileSync(url, options); - return typeof buffer !== 'string' ? toArrayBuffer(buffer) : buffer; - } - - // @ts-ignore - if (!options.nothrow) { - // throw new Error('Cant load URI synchronously'); - assert(false); - } - - return null; -} diff --git a/modules/core/src/lib/fetch/write-file.ts b/modules/core/src/lib/fetch/write-file.ts deleted file mode 100644 index 10502aae6a..0000000000 --- a/modules/core/src/lib/fetch/write-file.ts +++ /dev/null @@ -1,27 +0,0 @@ -// file write -import {isBrowser, assert, resolvePath} from '@loaders.gl/loader-utils'; -import {fs, toBuffer} from '@loaders.gl/loader-utils'; - -export async function writeFile( - filePath: string, - arrayBufferOrString: ArrayBuffer | string, - options? -): Promise { - filePath = resolvePath(filePath); - if (!isBrowser) { - await fs.writeFile(filePath, toBuffer(arrayBufferOrString), {flag: 'w'}); - } - assert(false); -} - -export function writeFileSync( - filePath: string, - arrayBufferOrString: ArrayBuffer | string, - options? -): void { - filePath = resolvePath(filePath); - if (!isBrowser) { - fs.writeFileSync(filePath, toBuffer(arrayBufferOrString), {flag: 'w'}); - } - assert(false); -} diff --git a/modules/core/test/index.ts b/modules/core/test/index.ts index 4e6264d3a8..94ec84744e 100644 --- a/modules/core/test/index.ts +++ b/modules/core/test/index.ts @@ -17,7 +17,7 @@ import './lib/fetch/fetch-error-message.spec'; import './lib/fetch/fetch-file.spec'; import './lib/fetch/fetch-file.browser.spec'; import './lib/fetch/fetch-file.node.spec'; -import './lib/fetch/read-file.spec'; +// import './lib/fetch/read-file.spec'; import './lib/api/set-loader-options.spec'; import './lib/api/register-loaders.spec'; diff --git a/modules/core/test/lib/fetch/read-file.spec.ts b/modules/core/test/lib/fetch/read-file.spec.ts.disabled similarity index 100% rename from modules/core/test/lib/fetch/read-file.spec.ts rename to modules/core/test/lib/fetch/read-file.spec.ts.disabled diff --git a/modules/loader-utils/src/index.ts b/modules/loader-utils/src/index.ts index 3c7fa6f1a5..8d985927bf 100644 --- a/modules/loader-utils/src/index.ts +++ b/modules/loader-utils/src/index.ts @@ -110,23 +110,12 @@ export {promisify1, promisify2} from './lib/node/promisify'; import * as path from './lib/path-utils/path'; export {path}; -// Use instead of importing 'fs' to avoid node dependencies` -import * as fs from './lib/node/fs'; -export {fs}; - // Use instead of importing 'stream' to avoid node dependencies` import * as stream from './lib/node/stream'; export {stream}; // EXPERIMENTAL: FILE SYSTEMS -export type {FileProvider} from './lib/file-provider/file-provider'; -export {isFileProvider} from './lib/file-provider/file-provider'; - -export {FileHandle} from './lib/file-provider/file-handle'; -export {FileHandleFile} from './lib/file-provider/file-handle-file'; -export {DataViewFile} from './lib/file-provider/data-view-file'; - export type {ReadableFile, WritableFile, Stat} from './lib/files/file'; export {BlobFile} from './lib/files/blob-file'; export {HttpFile} from './lib/files/http-file'; @@ -135,6 +124,12 @@ export {NodeFileFacade as NodeFile} from './lib/files/node-file-facade'; export type {FileSystem, RandomAccessFileSystem} from './lib/filesystems/filesystem'; export {NodeFileSystemFacade as NodeFilesystem} from './lib/filesystems/node-filesystem-facade'; +// TODO - replace with ReadableFile +export type {FileProvider} from './lib/file-provider/file-provider'; +export {isFileProvider} from './lib/file-provider/file-provider'; +export {FileHandleFile} from './lib/file-provider/file-handle-file'; +export {DataViewFile} from './lib/file-provider/data-view-file'; + // EXPERIMENTAL: DATA SOURCES export type {Service} from './service-types'; diff --git a/modules/loader-utils/src/lib/file-provider/data-view-file.ts b/modules/loader-utils/src/lib/file-provider/data-view-file.ts index a08e12ec33..5f38a6cd5c 100644 --- a/modules/loader-utils/src/lib/file-provider/data-view-file.ts +++ b/modules/loader-utils/src/lib/file-provider/data-view-file.ts @@ -12,7 +12,10 @@ const toNumber = (bigint: bigint) => { return Number(bigint); }; -/** Provides file data using DataView */ +/** + * Provides file data using DataView + * @deprecated - will be replaced with ReadableFile + */ export class DataViewFile implements FileProvider { /** The DataView from which data is provided */ private file: DataView; diff --git a/modules/loader-utils/src/lib/file-provider/file-handle-file.ts b/modules/loader-utils/src/lib/file-provider/file-handle-file.ts index 1a689b78b8..02179961e4 100644 --- a/modules/loader-utils/src/lib/file-provider/file-handle-file.ts +++ b/modules/loader-utils/src/lib/file-provider/file-handle-file.ts @@ -1,49 +1,37 @@ +// loaders.gl, MIT license + import {FileProvider} from './file-provider'; -import {FileHandle} from './file-handle'; -import {resolvePath} from '../path-utils/file-aliases'; +import {NodeFileFacade as NodeFile} from '../files/node-file-facade'; /** * Provides file data using node fs library + * @deprecated - will be replaced with ReadableFile */ export class FileHandleFile implements FileProvider { - /** - * Returns a new copy of FileHandleFile - * @param path The path to the file in file system - */ - static async from(path: string): Promise { - path = resolvePath(path); - const fileDescriptor = await FileHandle.open(path); - return new FileHandleFile(fileDescriptor, fileDescriptor.stat.size); - } + /** The FileHandle from which data is provided */ + private file: NodeFile; - /** - * The FileHandle from which data is provided - */ - private fileDescriptor: FileHandle; - - /** - * The file length in bytes - */ + /** The file length in bytes */ private size: bigint; - private constructor(fileDescriptor: FileHandle, size: bigint) { - this.fileDescriptor = fileDescriptor; - this.size = size; + /** Create a new FileHandleFile */ + constructor(path: string) { + this.file = new NodeFile(path, 'r'); + this.size = this.file.bigsize; } /** Close file */ async destroy(): Promise { - await this.fileDescriptor.close(); + await this.file.close(); } /** * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - async getUint8(offset: bigint): Promise { - const val = new Uint8Array( - (await this.fileDescriptor.read(Buffer.alloc(1), 0, 1, offset)).buffer.buffer - ).at(0); + async getUint8(offset: number | bigint): Promise { + const arrayBuffer = await this.file.read(offset, 1); + const val = new Uint8Array(arrayBuffer).at(0); if (val === undefined) { throw new Error('something went wrong'); } @@ -54,10 +42,9 @@ export class FileHandleFile implements FileProvider { * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - async getUint16(offset: bigint): Promise { - const val = new Uint16Array( - (await this.fileDescriptor.read(Buffer.alloc(2), 0, 2, offset)).buffer.buffer - ).at(0); + async getUint16(offset: number | bigint): Promise { + const arrayBuffer = await this.file.read(offset, 2); + const val = new Uint16Array(arrayBuffer).at(0); if (val === undefined) { throw new Error('something went wrong'); } @@ -68,10 +55,9 @@ export class FileHandleFile implements FileProvider { * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - async getUint32(offset: bigint): Promise { - const val = new Uint32Array( - (await this.fileDescriptor.read(Buffer.alloc(4), 0, 4, offset)).buffer.buffer - ).at(0); + async getUint32(offset: number | bigint): Promise { + const arrayBuffer = await this.file.read(offset, 4); + const val = new Uint32Array(arrayBuffer).at(0); if (val === undefined) { throw new Error('something went wrong'); } @@ -82,10 +68,9 @@ export class FileHandleFile implements FileProvider { * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file. * @param offset The offset, in bytes, from the start of the file where to read the data. */ - async getBigUint64(offset: bigint): Promise { - const val = new BigInt64Array( - (await this.fileDescriptor.read(Buffer.alloc(8), 0, 8, offset)).buffer.buffer - ).at(0); + async getBigUint64(offset: number | bigint): Promise { + const arrayBuffer = await this.file.read(offset, 8); + const val = new BigInt64Array(arrayBuffer).at(0); if (val === undefined) { throw new Error('something went wrong'); } @@ -94,17 +79,16 @@ export class FileHandleFile implements FileProvider { /** * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive. - * @param startOffsset The offset, in byte, from the start of the file where to start reading the data. + * @param startOffset The offset, in byte, from the start of the file where to start reading the data. * @param endOffset The offset, in bytes, from the start of the file where to end reading the data. */ - async slice(startOffsset: bigint, endOffset: bigint): Promise { - const bigLength = endOffset - startOffsset; + async slice(startOffset: bigint, endOffset: bigint): Promise { + const bigLength = endOffset - startOffset; if (bigLength > Number.MAX_SAFE_INTEGER) { throw new Error('too big slice'); } const length = Number(bigLength); - return (await this.fileDescriptor.read(Buffer.alloc(length), 0, length, startOffsset)).buffer - .buffer; + return await this.file.read(startOffset, length); } /** diff --git a/modules/loader-utils/src/lib/file-provider/file-handle.ts b/modules/loader-utils/src/lib/file-provider/file-handle.ts deleted file mode 100644 index 42d0daebc4..0000000000 --- a/modules/loader-utils/src/lib/file-provider/file-handle.ts +++ /dev/null @@ -1,79 +0,0 @@ -import * as fs from '../node/fs'; - -/** file reading result */ -export type FileReadResult = { - /** amount of the bytes read */ - bytesRead: number; - /** the buffer filled with data from file*/ - buffer: Buffer; -}; - -/** Object handling file info */ -export class FileHandle { - private fileDescriptor: number; - private stats: fs.BigIntStats; - - private constructor(fileDescriptor: number, stats: fs.BigIntStats) { - this.fileDescriptor = fileDescriptor; - this.stats = stats; - } - /** - * Opens a `FileHandle`. - * - * @param path path to the file - * @return Fulfills with a {FileHandle} object. - */ - - static async open(path: string): Promise { - const [fd, stats] = await Promise.all([ - new Promise((resolve, reject) => { - fs.open(path, undefined, undefined, (_err, fd) => (_err ? reject(_err) : resolve(fd))); - }), - fs.stat(path, {bigint: true}) - // new Promise((resolve, reject) => { - // console.error(fs.stat) - // fs.stat(path, {bigint: true}, (_err, stats) => (_err ? reject(_err) : resolve(stats))); - // console.error(fs.open) - // }) - ]); - return new FileHandle(fd, stats); - } - - /** Close file */ - async close(): Promise { - return fs.close(this.fileDescriptor); - // return new Promise((resolve) => { - // // @ts-expect-error - // fs.close(this.fileDescriptor, (_err) => resolve()); - // }); - } - - /** - * Reads data from the file and stores that in the given buffer. - * - * If the file is not modified concurrently, the end-of-file is reached when the - * number of bytes read is zero. - * @param buffer A buffer that will be filled with the file data read. - * @param offset The location in the buffer at which to start filling. - * @param length The number of bytes to read. - * @param position The location where to begin reading data from the file. If `null`, data will be read from the current file position, and the position will be updated. If `position` is an - * integer, the current file position will remain unchanged. - * @return Fulfills upon success with a FileReadResult object - */ - read = ( - buffer: Buffer, - offset: number, - length: number, - position: number | bigint - ): Promise => { - return new Promise((s) => { - fs.read(this.fileDescriptor, buffer, offset, length, position, (_err, bytesRead, buffer) => - s({bytesRead, buffer}) - ); - }); - }; - - get stat(): fs.BigIntStats { - return this.stats; - } -} diff --git a/modules/loader-utils/src/lib/file-provider/file-provider.ts b/modules/loader-utils/src/lib/file-provider/file-provider.ts index a0508ca869..f7ed7f34c0 100644 --- a/modules/loader-utils/src/lib/file-provider/file-provider.ts +++ b/modules/loader-utils/src/lib/file-provider/file-provider.ts @@ -1,5 +1,6 @@ /** * Interface for providing file data + * @deprecated - will be replaced with ReadableFile */ export interface FileProvider { /** diff --git a/modules/loader-utils/src/lib/files/blob-file.ts b/modules/loader-utils/src/lib/files/blob-file.ts index 09eb3d050d..158f535cd8 100644 --- a/modules/loader-utils/src/lib/files/blob-file.ts +++ b/modules/loader-utils/src/lib/files/blob-file.ts @@ -5,11 +5,13 @@ import {ReadableFile} from './file'; export class BlobFile implements ReadableFile { readonly handle: Blob; readonly size: number; + readonly bigsize: bigint; readonly url: string; constructor(blob: Blob | File | ArrayBuffer) { this.handle = blob instanceof ArrayBuffer ? new Blob([blob]) : blob; this.size = blob instanceof ArrayBuffer ? blob.byteLength : blob.size; + this.bigsize = BigInt(this.size); this.url = blob instanceof File ? blob.name : ''; } @@ -18,6 +20,7 @@ export class BlobFile implements ReadableFile { async stat() { return { size: this.handle.size, + bigsize: BigInt(this.handle.size), isDirectory: false }; } diff --git a/modules/loader-utils/src/lib/files/file.ts b/modules/loader-utils/src/lib/files/file.ts index 7c0e31ed41..a7788ced2f 100644 --- a/modules/loader-utils/src/lib/files/file.ts +++ b/modules/loader-utils/src/lib/files/file.ts @@ -1,17 +1,25 @@ +// loaders.gl, MIT license + export type Stat = { size: number; + bigsize: bigint; isDirectory: boolean; }; export interface ReadableFile { /** The underlying file handle (Blob, Node.js file descriptor etc) */ - handle: unknown; + readonly handle: unknown; /** Length of file in bytes, if available */ - size: number; + readonly size: number; + /** Length of file in bytes, if available */ + readonly bigsize: bigint; + /** Url, if available */ + readonly url: string; + /** Read data */ - read(start?: number, end?: number): Promise; + read(start?: number | bigint, length?: number): Promise; /** Read data */ - fetchRange?(offset: number, length: number, signal?: AbortSignal): Promise; + fetchRange?(offset: number | bigint, length: number, signal?: AbortSignal): Promise; /** Get information about file */ stat?(): Promise; /** Close the file */ @@ -21,7 +29,7 @@ export interface ReadableFile { export interface WritableFile { handle: unknown; /** Write to file. The number of bytes written will be returned */ - write: (arrayBuffer: ArrayBuffer, offset?: number, length?: number) => Promise; + write: (arrayBuffer: ArrayBuffer, offset?: number | bigint, length?: number) => Promise; /** Get information about the file */ stat?(): Promise; /** Close the file */ diff --git a/modules/loader-utils/src/lib/files/http-file.ts b/modules/loader-utils/src/lib/files/http-file.ts index 433d281c80..44a1dd4d2e 100644 --- a/modules/loader-utils/src/lib/files/http-file.ts +++ b/modules/loader-utils/src/lib/files/http-file.ts @@ -5,6 +5,7 @@ import {ReadableFile, Stat} from './file'; export class HttpFile implements ReadableFile { readonly handle: string; readonly size: number = 0; + readonly bigsize: bigint = 0n; readonly url: string; constructor(url: string) { @@ -19,13 +20,15 @@ export class HttpFile implements ReadableFile { if (!response.ok) { throw new Error(`Failed to fetch HEAD ${this.handle}`); } + const size = parseInt(response.headers.get('Content-Length') || '0'); return { - size: parseInt(response.headers.get('Content-Length') || '0'), + size, + bigsize: BigInt(size), isDirectory: false }; } - async read(offset: number, length: number): Promise { + async read(offset: number | bigint, length: number): Promise { const response = await this.fetchRange(offset, length); const arrayBuffer = await response.arrayBuffer(); return arrayBuffer; @@ -40,7 +43,14 @@ export class HttpFile implements ReadableFile { * @see https://github.com/protomaps/PMTiles */ // eslint-disable-next-line complexity - async fetchRange(offset: number, length: number, signal?: AbortSignal): Promise { + async fetchRange( + offset: number | bigint, + length: number, + signal?: AbortSignal + ): Promise { + const nOffset = Number(offset); + const nLength = Number(length); + let controller: AbortController | undefined; if (!signal) { // ToDO why is it so important to abort in case 200? @@ -52,7 +62,7 @@ export class HttpFile implements ReadableFile { const url = this.handle; let response = await fetch(url, { signal, - headers: {Range: `bytes=${offset}-${offset + length - 1}`} + headers: {Range: `bytes=${nOffset}-${nOffset + nLength - 1}`} }); switch (response.status) { diff --git a/modules/loader-utils/src/lib/files/node-file-facade.ts b/modules/loader-utils/src/lib/files/node-file-facade.ts index 73c9155c06..d40ad2bdf4 100644 --- a/modules/loader-utils/src/lib/files/node-file-facade.ts +++ b/modules/loader-utils/src/lib/files/node-file-facade.ts @@ -7,15 +7,15 @@ const NOT_IMPLEMENTED = new Error('Not implemented'); /** This class is a facade that gets replaced with an actual NodeFile instance */ export class NodeFileFacade implements ReadableFile, WritableFile { - /** The underlying file handle (Blob, Node.js file descriptor etc) */ handle: unknown; - /** Length of file in bytes, if available */ size: number = 0; + bigsize: bigint = 0n; + url: string = ''; - constructor(options) { + constructor(url: string, flags?: 'r' | 'w' | 'wx', mode?: number) { // Return the actual implementation instance if (globalThis.loaders?.NodeFile) { - return new globalThis.loaders.NodeFile(options); + return new globalThis.loaders.NodeFile(url, flags, mode); } if (isBrowser) { throw new Error('Can\'t instantiate NodeFile in browser.'); @@ -23,11 +23,11 @@ export class NodeFileFacade implements ReadableFile, WritableFile { throw new Error('Can\'t instantiate NodeFile. Make sure to import @loaders.gl/polyfills first.'); } /** Read data */ - async read(start?: number, end?: number): Promise { + async read(start?: number | bigint, end?: number | bigint): Promise { throw NOT_IMPLEMENTED; } /** Write to file. The number of bytes written will be returned */ - async write(arrayBuffer: ArrayBuffer, offset?: number, length?: number): Promise { + async write(arrayBuffer: ArrayBuffer, offset?: number | bigint, length?: number | bigint): Promise { throw NOT_IMPLEMENTED; } /** Get information about file */ diff --git a/modules/loader-utils/src/lib/filesystems/filesystem.ts b/modules/loader-utils/src/lib/filesystems/filesystem.ts index fe1f8837fb..6ba0989a73 100644 --- a/modules/loader-utils/src/lib/filesystems/filesystem.ts +++ b/modules/loader-utils/src/lib/filesystems/filesystem.ts @@ -31,7 +31,7 @@ export interface RandomAccessFileSystem extends FileSystem { readonly writable: boolean; /** Open a readable file */ - openReadableFile(path: string, flags?: 'a'): Promise; + openReadableFile(path: string, flags?: 'r'): Promise; /** Open a writable file */ openWritableFile(path: string, flags?: 'w' | 'wx', mode?: number): Promise; diff --git a/modules/loader-utils/src/lib/node/fs.ts b/modules/loader-utils/src/lib/node/fs.ts deleted file mode 100644 index fe16a80f2d..0000000000 --- a/modules/loader-utils/src/lib/node/fs.ts +++ /dev/null @@ -1,49 +0,0 @@ -// fs wrapper (promisified fs + avoids bundling fs in browsers) -import * as fs from 'fs'; -import * as fsPromises from 'fs/promises'; -import {toArrayBuffer} from './buffer'; -import {promisify2, promisify3} from './promisify'; - -export type {BigIntStats, Stats} from 'fs'; -export type {ReadStream, WriteStream} from 'fs'; - -/** Wrapper for Node.js fs method */ -export const readdir: any = promisify2(fs.readdir); -/** Wrapper for Node.js fs method */ -export const stat: any = fsPromises.stat; -export const statSync = fs.statSync; - -/** Wrapper for Node.js fs method */ -export const readFile: any = fs.readFile; -/** Wrapper for Node.js fs method */ -export const readFileSync = fs.readFileSync; -/** Wrapper for Node.js fs method */ -export const writeFile: any = promisify3(fs.writeFile); -/** Wrapper for Node.js fs method */ -export const writeFileSync = fs.writeFileSync; - -// file descriptors - -/** Wrapper for Node.js fs method */ -export const open: any = fs.open; -/** Wrapper for Node.js fs method */ -export const close = (fd: number) => - new Promise((resolve, reject) => fs.close(fd, (err) => (err ? reject(err) : resolve()))); -/** Wrapper for Node.js fs method */ -export const read: any = fs.read; -/** Wrapper for Node.js fs method */ -export const fstat: any = fs.fstat; - -export const createReadStream = fs.createReadStream; -export const createWriteStream = fs.createWriteStream; - -export const isSupported = Boolean(fs); - -export async function _readToArrayBuffer(fd: number, start: number, length: number) { - const buffer = Buffer.alloc(length); - const {bytesRead} = await read(fd, buffer, 0, length, start); - if (bytesRead !== length) { - throw new Error('fs.read failed'); - } - return toArrayBuffer(buffer); -} diff --git a/modules/loader-utils/test/index.ts b/modules/loader-utils/test/index.ts index 772ccd5254..0aaabfb1f8 100644 --- a/modules/loader-utils/test/index.ts +++ b/modules/loader-utils/test/index.ts @@ -15,8 +15,10 @@ import './lib/path-utils/path.spec'; import './lib/request-utils/request-scheduler.spec'; +// import './lib/files/node-file-facade.spec'; +// import './lib/filesystems/node-filesystem-facade.spec'; + import './lib/file-provider/data-view-file.spec'; import './lib/file-provider/file-handle-file.spec'; -import './lib/file-provider/file-handle.spec'; import './lib/worker-loader-utils/parse-with-worker.spec'; diff --git a/modules/loader-utils/test/lib/file-provider/file-handle-file.spec.js b/modules/loader-utils/test/lib/file-provider/file-handle-file.spec.js index fe497aea08..3c31c27f74 100644 --- a/modules/loader-utils/test/lib/file-provider/file-handle-file.spec.js +++ b/modules/loader-utils/test/lib/file-provider/file-handle-file.spec.js @@ -8,7 +8,7 @@ const SLPKUrl = 'modules/i3s/test/data/DA12_subset.slpk'; test('FileHandleFile#slice', async (t) => { if (!isBrowser) { - const provider = await FileHandleFile.from(SLPKUrl); + const provider = new FileHandleFile(SLPKUrl); t.equals(Buffer.from(await provider.slice(0n, 4n)).compare(Buffer.from(signature)), 0); } t.end(); @@ -16,7 +16,7 @@ test('FileHandleFile#slice', async (t) => { test('FileHandleFile#getUint8', async (t) => { if (!isBrowser) { - const provider = await FileHandleFile.from(SLPKUrl); + const provider = new FileHandleFile(SLPKUrl); t.equals(await provider.getUint8(0n), 80); t.end(); } @@ -25,7 +25,7 @@ test('FileHandleFile#getUint8', async (t) => { test('FileHandleFile#getUint16', async (t) => { if (!isBrowser) { - const provider = await FileHandleFile.from(SLPKUrl); + const provider = new FileHandleFile(SLPKUrl); t.equals(await provider.getUint16(0n), 19280); } t.end(); @@ -33,7 +33,7 @@ test('FileHandleFile#getUint16', async (t) => { test('FileHandleFile#getUint32', async (t) => { if (!isBrowser) { - const provider = await FileHandleFile.from(SLPKUrl); + const provider = new FileHandleFile(SLPKUrl); t.equals(await provider.getUint32(0n), 67324752); } t.end(); @@ -41,7 +41,7 @@ test('FileHandleFile#getUint32', async (t) => { test('FileHandleFile#getBigUint64', async (t) => { if (!isBrowser) { - const provider = await FileHandleFile.from(SLPKUrl); + const provider = new FileHandleFile(SLPKUrl); t.equals(await provider.getBigUint64(0n), 193340853072n); } t.end(); diff --git a/modules/loader-utils/test/lib/file-provider/file-handle.spec.js b/modules/loader-utils/test/lib/file-provider/file-handle.spec.js deleted file mode 100644 index 6ea454a773..0000000000 --- a/modules/loader-utils/test/lib/file-provider/file-handle.spec.js +++ /dev/null @@ -1,20 +0,0 @@ -import test from 'tape-promise/tape'; -import {isBrowser} from '@loaders.gl/core'; -import {FileHandle} from '@loaders.gl/loader-utils'; -import {promises as fsPromises} from 'fs'; - -const SLPKUrl = 'modules/i3s/test/data/DA12_subset.slpk'; - -test('FileHandle#open and read', async (t) => { - if (!isBrowser) { - const provider = await FileHandle.open(SLPKUrl); - const fsHandler = await fsPromises.open(SLPKUrl); - t.equals( - (await provider.read(Buffer.alloc(4), 0, 4, 1)).buffer.compare( - (await fsHandler.read(Buffer.alloc(4), 0, 4, 1)).buffer - ), - 0 - ); - } - t.end(); -}); diff --git a/modules/parquet/src/parquetjs/utils/file-utils.ts b/modules/parquet/src/parquetjs/utils/file-utils.ts index f4dcf8a4f8..347540d0c0 100644 --- a/modules/parquet/src/parquetjs/utils/file-utils.ts +++ b/modules/parquet/src/parquetjs/utils/file-utils.ts @@ -1,5 +1,6 @@ // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.) -import {fs, stream} from '@loaders.gl/loader-utils'; +import {stream} from '@loaders.gl/loader-utils'; +import * as fs from 'fs'; export function load(name: string): any { return (module || (global as any)).require(name); diff --git a/modules/polyfills/src/filesystems/node-file.ts b/modules/polyfills/src/filesystems/node-file.ts index 353e6bc9a2..6d7e0e3f0d 100644 --- a/modules/polyfills/src/filesystems/node-file.ts +++ b/modules/polyfills/src/filesystems/node-file.ts @@ -1,13 +1,20 @@ -import {ReadableFile, WritableFile, Stat} from '@loaders.gl/loader-utils'; +import type {ReadableFile, WritableFile, Stat} from '@loaders.gl/loader-utils'; +import {resolvePath} from '@loaders.gl/loader-utils'; import fs from 'fs'; export class NodeFile implements ReadableFile, WritableFile { handle: number; size: number; + bigsize: bigint; + url: string; - constructor(path, flags: 'a' | 'w' | 'wx', mode?: number) { + constructor(path: string, flags: 'r' | 'w' | 'wx', mode?: number) { + path = resolvePath(path); this.handle = fs.openSync(path, flags, mode); - this.size = fs.fstatSync(this.handle).size; + const stats = fs.fstatSync(this.handle, {bigint: true}); + this.size = Number(stats.size); + this.bigsize = stats.size; + this.url = path; } async close(): Promise { @@ -18,14 +25,24 @@ export class NodeFile implements ReadableFile, WritableFile { async stat(): Promise { return await new Promise((resolve, reject) => - fs.fstat(this.handle, (err, info) => - err ? reject(err) : resolve({size: info.size, isDirectory: info.isDirectory()}) - ) + fs.fstat(this.handle, {bigint: true}, (err, info) => { + const stats: Stat = { + size: Number(info.size), + bigsize: info.size, + isDirectory: info.isDirectory() + }; + if (err) { + reject(err); + } else { + resolve(stats); + } + }) ); } - async read(offset: number, length: number): Promise { + async read(offset: number | bigint, length: number): Promise { const arrayBuffer = new ArrayBuffer(length); + let bigOffset = BigInt(offset); let totalBytesRead = 0; const uint8Array = new Uint8Array(arrayBuffer); @@ -33,7 +50,7 @@ export class NodeFile implements ReadableFile, WritableFile { let position; // Read in loop until we get required number of bytes while (length > 0) { - const bytesRead = await readBytes(this.handle, uint8Array, 0, length, offset); + const bytesRead = await readBytes(this.handle, uint8Array, 0, length, bigOffset); // Check if end of file reached if (bytesRead === 0) { @@ -41,7 +58,7 @@ export class NodeFile implements ReadableFile, WritableFile { } totalBytesRead += bytesRead; - offset += bytesRead; + bigOffset += BigInt(bytesRead); length -= bytesRead; // Advance position unless we are using built-in position advancement @@ -54,12 +71,14 @@ export class NodeFile implements ReadableFile, WritableFile { async write( arrayBuffer: ArrayBuffer, - offset: number = 0, + offset: number | bigint = 0, length: number = arrayBuffer.byteLength ): Promise { return new Promise((resolve, reject) => { - const uint8Array = new Uint8Array(arrayBuffer, offset, length); - fs.write(this.handle, uint8Array, (err, bytesWritten) => + // TODO - Node.js doesn't offer write with bigint offsets??? + const nOffset = Number(offset); + const uint8Array = new Uint8Array(arrayBuffer, Number(offset), length); + fs.write(this.handle, uint8Array, 0, length, nOffset, (err, bytesWritten) => err ? reject(err) : resolve(bytesWritten) ); }); @@ -71,7 +90,7 @@ async function readBytes( uint8Array: Uint8Array, offset: number, length: number, - position: number | null + position: number | bigint | null ): Promise { return await new Promise((resolve, reject) => fs.read(fd, uint8Array, offset, length, position, (err, bytesRead) => diff --git a/modules/polyfills/src/filesystems/node-filesystem.ts b/modules/polyfills/src/filesystems/node-filesystem.ts index d4039e5b16..562fc5503e 100644 --- a/modules/polyfills/src/filesystems/node-filesystem.ts +++ b/modules/polyfills/src/filesystems/node-filesystem.ts @@ -24,9 +24,13 @@ export class NodeFileSystem implements RandomAccessFileSystem { return await fsPromise.readdir(dirname, options); } - async stat(path: string, options?: {}): Promise { - const info = await fsPromise.stat(path, options); - return {size: Number(info.size), isDirectory: info.isDirectory()}; + async stat(path: string): Promise { + const info = await fsPromise.stat(path, {bigint: true}); + return { + size: Number(info.size), + bigsize: info.size, + isDirectory: info.isDirectory() + }; } async unlink(path: string): Promise { @@ -38,7 +42,7 @@ export class NodeFileSystem implements RandomAccessFileSystem { } // implements IRandomAccessFileSystem - async openReadableFile(path: string, flags: 'a' = 'a'): Promise { + async openReadableFile(path: string, flags: 'r' = 'r'): Promise { return new NodeFile(path, flags); } diff --git a/modules/polyfills/test/filesystems/node-file.spec.ts b/modules/polyfills/test/filesystems/node-file.spec.ts new file mode 100644 index 0000000000..8afac5f5de --- /dev/null +++ b/modules/polyfills/test/filesystems/node-file.spec.ts @@ -0,0 +1,19 @@ +import test from 'tape-promise/tape'; +import {isBrowser} from '@loaders.gl/core'; +import {NodeFile, resolvePath} from '@loaders.gl/loader-utils'; +import {promises as fsPromises} from 'fs'; + +const SLPK_URL = '@loaders.gl/i3s/test/data/DA12_subset.slpk'; + +test('NodeFile#open and read', async (t) => { + if (!isBrowser) { + const fsHandler = await fsPromises.open(resolvePath(SLPK_URL)); + const reference = await fsHandler.read(Buffer.alloc(4), 0, 4, 1); + + const provider = new NodeFile(SLPK_URL); + const arrayBuffer = await provider.read(1, 4); + + t.equals(reference.buffer.compare(Buffer.from(arrayBuffer)), 0); + } + t.end(); +}); diff --git a/modules/polyfills/test/index.ts b/modules/polyfills/test/index.ts index 334d808269..ed5f6259ef 100644 --- a/modules/polyfills/test/index.ts +++ b/modules/polyfills/test/index.ts @@ -11,5 +11,7 @@ import './file/blob-polyfill.spec'; import './file/file-polyfill.spec'; import './promise/all-settled.spec'; +import './filesystems/fetch-node.spec'; +import './filesystems/node-file.spec'; import './filesystems/node-filesystem.spec'; import './filesystems/fetch-node.spec'; diff --git a/modules/tile-converter/src/i3s-server/controllers/slpk-controller.ts b/modules/tile-converter/src/i3s-server/controllers/slpk-controller.ts index 302d28f8bc..ffed50486a 100644 --- a/modules/tile-converter/src/i3s-server/controllers/slpk-controller.ts +++ b/modules/tile-converter/src/i3s-server/controllers/slpk-controller.ts @@ -9,7 +9,7 @@ let slpkArchive: SLPKArchive; * @param fullLayerPath - full path to SLPK file */ export async function loadArchive(fullLayerPath: string): Promise { - slpkArchive = await parseSLPKArchive(await FileHandleFile.from(fullLayerPath), (msg) => + slpkArchive = await parseSLPKArchive(new FileHandleFile(fullLayerPath), (msg) => console.log(msg) ); console.log('The server is ready to use'); diff --git a/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts b/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts index 6335468466..ea6de7d097 100644 --- a/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts +++ b/modules/tile-converter/src/slpk-extractor/slpk-extractor.ts @@ -38,7 +38,7 @@ export default class SLPKExtractor { } const {inputUrl} = options; - const provider = await FileHandleFile.from(inputUrl); + const provider = new FileHandleFile(inputUrl); let localHeader = await parseZipLocalFileHeader(0n, provider); while (localHeader) { diff --git a/modules/zip/src/filesystems/zip-filesystem.ts b/modules/zip/src/filesystems/zip-filesystem.ts index 1f48d67013..b600ec2a8e 100644 --- a/modules/zip/src/filesystems/zip-filesystem.ts +++ b/modules/zip/src/filesystems/zip-filesystem.ts @@ -24,7 +24,7 @@ const COMPRESSION_METHODS: {[key: number]: CompressionHandler} = { */ export class ZipFileSystem implements FileSystem { /** FileProvider instance promise */ - protected fileProvider: Promise = Promise.resolve(null); + protected fileProvider: FileProvider | null = null; public fileName?: string; /** @@ -36,20 +36,19 @@ export class ZipFileSystem implements FileSystem { if (typeof file === 'string') { this.fileName = file; if (!isBrowser) { - this.fileProvider = FileHandleFile.from(file); + this.fileProvider = new FileHandleFile(file); } else { throw new Error('Cannot open file for random access in a WEB browser'); } } else if (isFileProvider(file)) { - this.fileProvider = Promise.resolve(file); + this.fileProvider = file; } } /** Clean up resources */ async destroy() { - const fileProvider = await this.fileProvider; - if (fileProvider) { - await fileProvider.destroy(); + if (this.fileProvider) { + await this.fileProvider.destroy(); } } @@ -58,12 +57,11 @@ export class ZipFileSystem implements FileSystem { * @returns array of file names */ async readdir(): Promise { - const fileProvider = await this.fileProvider; - if (!fileProvider) { + if (!this.fileProvider) { throw new Error('No data detected in the zip archive'); } const fileNames: string[] = []; - const zipCDIterator = makeZipCDHeaderIterator(fileProvider); + const zipCDIterator = makeZipCDHeaderIterator(this.fileProvider); for await (const cdHeader of zipCDIterator) { fileNames.push(cdHeader.fileName); } @@ -86,14 +84,13 @@ export class ZipFileSystem implements FileSystem { * @returns - Response with file data */ async fetch(filename: string): Promise { - const fileProvider = await this.fileProvider; - if (!fileProvider) { + if (!this.fileProvider) { throw new Error('No data detected in the zip archive'); } const cdFileHeader = await this.getCDFileHeader(filename); const localFileHeader = await parseZipLocalFileHeader( cdFileHeader.localHeaderOffset, - fileProvider + this.fileProvider ); if (!localFileHeader) { throw new Error('Local file header has not been found in the zip archive`'); @@ -104,7 +101,7 @@ export class ZipFileSystem implements FileSystem { throw Error('Only Deflation compression is supported'); } - const compressedFile = await fileProvider.slice( + const compressedFile = await this.fileProvider.slice( localFileHeader.fileDataOffset, localFileHeader.fileDataOffset + localFileHeader.compressedSize ); @@ -122,11 +119,10 @@ export class ZipFileSystem implements FileSystem { * @returns central directory file header */ private async getCDFileHeader(filename: string): Promise { - const fileProvider = await this.fileProvider; - if (!fileProvider) { + if (!this.fileProvider) { throw new Error('No data detected in the zip archive'); } - const zipCDIterator = makeZipCDHeaderIterator(fileProvider); + const zipCDIterator = makeZipCDHeaderIterator(this.fileProvider); let result: ZipCDFileHeader | null = null; for await (const cdHeader of zipCDIterator) { if (cdHeader.fileName === filename) { diff --git a/modules/zip/test/filesystems/zip-filesystem.spec.ts b/modules/zip/test/filesystems/zip-filesystem.spec.ts index 2c0f9c97f1..114ae4f2a1 100644 --- a/modules/zip/test/filesystems/zip-filesystem.spec.ts +++ b/modules/zip/test/filesystems/zip-filesystem.spec.ts @@ -82,7 +82,7 @@ const getFileProvider = async (fileName: string) => { const file = await fileResponse.arrayBuffer(); fileProvider = new DataViewFile(new DataView(file)); } else { - fileProvider = await FileHandleFile.from(ZIP_FILE_PATH); + fileProvider = new FileHandleFile(ZIP_FILE_PATH); } return fileProvider; }; diff --git a/yarn.lock b/yarn.lock index da5494b7cb..29e46f236e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1970,6 +1970,15 @@ "@loaders.gl/worker-utils" "3.4.14" "@probe.gl/stats" "^4.0.1" +"@loaders.gl/loader-utils@4.0.0-beta.1": + version "4.0.0-beta.1" + resolved "https://registry.yarnpkg.com/@loaders.gl/loader-utils/-/loader-utils-4.0.0-beta.1.tgz#4234d9d5f7227dc4324bfb1c9348a471f87fbb55" + integrity sha512-nf/fKHoZaEYsM/sOwGpWz+Gou+0wN/GQRp0KpLXRLn6OZIxbYiMsM0jjYfDzQQ3qzv02la3R809BfJ+miaY3pw== + dependencies: + "@babel/runtime" "^7.3.1" + "@loaders.gl/worker-utils" "4.0.0-beta.1" + "@probe.gl/stats" "^4.0.2" + "@loaders.gl/worker-utils@3.4.14": version "3.4.14" resolved "https://registry.yarnpkg.com/@loaders.gl/worker-utils/-/worker-utils-3.4.14.tgz#5391a416a3d60e03b9edcedb285af44312d40d2e" @@ -1977,6 +1986,13 @@ dependencies: "@babel/runtime" "^7.3.1" +"@loaders.gl/worker-utils@4.0.0-beta.1": + version "4.0.0-beta.1" + resolved "https://registry.yarnpkg.com/@loaders.gl/worker-utils/-/worker-utils-4.0.0-beta.1.tgz#64a8c3ddf54bbed0231bc0b0901aa8c1b6e08c2c" + integrity sha512-GCQqs//ehUXH4vSYC/CffZTY/T/j/dFNhoT3WEWgSXuOAVNh6ovYt/NnKNe6kLMw94dCMnafFKK0hc3XB0WIrQ== + dependencies: + "@babel/runtime" "^7.3.1" + "@luma.gl/constants@8.5.21", "@luma.gl/constants@^8.5.21", "@luma.gl/constants@^8.5.4": version "8.5.21" resolved "https://registry.yarnpkg.com/@luma.gl/constants/-/constants-8.5.21.tgz#81825e9bd9bdf4a9449bcface8b504389f65f634"