From 499c01dec69d4c34588a5158cb13dcbae0c940ca Mon Sep 17 00:00:00 2001 From: Ib Green Date: Sat, 7 Oct 2023 22:36:59 +0200 Subject: [PATCH] wip --- modules/core/src/index.ts | 5 +- modules/core/src/lib/fetch/fetch-file.ts | 5 +- .../src/lib/filesystems/browser-filesystem.ts | 33 +-- modules/loader-utils/src/index.ts | 13 +- .../loader-utils/src/lib/files/blob-file.ts | 29 +++ modules/loader-utils/src/lib/files/file.ts | 29 +++ .../loader-utils/src/lib/files/http-file.ts | 108 ++++++++ .../src/lib/files/node-file-facade.ts | 39 +++ .../src/lib/files}/sources.ts | 0 .../src/lib/filesystems/filesystem.ts | 114 ++------ .../src/lib/filesystems/make-readable-file.ts | 25 -- .../src/lib/filesystems/make-writable-file.ts | 5 - ...ilesystem.ts => node-filesystem-facade.ts} | 14 +- modules/pmtiles/src/pmtiles-source.ts | 40 +-- modules/polyfills/package.json | 1 + .../src/filesystems/fetch-node.ts} | 4 +- .../polyfills/src/filesystems/node-file.ts | 116 +++++++++ .../src/filesystems/node-filesystem.ts | 243 ++---------------- modules/polyfills/src/index.ts | 27 +- 19 files changed, 436 insertions(+), 414 deletions(-) create mode 100644 modules/loader-utils/src/lib/files/blob-file.ts create mode 100644 modules/loader-utils/src/lib/files/file.ts create mode 100644 modules/loader-utils/src/lib/files/http-file.ts create mode 100644 modules/loader-utils/src/lib/files/node-file-facade.ts rename modules/{pmtiles/src/lib => loader-utils/src/lib/files}/sources.ts (100%) delete mode 100644 modules/loader-utils/src/lib/filesystems/make-readable-file.ts delete mode 100644 modules/loader-utils/src/lib/filesystems/make-writable-file.ts rename modules/loader-utils/src/lib/filesystems/{node-filesystem.ts => node-filesystem-facade.ts} (75%) rename modules/{core/src/lib/fetch/fetch-file.node.ts => polyfills/src/filesystems/fetch-node.ts} (93%) create mode 100644 modules/polyfills/src/filesystems/node-file.ts diff --git a/modules/core/src/index.ts b/modules/core/src/index.ts index 935b4069ef..b25bc5f5b6 100644 --- a/modules/core/src/index.ts +++ b/modules/core/src/index.ts @@ -11,8 +11,11 @@ export type { DataType, SyncDataType, BatchableDataType, + ReadableFile, + WritableFile, + Stat, FileSystem, - RandomAccessReadFileSystem + RandomAccessFileSystem } from '@loaders.gl/loader-utils'; // FILE READING AND WRITING diff --git a/modules/core/src/lib/fetch/fetch-file.ts b/modules/core/src/lib/fetch/fetch-file.ts index aee84e108a..6a28626f40 100644 --- a/modules/core/src/lib/fetch/fetch-file.ts +++ b/modules/core/src/lib/fetch/fetch-file.ts @@ -2,7 +2,6 @@ import {resolvePath} from '@loaders.gl/loader-utils'; import {makeResponse} from '../utils/response-utils'; -import * as node from './fetch-file.node'; export function isNodePath(url: string): boolean { return !isRequestURL(url) && !isDataURL(url); @@ -29,8 +28,8 @@ export async function fetchFile( const url = resolvePath(urlOrData); // Support fetching from local file system - if (isNodePath(url) && node?.fetchFileNode) { - return node.fetchFileNode(url, fetchOptions); + if (isNodePath(url) && globalThis.loaders?.fetchNode) { + return globalThis.loaders?.fetchNode(url, fetchOptions); } // Call global fetch diff --git a/modules/core/src/lib/filesystems/browser-filesystem.ts b/modules/core/src/lib/filesystems/browser-filesystem.ts index 69f06574e9..f3018ee6e6 100644 --- a/modules/core/src/lib/filesystems/browser-filesystem.ts +++ b/modules/core/src/lib/filesystems/browser-filesystem.ts @@ -1,4 +1,5 @@ import type {FileSystem, ReadableFile} from '@loaders.gl/loader-utils'; +import {BlobFile} from '@loaders.gl/loader-utils'; type BrowserFileSystemOptions = { fetch?: typeof fetch; @@ -110,17 +111,29 @@ export class BrowserFileSystem implements FileSystem { // RANDOM ACCESS async openReadableFile(pathname: string, flags: unknown): Promise { - return this.files[pathname]; + return new BlobFile(this.files[pathname]); } - /** + // PRIVATE + + // Supports case independent paths, and file usage tracking + _getFile(path: string, used: boolean): File { + // Prefer case match, but fall back to case independent. + const file = this.files[path] || this.lowerCaseFiles[path]; + if (file && used) { + this.usedFiles[path] = true; + } + return file; + } +} +/* * Read a range into a buffer * @todo - handle position memory * @param buffer is the buffer that the data (read from the fd) will be written to. * @param offset is the offset in the buffer to start writing at. * @param length is an integer specifying the number of bytes to read. * @param position is an argument specifying where to begin reading from in the file. If position is null, data will be read from the current file position, and the file position will be updated. If position is an integer, the file position will remain unchanged. - */ + * async read( fd: any, buffer: ArrayBuffer, @@ -140,16 +153,4 @@ export class BrowserFileSystem implements FileSystem { } // fstat(fd: number): Promise; // Stat - - // PRIVATE - - // Supports case independent paths, and file usage tracking - _getFile(path: string, used: boolean): File { - // Prefer case match, but fall back to case independent. - const file = this.files[path] || this.lowerCaseFiles[path]; - if (file && used) { - this.usedFiles[path] = true; - } - return file; - } -} + */ diff --git a/modules/loader-utils/src/index.ts b/modules/loader-utils/src/index.ts index 1a2bad87bf..3c7fa6f1a5 100644 --- a/modules/loader-utils/src/index.ts +++ b/modules/loader-utils/src/index.ts @@ -120,9 +120,6 @@ export {stream}; // EXPERIMENTAL: FILE SYSTEMS -export type {FileSystem, RandomAccessFileSystem} from './lib/filesystems/filesystem'; -export {NodeFileSystem as _NodeFileSystem} from './lib/filesystems/node-filesystem'; - export type {FileProvider} from './lib/file-provider/file-provider'; export {isFileProvider} from './lib/file-provider/file-provider'; @@ -130,11 +127,13 @@ export {FileHandle} from './lib/file-provider/file-handle'; export {FileHandleFile} from './lib/file-provider/file-handle-file'; export {DataViewFile} from './lib/file-provider/data-view-file'; -export type {ReadableFile} from './lib/filesystems/filesystem'; -// export {makeReadableFile} from './lib/filesystems/make-readable-file'; +export type {ReadableFile, WritableFile, Stat} from './lib/files/file'; +export {BlobFile} from './lib/files/blob-file'; +export {HttpFile} from './lib/files/http-file'; +export {NodeFileFacade as NodeFile} from './lib/files/node-file-facade'; -export type {WritableFile} from './lib/filesystems/filesystem'; -// export {makeWritableFile} from './lib/filesystems/make-writable-file'; +export type {FileSystem, RandomAccessFileSystem} from './lib/filesystems/filesystem'; +export {NodeFileSystemFacade as NodeFilesystem} from './lib/filesystems/node-filesystem-facade'; // EXPERIMENTAL: DATA SOURCES export type {Service} from './service-types'; diff --git a/modules/loader-utils/src/lib/files/blob-file.ts b/modules/loader-utils/src/lib/files/blob-file.ts new file mode 100644 index 0000000000..5df548cee6 --- /dev/null +++ b/modules/loader-utils/src/lib/files/blob-file.ts @@ -0,0 +1,29 @@ +// loaders.gl, MIT license + +import {ReadableFile} from './file'; + +export class BlobFile implements ReadableFile { + readonly handle: Blob; + readonly size: number; + readonly url: string; + + constructor(blob: Blob | File) { + this.handle = blob; + this.size = blob.size; + this.url = (blob as File).name || ''; + } + + async close() {} + + async stat() { + return { + size: this.handle.size, + isDirectory: false + }; + } + + async read(start: number, length: number): Promise { + const arrayBuffer = await this.handle.slice(start, start + length).arrayBuffer(); + return arrayBuffer; + } +} diff --git a/modules/loader-utils/src/lib/files/file.ts b/modules/loader-utils/src/lib/files/file.ts new file mode 100644 index 0000000000..7c0e31ed41 --- /dev/null +++ b/modules/loader-utils/src/lib/files/file.ts @@ -0,0 +1,29 @@ +export type Stat = { + size: number; + isDirectory: boolean; +}; + +export interface ReadableFile { + /** The underlying file handle (Blob, Node.js file descriptor etc) */ + handle: unknown; + /** Length of file in bytes, if available */ + size: number; + /** Read data */ + read(start?: number, end?: number): Promise; + /** Read data */ + fetchRange?(offset: number, length: number, signal?: AbortSignal): Promise; + /** Get information about file */ + stat?(): Promise; + /** Close the file */ + close(): Promise; +} + +export interface WritableFile { + handle: unknown; + /** Write to file. The number of bytes written will be returned */ + write: (arrayBuffer: ArrayBuffer, offset?: number, length?: number) => Promise; + /** Get information about the file */ + stat?(): Promise; + /** Close the file */ + close(): Promise; +} diff --git a/modules/loader-utils/src/lib/files/http-file.ts b/modules/loader-utils/src/lib/files/http-file.ts new file mode 100644 index 0000000000..28624ff21d --- /dev/null +++ b/modules/loader-utils/src/lib/files/http-file.ts @@ -0,0 +1,108 @@ +// loaders.gl, MIT license + +import {ReadableFile, Stat} from './file'; + +export class HttpFile implements ReadableFile { + readonly handle: string; + readonly size: number = 0; + readonly url: string; + + constructor(url: string) { + this.handle = url; + this.url = url; + } + + async close(): Promise {} + + async stat(): Promise { + const response = await fetch(this.handle, {method: 'HEAD'}); + if (!response.ok) { + throw new Error(`Failed to fetch HEAD ${this.handle}`); + } + return { + size: parseInt(response.headers.get('Content-Length') || '0'), + isDirectory: false + }; + } + + async read(offset: number, length: number): Promise { + const response = await this.fetchRange(offset, length); + const arrayBuffer = await response.arrayBuffer(); + return arrayBuffer; + } + + /** + * + * @param offset + * @param length + * @param signal + * @returns + * @see https://github.com/protomaps/PMTiles + */ + async fetchRange(offset: number, length: number, signal?: AbortSignal): Promise { + let controller: AbortController | undefined; + if (!signal) { + // ToDO why is it so important to abort in case 200? + // TODO check this works or assert 206 + controller = new AbortController(); + signal = controller.signal; + } + + const url = this.handle; + let response = await fetch(url, { + signal, + headers: {Range: `bytes=${offset}-${offset + length - 1}`} + }); + + switch (response.status) { + case 206: // Partial Content success + // This is the expected success code for a range request + break; + + case 200: + // some well-behaved backends, e.g. DigitalOcean CDN, respond with 200 instead of 206 + // but we also need to detect no support for Byte Serving which is returning the whole file + const content_length = response.headers.get('Content-Length'); + if (!content_length || Number(content_length) > length) { + if (controller) { + controller.abort(); + } + throw Error( + 'content-length header missing or exceeding request. Server must support HTTP Byte Serving.' + ); + } + + case 416: // "Range Not Satisfiable" + // some HTTP servers don't accept ranges beyond the end of the resource. + // Retry with the exact length + // TODO: can return 416 with offset > 0 if content changed, which will have a blank etag. + // See https://github.com/protomaps/PMTiles/issues/90 + if (offset === 0) { + const content_range = response.headers.get('Content-Range'); + if (!content_range || !content_range.startsWith('bytes *')) { + throw Error('Missing content-length on 416 response'); + } + const actual_length = Number(content_range.substr(8)); + response = await fetch(this.url, { + signal, + headers: {Range: `bytes=0-${actual_length - 1}`} + }); + } + break; + + default: + if (response.status >= 300) { + throw Error(`Bad response code: ${response.status}`); + } + } + + return response; + // const data = await response.arrayBuffer(); + // return { + // data, + // etag: response.headers.get('ETag') || undefined, + // cacheControl: response.headers.get('Cache-Control') || undefined, + // expires: response.headers.get('Expires') || undefined + // }; + } +} diff --git a/modules/loader-utils/src/lib/files/node-file-facade.ts b/modules/loader-utils/src/lib/files/node-file-facade.ts new file mode 100644 index 0000000000..c80959b9a7 --- /dev/null +++ b/modules/loader-utils/src/lib/files/node-file-facade.ts @@ -0,0 +1,39 @@ +// loaders.gl, MIT license + +import {isBrowser} from '../env-utils/globals'; +import {ReadableFile, WritableFile, Stat} from './file'; + +const NOT_IMPLEMENTED = new Error('Not implemented'); + +/** This class is a facade that gets replaced with an actual NodeFile instance */ +export class NodeFileFacade implements ReadableFile, WritableFile { + /** The underlying file handle (Blob, Node.js file descriptor etc) */ + handle: unknown; + /** Length of file in bytes, if available */ + size: number = 0; + + constructor(options) { + // Return the actual implementation instance + if (globalThis.loaders?.NodeFile) { + return new globalThis.loaders.NodeFile(options); + } + if (isBrowser) { + throw new Error("Can't instantiate NodeFile in browser."); + } + throw new Error("Can't instantiate NodeFile. Make sure to import @loaders.gl/polyfills first."); + } + /** Read data */ + async read(start?: number, end?: number): Promise { + throw NOT_IMPLEMENTED; + } + /** Write to file. The number of bytes written will be returned */ + async write(arrayBuffer: ArrayBuffer, offset?: number, length?: number): Promise { + throw NOT_IMPLEMENTED; + } + /** Get information about file */ + async stat(): Promise { + throw NOT_IMPLEMENTED; + } + /** Close the file */ + async close(): Promise {} +} diff --git a/modules/pmtiles/src/lib/sources.ts b/modules/loader-utils/src/lib/files/sources.ts similarity index 100% rename from modules/pmtiles/src/lib/sources.ts rename to modules/loader-utils/src/lib/files/sources.ts diff --git a/modules/loader-utils/src/lib/filesystems/filesystem.ts b/modules/loader-utils/src/lib/filesystems/filesystem.ts index dd30e2ebf4..fe1f8837fb 100644 --- a/modules/loader-utils/src/lib/filesystems/filesystem.ts +++ b/modules/loader-utils/src/lib/filesystems/filesystem.ts @@ -1,122 +1,38 @@ // loaders.gl, MIT license -export type Stat = { - size: number; - isDirectory: boolean; -}; - -export interface ReadableFile { - /** The underlying file handle (Blob, Node.js file descriptor etc) */ - handle: unknown; - /** Length of file in bytes, if available */ - size?: number; - /** Read data */ - read(length: number, position?: number): Promise<{bytesRead: number; arrayBuffer: ArrayBuffer}>; - read( - target?: ArrayBuffer, - offset?: number, - length?: number, - position?: number - ): Promise<{bytesRead: number; arrayBuffer: ArrayBuffer}>; - /** Get information about file */ - stat?(): Promise; - /** Close the file */ - close(): Promise; -} - -export interface WritableFile { - handle: unknown; - /** Write to file. The number of bytes written will be returned */ - write: (arrayBuffer: ArrayBuffer, offset?: number, length?: number) => Promise; - /** Get information about the file */ - stat?(): Promise; - /** Close the file */ - close(): Promise; -} +import {ReadableFile, WritableFile} from '../files/file'; /** * A FileSystem interface can encapsulate various file sources, - * a FileList, a ZipFile, a GoogleDrive etc. + * a FileList, a Node.js filesystem, a ZipFile, a GoogleDrive etc. */ export interface FileSystem { - /** - * Return a list of file names - * @param dirname directory name. file system root directory if omitted - */ + /** Return a list of file names in a "directory" */ readdir(dirname?: string, options?: {recursive?: boolean}): Promise; - /** - * Gets information from a local file from the filesystem - * @param filename file name to stat - * @param options currently unused - * @throws if filename is not in local filesystem - */ + /** Gets information from a local file from the filesystem */ stat(filename: string, options?: object): Promise<{size: number}>; - unlink(path: string): Promise; + /** Removes a file from the file system */ + unlink?(path: string): Promise; - /** - * Fetches a local file from the filesystem (or a URL) - * @param filename - * @param options - */ - fetch(filename: RequestInfo, options?: RequestInit): Promise; + /** Fetches the full contents of a file from the filesystem (or a URL) */ + fetch(path: string, options?: RequestInit): Promise; } /** - * A random access file system, read-write + * A random access file system, open readable and/or writable files */ export interface RandomAccessFileSystem extends FileSystem { + /** Can open readable files */ readonly readable: boolean; + + /** Can open writable files */ readonly writable: boolean; + /** Open a readable file */ openReadableFile(path: string, flags?: 'a'): Promise; - openWritableFile(path: string, flags?: 'w' | 'wx' | 'w+' | 'wx+', mode?: number): Promise; - - // Node.js style API - - // // implements IRandomAccessFileSystem - // open(path: string, flags: 'r', mode?: any): Promise; - // open( - // path: string, - // flags: 'w' | 'wx' | 'w+' | 'x+', - // mode?: number - // ): Promise; - - // open( - // path: string, - // flags: 'r' | 'w' | 'wx' | 'w+' | 'x+', - // mode?: number - // ): Promise { - // return flags === 'r' - // ? this.openReadableFile(path, flags) - // : this.openWritableFile(path, flags, mode); - // } - - // async close(file: ReadableFile | WritableFile): Promise { - // return await file.close(); - // } - - // async fstat(file: ReadableFile | WritableFile): Promise { - // return (await file.stat?.()) || null; - // } - - // async read( - // file: ReadableFile, - // target?: ArrayBuffer, - // offset = 0, - // length = target?.byteLength, - // position = 0 - // ): Promise<{bytesRead: number; buffer: Uint8Array}> { - // return await file.read(target, offset, length, position); - // } - // async write( - // file: WritableFile, - // arrayBuffer: ArrayBuffer, - // offset = 0, - // length = arrayBuffer.byteLength - // ) { - // return await file.write(arrayBuffer, offset, length); - // } + /** Open a writable file */ + openWritableFile(path: string, flags?: 'w' | 'wx', mode?: number): Promise; } diff --git a/modules/loader-utils/src/lib/filesystems/make-readable-file.ts b/modules/loader-utils/src/lib/filesystems/make-readable-file.ts deleted file mode 100644 index 16589bdbbc..0000000000 --- a/modules/loader-utils/src/lib/filesystems/make-readable-file.ts +++ /dev/null @@ -1,25 +0,0 @@ -// loaders.gl, MIT license - - -/** Helper function to create an envelope reader for a binary memory input * -export function makeReadableFile(data: Blob | ArrayBuffer): ReadableFile { - if (data instanceof ArrayBuffer) { - const arrayBuffer: ArrayBuffer = data; - return { - read: async (start: number, length: number) => Buffer.from(data, start, length), - close: async () => {}, - size: arrayBuffer.byteLength - }; - } - - const blob: Blob = data; - return { - read: async (start: number, length: number) => { - const arrayBuffer = await blob.slice(start, start + length).arrayBuffer(); - return Buffer.from(arrayBuffer); - }, - close: async () => {}, - size: blob.size - }; -} -*/ diff --git a/modules/loader-utils/src/lib/filesystems/make-writable-file.ts b/modules/loader-utils/src/lib/filesystems/make-writable-file.ts deleted file mode 100644 index 52c11abeb8..0000000000 --- a/modules/loader-utils/src/lib/filesystems/make-writable-file.ts +++ /dev/null @@ -1,5 +0,0 @@ -// loaders.gl, MIT license -// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.) - -// import {isBrowser} from '../env-utils/globals'; -// import * as fs from '../node/fs'; \ No newline at end of file diff --git a/modules/loader-utils/src/lib/filesystems/node-filesystem.ts b/modules/loader-utils/src/lib/filesystems/node-filesystem-facade.ts similarity index 75% rename from modules/loader-utils/src/lib/filesystems/node-filesystem.ts rename to modules/loader-utils/src/lib/filesystems/node-filesystem-facade.ts index 47523b6995..4989fec79f 100644 --- a/modules/loader-utils/src/lib/filesystems/node-filesystem.ts +++ b/modules/loader-utils/src/lib/filesystems/node-filesystem-facade.ts @@ -1,7 +1,9 @@ // loaders.gl, MIT license -import {RandomAccessFileSystem, Stat, ReadableFile, WritableFile} from './filesystem'; import {isBrowser} from '../env-utils/globals'; +import {Stat} from '../files/file'; +import {NodeFileFacade as NodeFile} from '../files/node-file-facade'; +import {RandomAccessFileSystem} from './filesystem'; const NOT_IMPLEMENTED = new Error('Not implemented'); @@ -11,7 +13,7 @@ const NOT_IMPLEMENTED = new Error('Not implemented'); * @note Dummy implementation, not used (constructor returns a real NodeFileSystem instance) * @param options */ -export class NodeFileSystem implements RandomAccessFileSystem { +export class NodeFileSystemFacade implements RandomAccessFileSystem { // implements FileSystem constructor(options: {[key: string]: any}) { if (globalThis.loaders?.NodeFileSystem) { @@ -32,12 +34,12 @@ export class NodeFileSystem implements RandomAccessFileSystem { readonly readable = true; readonly writable = true; - async openReadableFile(path: string): Promise { + async openReadableFile(path: string, flags): Promise { throw NOT_IMPLEMENTED; } // implements RandomAccessWriteFileSystem - async openWritableFile(path: string, flags, mode): Promise { + async openWritableFile(path: string, flags, mode): Promise { throw NOT_IMPLEMENTED; } @@ -51,6 +53,10 @@ export class NodeFileSystem implements RandomAccessFileSystem { throw NOT_IMPLEMENTED; } + async unlink(path: string): Promise { + throw NOT_IMPLEMENTED; + } + async fetch(path: RequestInfo, options?: RequestInit): Promise { throw NOT_IMPLEMENTED; } diff --git a/modules/pmtiles/src/pmtiles-source.ts b/modules/pmtiles/src/pmtiles-source.ts index 53add8d641..a172145dd4 100644 --- a/modules/pmtiles/src/pmtiles-source.ts +++ b/modules/pmtiles/src/pmtiles-source.ts @@ -12,37 +12,37 @@ import type {PMTilesMetadata} from './lib/parse-pmtiles'; import {parsePMTilesHeader} from './lib/parse-pmtiles'; import {TileLoadParameters} from 'modules/loader-utils/src/lib/sources/tile-source'; -// export const PMTilesService: Service = { -// name: 'PMTiles', -// id: 'pmtiles', -// module: 'pmtiles', -// // version: VERSION, -// extensions: ['pmtiles'], -// mimeTypes: ['application/octet-stream'], -// options: { -// pmtiles: {} -// } -// }; +const VERSION = '1.0.0'; + +export type Service = { + name: string; + id: string; + module: string; + version: string; + extensions: string[]; + mimeTypes: string[]; + options: Record; +} -/** - * WIP - Loader for pmtiles metadata - * @note loads metadata only. To load individual tiles, use PMTilesSource -export const PMTilesLoader: LoaderWithParser = { +export type ServiceWithSource = Service & { + _source?: SourceT; + _sourceProps?: SourcePropsT; + createSource: (props: SourcePropsT) => SourceT; +} + + +export const PMTilesService: ServiceWithSource = { name: 'PMTiles', id: 'pmtiles', module: 'pmtiles', version: VERSION, - worker: true, extensions: ['pmtiles'], mimeTypes: ['application/octet-stream'], options: { pmtiles: {} }, - parse: async (arrayBuffer, options) => { - throw new Error('not implemented'); - } + createSource: (props: PMTilesSourceProps) => new PMTilesSource(props) }; - */ export type PMTilesSourceProps = DataSourceProps & { url: string | Blob; diff --git a/modules/polyfills/package.json b/modules/polyfills/package.json index 558a8ae2ab..03f53f2b8a 100644 --- a/modules/polyfills/package.json +++ b/modules/polyfills/package.json @@ -36,6 +36,7 @@ "build-bundle": "esbuild src/bundle.ts --bundle --outfile=dist/dist.min.js" }, "dependencies": { + "@loaders.gl/loader-utils": "4.0.0-beta.1", "@babel/runtime": "^7.3.1", "buffer": "^6.0.3", "get-pixels": "^3.3.2", diff --git a/modules/core/src/lib/fetch/fetch-file.node.ts b/modules/polyfills/src/filesystems/fetch-node.ts similarity index 93% rename from modules/core/src/lib/fetch/fetch-file.node.ts rename to modules/polyfills/src/filesystems/fetch-node.ts index 8be94ee224..d783352330 100644 --- a/modules/core/src/lib/fetch/fetch-file.node.ts +++ b/modules/polyfills/src/filesystems/fetch-node.ts @@ -1,6 +1,6 @@ // loaders.gl, MIT license -import {fs} from '@loaders.gl/loader-utils'; +import fs from 'fs'; /** * Enables @@ -8,7 +8,7 @@ import {fs} from '@loaders.gl/loader-utils'; * @param options * @returns */ -export async function fetchFileNode(url: string, options): Promise { +export async function fetchNode(url: string, options): Promise { // Support `file://` protocol const FILE_PROTOCOL_REGEX = /^file:\/\//; url.replace(FILE_PROTOCOL_REGEX, '/'); diff --git a/modules/polyfills/src/filesystems/node-file.ts b/modules/polyfills/src/filesystems/node-file.ts new file mode 100644 index 0000000000..98663609e3 --- /dev/null +++ b/modules/polyfills/src/filesystems/node-file.ts @@ -0,0 +1,116 @@ +import {ReadableFile, WritableFile, Stat} from '@loaders.gl/loader-utils'; +import fs from 'fs'; + +export class NodeFile implements ReadableFile, WritableFile { + handle: number; + size: number; + + constructor(path, flags: 'a' | 'w' | 'wx', mode?: number) { + this.handle = fs.openSync(path, flags, mode); + this.size = fs.fstatSync(this.handle).size; + } + + async close(): Promise { + return new Promise((resolve, reject) => { + fs.close(this.handle, (err) => (err ? reject(err) : resolve())); + }); + } + + async stat(): Promise { + return await new Promise((resolve, reject) => + fs.fstat(this.handle, (err, info) => (err ? reject(err) : resolve({size: info.size, isDirectory: info.isDirectory()}))), + ); + } + + async read(offset: number, length: number): Promise { + const arrayBuffer = new ArrayBuffer(length); + + let totalBytesRead = 0; + const uint8Array = new Uint8Array(arrayBuffer); + + let position + // Read in loop until we get required number of bytes + while (length > 0) { + const bytesRead = await readBytes(this.handle, uint8Array, 0, length, offset); + + // Check if end of file reached + if (bytesRead === 0) { + break; + } + + totalBytesRead += bytesRead; + offset + bytesRead; + length -= bytesRead; + + // Advance position unless we are using built-in position advancement + if (position !== undefined) { + position += bytesRead; + } + } + return totalBytesRead < length ? arrayBuffer.slice(0, totalBytesRead) : arrayBuffer; + } + + async write( + arrayBuffer: ArrayBuffer, + offset: number = 0, + length: number = arrayBuffer.byteLength + ): Promise { + return new Promise((resolve, reject) => { + const uint8Array = new Uint8Array(arrayBuffer, offset, length); + fs.write(this.handle, uint8Array, (err, bytesWritten) => + err ? reject(err) : resolve(bytesWritten) + ); + }); + } +} + +async function readBytes( + fd: number, + uint8Array: Uint8Array, + offset: number, + length: number, + position: number | null +): Promise { + return await new Promise((resolve, reject) => + fs.read(fd, uint8Array, offset, length, position, (err, bytesRead) => + err ? reject(err) : resolve(bytesRead) + ) + ); +} + +// TODO - implement streaming write +/* +export interface WriteStreamOptions { + flags?: string; + encoding?: 'utf8'; + fd?: number; + mode?: number; + autoClose?: boolean; + start?: number; +} + +export class NodeStreamWritableFile implements WritableFile { + outputStream: fs.WriteStream | Writable; + + constructor(pathOrStream: string | Writable, options?: WriteStreamOptions) { + this.outputStream = + typeof pathOrStream === 'string' ? fs.createWriteStream(pathOrStream, options) : pathOrStream; + } + + async write(buffer: ArrayBuffer): Promise { + return new Promise((resolve, reject) => { + const uint8Array = new Uint8Array(buffer); + this.outputStream.write(uint8Array, (err) => (err ? reject(err) : resolve())); + }); + } + + async close(): Promise { + if (this.outputStream instanceof fs.WriteStream) { + return new Promise((resolve, reject) => { + const stream = this.outputStream as fs.WriteStream; + stream.close((err) => (err ? reject(err) : resolve())); + }); + } + } +} +*/ diff --git a/modules/polyfills/src/filesystems/node-filesystem.ts b/modules/polyfills/src/filesystems/node-filesystem.ts index 726ce78eee..cdb9df61e4 100644 --- a/modules/polyfills/src/filesystems/node-filesystem.ts +++ b/modules/polyfills/src/filesystems/node-filesystem.ts @@ -1,170 +1,24 @@ -import { - FileSystem, - RandomAccessReadFileSystem, - ReadableFile, - WritableFile -} from '@loaders.gl/loader-utils'; -import fs from 'fs'; +// loaders.gl, MIT license + +import {Stat, RandomAccessFileSystem} from '@loaders.gl/loader-utils'; import fsPromise from 'fs/promises'; -import type {Writable} from 'stream'; +import {NodeFile} from './node-file'; +import {fetchNode} from './fetch-node'; // import {fetchFile} from "../fetch/fetch-file" // import {selectLoader} from "../api/select-loader"; -type Stat = { - size: number; - isDirectory: () => boolean; - info?: fs.Stats; -}; - -type ReadOptions = { - arraybuffer?: Buffer; - offset?: number; - length?: number; - position?: number; -}; - -export interface WriteStreamOptions { - flags?: string; - encoding?: 'utf8'; - fd?: number; - mode?: number; - autoClose?: boolean; - start?: number; -} - - - -export class ArrayBufferFile implements ReadableFile, WritableFile { - readonly handle: ArrayBuffer; - readonly size: number; - - constructor (arrayBuffer: ArrayBuffer): ReadableFile { - this.handle = arrayBuffer; - this.size = arrayBuffer.byteLength - } - - async close() {} - - async read(start: number, length: number) { - return Buffer.from(this.data, start, length) - } -} - -export class BlobFile implements ReadableFile, WritableFile { - readonly handle: Blob; - readonly size: number; - - constructor (blob: Blob): ReadableFile { - this.handle = blob; - this.size = blob.size; - } - - async close() {} - - async read(start: number, length: number) { - const arrayBuffer = await blob.slice(start, start + length).arrayBuffer(); - return Buffer.from(arrayBuffer); - } -} - -export class NodeFile implements ReadableFile, WritableFile { - handle: number; - - constructor(path, flags: 'w' | 'wx' | 'w+' | 'wx+', mode: number) { - this.handle = fs.openSync(path, flags, mode); - } - - async close(): Promise { - return new Promise((resolve, reject) => { - fs.close(this.handle, (err) => (err ? reject(err) : resolve())); - }); - } - - async read( - fd: number, - arrayBuffer: ArrayBuffer, - offset = 0, - length = arrayBuffer.byteLength, - position?: number - ): Promise { - - let totalBytesRead = 0; - const uint8Array = new Uint8Array(arrayBuffer); - - // Read in loop until we get required number of bytes - while (length > 0) { - const bytesRead = await readBytes(fd, uint8Array, offset, length, position ?? null); - - // Check if end of file reached - if (bytesRead === 0) { - return totalBytesRead; - } - - totalBytesRead += bytesRead; - offset + bytesRead; - length -= bytesRead; - - // Advance position unless we are using built-in position advancement - if (position !== undefined) { - position += bytesRead; - } - } - return totalBytesRead; - } - - async write( - arrayBuffer: ArrayBuffer, - offset: number = 0, - length: number = arrayBuffer.byteLength - ): Promise { - return new Promise((resolve, reject) => { - const uint8Array = new Uint8Array(arrayBuffer, offset, length); - fs.write(this.handle, uint8Array, (err, bytesWritten) => - err ? reject(err) : resolve(bytesWritten) - ); - }); - } -} - -/** - * -export class NodeStreamWritableFile implements WritableFile { - outputStream: fs.WriteStream | Writable; - - constructor(pathOrStream: string | Writable, options?: WriteStreamOptions) { - this.outputStream = - typeof pathOrStream === 'string' ? fs.createWriteStream(pathOrStream, options) : pathOrStream; - } - - async write(buffer: ArrayBuffer): Promise { - return new Promise((resolve, reject) => { - const uint8Array = new Uint8Array(buffer); - this.outputStream.write(uint8Array, (err) => (err ? reject(err) : resolve())); - }); - } - - async close(): Promise { - if (this.outputStream instanceof fs.WriteStream) { - return new Promise((resolve, reject) => { - const stream = this.outputStream as fs.WriteStream; - stream.close((err) => (err ? reject(err) : resolve())); - }); - } - } -} - */ - /** * FileSystem pass-through for Node.js * Compatible with BrowserFileSystem. * @param options */ -export class NodeFileSystem implements RandomAccessReadFileSystem { +export class NodeFileSystem implements RandomAccessFileSystem { + readable: boolean = true; + writable: boolean = true; + // implements FileSystem - constructor(options: {[key: string]: any}) { - this.fetch = options._fetch; - } + constructor() {} async readdir(dirname = '.', options?: {}): Promise { return await fsPromise.readdir(dirname, options); @@ -172,76 +26,27 @@ export class NodeFileSystem implements RandomAccessReadFileSystem { async stat(path: string, options?: {}): Promise { const info = await fsPromise.stat(path, options); - return {size: Number(info.size), isDirectory: () => false, info}; + return {size: Number(info.size), isDirectory: info.isDirectory()}; } - async fetch(path: string, options: {[key: string]: any}) { - // Falls back to handle https:/http:/data: etc fetches - // eslint-disable-next-line - const fallbackFetch = options.fetch || this.fetch; - return fallbackFetch(path, options); + async unlink(path: string): Promise { + return await fsPromise.unlink(path); } - // implements IRandomAccessFileSystem - async open(path: string, flags: string | number, mode?: any): Promise { - return (await fsPromise.open(path, flags)) as unknown as number; + async fetch(path: string, options: RequestInit): Promise { + return await fetchNode(path, options); } - /* - async close(fd: number): Promise { - fs.close(fd); - } - - async fstat(fd: number): Promise { - return await new Promise((resolve, reject) => - fs.fstat(fd, (err, info) => (err ? reject(err) : resolve(info))) - ); + // implements IRandomAccessFileSystem + async openReadableFile(path: string, flags: 'a' = 'a'): Promise { + return new NodeFile(path, flags); } - async read( - fd: number, - arrayBuffer: ArrayBuffer, - offset = 0, - length = arrayBuffer.byteLength, - position?: number - ): Promise { - - let totalBytesRead = 0; - const uint8Array = new Uint8Array(arrayBuffer); - - // Read in loop until we get required number of bytes - while (length > 0) { - const bytesRead = await readBytes(fd, uint8Array, offset, length, position ?? null); - - // Check if end of file reached - if (bytesRead === 0) { - return totalBytesRead; - } - - totalBytesRead += bytesRead; - offset + bytesRead; - length -= bytesRead; - - // Advance position unless we are using built-in position advancement - if (position !== undefined) { - position += bytesRead; - } - } - return totalBytesRead; + async openWritableFile( + path: string, + flags: 'w' | 'wx' = 'w', + mode?: any + ): Promise { + return new NodeFile(path, flags, mode); } - */ -} - -async function readBytes( - fd: number, - uint8Array: Uint8Array, - offset: number, - length: number, - position: number | null -): Promise { - return await new Promise((resolve, reject) => - fs.read(fd, uint8Array, offset, length, position, (err, bytesRead) => - err ? reject(err) : resolve(bytesRead) - ) - ); } diff --git a/modules/polyfills/src/index.ts b/modules/polyfills/src/index.ts index db71f3ad2e..c5cfa393f3 100644 --- a/modules/polyfills/src/index.ts +++ b/modules/polyfills/src/index.ts @@ -2,7 +2,6 @@ import {isBrowser} from './utils/globals'; import {TextDecoder, TextEncoder} from './lib/encoding'; -import {allSettled} from './promise/all-settled'; // Node specific import * as base64 from './node/buffer/btoa.node'; @@ -10,6 +9,14 @@ import * as base64 from './node/buffer/btoa.node'; import {encodeImageNode} from './node/images/encode-image.node'; import {parseImageNode, NODE_FORMAT_SUPPORT} from './node/images/parse-image.node'; +// FILESYSTEM POLYFILLS +export {NodeFile} from './filesystems/node-file'; +export {NodeFileSystem} from './filesystems/node-filesystem'; +export {fetchNode} from './filesystems/fetch-node'; +import {NodeFile} from './filesystems/node-file'; +import {NodeFileSystem} from './filesystems/node-filesystem'; +import {fetchNode} from './filesystems/fetch-node'; + // export {ReadableStreamPolyfill} from './node/file/readable-stream'; // export {BlobPolyfill} from './node/file/blob'; export {FileReaderPolyfill} from './node/file/file-reader'; @@ -23,6 +30,12 @@ if (isBrowser) { ); } +// FILESYSTEM POLYFILLS +globalThis.loaders = globalThis.loaders || {}; +globalThis.loaders.NodeFile = NodeFile; +globalThis.loaders.NodeFileSystem = NodeFileSystem; +globalThis.loaders.fetchNode = fetchNode; + // POLYFILLS: TextEncoder, TextDecoder // - Recent Node versions have these classes but virtually no encodings unless special build. // - Browser: Edge, IE11 do not have these @@ -48,12 +61,6 @@ if (!isBrowser && !('btoa' in globalThis) && base64.btoa) { globalThis['btoa'] = base64.btoa; } -globalThis.loaders = globalThis.loaders || {}; - -// FILESYSTEM POLYFILLS: -export {NodeFileSystem} from './filesystems/node-filesystem'; -import {NodeFileSystem} from './filesystems/node-filesystem'; -globalThis.loaders.NodeFileSystem = NodeFileSystem; // NODE IMAGE FUNCTIONS: // These are not official polyfills but used by the @loaders.gl/images module if installed @@ -68,11 +75,6 @@ if (!isBrowser && !('_parseImageNode' in globalThis) && parseImageNode) { globalThis['_imageFormatsNode'] = NODE_FORMAT_SUPPORT; } -if (!('allSettled' in Promise)) { - // @ts-ignore - Promise.allSettled = allSettled; -} - // DEPRECATED POLYFILL: // - Node v18+: No, not needed // - Node v16 and lower: Yes @@ -80,7 +82,6 @@ if (!('allSettled' in Promise)) { // - IE11: No. This polyfill is node only, install external polyfill import {Headers as HeadersNode} from './node/fetch/headers.node'; import {Response as ResponseNode} from './node/fetch/response.node'; -import {fetchNode as fetchNode} from './node/fetch/fetch.node'; if (!isBrowser && !('Headers' in globalThis) && HeadersNode) { // @ts-expect-error