From ece5138422ac7aae5fc9c9641bb2fc7116738268 Mon Sep 17 00:00:00 2001 From: Ib Green Date: Mon, 2 Oct 2023 17:50:56 +0200 Subject: [PATCH] chore(polyfills): Start moving Node.js code into polyfills (#2669) --- .prettierignore | 1 + .../filesystems/node-filesystem.browser.ts | 61 ------------- .../src/lib/filesystems/node-filesystem.ts | 42 ++++----- modules/polyfills/package.json | 68 +------------- .../src/filesystems/node-filesystem.ts | 87 ++++++++++++++++++ modules/polyfills/src/index.browser.ts | 14 +++ modules/polyfills/src/index.ts | 88 +++++++++++-------- .../test/filesystems/node-filesystem.spec.ts | 14 +++ modules/polyfills/test/index.ts | 2 + 9 files changed, 189 insertions(+), 188 deletions(-) delete mode 100644 modules/loader-utils/src/lib/filesystems/node-filesystem.browser.ts create mode 100644 modules/polyfills/src/filesystems/node-filesystem.ts create mode 100644 modules/polyfills/src/index.browser.ts create mode 100644 modules/polyfills/test/filesystems/node-filesystem.spec.ts diff --git a/.prettierignore b/.prettierignore index f38eb8d775..ec9ac93cff 100644 --- a/.prettierignore +++ b/.prettierignore @@ -10,6 +10,7 @@ public modules/core/src/iterators/make-stream/make-node-stream.ts modules/loader-utils/src/lib/filesystems/node-filesystem.browser.ts +modules/loader-utils/src/lib/filesystems/node-filesystem.ts modules/3d-tiles/test/lib/classes/tile-3d-batch-table-hierarchy.spec.ts diff --git a/modules/loader-utils/src/lib/filesystems/node-filesystem.browser.ts b/modules/loader-utils/src/lib/filesystems/node-filesystem.browser.ts deleted file mode 100644 index bbee53815d..0000000000 --- a/modules/loader-utils/src/lib/filesystems/node-filesystem.browser.ts +++ /dev/null @@ -1,61 +0,0 @@ -import * as fs from '../node/fs'; -import {FileSystem, RandomAccessReadFileSystem} from './filesystem'; -// import {fetchFile} from "../fetch/fetch-file" -// import {selectLoader} from "../api/select-loader"; - -type Stat = { - size: number; - isDirectory: () => boolean; - info?: fs.Stats; -}; - -type ReadOptions = { - buffer?: Buffer; - offset?: number; - length?: number; - position?: number; -}; - -/** - * FileSystem pass-through for Node.js - * Compatible with BrowserFileSystem. - * @param options - */ -export class NodeFileSystem implements FileSystem, RandomAccessReadFileSystem { - // implements FileSystem - constructor(options: {[key: string]: any}) { - throw new Error('Can\'t instantiate NodeFileSystem in browser'); - } - - async readdir(dirname = '.', options?: {}): Promise { - return []; - } - - async stat(path: string, options?: {}): Promise { - return {size: 0, isDirectory: () => false}; - } - - async fetch(path: string, options: {[key: string]: any}) { - return globalThis.fetch(path, options); - } - - // implements IRandomAccessFileSystem - async open(path: string, flags: string | number, mode?: any): Promise { - return 0; - } - - // eslint-disable-next-line @typescript-eslint/no-empty-function - async close(fd: number): Promise {} - - async fstat(fd: number): Promise { - return {size: 0, isDirectory: () => false}; - } - - async read( - fd: number, - // @ts-ignore Possibly null - {buffer = null, offset = 0, length = buffer.byteLength, position = null}: ReadOptions - ): Promise<{bytesRead: number; buffer: Uint8Array}> { - return {bytesRead: 0, buffer: new Uint8Array(0)}; - } -} diff --git a/modules/loader-utils/src/lib/filesystems/node-filesystem.ts b/modules/loader-utils/src/lib/filesystems/node-filesystem.ts index 673a08bdb3..1ddae25083 100644 --- a/modules/loader-utils/src/lib/filesystems/node-filesystem.ts +++ b/modules/loader-utils/src/lib/filesystems/node-filesystem.ts @@ -24,37 +24,37 @@ type ReadOptions = { export class NodeFileSystem implements FileSystem, RandomAccessReadFileSystem { // implements FileSystem constructor(options: {[key: string]: any}) { - this.fetch = options._fetch; + if (globalThis.loaders?.NodeFileSystem) { + return new globalThis.loaders.NodeFileSystem(options); + } + throw new Error( + 'Can\'t instantiate NodeFileSystem in browser. Make sure to import @loaders.gl/polyfills first.' + ); } async readdir(dirname = '.', options?: {}): Promise { - return await fs.readdir(dirname, options); + return []; } async stat(path: string, options?: {}): Promise { - const info = await fs.stat(path, options); - return {size: Number(info.size), isDirectory: () => false, info}; + return {size: 0, isDirectory: () => false}; } async fetch(path: string, options: {[key: string]: any}) { - // Falls back to handle https:/http:/data: etc fetches - // eslint-disable-next-line - const fallbackFetch = options.fetch || this.fetch; - return fallbackFetch(path, options); + return globalThis.fetch(path, options); } // implements IRandomAccessFileSystem + async open(path: string, flags: string | number, mode?: any): Promise { - return await fs.open(path, flags); + return 0; } - async close(fd: number): Promise { - return await fs.close(fd); - } + // eslint-disable-next-line @typescript-eslint/no-empty-function + async close(fd: number): Promise {} async fstat(fd: number): Promise { - const info = await fs.fstat(fd); - return info; + return {size: 0, isDirectory: () => false}; } async read( @@ -62,18 +62,6 @@ export class NodeFileSystem implements FileSystem, RandomAccessReadFileSystem { // @ts-ignore Possibly null {buffer = null, offset = 0, length = buffer.byteLength, position = null}: ReadOptions ): Promise<{bytesRead: number; buffer: Uint8Array}> { - let totalBytesRead = 0; - // Read in loop until we get required number of bytes - while (totalBytesRead < length) { - const {bytesRead} = await fs.read( - fd, - buffer, - offset + totalBytesRead, - length - totalBytesRead, - position + totalBytesRead - ); - totalBytesRead += bytesRead; - } - return {bytesRead: totalBytesRead, buffer}; + return {bytesRead: 0, buffer: new Uint8Array(0)}; } } diff --git a/modules/polyfills/package.json b/modules/polyfills/package.json index de88db3dac..558a8ae2ab 100644 --- a/modules/polyfills/package.json +++ b/modules/polyfills/package.json @@ -26,70 +26,10 @@ "README.md" ], "browser": { - "./src/node/fetch/utils/decode-data-uri.node.js": false, - "./src/node/fetch/utils/decode-data-uri.node.ts": false, - "./dist/es5/node/fetch/utils/decode-data-uri.node.js": false, - "./dist/esm/node/fetch/utils/decode-data-uri.node.js": false, - "./src/node/fetch/utils/stream-utils.node.js": false, - "./src/node/fetch/utils/stream-utils.node.ts": false, - "./dist/es5/node/fetch/utils/stream-utils.node.js": false, - "./dist/esm/node/fetch/utils/stream-utils.node.js": false, - "./src/node/fetch/fetch.node.js": false, - "./src/node/fetch/fetch.node.ts": false, - "./dist/es5/node/fetch/fetch.node.js": false, - "./dist/esm/node/fetch/fetch.node.js": false, - "./src/node/fetch/headers.node.js": false, - "./src/node/fetch/headers.node.ts": false, - "./dist/es5/node/fetch/headers.node.js": false, - "./dist/esm/node/fetch/headers.node.js": false, - "./src/node/fetch/response.node.js": false, - "./src/node/fetch/response.node.ts": false, - "./dist/es5/node/fetch/response.node.js": false, - "./dist/esm/node/fetch/response.node.js": false, - "./src/node/images/encode-image.node.js": false, - "./src/node/images/encode-image.node.ts": false, - "./dist/es5/node/images/encode-image.node.js": false, - "./dist/esm/node/images/encode-image.node.js": false, - "./src/node/images/parse-image.node.js": false, - "./src/node/images/parse-image.node.ts": false, - "./dist/es5/node/images/parse-image.node.js": false, - "./dist/esm/node/images/parse-image.node.js": false, - "./src/node/buffer/to-array-buffer.node.js": false, - "./src/node/buffer/to-array-buffer.node.ts": false, - "./dist/es5/node/buffer/to-array-buffer.node.js": false, - "./dist/esm/node/buffer/to-array-buffer.node.js": false, - "./src/node/buffer/btoa.node.js": false, - "./src/node/buffer/btoa.node.ts": false, - "./dist/es5/node/buffer/btoa.node.js": false, - "./dist/esm/node/buffer/btoa.node.js": false, - "./src/node/file/blob.js": false, - "./src/node/file/blob.ts": false, - "./dist/es5/node/file/blob.js": false, - "./dist/esm/node/file/blob.js": false, - "./src/node/file/file.js": false, - "./src/node/file/file.ts": false, - "./dist/es5/node/file/file.js": false, - "./dist/esm/node/file/file.js": false, - "./src/node/file/readable-stream.js": false, - "./src/node/file/readable-stream.ts": false, - "./dist/es5/node/file/readable-stream.js": false, - "./dist/esm/node/file/readable-stream.js": false, - "./src/libs/encoding-indices.js": false, - "./src/libs/encoding-indices.ts": false, - "./dist/es5/libs/encoding-indices.js": false, - "./dist/esm/libs/encoding-indices.js": false, - "fs": false, - "http": false, - "https": false, - "stream": false, - "get-pixels": false, - "ndarray": false, - "save-pixels": false, - "stream-to-async-iterator": false, - "through": false, - "util": false, - "zlib": false, - "web-streams-polyfill": false + "./src/index.ts": "./src/index.browser.ts", + "./dist/index.js": "./dist/index.browser.js", + "./dist/es5/index.js": "./dist/es5/index.browser.js", + "./dist/esm/index.js": "./dist/es5/index.browser.js" }, "scripts": { "pre-build": "npm run build-bundle", diff --git a/modules/polyfills/src/filesystems/node-filesystem.ts b/modules/polyfills/src/filesystems/node-filesystem.ts new file mode 100644 index 0000000000..496c469cf2 --- /dev/null +++ b/modules/polyfills/src/filesystems/node-filesystem.ts @@ -0,0 +1,87 @@ +import {FileSystem, RandomAccessReadFileSystem} from '@loaders.gl/loader-utils'; +import fs from 'fs'; +import fsPromise from 'fs/promises'; + +// import {fetchFile} from "../fetch/fetch-file" +// import {selectLoader} from "../api/select-loader"; + +type Stat = { + size: number; + isDirectory: () => boolean; + info?: fs.Stats; +}; + +type ReadOptions = { + buffer?: Buffer; + offset?: number; + length?: number; + position?: number; +}; + +/** + * FileSystem pass-through for Node.js + * Compatible with BrowserFileSystem. + * @param options + */ +export class NodeFileSystem implements FileSystem, RandomAccessReadFileSystem { + // implements FileSystem + constructor(options: {[key: string]: any}) { + this.fetch = options._fetch; + } + + async readdir(dirname = '.', options?: {}): Promise { + return await fsPromise.readdir(dirname, options); + } + + async stat(path: string, options?: {}): Promise { + const info = await fsPromise.stat(path, options); + return {size: Number(info.size), isDirectory: () => false, info}; + } + + async fetch(path: string, options: {[key: string]: any}) { + // Falls back to handle https:/http:/data: etc fetches + // eslint-disable-next-line + const fallbackFetch = options.fetch || this.fetch; + return fallbackFetch(path, options); + } + + // implements IRandomAccessFileSystem + async open(path: string, flags: string | number, mode?: any): Promise { + return (await fsPromise.open(path, flags)) as unknown as number; + } + + async close(fd: number): Promise { + fs.close(fd); + } + + async fstat(fd: number): Promise { + return await new Promise((resolve, reject) => + fs.fstat(fd, (err, info) => (err ? reject(err) : resolve(info))) + ); + } + + async read( + fd: number, + // @ts-ignore Possibly null + {buffer = null, offset = 0, length = buffer.byteLength, position = null}: ReadOptions + ): Promise<{bytesRead: number; buffer: Uint8Array}> { + let totalBytesRead = 0; + // Read in loop until we get required number of bytes + while (totalBytesRead < length) { + const {bytesRead} = await new Promise<{bytesRead: number; buffer: Buffer}>( + // eslint-disable-next-line no-loop-func + (resolve, reject) => + fs.read( + fd, + buffer, + offset + totalBytesRead, + length - totalBytesRead, + position + totalBytesRead, + (err, bytesRead, buffer) => (err ? reject(err) : resolve({bytesRead, buffer})) + ) + ); + totalBytesRead += bytesRead; + } + return {bytesRead: totalBytesRead, buffer}; + } +} diff --git a/modules/polyfills/src/index.browser.ts b/modules/polyfills/src/index.browser.ts new file mode 100644 index 0000000000..ee02ee0930 --- /dev/null +++ b/modules/polyfills/src/index.browser.ts @@ -0,0 +1,14 @@ +// loaders.gl, MIT License + +import {allSettled} from './promise/all-settled'; + +if (!('allSettled' in Promise)) { + // @ts-ignore + Promise.allSettled = allSettled; +} + +// eslint-disable-next-line @typescript-eslint/no-empty-function +export function installFilePolyfills() {} + +// Dummy export to avoid import errors in browser tests +export const NodeFileSystem = null; diff --git a/modules/polyfills/src/index.ts b/modules/polyfills/src/index.ts index 7150e0660c..db71f3ad2e 100644 --- a/modules/polyfills/src/index.ts +++ b/modules/polyfills/src/index.ts @@ -1,5 +1,5 @@ /* eslint-disable dot-notation */ -import {isBrowser, global} from './utils/globals'; +import {isBrowser} from './utils/globals'; import {TextDecoder, TextEncoder} from './lib/encoding'; import {allSettled} from './promise/all-settled'; @@ -7,10 +7,6 @@ import {allSettled} from './promise/all-settled'; // Node specific import * as base64 from './node/buffer/btoa.node'; -import {Headers as HeadersNode} from './node/fetch/headers.node'; -import {Response as ResponseNode} from './node/fetch/response.node'; -import {fetchNode as fetchNode} from './node/fetch/fetch.node'; - import {encodeImageNode} from './node/images/encode-image.node'; import {parseImageNode, NODE_FORMAT_SUPPORT} from './node/images/parse-image.node'; @@ -20,63 +16,83 @@ export {FileReaderPolyfill} from './node/file/file-reader'; export {FilePolyfill} from './node/file/file'; export {installFilePolyfills} from './node/file/install-file-polyfills'; +if (isBrowser) { + // eslint-disable-next-line no-console + console.error( + 'loaders.gl: The @loaders.gl/polyfills should only be used in Node.js environments' + ); +} + // POLYFILLS: TextEncoder, TextDecoder // - Recent Node versions have these classes but virtually no encodings unless special build. // - Browser: Edge, IE11 do not have these -const installTextEncoder = !isBrowser || !('TextEncoder' in global); -if (installTextEncoder) { - global['TextEncoder'] = TextEncoder; +if (!globalThis.TextEncoder) { + // @ts-expect-error + globalThis.TextEncoder = TextEncoder; } -const installTextDecoder = !isBrowser || !('TextDecoder' in global); -if (installTextDecoder) { - global['TextDecoder'] = TextDecoder; +if (!globalThis.TextDecoder) { + // @ts-expect-error + globalThis.TextDecoder = TextDecoder; } // POLYFILLS: btoa, atob // - Node: Yes // - Browser: No -if (!isBrowser && !('atob' in global) && base64.atob) { - global['atob'] = base64.atob; +if (!isBrowser && !('atob' in globalThis) && base64.atob) { + globalThis['atob'] = base64.atob; } -if (!isBrowser && !('btoa' in global) && base64.btoa) { - global['btoa'] = base64.btoa; -} - -// DEPRECATED POLYFILL: -// - Node v18+: No, not needed -// - Node v16 and lower: Yes -// - Browsers (evergreen): Not needed. -// - IE11: No. This polyfill is node only, install external polyfill - -if (!isBrowser && !('Headers' in global) && HeadersNode) { - global['Headers'] = HeadersNode; +if (!isBrowser && !('btoa' in globalThis) && base64.btoa) { + globalThis['btoa'] = base64.btoa; } -if (!isBrowser && !('Response' in global) && ResponseNode) { - global['Response'] = ResponseNode; -} +globalThis.loaders = globalThis.loaders || {}; -if (!isBrowser && !('fetch' in global) && fetchNode) { - global['fetch'] = fetchNode; -} +// FILESYSTEM POLYFILLS: +export {NodeFileSystem} from './filesystems/node-filesystem'; +import {NodeFileSystem} from './filesystems/node-filesystem'; +globalThis.loaders.NodeFileSystem = NodeFileSystem; // NODE IMAGE FUNCTIONS: // These are not official polyfills but used by the @loaders.gl/images module if installed // TODO - is there an appropriate Image API we could polyfill using an adapter? -if (!isBrowser && !('_encodeImageNode' in global) && encodeImageNode) { - global['_encodeImageNode'] = encodeImageNode; +if (!isBrowser && !('_encodeImageNode' in globalThis) && encodeImageNode) { + globalThis['_encodeImageNode'] = encodeImageNode; } -if (!isBrowser && !('_parseImageNode' in global) && parseImageNode) { - global['_parseImageNode'] = parseImageNode; - global['_imageFormatsNode'] = NODE_FORMAT_SUPPORT; +if (!isBrowser && !('_parseImageNode' in globalThis) && parseImageNode) { + globalThis['_parseImageNode'] = parseImageNode; + globalThis['_imageFormatsNode'] = NODE_FORMAT_SUPPORT; } if (!('allSettled' in Promise)) { // @ts-ignore Promise.allSettled = allSettled; } + +// DEPRECATED POLYFILL: +// - Node v18+: No, not needed +// - Node v16 and lower: Yes +// - Browsers (evergreen): Not needed. +// - IE11: No. This polyfill is node only, install external polyfill +import {Headers as HeadersNode} from './node/fetch/headers.node'; +import {Response as ResponseNode} from './node/fetch/response.node'; +import {fetchNode as fetchNode} from './node/fetch/fetch.node'; + +if (!isBrowser && !('Headers' in globalThis) && HeadersNode) { + // @ts-expect-error + globalThis.Headers = HeadersNode; +} + +if (!isBrowser && !('Response' in globalThis) && ResponseNode) { + // @ts-expect-error + globalThis.Response = ResponseNode; +} + +if (!isBrowser && !('fetch' in globalThis) && fetchNode) { + // @ts-expect-error + globalThis.fetch = fetchNode; +} diff --git a/modules/polyfills/test/filesystems/node-filesystem.spec.ts b/modules/polyfills/test/filesystems/node-filesystem.spec.ts new file mode 100644 index 0000000000..514b8f7751 --- /dev/null +++ b/modules/polyfills/test/filesystems/node-filesystem.spec.ts @@ -0,0 +1,14 @@ +// loaders.gl, MIT license + +import test from 'tape-promise/tape'; +import {NodeFileSystem} from '@loaders.gl/polyfills'; + +test('NodeFileSystem#import', (t) => { + if (!NodeFileSystem) { + t.comment('NodeFileSystem not defined'); + t.end(); + return; + } + t.ok(NodeFileSystem, 'NodeFileSystem defined'); + t.end(); +}); diff --git a/modules/polyfills/test/index.ts b/modules/polyfills/test/index.ts index 8d176f66f5..c1d914093f 100644 --- a/modules/polyfills/test/index.ts +++ b/modules/polyfills/test/index.ts @@ -10,3 +10,5 @@ import './images-node/images-node.spec'; import './file/blob-polyfill.spec'; import './file/file-polyfill.spec'; import './promise/all-settled.spec'; + +import './filesystems/node-filesystem.spec';