From 9f6d110a4d693309ce84f7c68b8be55b8f5f0a3c Mon Sep 17 00:00:00 2001 From: Ib Green Date: Sun, 15 Oct 2023 07:43:16 -0400 Subject: [PATCH 1/2] chore(worker-utils): Move Node require utils to polyfills --- .../polyfills/src/images/encode-image.node.ts | 41 ------- .../polyfills/src/images/parse-image.node.ts | 53 ---------- .../test/load-library/require-utils.spec.ts | 6 +- .../wip/fetch-browser/fetch-file.browser.d.ts | 30 ------ modules/worker-utils/package.json | 6 -- .../worker-utils/src/lib/env-utils/globals.ts | 15 --- .../src/lib/library-utils/library-utils.ts | 24 ++--- .../src/lib/node/require-utils.node.ts | 100 ------------------ modules/worker-utils/test/index.ts | 6 +- .../lib/library-utils/require-utils.spec.ts | 93 ---------------- 10 files changed, 20 insertions(+), 354 deletions(-) delete mode 100644 modules/polyfills/src/images/encode-image.node.ts delete mode 100644 modules/polyfills/src/images/parse-image.node.ts delete mode 100644 modules/polyfills/wip/fetch-browser/fetch-file.browser.d.ts delete mode 100644 modules/worker-utils/src/lib/node/require-utils.node.ts delete mode 100644 modules/worker-utils/test/lib/library-utils/require-utils.spec.ts diff --git a/modules/polyfills/src/images/encode-image.node.ts b/modules/polyfills/src/images/encode-image.node.ts deleted file mode 100644 index 12e2512106..0000000000 --- a/modules/polyfills/src/images/encode-image.node.ts +++ /dev/null @@ -1,41 +0,0 @@ -// Use stackgl modules for DOM-less reading and writing of images - -import savePixels from 'save-pixels'; -import ndarray from 'ndarray'; -import {bufferToArrayBuffer} from '../buffer/to-array-buffer.node'; - -/** - * Returns data bytes representing a compressed image in PNG or JPG format, - * This data can be saved using file system (f) methods or - * used in a request. - * @param image to save - * @param options - * @param options.type='png' - png, jpg or image/png, image/jpg are valid - * @param options.dataURI - Whether to include a data URI header - * @return {*} bytes - */ -export function encodeImageToStreamNode( - image: {data: any; width: number; height: number}, - options: {type?: string; dataURI?: string} -) { - // Support MIME type strings - const type = options.type ? options.type.replace('image/', '') : 'jpeg'; - const pixels = ndarray(image.data, [image.width, image.height, 4], [4, image.width * 4, 1], 0); - - // Note: savePixels returns a stream - return savePixels(pixels, type, options); -} - -export function encodeImageNode(image, options) { - const imageStream = encodeImageToStreamNode(image, options); - - return new Promise((resolve) => { - const buffers: any[] = []; - imageStream.on('data', (buffer) => buffers.push(buffer)); - // TODO - convert to arraybuffer? - imageStream.on('end', () => { - const buffer = Buffer.concat(buffers); - resolve(bufferToArrayBuffer(buffer)); - }); - }); -} diff --git a/modules/polyfills/src/images/parse-image.node.ts b/modules/polyfills/src/images/parse-image.node.ts deleted file mode 100644 index 68959e6245..0000000000 --- a/modules/polyfills/src/images/parse-image.node.ts +++ /dev/null @@ -1,53 +0,0 @@ -// loaders.gl, MIT license - -import getPixels from 'get-pixels'; - -/** Declares which image format mime types this loader polyfill supports */ -export const NODE_FORMAT_SUPPORT = ['image/png', 'image/jpeg', 'image/gif']; - -// Note: These types are also defined in @loaders.gl/images and need to be kept in sync -type NDArray = { - shape: number[]; - data: Uint8Array; - width: number; - height: number; - components: number; - layers: number[]; -}; - -export async function parseImageNode(arrayBuffer: ArrayBuffer, mimeType: string): Promise { - if (!mimeType) { - throw new Error('MIMEType is required to parse image under Node.js'); - } - - const buffer = arrayBuffer instanceof Buffer ? arrayBuffer : Buffer.from(arrayBuffer); - const ndarray = await getPixelsAsync(buffer, mimeType); - return ndarray; -} - -// TODO - check if getPixels callback is asynchronous if provided with buffer input -// if not, parseImage can be a sync function -function getPixelsAsync(buffer: Buffer, mimeType: string): Promise { - return new Promise((resolve) => - getPixels(buffer, mimeType, (err, ndarray) => { - if (err) { - throw err; - } - - const shape = [...ndarray.shape]; - const layers = ndarray.shape.length === 4 ? ndarray.shape.shift() : 1; - const data = ndarray.data instanceof Buffer ? new Uint8Array(ndarray.data) : ndarray.data; - - // extract width/height etc - resolve({ - shape, - data, - width: ndarray.shape[0], - height: ndarray.shape[1], - components: ndarray.shape[2], - // TODO - error - layers: layers ? [layers] : [] - }); - }) - ); -} diff --git a/modules/polyfills/test/load-library/require-utils.spec.ts b/modules/polyfills/test/load-library/require-utils.spec.ts index 0a9b2c82a3..3d13a69913 100644 --- a/modules/polyfills/test/load-library/require-utils.spec.ts +++ b/modules/polyfills/test/load-library/require-utils.spec.ts @@ -18,7 +18,7 @@ test('polyfills#require-utils', (tt) => { return; } - test.skip('polyfills#requireFromFile#', (t) => { + test('polyfills#requireFromFile#', (t) => { t.ok(requireFromFile(MODULE_URL), 'Require from file worked'); t.ok(requireFromFile(SUBMODULE_URL), 'Require from file worked'); t.end(); @@ -50,7 +50,11 @@ test('polyfills#require-utils', (tt) => { t.end(); }); +<<<<<<< HEAD test.skip('polyfills#requireFromString#should have appended and preppended paths', (t) => { +======= + test.only('polyfills#requireFromString#should have appended and preppended paths', (t) => { +>>>>>>> 1b9bf5ad0 (chore(worker-utils): Move Node require utils to polyfills) const code = fs.readFileSync(SUBMODULE_URL, 'utf8'); const result = requireFromString(code, SUBMODULE_URL, { appendPaths: ['append'], diff --git a/modules/polyfills/wip/fetch-browser/fetch-file.browser.d.ts b/modules/polyfills/wip/fetch-browser/fetch-file.browser.d.ts deleted file mode 100644 index a5cb0b7417..0000000000 --- a/modules/polyfills/wip/fetch-browser/fetch-file.browser.d.ts +++ /dev/null @@ -1,30 +0,0 @@ -// File reader fetch "polyfill" for the browser -export class FileReadableResponse { - constructor(fileOrBlob, options); - readonly headers: Headers; - - get ok(): boolean; - get status(): number; - - // Note: This is just the file name without path information - // Note: File has `name` field but the Blob baseclass does not - get url(): string; - - arrayBuffer(): Promise; - - text(): Promise; - - json(): Promise; - - // TODO - body, how to support stream? - // Can this be portable? - // eslint-disable-next-line - // https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#Creating_your_own_custom_readable_stream - // get body() { - // assert(false); - // } -} - -// @param {File|Blob} file HTML File or Blob object to read as string -// @returns {Promise.string} Resolves to a string containing file contents -export default function fetchFileReadable(fileOrBlob: File | Blob, options?: object); diff --git a/modules/worker-utils/package.json b/modules/worker-utils/package.json index 8a74751845..870b27a238 100644 --- a/modules/worker-utils/package.json +++ b/modules/worker-utils/package.json @@ -34,15 +34,9 @@ ], "browser": { "child_process": false, - "module": false, - "path": false, - "fs": false, "./src/lib/node/worker_threads.ts": "./src/lib/node/worker_threads-browser.ts", "./dist/es5/lib/node/worker_threads.js": "./dist/es5/lib/node/worker_threads-browser.js", "./dist/esm/lib/node/worker_threads.js": "./dist/esm/lib/node/worker_threads-browser.js", - "./src/lib/node/require-utils.node.ts": false, - "./dist/es5/lib/node/require-utils.node.js": false, - "./dist/esm/lib/node/require-utils.node.js": false, "./src/lib/process-utils/child-process-proxy.ts": false, "./dist/es5/lib/process-utils/child-process-proxy.js": false, "./dist/esm/lib/process-utils/child-process-proxy.js": false diff --git a/modules/worker-utils/src/lib/env-utils/globals.ts b/modules/worker-utils/src/lib/env-utils/globals.ts index 277b946485..6f31e95316 100644 --- a/modules/worker-utils/src/lib/env-utils/globals.ts +++ b/modules/worker-utils/src/lib/env-utils/globals.ts @@ -1,21 +1,6 @@ // Purpose: include this in your module to avoids adding dependencies on // micro modules like 'global' and 'is-browser'; -/* eslint-disable no-restricted-globals */ -const globals = { - self: typeof self !== 'undefined' && self, - window: typeof window !== 'undefined' && window, - global: typeof global !== 'undefined' && global, - document: typeof document !== 'undefined' && document -}; - -const self_: {[key: string]: any} = globals.self || globals.window || globals.global || {}; -const window_: {[key: string]: any} = globals.window || globals.self || globals.global || {}; -const global_: {[key: string]: any} = globals.global || globals.self || globals.window || {}; -const document_: {[key: string]: any} = globals.document || {}; - -export {self_ as self, window_ as window, global_ as global, document_ as document}; - /** true if running in the browser, false if running in Node.js */ export const isBrowser: boolean = // @ts-ignore process.browser diff --git a/modules/worker-utils/src/lib/library-utils/library-utils.ts b/modules/worker-utils/src/lib/library-utils/library-utils.ts index 6cd13aed65..72662b121d 100644 --- a/modules/worker-utils/src/lib/library-utils/library-utils.ts +++ b/modules/worker-utils/src/lib/library-utils/library-utils.ts @@ -1,6 +1,5 @@ /* global importScripts */ import {isBrowser, isWorker} from '../env-utils/globals'; -import * as node from '../node/require-utils.node'; import {assert} from '../env-utils/assert'; import {VERSION} from '../env-utils/version'; @@ -84,14 +83,10 @@ async function loadLibraryFromFile(libraryUrl: string): Promise { } if (!isBrowser) { - // TODO - Node doesn't yet support dynamic import from https URLs - // try { - // return await import(libraryUrl); - // } catch (error) { - // console.error(error); - // } try { - return node && node.requireFromFile && (await node.requireFromFile(libraryUrl)); + // TODO - Node doesn't yet support dynamic import from https URLs + // return await import(libraryUrl); + return await globalThis.loaders?.requireFromFile?.(libraryUrl); } catch (error) { console.error(error); // eslint-disable-line no-console return null; @@ -129,7 +124,10 @@ async function loadScriptFromFile(libraryUrl) { // we could create a`LibraryLoader` or`ModuleLoader` function loadLibraryFromString(scriptSource: string, id: string): null | any { if (!isBrowser) { - return node.requireFromString && node.requireFromString(scriptSource, id); + return ( + globalThis.loaders?.requireFromString && + globalThis.loaders?.requireFromString(scriptSource, id) + ); } if (isWorker) { @@ -168,11 +166,11 @@ function combineWorkerWithLibrary(worker, jsContent) { */ async function loadAsArrayBuffer(url: string): Promise { - if (!node.readFileAsArrayBuffer || url.startsWith('http')) { + if (!globalThis.loaders?.readFileAsArrayBuffer || url.startsWith('http')) { const response = await fetch(url); return await response.arrayBuffer(); } - return await node.readFileAsArrayBuffer(url); + return await globalThis.loaders?.readFileAsArrayBuffer(url); } /** @@ -181,9 +179,9 @@ async function loadAsArrayBuffer(url: string): Promise { * @returns */ async function loadAsText(url: string): Promise { - if (!node.readFileAsText || url.startsWith('http')) { + if (!globalThis.loaders?.readFileAsText || url.startsWith('http')) { const response = await fetch(url); return await response.text(); } - return await node.readFileAsText(url); + return await globalThis.loaders?.readFileAsText(url); } diff --git a/modules/worker-utils/src/lib/node/require-utils.node.ts b/modules/worker-utils/src/lib/node/require-utils.node.ts deleted file mode 100644 index 8c4d9d552f..0000000000 --- a/modules/worker-utils/src/lib/node/require-utils.node.ts +++ /dev/null @@ -1,100 +0,0 @@ -// Fork of https://github.com/floatdrop/require-from-string/blob/master/index.js -// Copyright (c) Vsevolod Strukchinsky (github.com/floatdrop) -// MIT license - -// this file is not visible to webpack (it is excluded in the package.json "browser" field). - -import Module from 'module'; -import path from 'path'; -import fs from 'fs'; - -/** - * Load a file from local file system - * @param filename - * @returns - */ -export async function readFileAsArrayBuffer(filename: string): Promise { - if (filename.startsWith('http')) { - const response = await fetch(filename); - return await response.arrayBuffer(); - } - const buffer = fs.readFileSync(filename); - return buffer.buffer; -} - -/** - * Load a file from local file system - * @param filename - * @returns - */ -export async function readFileAsText(filename: string): Promise { - if (filename.startsWith('http')) { - const response = await fetch(filename); - return await response.text(); - } - const text = fs.readFileSync(filename, 'utf8'); - return text; -} - -// Node.js Dynamically require from file -// Relative names are resolved relative to cwd -// This indirect function is provided because webpack will try to bundle `module.require`. -// this file is not visible to webpack (it is excluded in the package.json "browser" field). -export async function requireFromFile(filename: string): Promise { - if (filename.startsWith('http')) { - const response = await fetch(filename); - const code = await response.text(); - return requireFromString(code); - } - - if (!filename.startsWith('/')) { - filename = `${process.cwd()}/${filename}`; - } - const code = await fs.readFileSync(filename, 'utf8'); - return requireFromString(code); -} - -// Dynamically require from string -// - `code` - Required - Type: string - Module code. -// - `filename` - Type: string - Default: '' - Optional filename. -// - `options.appendPaths` Type: Array List of paths, that will be appended to module paths. -// Useful, when you want to be able require modules from these paths. -// - `options.prependPaths` Type: Array Same as appendPaths, but paths will be prepended. -export function requireFromString( - code: string, - filename = '', - options?: { - prependPaths?: string[]; - appendPaths?: string[]; - } -): any { - if (typeof filename === 'object') { - options = filename; - filename = ''; - } - - if (typeof code !== 'string') { - throw new Error(`code must be a string, not ${typeof code}`); - } - - // @ts-ignore - const paths = Module._nodeModulePaths(path.dirname(filename)); - - const parent = typeof module !== 'undefined' && module?.parent; - - // @ts-ignore - const newModule = new Module(filename, parent); - newModule.filename = filename; - newModule.paths = ([] as string[]) - .concat(options?.prependPaths || []) - .concat(paths) - .concat(options?.appendPaths || []); - // @ts-ignore - newModule._compile(code, filename); - - if (parent && parent.children) { - parent.children.splice(parent.children.indexOf(newModule), 1); - } - - return newModule.exports; -} diff --git a/modules/worker-utils/test/index.ts b/modules/worker-utils/test/index.ts index 57a10d4fca..4aab417af3 100644 --- a/modules/worker-utils/test/index.ts +++ b/modules/worker-utils/test/index.ts @@ -1,7 +1,9 @@ +// loaders.gl, MIT license + import './lib/async-queue/async-queue.spec'; -// import './lib/library-utils/require-utils.spec'; -import './lib/library-utils/library-utils.spec'; +// TESTED by polyfills? +// import './lib/library-utils/library-utils.spec'; import './lib/worker-utils/get-transfer-list.spec'; import './lib/worker-utils/get-loadable-worker-url.spec'; diff --git a/modules/worker-utils/test/lib/library-utils/require-utils.spec.ts b/modules/worker-utils/test/lib/library-utils/require-utils.spec.ts deleted file mode 100644 index fb34cc0c09..0000000000 --- a/modules/worker-utils/test/lib/library-utils/require-utils.spec.ts +++ /dev/null @@ -1,93 +0,0 @@ -// Fork of https://github.com/floatdrop/require-from-string/blob/master/index.js -// Copyright (c) Vsevolod Strukchinsky (github.com/floatdrop) -// MIT license - -import test from 'tape-promise/tape'; -import * as fs from 'fs'; -import * as path from 'path'; -import {isBrowser} from '@loaders.gl/worker-utils'; -import {requireFromFile, requireFromString} from '../../../src/lib/node/require-utils.node'; - -const DIR = path?.dirname?.(import.meta.url) || '.'; -const MODULE_URL = `${DIR}/fixture/module.js`; -const SUBMODULE_URL = `${DIR}/fixture/submodule.js`; - -test('require-utils', (tt) => { - if (isBrowser) { - tt.end(); - return; - } - - test('requireFromFile#', (t) => { - t.ok(requireFromFile(MODULE_URL), 'Require from file worked'); - t.ok(requireFromFile(SUBMODULE_URL), 'Require from file worked'); - t.end(); - }); - - test('requireFromString#should accept only string as code', (t) => { - // @ts-expect-error - t.throws(() => requireFromString(), /code must be a string, not undefined/); - t.end(); - }); - - test('requireFromString#should require from string', (t) => { - t.equal(requireFromString('module.exports = 1;'), 1); - t.end(); - }); - - test('requireFromString#should accept filename', (t) => { - t.throws(() => requireFromString('module.exports = ', 'bug.js'), /bug\.js|Unexpected/); - t.end(); - }); - - test('requireFromString#should work with relative require in file', (t) => { - const code = fs.readFileSync(MODULE_URL, 'utf8'); - const result = requireFromString(code, MODULE_URL); - - t.ok(result); - // TODO - // t.equal(module, result.parent.parent); - t.end(); - }); - - test('requireFromString#should have appended and preppended paths', (t) => { - const code = fs.readFileSync(SUBMODULE_URL, 'utf8'); - const result = requireFromString(code, SUBMODULE_URL, { - appendPaths: ['append'], - prependPaths: ['prepend'] - }); - - t.ok(result); - t.equal(result.paths.indexOf('append'), result.paths.length - 1); - t.equal(result.paths.indexOf('prepend'), 0); - t.end(); - }); - - // TODO - test.skip('requireFromString#should have meaningful error message', (t) => { - try { - requireFromString('throw new Error("Boom!");'); - } catch (error) { - // @ts-ignore - t.ok(/\(:1:69\)/.test(error.stack), 'should contain (:1:69) in stack'); - } - - try { - requireFromString('throw new Error("Boom!");', ''); - } catch (error) { - // @ts-ignore - t.ok(/\(:1:69\)/.test(error.stack), 'should contain (:1:69) in stack'); - } - t.end(); - }); - - test('requireFromString#should cleanup parent.children', (t) => { - const code = fs.readFileSync(SUBMODULE_URL, 'utf8'); - const result = requireFromString(code, SUBMODULE_URL); - - t.ok(module.children.indexOf(result) === -1); - t.end(); - }); - - tt.end(); -}); From 5fe02779d2baa5c768593a85f123abfedf310560 Mon Sep 17 00:00:00 2001 From: Ib Green Date: Mon, 16 Oct 2023 15:05:06 -0400 Subject: [PATCH 2/2] wip --- modules/parquet/src/buffer-polyfill/buffer.ts | 3 --- modules/polyfills/test/load-library/require-utils.spec.ts | 4 ---- 2 files changed, 7 deletions(-) diff --git a/modules/parquet/src/buffer-polyfill/buffer.ts b/modules/parquet/src/buffer-polyfill/buffer.ts index ab10939f90..c44485e125 100644 --- a/modules/parquet/src/buffer-polyfill/buffer.ts +++ b/modules/parquet/src/buffer-polyfill/buffer.ts @@ -1899,7 +1899,6 @@ function writeDouble(buf: Buffer, value, offset, littleEndian, noAssert): number } // CUSTOM ERRORS -// ============= // Simplified versions from Node, changed for Buffer-only usage const errors: Record = {}; @@ -1991,7 +1990,6 @@ function addNumericalSeparator(val) { } // CHECK FUNCTIONS -// =============== function checkBounds(buf, offset, byteLength) { validateNumber(offset, 'offset'); @@ -2040,7 +2038,6 @@ function boundsError(value, length, type?) { } // HELPER FUNCTIONS -// ================ const INVALID_BASE64_RE = /[^+/0-9A-Za-z-_]/g; diff --git a/modules/polyfills/test/load-library/require-utils.spec.ts b/modules/polyfills/test/load-library/require-utils.spec.ts index 3d13a69913..03de9bcec7 100644 --- a/modules/polyfills/test/load-library/require-utils.spec.ts +++ b/modules/polyfills/test/load-library/require-utils.spec.ts @@ -50,11 +50,7 @@ test('polyfills#require-utils', (tt) => { t.end(); }); -<<<<<<< HEAD test.skip('polyfills#requireFromString#should have appended and preppended paths', (t) => { -======= - test.only('polyfills#requireFromString#should have appended and preppended paths', (t) => { ->>>>>>> 1b9bf5ad0 (chore(worker-utils): Move Node require utils to polyfills) const code = fs.readFileSync(SUBMODULE_URL, 'utf8'); const result = requireFromString(code, SUBMODULE_URL, { appendPaths: ['append'],