Skip to content

Commit

Permalink
chore(polyfills): Start moving Node.js code into polyfills (#2669)
Browse files Browse the repository at this point in the history
  • Loading branch information
ibgreen authored Oct 2, 2023
1 parent 35c625e commit ece5138
Show file tree
Hide file tree
Showing 9 changed files with 189 additions and 188 deletions.
1 change: 1 addition & 0 deletions .prettierignore
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ public
modules/core/src/iterators/make-stream/make-node-stream.ts

modules/loader-utils/src/lib/filesystems/node-filesystem.browser.ts
modules/loader-utils/src/lib/filesystems/node-filesystem.ts

modules/3d-tiles/test/lib/classes/tile-3d-batch-table-hierarchy.spec.ts

Expand Down

This file was deleted.

42 changes: 15 additions & 27 deletions modules/loader-utils/src/lib/filesystems/node-filesystem.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,56 +24,44 @@ type ReadOptions = {
export class NodeFileSystem implements FileSystem, RandomAccessReadFileSystem {
// implements FileSystem
constructor(options: {[key: string]: any}) {
this.fetch = options._fetch;
if (globalThis.loaders?.NodeFileSystem) {
return new globalThis.loaders.NodeFileSystem(options);
}
throw new Error(
'Can\'t instantiate NodeFileSystem in browser. Make sure to import @loaders.gl/polyfills first.'
);
}

async readdir(dirname = '.', options?: {}): Promise<any[]> {
return await fs.readdir(dirname, options);
return [];
}

async stat(path: string, options?: {}): Promise<Stat> {
const info = await fs.stat(path, options);
return {size: Number(info.size), isDirectory: () => false, info};
return {size: 0, isDirectory: () => false};
}

async fetch(path: string, options: {[key: string]: any}) {
// Falls back to handle https:/http:/data: etc fetches
// eslint-disable-next-line
const fallbackFetch = options.fetch || this.fetch;
return fallbackFetch(path, options);
return globalThis.fetch(path, options);
}

// implements IRandomAccessFileSystem

async open(path: string, flags: string | number, mode?: any): Promise<number> {
return await fs.open(path, flags);
return 0;
}

async close(fd: number): Promise<void> {
return await fs.close(fd);
}
// eslint-disable-next-line @typescript-eslint/no-empty-function
async close(fd: number): Promise<void> {}

async fstat(fd: number): Promise<Stat> {
const info = await fs.fstat(fd);
return info;
return {size: 0, isDirectory: () => false};
}

async read(
fd: number,
// @ts-ignore Possibly null
{buffer = null, offset = 0, length = buffer.byteLength, position = null}: ReadOptions
): Promise<{bytesRead: number; buffer: Uint8Array}> {
let totalBytesRead = 0;
// Read in loop until we get required number of bytes
while (totalBytesRead < length) {
const {bytesRead} = await fs.read(
fd,
buffer,
offset + totalBytesRead,
length - totalBytesRead,
position + totalBytesRead
);
totalBytesRead += bytesRead;
}
return {bytesRead: totalBytesRead, buffer};
return {bytesRead: 0, buffer: new Uint8Array(0)};
}
}
68 changes: 4 additions & 64 deletions modules/polyfills/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,70 +26,10 @@
"README.md"
],
"browser": {
"./src/node/fetch/utils/decode-data-uri.node.js": false,
"./src/node/fetch/utils/decode-data-uri.node.ts": false,
"./dist/es5/node/fetch/utils/decode-data-uri.node.js": false,
"./dist/esm/node/fetch/utils/decode-data-uri.node.js": false,
"./src/node/fetch/utils/stream-utils.node.js": false,
"./src/node/fetch/utils/stream-utils.node.ts": false,
"./dist/es5/node/fetch/utils/stream-utils.node.js": false,
"./dist/esm/node/fetch/utils/stream-utils.node.js": false,
"./src/node/fetch/fetch.node.js": false,
"./src/node/fetch/fetch.node.ts": false,
"./dist/es5/node/fetch/fetch.node.js": false,
"./dist/esm/node/fetch/fetch.node.js": false,
"./src/node/fetch/headers.node.js": false,
"./src/node/fetch/headers.node.ts": false,
"./dist/es5/node/fetch/headers.node.js": false,
"./dist/esm/node/fetch/headers.node.js": false,
"./src/node/fetch/response.node.js": false,
"./src/node/fetch/response.node.ts": false,
"./dist/es5/node/fetch/response.node.js": false,
"./dist/esm/node/fetch/response.node.js": false,
"./src/node/images/encode-image.node.js": false,
"./src/node/images/encode-image.node.ts": false,
"./dist/es5/node/images/encode-image.node.js": false,
"./dist/esm/node/images/encode-image.node.js": false,
"./src/node/images/parse-image.node.js": false,
"./src/node/images/parse-image.node.ts": false,
"./dist/es5/node/images/parse-image.node.js": false,
"./dist/esm/node/images/parse-image.node.js": false,
"./src/node/buffer/to-array-buffer.node.js": false,
"./src/node/buffer/to-array-buffer.node.ts": false,
"./dist/es5/node/buffer/to-array-buffer.node.js": false,
"./dist/esm/node/buffer/to-array-buffer.node.js": false,
"./src/node/buffer/btoa.node.js": false,
"./src/node/buffer/btoa.node.ts": false,
"./dist/es5/node/buffer/btoa.node.js": false,
"./dist/esm/node/buffer/btoa.node.js": false,
"./src/node/file/blob.js": false,
"./src/node/file/blob.ts": false,
"./dist/es5/node/file/blob.js": false,
"./dist/esm/node/file/blob.js": false,
"./src/node/file/file.js": false,
"./src/node/file/file.ts": false,
"./dist/es5/node/file/file.js": false,
"./dist/esm/node/file/file.js": false,
"./src/node/file/readable-stream.js": false,
"./src/node/file/readable-stream.ts": false,
"./dist/es5/node/file/readable-stream.js": false,
"./dist/esm/node/file/readable-stream.js": false,
"./src/libs/encoding-indices.js": false,
"./src/libs/encoding-indices.ts": false,
"./dist/es5/libs/encoding-indices.js": false,
"./dist/esm/libs/encoding-indices.js": false,
"fs": false,
"http": false,
"https": false,
"stream": false,
"get-pixels": false,
"ndarray": false,
"save-pixels": false,
"stream-to-async-iterator": false,
"through": false,
"util": false,
"zlib": false,
"web-streams-polyfill": false
"./src/index.ts": "./src/index.browser.ts",
"./dist/index.js": "./dist/index.browser.js",
"./dist/es5/index.js": "./dist/es5/index.browser.js",
"./dist/esm/index.js": "./dist/es5/index.browser.js"
},
"scripts": {
"pre-build": "npm run build-bundle",
Expand Down
87 changes: 87 additions & 0 deletions modules/polyfills/src/filesystems/node-filesystem.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
import {FileSystem, RandomAccessReadFileSystem} from '@loaders.gl/loader-utils';
import fs from 'fs';
import fsPromise from 'fs/promises';

// import {fetchFile} from "../fetch/fetch-file"
// import {selectLoader} from "../api/select-loader";

type Stat = {
size: number;
isDirectory: () => boolean;
info?: fs.Stats;
};

type ReadOptions = {
buffer?: Buffer;
offset?: number;
length?: number;
position?: number;
};

/**
* FileSystem pass-through for Node.js
* Compatible with BrowserFileSystem.
* @param options
*/
export class NodeFileSystem implements FileSystem, RandomAccessReadFileSystem {
// implements FileSystem
constructor(options: {[key: string]: any}) {
this.fetch = options._fetch;
}

async readdir(dirname = '.', options?: {}): Promise<any[]> {
return await fsPromise.readdir(dirname, options);
}

async stat(path: string, options?: {}): Promise<Stat> {
const info = await fsPromise.stat(path, options);
return {size: Number(info.size), isDirectory: () => false, info};
}

async fetch(path: string, options: {[key: string]: any}) {
// Falls back to handle https:/http:/data: etc fetches
// eslint-disable-next-line
const fallbackFetch = options.fetch || this.fetch;
return fallbackFetch(path, options);
}

// implements IRandomAccessFileSystem
async open(path: string, flags: string | number, mode?: any): Promise<number> {
return (await fsPromise.open(path, flags)) as unknown as number;
}

async close(fd: number): Promise<void> {
fs.close(fd);
}

async fstat(fd: number): Promise<Stat> {
return await new Promise((resolve, reject) =>
fs.fstat(fd, (err, info) => (err ? reject(err) : resolve(info)))
);
}

async read(
fd: number,
// @ts-ignore Possibly null
{buffer = null, offset = 0, length = buffer.byteLength, position = null}: ReadOptions
): Promise<{bytesRead: number; buffer: Uint8Array}> {
let totalBytesRead = 0;
// Read in loop until we get required number of bytes
while (totalBytesRead < length) {
const {bytesRead} = await new Promise<{bytesRead: number; buffer: Buffer}>(
// eslint-disable-next-line no-loop-func
(resolve, reject) =>
fs.read(
fd,
buffer,
offset + totalBytesRead,
length - totalBytesRead,
position + totalBytesRead,
(err, bytesRead, buffer) => (err ? reject(err) : resolve({bytesRead, buffer}))
)
);
totalBytesRead += bytesRead;
}
return {bytesRead: totalBytesRead, buffer};
}
}
14 changes: 14 additions & 0 deletions modules/polyfills/src/index.browser.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
// loaders.gl, MIT License

import {allSettled} from './promise/all-settled';

if (!('allSettled' in Promise)) {
// @ts-ignore
Promise.allSettled = allSettled;
}

// eslint-disable-next-line @typescript-eslint/no-empty-function
export function installFilePolyfills() {}

// Dummy export to avoid import errors in browser tests
export const NodeFileSystem = null;
Loading

0 comments on commit ece5138

Please sign in to comment.