feat(loader-utils): Refactor FileSystem to be independent of fs

.eslintrc.js

@@ -58,7 +58,7 @@ const config = deepMerge(defaultConfig, {
         // '@typescript-eslint/no-floating-promises': ['warn'],
         // '@typescript-eslint/await-thenable': ['warn'],
         // '@typescript-eslint/no-misused-promises': ['warn'],
-        '@typescript-eslint/no-empty-function': ['warn', {allow: ['arrowFunctions']}],
+        '@typescript-eslint/no-empty-function': 0,
         // We use function hoisting
         '@typescript-eslint/no-use-before-define': 0,
         // We always want explicit typing, e.g `field: string = ''`

modules/core/src/lib/fetch/fetch-file.ts

@@ -28,7 +28,12 @@ export async function fetchFile(
     const url = resolvePath(urlOrData);
 
     // Support fetching from local file system
-    if (isNodePath(url) && globalThis.loaders?.fetchNode) {
+    if (isNodePath(url)) {
+      if (!globalThis.loaders?.fetchNode) {
+        throw new Error(
+          'fetchFile: globalThis.loaders.fetchNode not defined. Install @loaders.gl/polyfills'
+        );
+      }
       return globalThis.loaders?.fetchNode(url, fetchOptions);
     }
 

modules/csv/test/papaparse/papaparse.spec.ts

@@ -488,7 +488,6 @@ const CUSTOM_TESTS = [
       Papa.parse(BASE_PATH + 'long-sample.csv', {
         download: true,
         chunkSize: 500,
-        // eslint-disable-next-line @typescript-eslint/no-empty-function
         beforeFirstChunk(chunk) {},
         step(response) {
           updates++;

modules/loader-utils/src/lib/file-provider/data-view-file.ts

@@ -21,7 +21,6 @@ export class DataViewFile implements FileProvider {
     this.file = file;
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-empty-function
   async destroy(): Promise<void> {}
 
   /**

modules/loader-utils/src/lib/files/blob-file.ts

@@ -7,10 +7,10 @@ export class BlobFile implements ReadableFile {
   readonly size: number;
   readonly url: string;
 
-  constructor(blob: Blob | File) {
-    this.handle = blob;
-    this.size = blob.size;
-    this.url = (blob as File).name || '';
+  constructor(blob: Blob | File | ArrayBuffer) {
+    this.handle = blob instanceof ArrayBuffer ? new Blob([blob]) : blob;
+    this.size = blob instanceof ArrayBuffer ? blob.byteLength : blob.size;
+    this.url = blob instanceof File ? blob.name : '';
   }
 
   async close() {}

modules/loader-utils/src/lib/files/http-file.ts

@@ -32,13 +32,14 @@ export class HttpFile implements ReadableFile {
   }
 
   /**
-   * 
-   * @param offset 
-   * @param length 
-   * @param signal 
+   *
+   * @param offset
+   * @param length
+   * @param signal
    * @returns
-   * @see https://github.com/protomaps/PMTiles 
+   * @see https://github.com/protomaps/PMTiles
    */
+  // eslint-disable-next-line complexity
   async fetchRange(offset: number, length: number, signal?: AbortSignal): Promise<Response> {
     let controller: AbortController | undefined;
     if (!signal) {
@@ -62,8 +63,8 @@ export class HttpFile implements ReadableFile {
       case 200:
         // some well-behaved backends, e.g. DigitalOcean CDN, respond with 200 instead of 206
         // but we also need to detect no support for Byte Serving which is returning the whole file
-        const content_length = response.headers.get('Content-Length');
-        if (!content_length || Number(content_length) > length) {
+        const contentLength = response.headers.get('Content-Length');
+        if (!contentLength || Number(contentLength) > length) {
           if (controller) {
             controller.abort();
           }
@@ -72,20 +73,21 @@ export class HttpFile implements ReadableFile {
           );
         }
 
+      // @eslint-disable-next-line no-fallthrough
       case 416: // "Range Not Satisfiable"
         // some HTTP servers don't accept ranges beyond the end of the resource.
         // Retry with the exact length
         // TODO: can return 416 with offset > 0 if content changed, which will have a blank etag.
         // See https://github.com/protomaps/PMTiles/issues/90
         if (offset === 0) {
-          const content_range = response.headers.get('Content-Range');
-          if (!content_range || !content_range.startsWith('bytes *')) {
+          const contentRange = response.headers.get('Content-Range');
+          if (!contentRange || !contentRange.startsWith('bytes *')) {
             throw Error('Missing content-length on 416 response');
           }
-          const actual_length = Number(content_range.substr(8));
+          const actualLength = Number(contentRange.substr(8));
           response = await fetch(this.url, {
             signal,
-            headers: {Range: `bytes=0-${actual_length - 1}`}
+            headers: {Range: `bytes=0-${actualLength - 1}`}
           });
         }
         break;

modules/loader-utils/src/lib/files/node-file-facade.ts

@@ -18,9 +18,9 @@ export class NodeFileFacade implements ReadableFile, WritableFile {
       return new globalThis.loaders.NodeFile(options);
     }
     if (isBrowser) {
-      throw new Error("Can't instantiate NodeFile in browser.");
+      throw new Error('Can\'t instantiate NodeFile in browser.');
     }
-    throw new Error("Can't instantiate NodeFile. Make sure to import @loaders.gl/polyfills first.");
+    throw new Error('Can\'t instantiate NodeFile. Make sure to import @loaders.gl/polyfills first.');
   }
   /** Read data */
   async read(start?: number, end?: number): Promise<ArrayBuffer> {

modules/loader-utils/src/lib/filesystems/node-filesystem-facade.ts

@@ -20,10 +20,10 @@ export class NodeFileSystemFacade implements RandomAccessFileSystem {
       return new globalThis.loaders.NodeFileSystem(options);
     }
     if (isBrowser) {
-      throw new Error("Can't instantiate NodeFileSystem in browser.");
+      throw new Error('Can\'t instantiate NodeFileSystem in browser.');
     }
     throw new Error(
-      "Can't instantiate NodeFileSystem. Make sure to import @loaders.gl/polyfills first."
+      'Can\'t instantiate NodeFileSystem. Make sure to import @loaders.gl/polyfills first.'
     );
   }
 

modules/mvt/test/lib/geojson-tiler/get-tile.spec.ts

@@ -28,7 +28,6 @@ const square = [
 
 test('GeoJSONVT#getTile#us-states.json', async (t) => {
   const log = console.log;
-  // eslint-disable-next-line @typescript-eslint/no-empty-function
   console.log = function () {};
 
   const geojson = await getJSON('us-states.json');

modules/parquet/src/parquetjs/parser/parquet-reader.ts

@@ -150,7 +150,8 @@ export class ParquetReader {
   /** Metadata is stored in the footer */
   async readFooter(): Promise<FileMetaData> {
     const trailerLen = PARQUET_MAGIC.length + 4;
-    const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);
+    const arrayBuffer = await this.file.read(this.file.size - trailerLen, trailerLen);
+    const trailerBuf = Buffer.from(arrayBuffer);
 
     const magic = trailerBuf.slice(4).toString();
     if (magic !== PARQUET_MAGIC) {
@@ -163,7 +164,8 @@ export class ParquetReader {
       throw new Error(`Invalid metadata size ${metadataOffset}`);
     }
 
-    const metadataBuf = await this.file.read(metadataOffset, metadataSize);
+    const arrayBuffer2 = await this.file.read(metadataOffset, metadataSize);
+    const metadataBuf = Buffer.from(arrayBuffer2);
     // let metadata = new parquet_thrift.FileMetaData();
     // parquet_util.decodeThrift(metadata, metadataBuf);
 
@@ -245,7 +247,8 @@ export class ParquetReader {
     }
 
     dictionary = context.dictionary?.length ? context.dictionary : dictionary;
-    const pagesBuf = await this.file.read(pagesOffset, pagesSize);
+    const arrayBuffer = await this.file.read(pagesOffset, pagesSize);
+    const pagesBuf = Buffer.from(arrayBuffer);
     return await decodeDataPages(pagesBuf, {...context, dictionary});
   }
 
@@ -276,7 +279,8 @@ export class ParquetReader {
       this.file.size - dictionaryPageOffset,
       this.props.defaultDictionarySize
     );
-    const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);
+    const arrayBuffer = await this.file.read(dictionaryPageOffset, dictionarySize);
+    const pagesBuf = Buffer.from(arrayBuffer);
 
     const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};
     const decodedPage = await decodePage(cursor, context);

modules/parquet/test/parquetjs/integration.spec.ts

@@ -1,7 +1,7 @@
 /* eslint-disable camelcase */
 import test, {Test} from 'tape-promise/tape';
 import {isBrowser, fetchFile} from '@loaders.gl/core';
-import {makeReadableFile} from '@loaders.gl/loader-utils';
+import {BlobFile} from '@loaders.gl/loader-utils';
 import {ParquetSchema, ParquetReader, ParquetEncoder} from '@loaders.gl/parquet';
 
 const FRUITS_URL = '@loaders.gl/parquet/test/data/fruits.parquet';
@@ -115,7 +115,7 @@ async function writeTestFile(opts) {
 async function readTestFile(t: Test) {
   const response = await fetchFile(FRUITS_URL);
   const arrayBuffer = await response.arrayBuffer();
-  const reader = new ParquetReader(makeReadableFile(arrayBuffer));
+  const reader = new ParquetReader(new BlobFile(arrayBuffer));
   t.equal(reader.getRowCount(), TEST_NUM_ROWS * 4);
   t.deepEqual(reader.getSchemaMetadata(), {myuid: '420', fnord: 'dronf'});
 

modules/parquet/test/parquetjs/reader.spec.ts

@@ -1,6 +1,6 @@
 /* eslint-disable camelcase */
 import test from 'tape-promise/tape';
-import {makeReadableFile} from '@loaders.gl/loader-utils';
+import {BlobFile} from '@loaders.gl/loader-utils';
 import {ParquetReader} from '@loaders.gl/parquet';
 import {fetchFile} from '@loaders.gl/core';
 
@@ -12,7 +12,7 @@ const FRUITS_URL = '@loaders.gl/parquet/test/data/fruits.parquet';
 test('ParquetReader#fruits.parquet', async t => {
   const response = await fetchFile(FRUITS_URL);
   const arrayBuffer = await response.arrayBuffer();
-  const reader = new ParquetReader(makeReadableFile(arrayBuffer));
+  const reader = new ParquetReader(new BlobFile(arrayBuffer));
   // t.equal(reader.getRowCount(), TEST_NUM_ROWS * 4, 'rowCount');
 
   const metadata = await reader.getSchemaMetadata();

modules/pmtiles/src/pmtiles-source.ts

@@ -22,14 +22,13 @@ export type Service = {
   extensions: string[];
   mimeTypes: string[];
   options: Record<string, unknown>;
-}
+};
 
 export type ServiceWithSource<SourceT, SourcePropsT> = Service & {
   _source?: SourceT;
   _sourceProps?: SourcePropsT;
   createSource: (props: SourcePropsT) => SourceT;
-}
-
+};
 
 export const PMTilesService: ServiceWithSource<PMTilesSource, PMTilesSourceProps> = {
   name: 'PMTiles',

modules/polyfills/src/filesystems/fetch-node.ts

@@ -1,19 +1,42 @@
 // loaders.gl, MIT license
 
 import fs from 'fs';
+import {Readable} from 'stream';
+import {resolvePath} from '@loaders.gl/loader-utils';
+import {decompressReadStream} from './stream-utils.node';
+
+const isBoolean = (x) => typeof x === 'boolean';
+const isFunction = (x) => typeof x === 'function';
+const isObject = (x) => x !== null && typeof x === 'object';
+const isReadableNodeStream = (x) =>
+  isObject(x) && isFunction(x.read) && isFunction(x.pipe) && isBoolean(x.readable);
 
 /**
  * Enables
  * @param url
  * @param options
  * @returns
  */
-export async function fetchNode(url: string, options): Promise<Response> {
+// eslint-disable-next-line max-statements
+export async function fetchNode(url: string, options?: RequestInit): Promise<Response> {
   // Support `file://` protocol
   const FILE_PROTOCOL_REGEX = /^file:\/\//;
   url.replace(FILE_PROTOCOL_REGEX, '/');
 
-  const noqueryUrl = url.split('?')[0];
+  // Remove any query parameters, as they have no meaning
+  let noqueryUrl = url.split('?')[0];
+  noqueryUrl = resolvePath(noqueryUrl);
+
+  const responseHeaders = new Headers();
+  // Automatically decompress gzipped files with .gz extension
+  if (url.endsWith('.gz')) {
+    // url = url.slice(0, -3);
+    responseHeaders['content-encoding'] = 'gzip';
+  }
+  if (url.endsWith('.br')) {
+    // url = url.slice(0, -3);
+    responseHeaders['content-encoding'] = 'br';
+  }
 
   try {
     // Now open the stream
@@ -24,14 +47,26 @@ export async function fetchNode(url: string, options): Promise<Response> {
       stream.on('error', (error) => reject(error));
     });
 
+    let bodyStream: Readable = body;
+
+    // Check for content-encoding and create a decompression stream
+    if (isReadableNodeStream(body)) {
+      bodyStream = decompressReadStream(body, responseHeaders);
+    } else if (typeof body === 'string') {
+      bodyStream = Readable.from([new TextEncoder().encode(body)]);
+    } else {
+      bodyStream = Readable.from([body || new ArrayBuffer(0)]);
+    }
+
     const status = 200;
     const statusText = 'OK';
     const headers = getHeadersForFile(noqueryUrl);
     // @ts-expect-error
-    const response = new Response(body, {headers, status, statusText});
+    const response = new Response(bodyStream, {headers, status, statusText});
     Object.defineProperty(response, 'url', {value: url});
     return response;
   } catch (error) {
+    // console.error(error);
     const errorMessage = (error as Error).message;
     const status = 400;
     const statusText = errorMessage;

modules/polyfills/src/filesystems/node-file.ts

@@ -18,7 +18,9 @@ export class NodeFile implements ReadableFile, WritableFile {
 
   async stat(): Promise<Stat> {
     return await new Promise((resolve, reject) =>
-      fs.fstat(this.handle, (err, info) => (err ? reject(err) : resolve({size: info.size, isDirectory: info.isDirectory()}))), 
+      fs.fstat(this.handle, (err, info) =>
+        err ? reject(err) : resolve({size: info.size, isDirectory: info.isDirectory()})
+      )
     );
   }
 
@@ -28,7 +30,7 @@ export class NodeFile implements ReadableFile, WritableFile {
     let totalBytesRead = 0;
     const uint8Array = new Uint8Array(arrayBuffer);
 
-    let position
+    let position;
     // Read in loop until we get required number of bytes
     while (length > 0) {
       const bytesRead = await readBytes(this.handle, uint8Array, 0, length, offset);
@@ -39,7 +41,7 @@ export class NodeFile implements ReadableFile, WritableFile {
       }
 
       totalBytesRead += bytesRead;
-      offset + bytesRead;
+      offset += bytesRead;
       length -= bytesRead;
 
       // Advance position unless we are using built-in position advancement

modules/polyfills/src/filesystems/node-filesystem.ts

@@ -42,11 +42,7 @@ export class NodeFileSystem implements RandomAccessFileSystem {
     return new NodeFile(path, flags);
   }
 
-  async openWritableFile(
-    path: string,
-    flags: 'w' | 'wx' = 'w',
-    mode?: any
-  ): Promise<NodeFile> {
+  async openWritableFile(path: string, flags: 'w' | 'wx' = 'w', mode?: any): Promise<NodeFile> {
     return new NodeFile(path, flags, mode);
   }
 }