wip

modules/core/src/index.ts

@@ -11,8 +11,11 @@ export type {
   DataType,
   SyncDataType,
   BatchableDataType,
+  ReadableFile,
+  WritableFile,
+  Stat,
   FileSystem,
-  RandomAccessReadFileSystem
+  RandomAccessFileSystem
 } from '@loaders.gl/loader-utils';
 
 // FILE READING AND WRITING

modules/core/src/lib/fetch/fetch-file.ts

@@ -2,7 +2,6 @@
 
 import {resolvePath} from '@loaders.gl/loader-utils';
 import {makeResponse} from '../utils/response-utils';
-import * as node from './fetch-file.node';
 
 export function isNodePath(url: string): boolean {
   return !isRequestURL(url) && !isDataURL(url);
@@ -29,8 +28,8 @@ export async function fetchFile(
     const url = resolvePath(urlOrData);
 
     // Support fetching from local file system
-    if (isNodePath(url) && node?.fetchFileNode) {
-      return node.fetchFileNode(url, fetchOptions);
+    if (isNodePath(url) && globalThis.loaders?.fetchNode) {
+      return globalThis.loaders?.fetchNode(url, fetchOptions);
     }
 
     // Call global fetch

modules/core/src/lib/filesystems/browser-filesystem.ts

@@ -1,4 +1,5 @@
 import type {FileSystem, ReadableFile} from '@loaders.gl/loader-utils';
+import {BlobFile} from '@loaders.gl/loader-utils';
 
 type BrowserFileSystemOptions = {
   fetch?: typeof fetch;
@@ -110,17 +111,29 @@ export class BrowserFileSystem implements FileSystem {
 
   // RANDOM ACCESS
   async openReadableFile(pathname: string, flags: unknown): Promise<ReadableFile> {
-    return this.files[pathname];
+    return new BlobFile(this.files[pathname]);
   }
 
-  /**
+  // PRIVATE
+
+  // Supports case independent paths, and file usage tracking
+  _getFile(path: string, used: boolean): File {
+    // Prefer case match, but fall back to case independent.
+    const file = this.files[path] || this.lowerCaseFiles[path];
+    if (file && used) {
+      this.usedFiles[path] = true;
+    }
+    return file;
+  }
+}
+/*
    * Read a range into a buffer
    * @todo - handle position memory
    * @param buffer is the buffer that the data (read from the fd) will be written to.
    * @param offset is the offset in the buffer to start writing at.
    * @param length is an integer specifying the number of bytes to read.
    * @param position is an argument specifying where to begin reading from in the file. If position is null, data will be read from the current file position, and the file position will be updated. If position is an integer, the file position will remain unchanged.
-   */
+   *
   async read(
     fd: any,
     buffer: ArrayBuffer,
@@ -140,16 +153,4 @@ export class BrowserFileSystem implements FileSystem {
   }
 
   // fstat(fd: number): Promise<object>; // Stat
-
-  // PRIVATE
-
-  // Supports case independent paths, and file usage tracking
-  _getFile(path: string, used: boolean): File {
-    // Prefer case match, but fall back to case independent.
-    const file = this.files[path] || this.lowerCaseFiles[path];
-    if (file && used) {
-      this.usedFiles[path] = true;
-    }
-    return file;
-  }
-}
+  */

modules/loader-utils/src/index.ts

@@ -120,21 +120,20 @@ export {stream};
 
 // EXPERIMENTAL: FILE SYSTEMS
 
-export type {FileSystem, RandomAccessFileSystem} from './lib/filesystems/filesystem';
-export {NodeFileSystem as _NodeFileSystem} from './lib/filesystems/node-filesystem';
-
 export type {FileProvider} from './lib/file-provider/file-provider';
 export {isFileProvider} from './lib/file-provider/file-provider';
 
 export {FileHandle} from './lib/file-provider/file-handle';
 export {FileHandleFile} from './lib/file-provider/file-handle-file';
 export {DataViewFile} from './lib/file-provider/data-view-file';
 
-export type {ReadableFile} from './lib/filesystems/filesystem';
-// export {makeReadableFile} from './lib/filesystems/make-readable-file';
+export type {ReadableFile, WritableFile, Stat} from './lib/files/file';
+export {BlobFile} from './lib/files/blob-file';
+export {HttpFile} from './lib/files/http-file';
+export {NodeFileFacade as NodeFile} from './lib/files/node-file-facade';
 
-export type {WritableFile} from './lib/filesystems/filesystem';
-// export {makeWritableFile} from './lib/filesystems/make-writable-file';
+export type {FileSystem, RandomAccessFileSystem} from './lib/filesystems/filesystem';
+export {NodeFileSystemFacade as NodeFilesystem} from './lib/filesystems/node-filesystem-facade';
 
 // EXPERIMENTAL: DATA SOURCES
 export type {Service} from './service-types';

modules/loader-utils/src/lib/files/blob-file.ts

@@ -0,0 +1,29 @@
+// loaders.gl, MIT license
+
+import {ReadableFile} from './file';
+
+export class BlobFile implements ReadableFile {
+  readonly handle: Blob;
+  readonly size: number;
+  readonly url: string;
+
+  constructor(blob: Blob | File) {
+    this.handle = blob;
+    this.size = blob.size;
+    this.url = (blob as File).name || '';
+  }
+
+  async close() {}
+
+  async stat() {
+    return {
+      size: this.handle.size,
+      isDirectory: false
+    };
+  }
+
+  async read(start: number, length: number): Promise<ArrayBuffer> {
+    const arrayBuffer = await this.handle.slice(start, start + length).arrayBuffer();
+    return arrayBuffer;
+  }
+}

modules/loader-utils/src/lib/files/file.ts

@@ -0,0 +1,29 @@
+export type Stat = {
+  size: number;
+  isDirectory: boolean;
+};
+
+export interface ReadableFile {
+  /** The underlying file handle (Blob, Node.js file descriptor etc) */
+  handle: unknown;
+  /** Length of file in bytes, if available */
+  size: number;
+  /** Read data */
+  read(start?: number, end?: number): Promise<ArrayBuffer>;
+  /** Read data */
+  fetchRange?(offset: number, length: number, signal?: AbortSignal): Promise<Response>;
+  /** Get information about file */
+  stat?(): Promise<Stat>;
+  /** Close the file */
+  close(): Promise<void>;
+}
+
+export interface WritableFile {
+  handle: unknown;
+  /** Write to file. The number of bytes written will be returned */
+  write: (arrayBuffer: ArrayBuffer, offset?: number, length?: number) => Promise<number>;
+  /** Get information about the file */
+  stat?(): Promise<Stat>;
+  /** Close the file */
+  close(): Promise<void>;
+}

modules/loader-utils/src/lib/files/http-file.ts

@@ -0,0 +1,108 @@
+// loaders.gl, MIT license
+
+import {ReadableFile, Stat} from './file';
+
+export class HttpFile implements ReadableFile {
+  readonly handle: string;
+  readonly size: number = 0;
+  readonly url: string;
+
+  constructor(url: string) {
+    this.handle = url;
+    this.url = url;
+  }
+
+  async close(): Promise<void> {}
+
+  async stat(): Promise<Stat> {
+    const response = await fetch(this.handle, {method: 'HEAD'});
+    if (!response.ok) {
+      throw new Error(`Failed to fetch HEAD ${this.handle}`);
+    }
+    return {
+      size: parseInt(response.headers.get('Content-Length') || '0'),
+      isDirectory: false
+    };
+  }
+
+  async read(offset: number, length: number): Promise<ArrayBuffer> {
+    const response = await this.fetchRange(offset, length);
+    const arrayBuffer = await response.arrayBuffer();
+    return arrayBuffer;
+  }
+
+  /**
+   * 
+   * @param offset 
+   * @param length 
+   * @param signal 
+   * @returns
+   * @see https://github.com/protomaps/PMTiles 
+   */
+  async fetchRange(offset: number, length: number, signal?: AbortSignal): Promise<Response> {
+    let controller: AbortController | undefined;
+    if (!signal) {
+      // ToDO why is it so important to abort in case 200?
+      // TODO check this works or assert 206
+      controller = new AbortController();
+      signal = controller.signal;
+    }
+
+    const url = this.handle;
+    let response = await fetch(url, {
+      signal,
+      headers: {Range: `bytes=${offset}-${offset + length - 1}`}
+    });
+
+    switch (response.status) {
+      case 206: // Partial Content success
+        // This is the expected success code for a range request
+        break;
+
+      case 200:
+        // some well-behaved backends, e.g. DigitalOcean CDN, respond with 200 instead of 206
+        // but we also need to detect no support for Byte Serving which is returning the whole file
+        const content_length = response.headers.get('Content-Length');
+        if (!content_length || Number(content_length) > length) {
+          if (controller) {
+            controller.abort();
+          }
+          throw Error(
+            'content-length header missing or exceeding request. Server must support HTTP Byte Serving.'
+          );
+        }
+
+      case 416: // "Range Not Satisfiable"
+        // some HTTP servers don't accept ranges beyond the end of the resource.
+        // Retry with the exact length
+        // TODO: can return 416 with offset > 0 if content changed, which will have a blank etag.
+        // See https://github.com/protomaps/PMTiles/issues/90
+        if (offset === 0) {
+          const content_range = response.headers.get('Content-Range');
+          if (!content_range || !content_range.startsWith('bytes *')) {
+            throw Error('Missing content-length on 416 response');
+          }
+          const actual_length = Number(content_range.substr(8));
+          response = await fetch(this.url, {
+            signal,
+            headers: {Range: `bytes=0-${actual_length - 1}`}
+          });
+        }
+        break;
+
+      default:
+        if (response.status >= 300) {
+          throw Error(`Bad response code: ${response.status}`);
+        }
+    }
+
+    return response;
+    // const data = await response.arrayBuffer();
+    // return {
+    //   data,
+    //   etag: response.headers.get('ETag') || undefined,
+    //   cacheControl: response.headers.get('Cache-Control') || undefined,
+    //   expires: response.headers.get('Expires') || undefined
+    // };
+  }
+}

modules/loader-utils/src/lib/files/node-file-facade.ts

@@ -0,0 +1,39 @@
+// loaders.gl, MIT license
+
+import {isBrowser} from '../env-utils/globals';
+import {ReadableFile, WritableFile, Stat} from './file';
+
+const NOT_IMPLEMENTED = new Error('Not implemented');
+
+/** This class is a facade that gets replaced with an actual NodeFile instance  */
+export class NodeFileFacade implements ReadableFile, WritableFile {
+  /** The underlying file handle (Blob, Node.js file descriptor etc) */
+  handle: unknown;
+  /** Length of file in bytes, if available */
+  size: number = 0;
+
+  constructor(options) {
+    // Return the actual implementation instance
+    if (globalThis.loaders?.NodeFile) {
+      return new globalThis.loaders.NodeFile(options);
+    }
+    if (isBrowser) {
+      throw new Error("Can't instantiate NodeFile in browser.");
+    }
+    throw new Error("Can't instantiate NodeFile. Make sure to import @loaders.gl/polyfills first.");
+  }
+  /** Read data */
+  async read(start?: number, end?: number): Promise<ArrayBuffer> {
+    throw NOT_IMPLEMENTED;
+  }
+  /** Write to file. The number of bytes written will be returned */
+  async write(arrayBuffer: ArrayBuffer, offset?: number, length?: number): Promise<number> {
+    throw NOT_IMPLEMENTED;
+  }
+  /** Get information about file */
+  async stat(): Promise<Stat> {
+    throw NOT_IMPLEMENTED;
+  }
+  /** Close the file */
+  async close(): Promise<void> {}
+}