Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
  • Loading branch information
OmarShehata committed Oct 9, 2023
2 parents b1a5fa6 + 84df1e9 commit 3db593c
Show file tree
Hide file tree
Showing 79 changed files with 1,094 additions and 987 deletions.
2 changes: 1 addition & 1 deletion .eslintrc.js
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ const config = deepMerge(defaultConfig, {
// '@typescript-eslint/no-floating-promises': ['warn'],
// '@typescript-eslint/await-thenable': ['warn'],
// '@typescript-eslint/no-misused-promises': ['warn'],
'@typescript-eslint/no-empty-function': ['warn', {allow: ['arrowFunctions']}],
'@typescript-eslint/no-empty-function': 0,
// We use function hoisting
'@typescript-eslint/no-use-before-define': 0,
// We always want explicit typing, e.g `field: string = ''`
Expand Down
4 changes: 2 additions & 2 deletions .prettierignore
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ public

modules/core/src/iterators/make-stream/make-node-stream.ts

modules/loader-utils/src/lib/filesystems/node-filesystem.browser.ts
modules/loader-utils/src/lib/filesystems/node-filesystem.ts
modules/loader-utils/src/lib/files/node-file-facade.ts
modules/loader-utils/src/lib/filesystems/node-filesystem-facade.ts

modules/3d-tiles/test/lib/classes/tile-3d-batch-table-hierarchy.spec.ts

Expand Down
2 changes: 1 addition & 1 deletion docs/upgrade-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ and loaders.gl v4.0 aligns with this practice.

**@loaders.gl/crypto**

- All hashes now require an encoding parameter. To get previous behavior, just specify `'base64'`.
- All hashes now require an encoding parameter. To get previous behavior, just specify `.hash...(..., 'base64')`.

**@loaders.gl/arrow**

Expand Down
23 changes: 12 additions & 11 deletions modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-archive.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import md5 from 'md5';
import {FileProvider} from '@loaders.gl/loader-utils';
import {parseZipLocalFileHeader, HashElement, findBin} from '@loaders.gl/zip';
import {MD5Hash} from '@loaders.gl/crypto';
import {DeflateCompression, NoCompression} from '@loaders.gl/compression';
import {parseZipLocalFileHeader} from '@loaders.gl/zip';

type CompressionHandler = (compressedFile: ArrayBuffer) => Promise<ArrayBuffer>;

Expand All @@ -22,16 +22,16 @@ export class Tiles3DArchive {
/** FileProvider with whe whole file */
private fileProvider: FileProvider;
/** hash info */
private hashArray: HashElement[];
private hashTable: Record<string, bigint>;

/**
* creates Tiles3DArchive handler
* @param fileProvider - FileProvider with the whole file
* @param hashFile - hash info
* @param hashTable - hash info
*/
constructor(fileProvider: FileProvider, hashFile: HashElement[]) {
constructor(fileProvider: FileProvider, hashTable: Record<string, bigint>) {
this.fileProvider = fileProvider;
this.hashArray = hashFile;
this.hashTable = hashTable;
}

/**
Expand All @@ -47,7 +47,7 @@ export class Tiles3DArchive {
data = await this.getFileBytes(path);
}
if (!data) {
throw new Error('No such file in the archive');
throw new Error(`No such file in the archive: ${path}`);
}

return data;
Expand All @@ -59,13 +59,14 @@ export class Tiles3DArchive {
* @returns buffer with the raw file data
*/
private async getFileBytes(path: string): Promise<ArrayBuffer | null> {
const nameHash = Buffer.from(md5(path), 'hex');
const fileInfo = findBin(nameHash, this.hashArray); // implement binary search
if (!fileInfo) {
const arrayBuffer = new TextEncoder().encode(path).buffer;
const nameHash = await new MD5Hash().hash(arrayBuffer, 'hex');
const byteOffset = this.hashTable[nameHash];
if (byteOffset === undefined) {
return null;
}

const localFileHeader = await parseZipLocalFileHeader(fileInfo.offset, this.fileProvider);
const localFileHeader = await parseZipLocalFileHeader(byteOffset, this.fileProvider);
if (!localFileHeader) {
return null;
}
Expand Down
22 changes: 11 additions & 11 deletions modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-parser.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
import {FileProvider} from '@loaders.gl/loader-utils';
import {
HashElement,
cdSignature as cdHeaderSignature,
generateHashInfo,
parseHashFile,
makeHashTableFromZipHeaders,
parseHashTable,
parseZipCDFileHeader,
parseZipLocalFileHeader,
searchFromTheEnd
Expand All @@ -24,19 +23,20 @@ export const parse3DTilesArchive = async (

const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider);

let hashData: HashElement[];
let hashTable: Record<string, bigint>;
if (cdFileHeader?.fileName !== '@3dtilesIndex1@') {
cb?.('3tz doesnt contain hash file');
hashData = await generateHashInfo(fileProvider);
cb?.('hash info has been composed according to central directory records');
hashTable = await makeHashTableFromZipHeaders(fileProvider);
cb?.(
'3tz doesnt contain hash file, hash info has been composed according to zip archive headers'
);
} else {
cb?.('3tz contains hash file');
// cb?.('3tz contains hash file');
const localFileHeader = await parseZipLocalFileHeader(
cdFileHeader.localHeaderOffset,
fileProvider
);
if (!localFileHeader) {
throw new Error('corrupted 3tz');
throw new Error('corrupted 3tz zip archive');
}

const fileDataOffset = localFileHeader.fileDataOffset;
Expand All @@ -45,8 +45,8 @@ export const parse3DTilesArchive = async (
fileDataOffset + localFileHeader.compressedSize
);

hashData = parseHashFile(hashFile);
hashTable = parseHashTable(hashFile);
}

return new Tiles3DArchive(fileProvider, hashData);
return new Tiles3DArchive(fileProvider, hashTable);
};
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@ import {
cdSignature as cdHeaderSignature,
searchFromTheEnd,
parseZipCDFileHeader,
HashElement,
parseHashFile,
parseHashTable,
parseZipLocalFileHeader
} from '@loaders.gl/zip';
import {Tiles3DArchive} from '../../3d-tiles-archive/3d-tiles-archive-archive';
Expand All @@ -18,7 +17,7 @@ import {Tiles3DArchive} from '../../3d-tiles-archive/3d-tiles-archive-archive';
* @see https://github.com/erikdahlstrom/3tz-specification/blob/master/Specification.md
*/
export class Tiles3DArchiveFileSystem extends ZipFileSystem {
hashData?: HashElement[] | null;
hashTable?: Record<string, bigint> | null;

/**
* Constructor
Expand All @@ -36,13 +35,13 @@ export class Tiles3DArchiveFileSystem extends ZipFileSystem {
* @returns - Response with file data
*/
async fetch(filename: string): Promise<Response> {
const fileProvider = await this.fileProvider;
const fileProvider = this.fileProvider;
if (!fileProvider) {
throw new Error('No data detected in the zip archive');
}
await this.parseHashFile();
if (this.hashData) {
const archive = new Tiles3DArchive(fileProvider, this.hashData);
await this.parseHashTable();
if (this.hashTable) {
const archive = new Tiles3DArchive(fileProvider, this.hashTable);

const fileData = await archive.getFile(filename);
const response = new Response(fileData);
Expand All @@ -57,12 +56,12 @@ export class Tiles3DArchiveFileSystem extends ZipFileSystem {
* to files inside the archive
* @returns void
*/
private async parseHashFile(): Promise<void> {
if (this.hashData !== undefined) {
private async parseHashTable(): Promise<void> {
if (this.hashTable !== undefined) {
return;
}

const fileProvider = await this.fileProvider;
const fileProvider = this.fileProvider;
if (!fileProvider) {
throw new Error('No data detected in the zip archive');
}
Expand All @@ -89,9 +88,9 @@ export class Tiles3DArchiveFileSystem extends ZipFileSystem {
fileDataOffset + localFileHeader.compressedSize
);

this.hashData = parseHashFile(hashFile);
this.hashTable = parseHashTable(hashFile);
} else {
this.hashData = null;
this.hashTable = null;
}
}
}
8 changes: 4 additions & 4 deletions modules/3d-tiles/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -301,9 +301,9 @@ export type Tiles3DTileContent = {
*/
export type Subtree = {
/** An array of buffers. */
buffers: Buffer[];
buffers: GLTFStyleBuffer[];
/** An array of buffer views. */
bufferViews: BufferView[];
bufferViews: GLTFStyleBufferView[];
/** The availability of tiles in the subtree. The availability bitstream is a 1D boolean array where tiles are ordered by their level in the subtree and Morton index
* within that level. A tile's availability is determined by a single bit, 1 meaning a tile exists at that spatial index, and 0 meaning it does not.
* The number of elements in the array is `(N^subtreeLevels - 1)/(N - 1)` where N is 4 for subdivision scheme `QUADTREE` and 8 for `OCTREE`.
Expand Down Expand Up @@ -356,14 +356,14 @@ export type ExplicitBitstream = Uint8Array;
*/
export type SubdivisionScheme = 'QUADTREE' | 'OCTREE';

type Buffer = {
type GLTFStyleBuffer = {
name: string;
uri?: string;
byteLength: number;
};

/** Subtree buffer view */
export type BufferView = {
export type GLTFStyleBufferView = {
buffer: number;
byteOffset: number;
byteLength: number;
Expand Down
10 changes: 6 additions & 4 deletions modules/core/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,19 @@ export type {
DataType,
SyncDataType,
BatchableDataType,
ReadableFile,
WritableFile,
Stat,
FileSystem,
RandomAccessReadFileSystem
RandomAccessFileSystem
} from '@loaders.gl/loader-utils';

// FILE READING AND WRITING
export {fetchFile} from './lib/fetch/fetch-file';

export {readArrayBuffer} from './lib/fetch/read-array-buffer';
export {readFileSync} from './lib/fetch/read-file';
export {writeFile, writeFileSync} from './lib/fetch/write-file';
// export {readFileSync} from './lib/fetch/read-file';
// export {writeFile, writeFileSync} from './lib/fetch/write-file';

// CONFIGURATION
export {setLoaderOptions, getLoaderOptions} from './lib/api/loader-options';
Expand All @@ -39,7 +42,6 @@ export {loadInBatches} from './lib/api/load-in-batches';
export {encodeTable, encodeTableAsText, encodeTableInBatches} from './lib/api/encode-table';
export {encode, encodeSync, encodeInBatches, encodeURLtoURL} from './lib/api/encode';
export {encodeText, encodeTextSync} from './lib/api/encode';
export {save, saveSync} from './lib/api/save';

// CORE UTILS SHARED WITH LOADERS (RE-EXPORTED FROM LOADER-UTILS)
export {setPathPrefix, getPathPrefix, resolvePath} from '@loaders.gl/loader-utils';
Expand Down
6 changes: 3 additions & 3 deletions modules/core/src/lib/api/encode.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import {Writer, WriterOptions, canEncodeWithWorker} from '@loaders.gl/loader-utils';
import {concatenateArrayBuffers, resolvePath, NodeFile} from '@loaders.gl/loader-utils';
import {processOnWorker} from '@loaders.gl/worker-utils';
import {concatenateArrayBuffers, resolvePath} from '@loaders.gl/loader-utils';
import {isBrowser} from '@loaders.gl/loader-utils';
import {writeFile} from '../fetch/write-file';
import {fetchFile} from '../fetch/fetch-file';
import {getLoaderOptions} from './loader-options';

Expand Down Expand Up @@ -51,7 +50,8 @@ export async function encode(
if (!isBrowser && writer.encodeURLtoURL) {
// TODO - how to generate filenames with correct extensions?
const tmpInputFilename = getTemporaryFilename('input');
await writeFile(tmpInputFilename, data as ArrayBuffer);
const file = new NodeFile(tmpInputFilename, 'w');
await file.write(data as ArrayBuffer);

const tmpOutputFilename = getTemporaryFilename('output');

Expand Down
13 changes: 0 additions & 13 deletions modules/core/src/lib/api/save.ts

This file was deleted.

61 changes: 0 additions & 61 deletions modules/core/src/lib/fetch/fetch-file.node.ts

This file was deleted.

10 changes: 7 additions & 3 deletions modules/core/src/lib/fetch/fetch-file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import {resolvePath} from '@loaders.gl/loader-utils';
import {makeResponse} from '../utils/response-utils';
import * as node from './fetch-file.node';

export function isNodePath(url: string): boolean {
return !isRequestURL(url) && !isDataURL(url);
Expand All @@ -29,8 +28,13 @@ export async function fetchFile(
const url = resolvePath(urlOrData);

// Support fetching from local file system
if (isNodePath(url) && node?.fetchFileNode) {
return node.fetchFileNode(url, fetchOptions);
if (isNodePath(url)) {
if (globalThis.loaders?.fetchNode) {
return globalThis.loaders?.fetchNode(url, fetchOptions);
}
// throw new Error(
// 'fetchFile: globalThis.loaders.fetchNode not defined. Install @loaders.gl/polyfills'
// );
}

// Call global fetch
Expand Down
Loading

0 comments on commit 3db593c

Please sign in to comment.