From e14500324432037b8358cad0d92d279e685de933 Mon Sep 17 00:00:00 2001 From: mspivak-actionengine <118991512+mspivak-actionengine@users.noreply.github.com> Date: Sat, 14 Oct 2023 09:43:48 +0300 Subject: [PATCH] fix(gltf): fix of getTypedArrayForAccessor in gltf-scenegraph (#2683) --- modules/gltf/src/lib/api/gltf-scenegraph.ts | 22 ++---- .../src/lib/extensions/EXT_mesh_features.ts | 6 +- .../lib/extensions/utils/3d-tiles-utils.ts | 34 +++++----- .../src/lib/gltf-utils/get-typed-array.ts | 67 +++++++++++++++---- modules/gltf/src/lib/gltf-utils/gltf-utils.ts | 40 +---------- .../types/gltf-ext-mesh-features-schema.ts | 4 +- .../gltf/src/lib/types/gltf-json-schema.ts | 7 ++ modules/gltf/src/lib/types/gltf-types.ts | 2 +- .../lib/api/gltf-scenegraph-accessors.spec.ts | 56 +++++++++++++++- 9 files changed, 147 insertions(+), 91 deletions(-) diff --git a/modules/gltf/src/lib/api/gltf-scenegraph.ts b/modules/gltf/src/lib/api/gltf-scenegraph.ts index 801914c4f2..8ba2242bdb 100644 --- a/modules/gltf/src/lib/api/gltf-scenegraph.ts +++ b/modules/gltf/src/lib/api/gltf-scenegraph.ts @@ -19,11 +19,9 @@ import type { import {getBinaryImageMetadata} from '@loaders.gl/images'; import {padToNBytes, copyToArray} from '@loaders.gl/loader-utils'; import {assert} from '../utils/assert'; -import { - getAccessorArrayTypeAndLength, - getAccessorTypeFromSize, - getComponentTypeFromArray -} from '../gltf-utils/gltf-utils'; +import {getAccessorTypeFromSize, getComponentTypeFromArray} from '../gltf-utils/gltf-utils'; + +import {getTypedArrayForAccessor as _getTypedArrayForAccessor} from '../gltf-utils/get-typed-array'; type Extension = {[key: string]: any}; @@ -205,18 +203,8 @@ export class GLTFScenegraph { */ getTypedArrayForAccessor(accessor: number | object): any { // @ts-ignore - accessor = this.getAccessor(accessor); - // @ts-ignore - const bufferView = this.getBufferView(accessor.bufferView); - const buffer = this.getBuffer(bufferView.buffer); - // @ts-ignore - const arrayBuffer = buffer.data; - - // Create a new typed array as a view into the combined buffer - const {ArrayType, length} = getAccessorArrayTypeAndLength(accessor, bufferView); - // @ts-ignore - const byteOffset = bufferView.byteOffset + accessor.byteOffset; - return new ArrayType(arrayBuffer, byteOffset, length); + const gltfAccessor = this.getAccessor(accessor); + return _getTypedArrayForAccessor(this.gltf.json, this.gltf.buffers, gltfAccessor); } /** accepts accessor index or accessor object diff --git a/modules/gltf/src/lib/extensions/EXT_mesh_features.ts b/modules/gltf/src/lib/extensions/EXT_mesh_features.ts index 38452d007c..6510de8b3a 100644 --- a/modules/gltf/src/lib/extensions/EXT_mesh_features.ts +++ b/modules/gltf/src/lib/extensions/EXT_mesh_features.ts @@ -1,6 +1,7 @@ // GLTF EXTENSION: EXT_mesh_features // https://github.com/CesiumGS/glTF/tree/3d-tiles-next/extensions/2.0/Vendor/EXT_mesh_features /* eslint-disable camelcase */ +import type {NumericArray} from '@loaders.gl/loader-utils'; import type {GLTF, GLTFMeshPrimitive} from '../types/gltf-json-schema'; import {GLTFLoaderOptions} from '../../gltf-loader'; import type { @@ -63,12 +64,12 @@ function processMeshPrimitiveFeatures( } for (const featureId of featureIds) { - let featureIdData: number[] | null = null; + let featureIdData: NumericArray; // Process "Feature ID by Vertex" if (typeof featureId.attribute !== 'undefined') { const accessorKey = `_FEATURE_ID_${featureId.attribute}`; const accessorIndex = primitive.attributes[accessorKey]; - featureIdData = scenegraph.getTypedArrayForAccessor(accessorIndex) as number[]; + featureIdData = scenegraph.getTypedArrayForAccessor(accessorIndex); } // Process "Feature ID by Texture Coordinates" @@ -84,6 +85,7 @@ function processMeshPrimitiveFeatures( In this case, the featureCount must match the number of vertices of the mesh primitive. */ // TODO: At the moment of writing we don't have a tileset with the data of that kind. Implement it later. + featureIdData = []; } featureId.data = featureIdData; diff --git a/modules/gltf/src/lib/extensions/utils/3d-tiles-utils.ts b/modules/gltf/src/lib/extensions/utils/3d-tiles-utils.ts index 7568ba01e3..4053251389 100644 --- a/modules/gltf/src/lib/extensions/utils/3d-tiles-utils.ts +++ b/modules/gltf/src/lib/extensions/utils/3d-tiles-utils.ts @@ -12,7 +12,7 @@ import type {BigTypedArray, TypedArray} from '@loaders.gl/schema'; import type {ImageType} from '@loaders.gl/images'; import {GLTFScenegraph} from '../../api/gltf-scenegraph'; -import {getComponentTypeFromArray, getFloat32ArrayForAccessor} from '../../gltf-utils/gltf-utils'; +import {getComponentTypeFromArray} from '../../gltf-utils/gltf-utils'; import {getImageData} from '@loaders.gl/images'; import {emod} from '@loaders.gl/math'; @@ -151,7 +151,7 @@ export function getPrimitiveTextureData( scenegraph: GLTFScenegraph, textureInfo: GLTFTextureInfoMetadata, primitive: GLTFMeshPrimitive -): number[] | null { +): number[] { /* texture.index is an index for the "textures" array. The texture object referenced by this index looks like this: @@ -160,24 +160,20 @@ export function getPrimitiveTextureData( "source": 0 } "sampler" is an index for the "samplers" array - "source" is an index for the "images" array that contains data. These data are stored in rgba channels of the image. + "source" is an index for the "images" array that contains data stored in rgba channels of the image. texture.texCoord is a number-suffix (like 1) for an attribute like "TEXCOORD_1" in meshes.primitives - The value of "TEXCOORD_1" is an accessor that is used to get coordinates. These coordinates are being used to get data from the image. + The value of "TEXCOORD_1" is an accessor that is used to get coordinates. + These coordinates are being used to get data from the image. + + Default for texture.texCoord is 0 + @see https://github.com/CesiumGS/glTF/blob/3d-tiles-next/specification/2.0/schema/textureInfo.schema.json */ - const json = scenegraph.gltf.json; - const texCoordAccessorKey = `TEXCOORD_${textureInfo.texCoord || 0}`; const texCoordAccessorIndex = primitive.attributes[texCoordAccessorKey]; + const textureCoordinates: TypedArray = scenegraph.getTypedArrayForAccessor(texCoordAccessorIndex); - const textureCoordinates: Float32Array | null = getFloat32ArrayForAccessor( - scenegraph.gltf, - texCoordAccessorIndex - ); - if (!textureCoordinates) { - return null; - } - + const json = scenegraph.gltf.json; const textureIndex: number = textureInfo.index; const imageIndex = json.textures?.[textureIndex]?.source; if (typeof imageIndex !== 'undefined') { @@ -199,7 +195,7 @@ export function getPrimitiveTextureData( return textureData; } } - return null; + return []; } /** @@ -260,13 +256,17 @@ export function primitivePropertyDataToAttributes( * @param mimeType - MIME type. * @param textureCoordinates - uv coordinates to access data in the image. * @param index - Index of uv coordinates in the array textureCoordinates. - * @param channels - Image channels where data are stored. Channels of an RGBA texture are numbered 0..3 respectively. + * @param channels - Image channels where data are stored. + * Channels of an RGBA texture are numbered 0..3 respectively. + * For Ext_mesh_features and EXT_strucural_metadata the channels default is [0] + * @see https://github.com/CesiumGS/glTF/blob/3d-tiles-next/extensions/2.0/Vendor/EXT_mesh_features/schema/featureIdTexture.schema.json + * @see https://github.com/CesiumGS/glTF/blob/3d-tiles-next/extensions/2.0/Vendor/EXT_structural_metadata/schema/propertyTexture.property.schema.json * @returns Value taken from the image. */ function getImageValueByCoordinates( parsedImage: ImageType, mimeType: string | undefined, - textureCoordinates: Float32Array, + textureCoordinates: TypedArray, index: number, channels: number[] | string = [0] ) { diff --git a/modules/gltf/src/lib/gltf-utils/get-typed-array.ts b/modules/gltf/src/lib/gltf-utils/get-typed-array.ts index be4d5ad03d..26a9282259 100644 --- a/modules/gltf/src/lib/gltf-utils/get-typed-array.ts +++ b/modules/gltf/src/lib/gltf-utils/get-typed-array.ts @@ -1,5 +1,8 @@ // TODO - GLTFScenegraph should use these import {assert} from '../utils/assert'; +import type {TypedArray} from '@loaders.gl/schema'; +import type {GLTF, GLTFExternalBuffer, GLTFAccessor} from '../types/gltf-types'; +import {getAccessorArrayTypeAndLength} from './gltf-utils'; // accepts buffer view index or buffer view object // returns a `Uint8Array` @@ -24,18 +27,54 @@ export function getTypedArrayForImageData(json, buffers, imageIndex) { return getTypedArrayForBufferView(json, buffers, bufferViewIndex); } -/* -// accepts accessor index or accessor object -// returns a typed array with type that matches the types -export function getTypedArrayForAccessor(accessor) { - accessor = this.getAccessor(accessor); - const bufferView = this.getBufferView(accessor.bufferView); - const buffer = this.getBuffer(bufferView.buffer); - const arrayBuffer = buffer.data; - - // Create a new typed array as a view into the combined buffer - const {ArrayType, length} = getAccessorArrayTypeAndLength(accessor, bufferView); - const byteOffset = bufferView.byteOffset + accessor.byteOffset; - return new ArrayType(arrayBuffer, byteOffset, length); +/** + * Gets data pointed by the accessor. + * @param json - json part of gltf content of a GLTF tile. + * @param buffers - Array containing buffers of data. + * @param accessor - accepts accessor index or accessor object. + * @returns {TypedArray} Typed array with type matching the type of data poited by the accessor. + */ +// eslint-disable-next-line complexity +export function getTypedArrayForAccessor( + json: GLTF, + buffers: GLTFExternalBuffer[], + accessor: GLTFAccessor | number +): TypedArray { + const gltfAccessor = typeof accessor === 'number' ? json.accessors?.[accessor] : accessor; + if (!gltfAccessor) { + throw new Error(`No gltf accessor ${accessor}`); + } + const bufferView = json.bufferViews?.[gltfAccessor.bufferView || 0]; + if (!bufferView) { + throw new Error(`No gltf buffer view for accessor ${bufferView}`); + } + // Get `arrayBuffer` the `bufferView` looks at + const {arrayBuffer, byteOffset: bufferByteOffset} = buffers[bufferView.buffer]; + // Resulting byteOffset is sum of the buffer, accessor and bufferView byte offsets + const byteOffset = + (bufferByteOffset || 0) + (gltfAccessor.byteOffset || 0) + (bufferView.byteOffset || 0); + // Deduce TypedArray type and its length from `accessor` and `bufferView` data + const {ArrayType, length, componentByteSize, numberOfComponentsInElement} = + getAccessorArrayTypeAndLength(gltfAccessor, bufferView); + // 'length' is a whole number of components of all elements in the buffer pointed by the accessor + // Multiplier to calculate the address of the element in the arrayBuffer + const elementByteSize = componentByteSize * numberOfComponentsInElement; + const elementAddressScale = bufferView.byteStride || elementByteSize; + // Creare an array of component's type where all components (not just elements) will reside + if (typeof bufferView.byteStride === 'undefined' || bufferView.byteStride === elementByteSize) { + // No iterleaving + const result: TypedArray = new ArrayType(arrayBuffer, byteOffset, length); + return result; + } + // Iterleaving + const result: TypedArray = new ArrayType(length); + for (let i = 0; i < gltfAccessor.count; i++) { + const values = new ArrayType( + arrayBuffer, + byteOffset + i * elementAddressScale, + numberOfComponentsInElement + ); + result.set(values, i * numberOfComponentsInElement); + } + return result; } -*/ diff --git a/modules/gltf/src/lib/gltf-utils/gltf-utils.ts b/modules/gltf/src/lib/gltf-utils/gltf-utils.ts index dfaea695a7..f54b95693d 100644 --- a/modules/gltf/src/lib/gltf-utils/gltf-utils.ts +++ b/modules/gltf/src/lib/gltf-utils/gltf-utils.ts @@ -1,6 +1,5 @@ import {assert} from '../utils/assert'; -import type {GLTFWithBuffers} from '../types/gltf-types'; import type {GLTFPostprocessed} from '../types/gltf-postprocessed-schema'; import {BYTES, COMPONENTS} from '../gltf-utils/gltf-constants'; @@ -85,42 +84,9 @@ export function getAccessorArrayTypeAndLength(accessor, bufferView) { const length = accessor.count * components; const byteLength = accessor.count * components * bytesPerComponent; assert(byteLength >= 0 && byteLength <= bufferView.byteLength); - return {ArrayType, length, byteLength}; -} - -export function getFloat32ArrayForAccessor( - gltfData: GLTFWithBuffers, - texCoordAccessor: number -): Float32Array | null { - const accessor = gltfData.json.accessors?.[texCoordAccessor]; - if (accessor && typeof accessor.bufferView !== 'undefined') { - // Get `bufferView` of the `accessor` - const bufferView = gltfData.json.bufferViews?.[accessor.bufferView]; - if (bufferView) { - // Get `arrayBuffer` the `bufferView` look at - const {arrayBuffer, byteOffset: bufferByteOffset} = gltfData.buffers[bufferView.buffer]; - // Resulting byteOffset is sum of the buffer, accessor and bufferView byte offsets - const byteOffset = - (bufferByteOffset || 0) + (accessor.byteOffset || 0) + (bufferView.byteOffset || 0); - // Deduce TypedArray type and its length from `accessor` and `bufferView` data - const {ArrayType, length} = getAccessorArrayTypeAndLength(accessor, bufferView); - // Number of bytes each component occupies - const bytes = BYTES[accessor.componentType]; - // Number of components. For the `TEXCOORD_0` with `VEC2` type, it must return 2 - const components = COMPONENTS[accessor.type]; - // Multiplier to calculate the address of the `TEXCOORD_0` element in the arrayBuffer - const elementAddressScale = bufferView.byteStride || bytes * components; - // Data transform to Float32Array - const result = new Float32Array(length); - for (let i = 0; i < accessor.count; i++) { - // Take [u, v] couple from the arrayBuffer - const uv = new ArrayType(arrayBuffer, byteOffset + i * elementAddressScale, 2); - result.set(uv, i * components); - } - return result; - } - } - return null; + const componentByteSize = BYTES[accessor.componentType]; + const numberOfComponentsInElement = COMPONENTS[accessor.type]; + return {ArrayType, length, byteLength, componentByteSize, numberOfComponentsInElement}; } /** diff --git a/modules/gltf/src/lib/types/gltf-ext-mesh-features-schema.ts b/modules/gltf/src/lib/types/gltf-ext-mesh-features-schema.ts index 5fa662c4a1..930819df1b 100644 --- a/modules/gltf/src/lib/types/gltf-ext-mesh-features-schema.ts +++ b/modules/gltf/src/lib/types/gltf-ext-mesh-features-schema.ts @@ -34,7 +34,9 @@ export type GLTF_EXT_mesh_features_featureId = { * (e.g. a value of `0` corresponds to `_FEATURE_ID_0`). */ attribute?: number; - /** A texture containing feature IDs. */ + /** A texture containing feature IDs. + * @see https://github.com/CesiumGS/glTF/blob/c38f7f37e894004353c15cd0481bc5b7381ce841/extensions/2.0/Vendor/EXT_mesh_features/schema/featureIdTexture.schema.json + */ texture?: GLTFTextureInfoMetadata; /** The index of the property table containing per-feature property values. Only applicable when using the `EXT_structural_metadata` extension. */ propertyTable?: number; diff --git a/modules/gltf/src/lib/types/gltf-json-schema.ts b/modules/gltf/src/lib/types/gltf-json-schema.ts index 0bd5b0974e..061694ee7d 100644 --- a/modules/gltf/src/lib/types/gltf-json-schema.ts +++ b/modules/gltf/src/lib/types/gltf-json-schema.ts @@ -358,6 +358,8 @@ export type GLTFTextureInfo = { index: GLTFId; /** * The set index of texture's TEXCOORD attribute used for texture coordinate mapping. + * Default is 0 + * @see https://github.com/CesiumGS/glTF/blob/3d-tiles-next/specification/2.0/schema/textureInfo.schema.json */ texCoord?: number; extensions?: Record; @@ -371,6 +373,11 @@ export type GLTFTextureInfo = { * https://github.com/CesiumGS/glTF/tree/3d-tiles-next/extensions/2.0/Vendor/EXT_mesh_features */ export type GLTFTextureInfoMetadata = GLTFTextureInfo & { + /** + * For EXT_structural_metadata and Ext_mesh_features the channels default is [0] + * @see https://github.com/CesiumGS/glTF/blob/3d-tiles-next/extensions/2.0/Vendor/EXT_mesh_features/schema/featureIdTexture.schema.json + * @see https://github.com/CesiumGS/glTF/blob/3d-tiles-next/extensions/2.0/Vendor/EXT_structural_metadata/schema/propertyTexture.property.schema.json + */ channels: number[] | string; /** For internal usage */ data?: unknown; diff --git a/modules/gltf/src/lib/types/gltf-types.ts b/modules/gltf/src/lib/types/gltf-types.ts index 480b744d91..3993f82066 100644 --- a/modules/gltf/src/lib/types/gltf-types.ts +++ b/modules/gltf/src/lib/types/gltf-types.ts @@ -13,7 +13,7 @@ export type GLTFWithBuffers = { images?: GLTFExternalImage[]; }; -type GLTFExternalBuffer = { +export type GLTFExternalBuffer = { arrayBuffer: ArrayBuffer; byteOffset: number; byteLength: number; diff --git a/modules/gltf/test/lib/api/gltf-scenegraph-accessors.spec.ts b/modules/gltf/test/lib/api/gltf-scenegraph-accessors.spec.ts index e504201b88..f97ef96673 100644 --- a/modules/gltf/test/lib/api/gltf-scenegraph-accessors.spec.ts +++ b/modules/gltf/test/lib/api/gltf-scenegraph-accessors.spec.ts @@ -1,9 +1,9 @@ /* eslint-disable max-len */ import test from 'tape-promise/tape'; -import {GLTFScenegraph} from '@loaders.gl/gltf'; -import {GLTFLoader} from '@loaders.gl/gltf'; +import {GLTFScenegraph, GLTFLoader, GLTFAccessor} from '@loaders.gl/gltf'; import {load} from '@loaders.gl/core'; +import type {TypedArray} from '@loaders.gl/schema'; // Extracted from Cesium 3D Tiles const GLB_TILE_WITH_DRACO_URL = '@loaders.gl/gltf/test/data/3d-tiles/143.glb'; @@ -79,3 +79,55 @@ test('GLTFScenegraph#BufferView indices resolve correctly', async (t) => { t.end(); }); + +test('GLTFScenegraph#Typed Arrays sgould be taken by Accessor', async (t) => { + const GLB_ACCESSOR_URL = '@loaders.gl/gltf/test/data/glb/DamagedHelmet.glb'; + const testDataSet = [ + { + accessorIndex: 0, + accessorCountExpected: 46356, + arrayExpected: [0, 1, 2, 2, 3, 0, 3, 2] + }, + { + accessorIndex: 1, + accessorCountExpected: 14556, + arrayExpected: [ + -0.6119945645332336, -0.03094087541103363, 0.48309004306793213, -0.5795046091079712, + 0.05627411603927612, 0.5217580199241638, -0.5735836029052734, 0.06353411078453064 + ] + } + ]; + const gltfWithBuffers = await load(GLB_ACCESSOR_URL, GLTFLoader); + const gltfScenegraph = new GLTFScenegraph(gltfWithBuffers); + + for (const testData of testDataSet) { + let typedArray: TypedArray = gltfScenegraph.getTypedArrayForAccessor(testData.accessorIndex); + t.deepEquals( + typedArray.slice(0, 8), + testData.arrayExpected, + 'typed array taken by accessor as a number' + ); + + const accessor: GLTFAccessor = gltfScenegraph.getAccessor(testData.accessorIndex); + t.equals(accessor.count, testData.accessorCountExpected, 'first accessor taken'); + + typedArray = gltfScenegraph.getTypedArrayForAccessor(accessor); + t.deepEquals( + typedArray.slice(0, 8), + testData.arrayExpected, + 'typed array taken by accessor as an object' + ); + + if (accessor.bufferView === 0) { + accessor.bufferView = undefined; + // default bufferView should be 0 + typedArray = gltfScenegraph.getTypedArrayForAccessor(accessor); + t.deepEquals( + typedArray.slice(0, 8), + testData.arrayExpected, + 'typed array taken by accessor as object with the bufferView set to undefined' + ); + } + } + t.end(); +});