diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml new file mode 100644 index 0000000..8ad2fa3 --- /dev/null +++ b/.github/workflows/audit.yml @@ -0,0 +1,33 @@ +# Audit dependencies against the RustSec Advisory DB (https://rustsec.org/advisories/) +name: Audit + +on: + push: + paths: + # Run if workflow changes + - '.github/workflows/audit.yml' + # Run on changed dependencies + - '**/Cargo.toml' + - '**/Cargo.lock' + # Run if the configuration file changes + - '**/audit.toml' + # Rerun periodicly to pick up new advisories + schedule: + - cron: '0 0 * * *' + # Run manually + workflow_dispatch: + +permissions: read-all + +jobs: + audit: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Audit Rust Dependencies + uses: actions-rust-lang/audit@v1 + with: + denyWarnings: true + createIssues: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..07fb9b0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,72 @@ +name: CI + +on: [push] + +permissions: + contents: read + +env: + RUST_BACKTRACE: 1 + +jobs: + check: + name: Check + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + components: rustfmt + + - name: Check + run: cargo check --all + + - name: Check featuresless + run: cargo check --all --no-default-features + + - name: Clippy + run: cargo clippy --all + + - name: Format + uses: actions-rust-lang/rustfmt@v1 + + test: + strategy: + fail-fast: false + matrix: + include: + - { rust: stable, os: ubuntu-latest } + - { rust: stable, os: windows-latest } + - { rust: stable, os: macos-latest } + - { rust: beta, os: ubuntu-latest } + # Turn this on once we have a MSRV (minimum supported rust version) + # - { rust: 1.70.0, os: ubuntu-latest } + name: Test Rust ${{ matrix.rust }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: ${{ matrix.rust }} + + - name: Test + run: cargo test --all + + - name: Build + run: cargo build --all --release + + - name: C Examples + if: runner.os != 'Windows' + working-directory: ./omf-c/examples + run: bash ./build.sh + + - name: C Examples (Windows) + if: runner.os == 'Windows' + working-directory: ./omf-c/examples + run: ./build.bat diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..ad5da75 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,51 @@ +name: Docs + +on: + push: + branches: ["main"] + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: true + +jobs: + docs: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + + - name: Build docs + run: | + cd docs + sh build.sh + + - name: Fix permissions + run: | + chmod -c -R +rX "./site/" | while read line; do + echo "::warning title=Invalid file permissions automatically fixed::$line" + done + + - name: Store artifact + uses: actions/upload-pages-artifact@v2 + with: + path: site/ + + deploy: + needs: docs + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - name: Deploy to pages + id: deployment + uses: actions/deploy-pages@v2 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..88a9392 --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +# Generated by Cargo +# will have compiled files and executables +target/ + +# Generated by CMake. +omf-c/examples/build/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb + +# Generated code. +/site/ +/docs/schema/ +/docs/parquet/ + +# Virtual environment for mkdocs +/venv/ diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..8468564 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,14 @@ +{ + "cSpell.language": "en", + "cSpell.words": [ + "colormap", + "colormaps", + "grayscale", + "octree", + "RGBA", + "struct", + "structs", + "subblock" + ], + "cmake.configureOnOpen": false +} \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..48e47be --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,51 @@ +[package] +name = "omf" +version = "0.1.0-beta.1" +description = "File reader and writer for Open Mining Format." +authors = ["Tim Evans "] +license = "MIT" +edition = "2021" +publish = true +exclude = ["/.github", "/.vscode"] + +[dependencies] +bytes = { workspace = true, optional = true } +chrono.workspace = true +flate2.workspace = true +image = { workspace = true, optional = true } +parquet = { workspace = true, optional = true } +schemars.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +zip.workspace = true + +[dev-dependencies] +bytes.workspace = true +regex.workspace = true + +[features] +default = ["image", "parquet", "omf1"] +image = ["dep:image"] +parquet = ["dep:parquet", "dep:bytes"] +omf1 = ["parquet"] + +[workspace] +members = ["omf-c"] + +[workspace.dependencies] +bytes = "1" +cbindgen = { version = "0.26", default-features = false } +chrono = { version = "0.4", default-features = false, features = ["serde"] } +flate2 = "1.0" +image = { version = "0.25", default-features = false, features = [ + "png", + "jpeg", +] } +parquet = { version = "51", default-features = false, features = ["flate2"] } +regex = "1" +schemars = { version = "0.8", features = ["chrono"] } +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["float_roundtrip"] } +thiserror = "1" +zip = { version = "2", default-features = false } diff --git a/README.md b/README.md index 732c199..bd5924b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,16 @@ +[![CI](https://github.com/gmggroup/omf-rust/actions/workflows/ci.yml/badge.svg)](https://github.com/gmggroup/omf-rust/actions/workflows/ci.yml) +[![Audit](https://github.com/gmggroup/omf-rust/actions/workflows/audit.yml/badge.svg)](https://github.com/gmggroup/omf-rust/actions/workflows/audit.yml) + # OMF -A library for reading a writing files in Open Mining Format 2.0. +A library for reading and writing files in Open Mining Format 2.0. +Also supports translating OMF 1 files to OMF 2. + +OMF file version: 2.0-beta.1 + +Crate version: 0.1.0-beta.1 + +**Warning:** this is pre-release code. ## What is OMF @@ -28,6 +38,7 @@ plus a wrapper to use that library from C. - Free-form sub-blocks that don't lie on any grid. - Composite elements made out of any of the above. + ### Attributes - Floating-point or signed integer values. @@ -44,3 +55,36 @@ Attributes values can be valid or null. They can be attached to different parts of each element type, such as the vertices vs. faces of a surface, or the parent blocks vs. sub-blocks of a block model. + +## Compiling + +First [install Rust](https://www.rust-lang.org/tools/install). +Run `cargo build --all --release` in the root directory to build the release version of the Rust +crate and C wrapper. +The C wrapper build will place `omf.h` and the platform-specific shared library files +(e.g.: `omfc.dll` and `omfc.dll.lib` for Windows) in `target/release`. + +You can the `--release` argument off to build a debug version. +This may be useful for debugging C code that calls into OMF for example, +but it will be slow. + +For the Rust tests, run `cargo test --all`. + +To build and run the C examples: + +1. Run `cargo build --all --release` first. +2. Change directory into `omf-c/examples/`. +3. Run `build.bat` on Windows or `build.sh` on Linux/MacOS. + +This will build all examples, run them, and compare the results to the benchmarks. + +## Documentation + +The documentation is built with [MkDocs](https://www.mkdocs.org/). +To build locally: + +1. Create and activate a Python virtual environment. +2. Change directory into `docs/`. +3. Run `build.bat` on Windows or `build.sh` on Linux/MacOS. + +This will install the required dependencies, then build the file format, Rust, and C documentation into `site/`. diff --git a/docs/build.bat b/docs/build.bat new file mode 100644 index 0000000..b993a88 --- /dev/null +++ b/docs/build.bat @@ -0,0 +1,8 @@ +@echo off +python -m pip install -r requirements.txt || exit /b +cd .. +cargo test -p omf --lib -- --ignored update_schema_docs || exit /b +cargo doc --no-deps || exit /b +python -m mkdocs build || exit /b +mv ./target/doc ./site/rust || exit /b +rm ./site/rust/.lock || exit /b diff --git a/docs/build.sh b/docs/build.sh new file mode 100644 index 0000000..e710ccc --- /dev/null +++ b/docs/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/sh +set -e +python -m pip install -r requirements.txt +cd .. +cargo test -p omf --lib -- --ignored update_schema_docs +cargo doc --no-deps +python -m mkdocs build +mv ./target/doc ./site/rust +rm ./site/rust/.lock diff --git a/docs/c/arrays.md b/docs/c/arrays.md new file mode 100644 index 0000000..f07f80d --- /dev/null +++ b/docs/c/arrays.md @@ -0,0 +1,144 @@ +# Arrays + +Types that describe data arrays within the OMF file. + +## OmfArrayType + +Contains the type of an array. + +```c +typedef enum { ... } OmfArrayType; +``` + +### Options + +OMF_ARRAY_TYPE_IMAGE +: An array that stores a single image. + +OMF_ARRAY_TYPE_SCALARS32 +: Scalar data in 32-bit floating-point. + +OMF_ARRAY_TYPE_SCALARS64 +: Scalar data in 64-bit floating-point. + +OMF_ARRAY_TYPE_VERTICES32 +: Vertex positions in 32-bit floating-point. + +OMF_ARRAY_TYPE_VERTICES64 +: Vertex positions in 64-bit floating-point. + +OMF_ARRAY_TYPE_SEGMENTS +: Segments as pairs of indices into a vertex array. + +OMF_ARRAY_TYPE_TRIANGLES +: Triangles as triples of indices into a vertex array. +Winding is counter-clockwise around an outward-pointing normal. + +OMF_ARRAY_TYPE_NAMES +: Category names. + +OMF_ARRAY_TYPE_GRADIENT +: Category or colormap colors. + +OMF_ARRAY_TYPE_TEXCOORDS32 +: UV texture coordinates in 32-bit floating-point. + +OMF_ARRAY_TYPE_TEXCOORDS64 +: UV texture coordinates in 64-bit floating-point. + +OMF_ARRAY_TYPE_BOUNDARIES_FLOAT32 +: Discrete colormap boundaries, storing a value and inclusive flag, in 32-bit floating-point. + +OMF_ARRAY_TYPE_BOUNDARIES_FLOAT64 +: Discrete colormap boundaries, storing a value and inclusive flag, in 64-bit floating-point. + +OMF_ARRAY_TYPE_BOUNDARIES_INT64 +: Discrete colormap boundaries, storing a value and inclusive flag, in 64-bit integer. + +OMF_ARRAY_TYPE_BOUNDARIES_DATE +: Discrete colormap boundaries, storing a value and inclusive flag, in days since the epoch. + +OMF_ARRAY_TYPE_BOUNDARIES_DATE_TIME +: Discrete colormap boundaries, storing a value and inclusive flag, in microseconds since the epoch. + +OMF_ARRAY_TYPE_REGULAR_SUBBLOCKS +: Regular sub-block parent indices and min/max corners. + +OMF_ARRAY_TYPE_FREEFORM_SUBBLOCKS32 +: Free-form sub-block parent indices, and min/max corners in 32-bit floating-point. + +OMF_ARRAY_TYPE_FREEFORM_SUBBLOCKS64 +: Free-form sub-block parent indices, and min/max corners in 64-bit floating-point. + +OMF_ARRAY_TYPE_NUMBERS_FLOAT32 +: Nullable number attribute values, in 32-bit floating-point. + +OMF_ARRAY_TYPE_NUMBERS_FLOAT64 +: Nullable number attribute values, in 64-bit floating-point. + +OMF_ARRAY_TYPE_NUMBERS_INT64 +: Nullable number attribute values, in 64-bit integer. + +OMF_ARRAY_TYPE_NUMBERS_DATE +: Nullable number attribute values, in days since the epoch. + +OMF_ARRAY_TYPE_NUMBERS_DATE_TIME +: Nullable number attribute values, in microseconds since the epoch. + +OMF_ARRAY_TYPE_INDICES +: Nullable index values for category attributes. + +OMF_ARRAY_TYPE_VECTORS32X2 +: Nullable 2D vectors in 32-bit floating-point. + +OMF_ARRAY_TYPE_VECTORS64X2 +: Nullable 2D vectors in 64-bit floating-point. + +OMF_ARRAY_TYPE_VECTORS32X3 +: Nullable 3D vectors in 32-bit floating-point. + +OMF_ARRAY_TYPE_VECTORS64X3 +: Nullable 3D vectors in 64-bit floating-point. + +OMF_ARRAY_TYPE_TEXT +: Nullable text in UTF-8 encoding and nul-terminated. +Applications may treat empty an null strings as the same thing. + +OMF_ARRAY_TYPE_BOOLEANS +: Nullable boolean values: true, false, or null. +Applications that don't support three-valued logic may treat null as false. + +OMF_ARRAY_TYPE_COLORS +: Nullable RGBA colors, 8-bits per channel. + + +## OmfArray + +References an array in the OMF file. +A single type is used to store references to all the different array types. +See [`OmfReader`](reader.md) and [`OmfWriter`](writer.md) for the functions that create and consume arrays. + +```c +typedef struct { /* private fields */ } OmfArray; +``` + +## OmfArrayInfo + +```c +typedef struct { + OmfArrayType array_type; + uint64_t item_count; + uint64_t compressed_size; +} OmfArrayInfo; +``` + +### Fields + +array_type: [`OmfArrayType`](#OmfArrayType) +: The type of data that the array stores. + +item_count: `uint64_t` +: The number of items in the array. + +compressed_size: `uint64_t` +: The compressed size of the array inside the Zip archive. diff --git a/docs/c/attribute.md b/docs/c/attribute.md new file mode 100644 index 0000000..9aac75b --- /dev/null +++ b/docs/c/attribute.md @@ -0,0 +1,284 @@ +# Attribute + +## OmfLocation + +```c +typedef enum { + OMF_LOCATION_VERTICES, + OMF_LOCATION_PRIMITIVES, + OMF_LOCATION_SUBBLOCKS, + OMF_LOCATION_ELEMENTS, + OMF_LOCATION_PROJECTED, + OMF_LOCATION_CATEGORIES, +} OmfLocation; +``` + +Defines what part of the geometry an attribute is applied to. +The documentation of each geometry includes what locations are valid for it. + +### Options + +OMF_LOCATION_VERTICES +: The attribute contains one value for each point, vertex, or block corner. + +OMF_LOCATION_PRIMITIVES +: The attribute contains one value for each line segment, triangle, or block. +For sub-blocked block models that means parent blocks. + +OMF_LOCATION_SUBBLOCKS +: The attribute contains one value for each sub-block in a block model. + +OMF_LOCATION_ELEMENTS +: The attribute contains one value for each sub-element in composite. + +OMF_LOCATION_PROJECTED +: Used for projected textures. +The texture is projected onto the element. + +OMF_LOCATION_CATEGORIES +: Used for category sub-attributes. +The attribute contains one value for each category. + + +## OmfAttribute + +```c +typedef struct { + const char *name; + const char *description; + const char *units; + size_t n_metadata; + const OmfValue *metadata; + OmfLocation location; + const OmfArray *boolean_data; + const OmfArray *vector_data; + const OmfArray *text_data; + const OmfArray *color_data; + const OmfNumberData *number_data; + const OmfCategoryData *category_data; + const OmfMappedTexture *mapped_texture_data; + const OmfProjectedTexture *projected_texture_data; +} OmfAttribute; +``` + +Contains attribute data and defines how it is applied to an element. +Exactly one data pointer must be non-null, defining the type of the attribute. + +### Fields + +name: `const char *` +: Attribute name, which should be unique within the containing element. + +description: `const char *` +: Optional attribute description or comments. + +units: `const char *` +: Optional attribute units, if applicable. +OMF does not currently attempt to standardize the strings you can use here, but our recommendations are: + + - Use full names, so "kilometers" rather than "km". + The abbreviations for non-metric units aren't consistent and complex units can be confusing. + - Use plurals, so "feet" rather than "foot". + - Avoid ambiguity, so "long tons" rather than just "tons". + - Accept American and British spellings, so "meter" and "metre" are the same. + +metadata: [`const OmfValue *`](metadata.md#omfvalue) +: Pointer to an array of `n_metadata` metadata items, forming a set of key/value pairs. + +n_attributes: `size_t ` +: Number of attributes. + +location: [`OmfLocation`](#omflocation) +: Defines where on the containing element this attribute is attached. + +boolean_data: [`const OmfArray *`](arrays.md#omfarray) +: Boolean array containing true/false/null data. +Applications that don't support three-valued logic may treat null as false. + +vector_data: [`const OmfArray *`](arrays.md#omfarray) +: Vector array. Items are nullable. + +text_data: [`const OmfArray *`](arrays.md#omfarray) +: Text array. Items are nullable. + +color_data: [`const OmfArray *`](arrays.md#omfarray) +: Color array. Items are nullable. + +number_data: [`const OmfNumberData *`](#omfnumberdata) +: Pointer to a number-data struct including the array and optional color-map. + +category_data: [`const OmfCategoryData *`](#omfcategorydata) +: Pointer to a category-data struct including the index array, plus category names, +optional colors, and sub-attributes. + +mapped_texture_data: [`const OmfMappedTexture *`](#omfmappedtexture) +: Pointer to a mapped texture struct. + +projected_texture_data: [`const OmfProjectedTexture *`](#omfprojectedtexture) +: Pointer to a projected texture struct. + + +### Methods + +#### omf_attribute_init + +```c +OmfAttribute omf_attribute_init(const char *name, OmfLocation location); +``` + +Initializes or resets an attribute struct. + + +## OmfNumberData + +```c +typedef struct { + const OmfArray *values; + const OmfContinuousColormap *continuous_colormap; + const OmfDiscreteColormap *discrete_colormap; +} OmfNumberData; +``` + +### Fields + +values: `const OmfArray *` +: Number array. Can have 32- or 64-bit floating-point, 64-bit signed integer, +date, or date-time type. + +continuous_colormap: `const OmfContinuousColormap *` +: Optional continuous colormap. +Only one of `continuous_colormap` and `discrete_colormap` may be non-null, +or both may be null. + +discrete_colormap: `const OmfDiscreteColormap *` +: Optional discrete colormap. + +### Methods + +#### omf_number_data_init + +```c +OmfNumberData omf_number_data_init(void); +``` + +Initializes or resets a number attribute data struct. + + +## OmfCategoryData + +```c +typedef struct { + const OmfArray *values; + const OmfArray *names; + const OmfArray *colors; + const OmfAttribute *attributes; + size_t n_attributes; +} OmfCategoryData; +``` + +Describes a category attribute. + +### Fields + +values: `const OmfArray *` +: Index array into `names`, `colors`, and other attributes. Indices are nullable. + +names: `const OmfArray *` +: Name array for category names or labels. + +colors: `const OmfArray *` +: Optional gradient array for category colors. +If non-null, must be the same length as `names`. + +attributes: `const OmfAttribute *` +: Pointer to an array of `n_attributes` attribute structures. +Each must use `OMF_LOCATION_CATEGORIES` and be the same length as `names`. +This can be used to add extra details to a category, such as density on a rock-type attribute. + +n_attributes: `size_t` +: The number of sub-attributes. + + +### Methods + +#### omf_category_data_init + +```c +OmfCategoryData omf_category_data_init(void); +``` + +Initializes or resets a category attribute data struct. + + +## OmfMappedTexture + +```c +typedef struct { + const OmfArray *image; + const OmfArray *texcoords; +} OmfMappedTexture; +``` + +A texture applied with [UV mapping](https://en.wikipedia.org/wiki/UV_mapping). +Typically applied to surface vertices; applications may ignore other locations. + +### Fields + +image: `const OmfArray *` +: Image array containing the texture image. + +texcoords: [`const OmfArray *`](arrays.md#omfarray) +: Texture coordinate array, +Values outside of the range 0–1 will cause the texture to wrap. + +### Methods + +#### omf_mapped_texture_init + +```c +OmfMappedTexture omf_mapped_texture_init(const OmfArray *image, + const OmfArray *texcoords); +``` + +Initializes or resets a mapped texture struct. + + +## OmfProjectedTexture + +```c +typedef struct { + const OmfArray *image; + OmfOrient2 orient; + double width; + double height; +} OmfProjectedTexture; +``` + +A texture that is orthographically projected through space. +Use this for maps and section images. +The fields define a rectangle in space and the texture is projected in both directions along its normal. +Typically applied to surface vertices; applications may ignore other locations. + +### Fields + +image: `const OmfArray *` +: Image array containing the texture image. + +orient: `OmfOrient2` +: The position and orientation of the texture rectangle in space. + +width: `double` +: The width of the texture rectangle in space. + +height: `double` +: The height of the texture rectangle in space. + +### Methods + +#### omf_projected_texture_init + +```c +OmfProjectedTexture omf_projected_texture_init(const OmfArray *image); +``` + +Initializes or resets a projected texture struct. diff --git a/docs/c/colormap.md b/docs/c/colormap.md new file mode 100644 index 0000000..22a5559 --- /dev/null +++ b/docs/c/colormap.md @@ -0,0 +1,113 @@ +# Colormaps + +## OmfContinuousColormap + +```c +typedef struct { + double min; + double max; + const OmfArray *gradient; +} OmfContinuousColormap; +``` + +A continuous colormap linearly samples a color gradient within a defined range. + +A value X% of way between `min` and `max` should use the color from X% way down gradient. When that X doesn't land directly on a color use the average of the colors on either side, inverse-weighted by the distance to each. + +Values below the minimum use the first color in the gradient array. Values above the maximum use the last. + +![Diagram of a continuous colormap](../images/colormap_continuous.svg "Continuous colormap") + +### Fields + +min: `double` +: Minimum value. + +max: `double` +: Maximum value. + +gradient: [`const OmfArray *`](arrays.md#omfarray) +: Gradient array. + +### Methods + +#### omf_continuous_colormap_init + +```c +OmfContinuousColormap omf_continuous_colormap_init(double min, double max, + const OmfArray *gradient); +``` + +Initializes or resets a continuous colormap struct. + + +## OmfIntegerContinuousColormap + +```c +typedef struct { + int64_t min; + int64_t max; + const OmfArray *gradient; +} OmfIntegerContinuousColormap; +``` + +The same as [`OmfContinuousColormap`](#omfcontinuouscolormap) except that the min and max are +`uint64_t` so they can be precise across the full range of possible values. + + +## OmfDiscreteColormap + +```c +typedef struct { + const OmfArray *boundaries; + const OmfArray *gradient; +} OmfIntegerContinuousColormap; +``` + +A discrete colormap divides the whole number line into non-overlapping ranges and gives a color to each range. +To calculate the color $c$ from the value $v$, boundaries $b$, and gradient $g$: + +$$ +c = \left\{ + \begin{array}{ll} + g_0 & \quad \text{if } v < b_0 \\ + g_1 & \quad \text{if } b_0 \le v < b_1 \\ + \vdots & \quad \vdots \\ + g_n & \quad \text{if } b_{n-1} \le v < b_n \\ + g_{n+1} & \quad \text{if } v \ge b_n \\ + \end{array} +\right. +$$ + +The inclusive array contains boolean values that, if true, change the match from $<$ to $\le$, +so values exactly equal to the boundary will use the color of the range below it rather than above. +Its length must match the length of the boundaries array, +or it can be omitted in which case all matches are exclusive. + +The color gradient array must be one element longer than boundaries, +with the extra color used for values above the last boundary. + +### Fields + +boundaries: [`const OmfArray *`](arrays.md#omfarray) +: Boundary array giving the value and inclusive flag for the edges of each discrete range, +in increasing order. +The boundary number type should match the number attribute type. +Values must increase along the array. + +gradient: [`const OmfArray *`](arrays.md#omfarray) +: Gradient array. +Must have length one larger than `boundaries`, +with the extra color being used for values above the last boundary. + +### Methods + +#### omf_discrete_colormap_init + +```c +OmfDiscreteColormap omf_discrete_colormap_init(const OmfArray *boundaries, + const OmfArray *inclusive, + const OmfArray *gradient); +``` + +Initializes or resets a discrete colormap struct. diff --git a/docs/c/element.md b/docs/c/element.md new file mode 100644 index 0000000..54d5de4 --- /dev/null +++ b/docs/c/element.md @@ -0,0 +1,72 @@ +## OmfElement + +```c +typedef struct { + const char *name; + const char *description; + bool color_set; + uint8_t color[3]; + double opacity; + size_t n_metadata; + const OmfValue *metadata; + size_t n_attributes; + const OmfAttribute *attributes; + const OmfPointSet *point_set; + const OmfLineSet *line_set; + const OmfSurface *surface; + const OmfGridSurface *grid_surface; + const OmfBlockModel *block_model; + const OmfComposite *composite; +} OmfElement; +``` + +Describes an object or shape within an OMF file, and links to attributes that attach data to it. + +### Fields + +name: `const char *` +: Element name, which should be unique. + +description: `const char *` +: Optional element description or comments. + +color_set: `bool ` +: Whether `color` should be used. + +color: `uint8_t[3] ` +: Optional solid RGB color of the element. + +opacity: `double ` +: Element opacity, from 0.0 for transparent to 1.0 for opaque. + +n_metadata: `size_t` +: Number of metadata items. + +metadata: [`const OmfValue *`](metadata.md#omfvalue) +: Pointer to an array of `n_metadata` metadata items, forming a set of key/value pairs. + +n_attributes: `size_t ` +: Number of attributes. + +attributes: [`const OmfAttribute *`](attribute.md#omfattribute) +: Pointer to an array of `n_attributes` attributes on this element. + +point_set: [`const OmfPointSet *`](geometry/pointset.md) +line_set: [`const OmfLineSet *`](geometry/lineset.md) +surface: [`const OmfSurface *`](geometry/surface.md) +grid_surface: [`const OmfGridSurface *`](geometry/gridsurface.md) +block_model: [`const OmfBlockModel *`](geometry/blockmodel.md) +composite: [`const OmfComposite *`](geometry/composite.md) +: Exactly one of these must be non-null. This defines the type of geometry the element has +and contains the details of it. + + +### Methods + +#### omf_element_init + +```c +struct OmfElement omf_element_init(const char *name); +``` + +Initializes or resets an element struct. diff --git a/docs/c/errors.md b/docs/c/errors.md new file mode 100644 index 0000000..63d7f7d --- /dev/null +++ b/docs/c/errors.md @@ -0,0 +1,186 @@ +# Errors + +## OmfError + +```c +typedef struct OmfError { + int32_t code; + int32_t detail; + const char *message; +} OmfError; +``` + +Stores an error code and message. + +### Fields + +code: `int32_t` +: An [`OmfStatus`](#OmfStatus) value for the error. + +detail: `int32_t` +: If `code` is `OMF_STATUS_IO_ERROR` this is the system error number, otherwise zero. + +message: `const char*` +: Human-readable error message in US-English. + +### Methods + +#### omf_error + +```c +OmfError* omf_error(void); +``` + +Returns and clears the error state of the current thread. +This is the first error that occurred since the last call to `omf_error` or `omf_error_clear`, +or null if no error occurred. + +Pass the returned pointer to `omf_error_free` once you're finished with it. + +#### omf_error_free + +```c +void omf_error_free(OmfError *error); +``` + +Frees an error pointer returned by `omf_error`. Does nothing if `error` is null. + +#### omf_error_clear + +```c +void omf_error_clear(void); +``` + +Clears the error state of the current thread, discarding an recorded error. + +#### omf_error_peek + +```c +int32_t omf_error_peek(void); +``` + +Returns the [`OmfStatus`](#OmfStatus) code if an error occurred on the current thread, +or zero if no error. +You can use this to check for errors before calling a handler that will retrieve the full error. + + +## OmfStatus + +```c +typedef enum { + OMF_STATUS_SUCCESS = 0, + ... +} OmfStatus; +``` + +This enum defines the error codes that the library can produce. +Errors will also come with a message string that gives more details. + +### Options + +OMF_STATUS_SUCCESS = 0 +: No error occurred. + +OMF_STATUS_PANIC +: Unexpected failure. + +OMF_STATUS_INVALID_ARGUMENT +: An invalid argument was passed, such as a pointer being null when that isn't allowed, +or a string not being in UTF-8 encoding. + +OMF_STATUS_INVALID_CALL +: A method call was invalid, such as trying to load the project twice from one reader. + +OMF_STATUS_OUT_OF_MEMORY +: Failed to allocate enough memory. + +OMF_STATUS_IO_ERROR +: File input or output error from the operating system. + +OMF_STATUS_NOT_OMF +: The file is not in OMF format. + +OMF_STATUS_NEWER_VERSION +: The file version is newer than what this library can load. + +OMF_STATUS_PRE_RELEASE +: The file has a pre-release version which can't be loaded. + +OMF_STATUS_DESERIALIZATION_FAILED +: JSON deserialization error when reading a file. + +OMF_STATUS_SERIALIZATION_FAILED +: JSON serialization error when writing a file. + +OMF_STATUS_VALIDATION_FAILED +: The file contains invalid info. + +OMF_STATUS_LIMIT_EXCEEDED +: A safety limit was exceeded when reading. + +OMF_STATUS_NOT_IMAGE_DATA +: Image bytes are not in PNG or JPEG format. + +OMF_STATUS_NOT_PARQUET_DATA +: Array bytes are not in Parquet format. + +OMF_STATUS_ARRAY_TYPE_WRONG +: An incorrect array type was used, such as passing triangles where segments are expected. +Note that even types that look superficially similar aren't the same, +such as 3D vectors and vertices different because the vectors are nullable. + +OMF_STATUS_BUFFER_LENGTH_WRONG +: Tried to read an array into a buffer with a different length. + +OMF_STATUS_INVALID_DATA +: Array data in the file is invalid, +such as having a triangle index that is larger than the number of vertices. + +OMF_STATUS_UNSAFE_CAST +: Attempted a cast that would lose data. +Most commonly 64-bit floating-point values to 32-bit, which would lose precision. + +OMF_STATUS_ZIP_MEMBER_MISSING +: A file referenced in the JSON index was not found in the Zip archive. + +OMF_STATUS_ZIP_ERROR +: The Zip-file sub-system failed. + +OMF_STATUS_PARQUET_SCHEMA_MISMATCH +: A Parquet array file did not have the expected schema. + +OMF_STATUS_PARQUET_ERROR +: The Parquet sub-system failed. + +OMF_STATUS_IMAGE_ERROR +: The image sub-system failed. + + +## OmfValidation + +```c +typedef struct { + size_t n_messages; + const char *const *messages; +} OmfValidation; +``` + +Used when reading or writing a file to return a list of validation errors and warnings. + +### Fields + +n_messages: `size_t` +: The number of messages. + +messages: `const char *const *` +: Array of messages, each a UTF-8 encoded and nul-terminated string in US-English. + +### Methods + +#### omf_validation_free + +```c +bool omf_validation_free(OmfValidation *ptr); +``` + +Frees an `OmfValidation` pointer. Does nothing if it is null. Returns false on error. diff --git a/docs/c/examples/attributes.md b/docs/c/examples/attributes.md new file mode 100644 index 0000000..2d5604e --- /dev/null +++ b/docs/c/examples/attributes.md @@ -0,0 +1,5 @@ +# Attributes Example + +```c +--8<-- "omf-c/examples/attributes.c" +``` diff --git a/docs/c/examples/geometries.md b/docs/c/examples/geometries.md new file mode 100644 index 0000000..29b30a0 --- /dev/null +++ b/docs/c/examples/geometries.md @@ -0,0 +1,5 @@ +# Geometries Example + +```c +--8<-- "omf-c/examples/geometries.c" +``` diff --git a/docs/c/examples/metadata.md b/docs/c/examples/metadata.md new file mode 100644 index 0000000..ace846e --- /dev/null +++ b/docs/c/examples/metadata.md @@ -0,0 +1,5 @@ +# Metadata Example + +```c +--8<-- "omf-c/examples/metadata.c" +``` diff --git a/docs/c/examples/pyramid.md b/docs/c/examples/pyramid.md new file mode 100644 index 0000000..f9f7e8b --- /dev/null +++ b/docs/c/examples/pyramid.md @@ -0,0 +1,5 @@ +# Pyramid Example + +```c +--8<-- "omf-c/examples/pyramid.c" +``` diff --git a/docs/c/examples/textures.md b/docs/c/examples/textures.md new file mode 100644 index 0000000..a548a7f --- /dev/null +++ b/docs/c/examples/textures.md @@ -0,0 +1,5 @@ +# Textures Example + +```c +--8<-- "omf-c/examples/textures.c" +``` diff --git a/docs/c/geometry/blockmodel.md b/docs/c/geometry/blockmodel.md new file mode 100644 index 0000000..0182eec --- /dev/null +++ b/docs/c/geometry/blockmodel.md @@ -0,0 +1,192 @@ +# Block Model + +## OmfBlockModel + +```c +typedef struct { + OmfOrient3 orient; + const OmfRegularGrid3 *regular_grid; + const OmfTensorGrid3 *tensor_grid; + const OmfRegularSubblocks *regular_subblocks; + const OmfFreeformSubblocks *freeform_subblocks; +} OmfBlockModel; +``` + +A 3D grid or block model. +Block sizes can be regular or tensor, +and may have regular or free-form sub-blocks inside each parent block. + +Valid attribute location are `OMF_LOCATION_VERTICES` for per-corner data, +`OMF_LOCATION_BLOCKS` for data on parent block or at parent block centroids, +or `OMF_LOCATION_SUBBLOCKS` for sub-blocks or sub-block centroids. + +### Attribute Locations + +- [`Vertices`](crate::Location::Vertices) puts attribute values on the corners of the + parent blocks. If the block count is $(N_0, N_1, N_2)$ then there must be + $(N_0 + 1) · (N_1 + 1) · (N_2 + 1)$ values. Ordering increases U first, then V, then W. + +- [`Blocks`](crate::Location::Primitives) puts attribute values on the centroids of the + parent block. If the block count is $(N_0, N_1, N_2)$ then there must be + $N_0 · N_1 · N_2$ values. Ordering increases U first, then V, then W. + +- [`Subblocks`](crate::Location::Subblocks) puts attribute values on sub-block centroids. + The number and values and their ordering matches the `parents` and `corners` arrays. + + To have attribute values on undivided parent blocks in this mode there must be a sub-block + that covers the whole parent block. + +### Fields + +orient: `OmfOrient3 ` +: Contains the position and orientation. + +regular_grid: [`const OmfRegularGrid3*`](../grids.md#OmfRegularGrid3) +tensor_grid: [`const OmfTensorGrid3*`](../grids.md#OmfTensorGrid3) +: Exactly one of these must be non-null, defining a grid with either regular or varied spacing. + +regular_subblocks: [`const OmfRegularSubblocks*`](#OmfRegularSubblocks) +freeform_subblocks: [`const OmfFreeformSubblocks*`](#OmfFreeformSubblocks) +: One or none may be non-null, defining either regular or free-form the sub-blocks. + +### Methods + +#### omf_block_model_init + +```c +OmfBlockModel omf_block_model_init(void); +``` + +Initializes or resets a block model struct. + + +## OmfRegularSubblocks + +```c +typedef struct { + uint32_t count[3]; + const OmfArray *subblocks; + OmfSubblockMode mode; +} OmfRegularSubblocks; +``` + +Divide each parent block into a regular grid of `count` cells. +Sub-blocks each covers a non-overlapping cuboid subset of that grid. + +Sub-blocks are described by the `parents` and `corners` arrays. +Those arrays must be the same length and matching rows in each describe the same sub-block. +Each row in `parents` is an IJK index on the block model grid. +Each row of `corners` is $(i_{min}, j_{min}, k_{min}, i_{max}, j_{max}, k_{max})$, +all integers, that refer to the *vertices* of the sub-block grid within the parent block. +For example: + +- A block with minimum size in the corner of the parent block would be (0, 0, 0, 1, 1, 1). +- If the `subblock_count` is (5, 5, 3) then a sub-block covering the whole parent would be (0, 0, 0, 5, 5, 3). + +Sub-blocks must stay within their parent, +must have a non-zero size in all directions, and should not overlap. Further restrictions can be applied by the `mode` field, +see [`OmfSubblockMode`](#omfsubblockmode) for details. + +![Example of regular sub-blocks](../../images/subblocks_regular.svg "Regular sub-bloks") + +### Fields + +count: `uint32_t[3]` +: The maximum number of sub-blocks in each direction. + +subblocks: [`const OmfArray*`](../arrays.md#omfarray) +: Regular sub-block array giving the parent block index and min/max corners of each sub-blocks. + +mode: [`OmfSubblockMode`](#omfsubblockmode) +: Describes any extra restrictions on the sub-block layout. + + +### Methods + +#### omf_regular_subblocks_init + +```c +OmfRegularSubblocks omf_regular_subblocks_init(uint32_t nu, + uint32_t nv, + uint32_t nw, + const OmfArray *parents, + const OmfArray *corners); +``` + +Initializes or resets a regular sub-blocks struct. + + +## OmfFreeformSubblocks + +```c +typedef struct { + const OmfArray *subblocks; +} OmfFreeformSubblocks; +``` + +Divide each parent block into any number and arrangement of non-overlapping cubiod regions. + +Sub-blocks are described by the `parents` and `corners` arrays. +Each row in `parents` is an IJK index on the block model grid. +Each row of `corners` is $(i_{min}, j_{min}, k_{min}, i_{max}, j_{max}, k_{max})$ +in floating-point and relative to the parent block, +running from 0.0 to 1.0 across the parent. +For example: + +- A sub-block covering the whole parent will be (0.0, 0.0, 0.0, 1.0, 1.0, 1.0) no matter the size of the parent. +- A sub-block covering the bottom third of the parent block would be (0.0, 0.0, 0.0, 1.0, 1.0, 0.3333) +and one covering the top two-thirds would be (0.0, 0.0, 0.3333, 1.0, 1.0, 1.0), +again no matter the size of the parent. + +Sub-blocks must stay within their parent, must have a non-zero size in all directions, and shouldn't overlap. + +### Fields + +subblocks: [`const OmfArray*`](../arrays.md#omfarray) +: Free-form sub-block array giving the parent block index and min/max corners of each sub-blocks. + +### Methods + +#### omf_freeform_subblocks_init + +```c +OmfFreeformSubblocks omf_freeform_subblocks_init(const OmfArray *subblocks); +``` + +Initializes or resets a free-form sub-blocks struct. + + +## OmfSubblockMode + +```c +typedef enum { + OMF_SUBBLOCK_MODE_NONE = 0, + OMF_SUBBLOCK_MODE_OCTREE, + OMF_SUBBLOCK_MODE_FULL, +} OmfSubblockMode; +``` + +Applies an optional extra restriction to sub-block layout within each parent block. + +### Options + +`OMF_SUBBLOCK_MODE_NONE` +: No restictions. + +`OMF_SUBBLOCK_MODE_OCTREE` +: Sub-blocks form a octree-like inside the parent block. + + To form this structure, cut the parent block in half in all directions to create eight child blocks. + Repeat that cut for some or all of those children, + and continue doing that until the limit on sub-block count is reached + or until the sub-blocks accurately model the inputs. + + The sub-block count must be a power of two in each direction. + This isn't strictly an octree because the sub-block count doesn't have to be the *same* in all directions. + For example you can have count (16, 16, 2) + and blocks will stop dividing the the W direction after the first split. + +`OMF_SUBBLOCK_MODE_FULL` +: Parent blocks are fully divided or not divided at all. + + Applications reading this mode may choose to merge sub-blocks with matching attributes to reduce the overall number of them. diff --git a/docs/c/geometry/composite.md b/docs/c/geometry/composite.md new file mode 100644 index 0000000..b8eea34 --- /dev/null +++ b/docs/c/geometry/composite.md @@ -0,0 +1,33 @@ +## OmfComposite + +```c +typedef struct { + size_t n_elements; + const OmfElement *elements; +} OmfComposite; +``` + +A container for a set of related sub-elements. + +### Attribute Locations + +- `OMF_LOCATION_ELEMENTS` to apply one value to each sub-element. + + +### Fields + +n_elements: `size_t` +: Number of sub-elements. + +elements: [`const OmfElement *`](../element.md#OmfElement) +: Pointer to an array of `n_elements` sub-elements. + +### Methods + +#### omf_composite_init + +```c +OmfComposite omf_composite_init(void); +``` + +Initializes or resets a composite struct. diff --git a/docs/c/geometry/gridsurface.md b/docs/c/geometry/gridsurface.md new file mode 100644 index 0000000..d184154 --- /dev/null +++ b/docs/c/geometry/gridsurface.md @@ -0,0 +1,43 @@ +# Grid Surface + +## OmfGridSurface + +```c +typedef OmfGridSurface { + OmfOrient2 orient; + const OmfRegularGrid2 *regular_grid; + const OmfTensorGrid2 *tensor_grid; + const OmfArray *heights; +} OmfGridSurface; +``` + +A 2D grid or surface positioned and oriented in 3D space. + +### Attribute Locations + +- `OMF_LOCATION_VERTICES` for per-corner data. +- `OMF_LOCATION_PRIMITIVES` for per-cell data. + +### Fields + +orient: [`OmfOrient2`](#omforient2) +: Contains the position and orientation. + +regular_grid: [`const OmfRegularGrid2 *`](../grids.md#omfregulargrid2) +tensor_grid: [`const OmfTensorGrid2 *`](../grids.md#omftensorgrid2) +: Exactly one of these must be non-null, defining a grid with either regular or varied spacing. + +heights: [`const OmfArray *`](../arrays.md#omfarray) +: Optional scalar array giving a signed offset from the plane for each grid corner. +If null then the grid is flat. + + +### Methods + +#### omf_grid_surface_init + +```c +OmfGridSurface omf_grid_surface_init(void); +``` + +Initializes or resets a grid surface struct. diff --git a/docs/c/geometry/lineset.md b/docs/c/geometry/lineset.md new file mode 100644 index 0000000..1b613f7 --- /dev/null +++ b/docs/c/geometry/lineset.md @@ -0,0 +1,39 @@ +# Line Set + +## OmfLineSet + +```c +typedef struct { + double origin[3]; + const OmfArray *vertices; + const OmfArray *segments; +} OmfLineSet; +``` + +A set of straight line segments. + +### Attribute Locations + +- `OMF_LOCATION_VERTICES` for per-vertex data. +- `OMF_LOCATION_PRIMITIVES` for per-segment data. + +### Fields + +origin: `double[3]` +: An offset to apply to all vertices, along with the [project](../project.md) origin. + +vertices: [`const OmfArray *`](../arrays.md#omfarray) +: Vertex array. + +segments: [`const OmfArray *`](../arrays.md#omfarray) +: Segment array. Values must be less than the length of `vertices`. + +### Methods + +#### omf_line_set_init + +```c +OmfLineSet omf_line_set_init(const OmfArray *vertices, const OmfArray *segments); +``` + +Initializes or resets a line-set struct. diff --git a/docs/c/geometry/pointset.md b/docs/c/geometry/pointset.md new file mode 100644 index 0000000..9e6d8e8 --- /dev/null +++ b/docs/c/geometry/pointset.md @@ -0,0 +1,35 @@ +# Point Set + +## OmfPointSet + +```c +typedef struct { + double origin[3]; + const OmfArray *vertices; +} OmfPointSet; +``` + +A set of point locations in space. + +### Attribute Locations + +- `OMF_LOCATION_VERTICES` for per-point data. + +### Fields + +origin: `double[3]` +: An offset to apply to all points, along with the [project](../project.md) origin. + +vertices: [`const OmfArray *`](../arrays.md#omfarray) +: Vertex array. + + +### Methods + +#### omf_point_set_init + +```c +OmfPointSet omf_point_set_init(const OmfArray *vertices); +``` + +Initializes or resets a point-set struct. diff --git a/docs/c/geometry/surface.md b/docs/c/geometry/surface.md new file mode 100644 index 0000000..b40cb9f --- /dev/null +++ b/docs/c/geometry/surface.md @@ -0,0 +1,40 @@ +## OmfSurface + +```c +typedef struct { + double origin[3]; + const OmfArray *vertices; + const OmfArray *triangles; +} OmfSurface; +``` + +A triangulated surface. + +### Attribute Locations + +- `OMF_LOCATION_VERTICES` for per-vertex data. +- `OMF_LOCATION_PRIMITIVES` for per-triangle data. + +### Fields + +origin: `double[3]` +: An offset to apply to all vertices, along with the [project](../project.md) origin. + +vertices: [`const OmfArray *`](../arrays.md#omfarray) +: Vertex array. + +segments: [`const OmfArray *`](../arrays.md#omfarray) +: Triangle array. +Each row contains the three vertex indices of one triangle, +ordered counter-clockwise around an outward-pointing normal vector. +Values must be less than the length of `vertices`. + +### Methods + +#### omf_surface_init + +```c +OmfSurface omf_surface_init(const OmfArray *vertices, const OmfArray *triangles); +``` + +Initializes or resets a surface struct. diff --git a/docs/c/grids.md b/docs/c/grids.md new file mode 100644 index 0000000..39ad4ca --- /dev/null +++ b/docs/c/grids.md @@ -0,0 +1,175 @@ +# Grid Structs + +Types used to define grid spacing and orientation. +These types are used by grid surfaces, block models, and projected textures. + + +## OmfOrient2 + +```c +typedef struct { + double origin[3]; + double u[3]; + double v[3]; +} OmfOrient2; +``` + +Defines a rotated and translated $(u, v)$ space within the $(x, y, z)$ project space. + +### Fields + +origin: `double[3]` +: The position of the minimum corner of the grid. Add the [project](project.md) origin too. + +u: `double[3]` +v: `double[3]` +: Directions of the $\mathbf{u}$ and $\mathbf{v}$ axes in project space. +Must both be unit vectors, and perpendicular. + + +## OmfOrient3 + +```c +typedef struct { + double origin[3]; + double u[3]; + double v[3]; + double w[3]; +} OmfOrient2; +``` + +Defines a rotated and translated $(u, v, w)$ space within the $(x, y, z)$ project space. + +### Fields + +origin: `double[3]` +: The position of the minimum corner of the grid. Add the [project](project.md) origin too. + +u: `double[3]` +v: `double[3]` +w: `double[3]` +: Directions of the $\mathbf{u}$ and $\mathbf{v}$ axes in project space. +Must all be unit vectors, and all perpendicular to each other. + + +## OmfRegularGrid2 + +```c +typedef struct { + double size[2]; + uint32_t count[2]; +} OmfRegularGrid2; +``` + +Defines a regularly spaced 2D grid. + +### Fields + +size: `double[2]` +: The size of each grid cell in the $\mathbf{u}$ and $\mathbf{v}$ directions. + +count: `uint32_t[2]` +: The number of grid cells in the $\mathbf{u}$ and $\mathbf{v}$ directions. + + +### Methods + +#### omf_regular_grid2_init + +```c +OmfRegularGrid2 omf_regular_grid2_init(double du, double dv, + uint32_t nu, uint32_t nv); +``` + +Initializes or resets a 2D regular grid struct, +setting the size to `{ du, dv }` and the count to `{ nu, nv }`. + + +## OmfTensorGrid2 + +```c +typedef struct { + const OmfArray *u; + const OmfArray *v; +} OmfTensorGrid2; +``` + +### Fields + +u: [`const OmfArray *`](arrays.md#omfarray) +v: [`const OmfArray *`](arrays.md#omfarray) +: Scalar arrays of cell sizes along the $\mathbf{u}$ and $\mathbf{v}$ directions. +All these sizes must be greater than zero. + + +### Methods + +#### omf_tensor_grid2_init + +```c +OmfTensorGrid2 omf_tensor_grid2_init(const OmfArray *u, const OmfArray *v); +``` + +Initializes or resets a 2D tensor grid struct. + + +## OmfRegularGrid3 + +```c +typedef struct { + double size[3]; + uint32_t count[3]; +} OmfRegularGrid3; +``` + +Defines a regularly spaced 3D grid. + +### Fields + +size: `double[3]` +: The size of each grid cell in the $\mathbf{u}$, $\mathbf{v}$, and $\mathbf{w}$ directions. + +count: `uint32_t[3]` +: The number of grid cells in the $\mathbf{u}$, $\mathbf{v}$, and $\mathbf{w}$ directions. + +### Methods + +#### omf_regular_grid3_init + +```c +OmfRegularGrid3 omf_regular_grid3_init(double du, double dv, double dw, + uint32_t nu, uint32_t nv, uint32_t nw); +``` + +Initializes or resets a 2D regular grid struct, +setting the size to `{ du, dv, dw }` and the count to `{ nu, nv, nw }`. + + +## OmfTensorGrid3 + +```c +typedef struct { + const OmfArray *u; + const OmfArray *v; + const OmfArray *w; +} OmfTensorGrid3; +``` + +### Fields + +u: [`const OmfArray *`](arrays.md#omfarray) +v: [`const OmfArray *`](arrays.md#omfarray) +w: [`const OmfArray *`](arrays.md#omfarray) +: Scalar arrays of cell sizes along the $\mathbf{u}$, $\mathbf{v}$, and $\mathbf{w}$ directions. +All these sizes must be greater than zero. + + +### Methods + +#### omf_tensor_grid3_init + +```c +OmfTensorGrid3 omf_tensor_grid3_init(const OmfArray *u, const OmfArray *v, const OmfArray *w); +``` + +Initializes or resets a 3D tensor grid struct. diff --git a/docs/c/images.md b/docs/c/images.md new file mode 100644 index 0000000..fa2019f --- /dev/null +++ b/docs/c/images.md @@ -0,0 +1,71 @@ +# Images + +Types related to image reading and writing. + + +## OmfImageMode + +```c +typedef enum { ... } OmfImageMode; +``` + +Describes what channels the image data has. + +### Options + +OMF_IMAGE_MODE_GRAY = 1 +: Grayscale. One channel. + +OMF_IMAGE_MODE_GRAY_ALPHA = 2 +: Grayscale with alpha. Two channels. + +OMF_IMAGE_MODE_RGB = 3 +: Red, green, and blue. Three channels. + +OMF_IMAGE_MODE_RGBA = 4 +: Red, green, blue, and alpha. Four channels. + + +## OmfImageData + +```c +typedef struct { + uint32_t width; + uint32_t height; + OmfImageMode mode; + const uint8_t *uint8; + const uint16_t *uint16; +} OmfImageData; +``` + +The type returned when reading image data from the file. +The image can have 8 or 16 bits-per-channel and be grayscale, grayscale-alpha, RGB, or RGBA channels. + +### Fields + +width: `uint32_t` +: The image width in pixels. + +height: `uint32_t` +: The image height in pixels. + +mode: [`OmfImageMode`](#OmfImageMode) +: What channels the image data has. + +uint8: `const uint8_t*` +: + +uint16: `const uint16_t*` +: Pixel data in 8 or 16 bits per channel. +Exactly one will be non-null. +There is no padding or row alignment. + +### Methods + +#### omf_image_data_free + +```c +bool omf_image_data_free(OmfImageData *data); +``` + +Call to free an `OmfImageData` pointer when you are finished with it. Returns false on error. diff --git a/docs/c/index.md b/docs/c/index.md new file mode 100644 index 0000000..fe8739e --- /dev/null +++ b/docs/c/index.md @@ -0,0 +1,43 @@ +# OMF C API + +## Examples + +C examples, from simplest to most complex: + +- [`pyramid.c`](./examples/pyramid.md) +writes a file containing a small surface and line-set of a square pyramid, +then reads it back and prints everything. +- [`metadata.c`](./examples/metadata.md) +stores and retrieves metadata, including nested structures. +- [`geometries.c`](./examples/geometries.md) +stores and retrieves the remaining element geometries: +point set, grid surface, block models, and composite. +- [`attributes.c`](./examples/attributes.md) +puts different types of attributes on a cube surface, +then reads back and prints a few of them. +- [`textures.c`](./examples/textures.md) +creates mapped and projected textures from a pre-existing image and reads it back as pixels. + +## By Section + +- [Errors](errors.md) +- [Metadata](metadata.md) +- [Arrays](arrays.md) +- [Images](images.md) +- [Project](project.md) +- [Element](element.md) +- Geometries: + - [Point Set](geometry/pointset.md) + - [Line Set](geometry/lineset.md) + - [Surface](geometry/surface.md) + - [Grid Surface](geometry/gridsurface.md) + - [Composite](geometry/composite.md) + - [Block Model](geometry/blockmodel.md) +- [Attribute](attribute.md) +- [Color maps](colormap.md) +- [Grid Position and Orientation](grids.md) +- [Reader](reader.md) + - [Reader Iterators](reader_iterators.md) +- [Writer](writer.md) + - [Writer Iterators](writer_iterators.md) +- [OMF v1 Conversion](omf1.md) diff --git a/docs/c/metadata.md b/docs/c/metadata.md new file mode 100644 index 0000000..5efa42f --- /dev/null +++ b/docs/c/metadata.md @@ -0,0 +1,57 @@ +# Metadata + +## OmfValueType + +Stores arbitrary metadata for OMF project, elements, and attributes. +Metadata values come in a variety of types and can be nested. + +```c +typedef enum { + OMF_VALUE_TYPE_NULL, + OMF_VALUE_TYPE_BOOLEAN, + OMF_VALUE_TYPE_NUMBER, + OMF_VALUE_TYPE_STRING, + OMF_VALUE_TYPE_ARRAY, + OMF_VALUE_TYPE_OBJECT, +} OmfValueType; +``` + +## OmfValue + +```c +typedef struct OmfValue { + const char *name; + OmfValueType type; + bool boolean; + double number; + const char *string; + const struct OmfValue *values; + size_t n_values; +} OmfValue; +``` + +### Fields + +name: `const char*` +: The value name. Ignored for values inside metadata arrays. + +type: `OmfValueType` +: The value type, which defines which is the following fields is to be used. + +boolean: `bool` +: The boolean value if `type` is `OMF_VALUE_TYPE_BOOLEAN`. + +number: `double` +: The number value if `type` is `OMF_VALUE_TYPE_NUMBER`. + +string: `const char*` +: The string value if `type` is `OMF_VALUE_TYPE_STRING`, in UTF-8 encoding. + +values: `const struct OmfValue*` +: The array of sub-values. +If `type` is `OMF_VALUE_TYPE_ARRAY` then the values in order form an array. +If `type` is `OMF_VALUE_TYPE_OBJECT` then the value names should be used to form a map of key/value pairs. +Otherwise this will be null and `n_values` will be zero. + +n_values: `size_t` +: The number of sub-values. diff --git a/docs/c/omf1.md b/docs/c/omf1.md new file mode 100644 index 0000000..696d0c7 --- /dev/null +++ b/docs/c/omf1.md @@ -0,0 +1,141 @@ +# OMF v1 Conversion + +This library can convert existing OMF v1 files to OMF v2. +This is a standalone process that reads the OMF v1 file and writes a new OMF v2 file. + +## Conversion details + +There are a few parts of OMF1 that don't map directly to OMF2. + +### Elements + +- The `date_created` and `date_modified` fields are moved into the metadata. +- The `subtype` field on point-sets and line-sets is moved into the metadata. + On other elements, where it only had one valid value, it is discarded. +- Line-sets and surfaces with invalid vertex indices will cause conversion to fail. +- Line-sets and surfaces with more than 4,294,967,295 vertices will cause conversion to fail. + +### Data to Attributes + +- Scalar data becomes a number attribute, preserving the float/int type of the array. +- In number data, NaN becomes null. +- In 2D or 3D vector data, if any component is NaN the vector becomes null. +- In string data, empty strings become nulls. + OMF2 supports both null and empty string so we can only guess which was intended. +- In date-time data, empty strings become null. +- Date-times outside the range of approximately ±262,000 years CE will cause conversion to fail. + +### Mapped Data to Category Attribute + +The exact layout of mapped data from OMF v1 can't be stored in OMF v2. +It is transformed to a category attribute by following these rules: + +- Indices equal to minus one become null. +- Indices outside the range 0 to 4,294,967,295 will cause conversion to fail. +- The most unique, least empty, and shortest string legend becomes the category names, + padded with empty strings if necessary. +- The most unique and least empty color legend becomes the category colors, padded with + gray if necessary. +- Other legends become extra attributes, padded with nulls if necessary. + +## omf_omf1_detect + +```c +bool omf_omf1_detect(const char *path); +``` + +Returns true if the file at the given path looks more like OMF v1 than OMF v2. +This is a very quick check and doesn't guarantee that the file will load successfully. + +Returns false on error, or if file open or read fails; call `omf_error()` to tell the difference. + +## OmfOmf1Converter + +```c +typedef struct { /* private fields */ } Omf1Converter; +``` + +The object that handles OMF v1 conversion. The general usage pattern is: + +1. Create the object. +1. Set parameters. +1. Convert one or more files. +1. Free the object. + +### Methods + +#### omf_omf1_converter_new + +```c +OmfOmf1Converter *omf_omf1_converter_new(void); +``` + +Creates a new OMF v1 converter with default parameters. +Returns null on error. +Pass the returned pointer to `omf_omf1_converter_free` when you're finished with it. + +#### omf_omf1_converter_free + +```c +bool omf_omf1_converter_free(OmfOmf1Converter *converter); +``` + +Frees a converter returned by `omf_omf1_converter_new`. +Returns false on error. + +#### omf_omf1_converter_compression + +```c +int32_t omf_omf1_converter_compression(struct OmfOmf1Converter *converter); +``` + +Returns the current compression level of the converter, or -1 on error. + +#### omf_omf1_converter_set_compression + +```c +bool omf_omf1_converter_set_compression(struct OmfOmf1Converter *converter, int32_t compression); +``` + +Sets the compression to use when writing the OMF v2 file. +Pass an integer between 1 for fastest and 9 for most compressed, or -1 to use the default. + +Returns false on error. + +#### omf_omf1_converter_limits + +```c +struct OmfLimits omf_omf1_converter_limits(struct OmfOmf1Converter *converter); +``` + +Returns the current limits to be used when reading the OMF v1 file. + +#### omf_omf1_converter_set_limits + +```c +bool omf_omf1_converter_set_limits(struct OmfOmf1Converter *converter, + const struct OmfLimits *limits); +``` + +Sets the limits to use when reading the OMF v1 file. +See [`OmfLimits`](./reader.md#omflimits). + +Currently only the `json_bytes` field applies to conversion. +All other parts of the file are streamed in and out so the amount of memory used +doesn't depend on the file contents. + +Returns false on error. + +#### omf_omf1_converter_convert + +```c +bool omf_omf1_converter_convert(struct OmfOmf1Converter *converter, + const char *input_path, + const char *output_path, + struct OmfValidation **validation); +``` + +Runs the actual conversion, reading from `input_path` and writing to `output_path`. +The output file will be created if it doesn't exist, or overwritten if it does. + +Returns false on error. diff --git a/docs/c/project.md b/docs/c/project.md new file mode 100644 index 0000000..9f4b089 --- /dev/null +++ b/docs/c/project.md @@ -0,0 +1,68 @@ +# Project + +## OmfProject + +```c +typedef struct { + const char *name; + const char *description; + const char *coordinate_reference_system; + const char *author; + int64_t date; + double origin[3]; + size_t n_metadata; + const OmfValue *metadata; + size_t n_elements; + const OmfElement *elements; +} OmfProject; +``` + +The root object of an OMF file. + +### Fields + +name: `const char*` +: Project name. + +description: `const char *` +: Optional project description or comments. + +coordinate_reference_system: `const char *` +: Optional coordinate reference system. + +units: `const char *` +: The spacial units used for positions and distances within this project if they aren't defined by +`coordinate_reference_system`. +If no unit is explicitly defined then assume meters. + +author: `const char *` +: The name or email address of the creating person. + +date: `int64_t` +: The creation date and time, in microseconds since the `1970-01-01T00:00:00Z` epoch. + +origin: `double[3]` +: An offset to apply to all locations in the file. + +n_metadata: `size_t` +: Number of metadata items. + +metadata: [`const OmfValue *`](metadata.md#omfvalue) +: Pointer to an array of `n_metadata` metadata items, forming a set of key/value pairs. + +n_elements: `size_t` +: Number of elements. + +elements: [`const OmfElement *`](element.md#omfelement) +: Pointer to an array of `n_elements` elements. + + +### Methods + +#### omf_project_init + +```c +OmfProject omf_project_init(const char *name); +``` + +Initializes or resets a project struct. diff --git a/docs/c/reader.md b/docs/c/reader.md new file mode 100644 index 0000000..1a4eada --- /dev/null +++ b/docs/c/reader.md @@ -0,0 +1,275 @@ +# Reader + +## OmfFileVersion + +```c +typedef struct { + uint32_t major; + uint32_t minor; +} OmfFileVersion; +``` + +The major and minor version numbers returned by [omf_reader_version](#omf_reader_version). + + +## OmfLimits + +```c +typedef struct { + uint64_t json_bytes; + uint64_t image_bytes; + uint32_t image_dim; + uint32_t validation; +} OmfLimits; +``` + +Contains the safety limits used by `OmfReader` when reading files. +For all fields zero means unlimited. + +Limits are set on a per-reader basis using [omf_reader_set_limits](#omf_reader_set_limits). + +> WARNING: +> Running without any limits is not recommended. +> A file could be maliciously crafted to consume excessive system resources when read and decompressed, +> leading to a denial of service attack. + +### Fields + +json_bytes: `uint64_t` +: Maximum uncompressed size for the JSON index. Default is 1 MB. + +image_bytes: `uint64_t` +: Maximum memory to use when decoding an image. +Default is 1 GB on 32-bit systems or 16 GB on 64-bit systems. + +image_dim: `uint32_t` +: Maximum image width or height in pixels, default unlimited. + +validation: `uint32_t` +: Maximum number of validation messages. +Errors beyond this limit will be discarded. +Default is 100. + + +## OmfReader + +The class used for reading OMF files. + +Typical usage pattern is: + +1. Create the reader object with [omf_reader_open](#omf_reader_open). +1. Optional: retrieve the file version with [omf_reader_version](#omf_reader_version). +1. Optional: adjust the limits with [omf_reader_set_limits](#omf_reader_set_limits). +1. Read the project from the file with [omf_reader_project](#omf_reader_project). +1. Iterate through the project's contents to find the elements and attributes you want to load. +1. For each of those items load the array or image data. + +### Methods + +#### omf_reader_open + +```c +OmfReader *omf_reader_open(const char *path); +``` + +Attempts to opens the given path as an OMF file. +The `path` string must be UTF-8 encoded. + +Returns a new reader object on success or null on error. +Pass the returned pointer to `omf_reader_close` when you're finished everything inside it. + +#### omf_reader_close + +```c +bool omf_reader_close(OmfReader *reader); +``` + +Closes and frees a reader returned by `omf_reader_open`. +Does nothing if `reader` is null. +Returns false on error. + +You mustn't use `reader` or anything belonging to it after this call. + +#### omf_reader_version + +```c +OmfFileVersion omf_reader_version(OmfReader *reader); +``` + +Returns the OMF version of the file opened with this reader. + +#### omf_reader_limits + +```c +struct OmfLimits omf_reader_limits(OmfReader *reader); +``` + +Returns the current safety limits, or the default limits if `reader` is null. + +#### omf_reader_set_limits + +```c +bool omf_reader_set_limits(OmfReader *reader, const OmfLimits *limits); +``` + +Sets the safety limits for this reader. +If `limits` is null then the default limits are restored. + +#### omf_reader_project + +```c +const OmfProject *omf_reader_project(OmfReader *reader, OmfValidation **validation); +``` + +Reads, validates, and returns the project from this OMF file. +Returns null on error. +The returned pointer belongs to the reader and does not need to be freed separately. +You may only call this function once on a given reader. + +If validation fails, or succeeds but produces warnings, +`validation` will be modified to point to a new `OmfValidation` struct containing the validation messages. +If `validation` is null then these messages will be printed to `stdout` instead. + +#### omf_reader_image + +```c +OmfImageData *omf_reader_image(OmfReader *reader, const OmfArray *image); +``` + +Reads and returns an image from the OMF file. Returns null on error. + +Pass the returned pointer to `omf_image_free` when you're finished with it. + +#### omf_reader_array_info + +```c +OmfArrayInfo omf_reader_array_info(OmfReader *reader, const OmfArray *array); +``` + +Returns information about an array. See [`OmfArrayInfo`](./arrays.md#OmfArrayInfo). +The `array_type` will be `-1` if an error occurs. + +#### omf_reader_array_bytes + +```c +bool omf_reader_array_bytes(struct OmfReader *reader, + const struct OmfArray *array, + char *output, + size_t n_output); +``` + +Reads an array without decompressing it. +Call `omf_reader_array_info` to get the compressed size so you can allocate a large enough buffer. +Useful if you want to use an alternative Parquet implementation, +or if you're copying arrays from one file to another. + +#### omf_reader_array_… + +```c +bool omf_reader_array_scalars64 (struct OmfReader *reader, + const struct OmfArray *array, + double *values, + size_t n_values); +bool omf_reader_array_scalars32 (struct OmfReader *reader, + const struct OmfArray *array, + float *values, + size_t n_values); +bool omf_reader_array_vertices64 (struct OmfReader *reader, + const struct OmfArray *array, + double (*values)[3], + size_t n_values); +bool omf_reader_array_vertices32 (struct OmfReader *reader, + const struct OmfArray *array, + float (*values)[3], + size_t n_values); +bool omf_reader_array_segments (struct OmfReader *reader, + const struct OmfArray *array, + uint32_t (*values)[2], + size_t n_values); +bool omf_reader_array_triangles (struct OmfReader *reader, + const struct OmfArray *array, + uint32_t (*values)[3], + size_t n_values); +bool omf_reader_array_gradient (struct OmfReader *reader, + const struct OmfArray *array, + uint8_t (*values)[4], + size_t n_values); +bool omf_reader_array_texcoords64 (struct OmfReader *reader, + const struct OmfArray *array, + double (*values)[2], + size_t n_values); +bool omf_reader_array_texcoords32 (struct OmfReader *reader, + const struct OmfArray *array, + float (*values)[2], + size_t n_values); +bool omf_reader_array_boundaries_float64 (struct OmfReader *reader, + const struct OmfArray *array, + double *values, + bool *inclusive, + size_t n_values); +bool omf_reader_array_boundaries_int64 (struct OmfReader *reader, + const struct OmfArray *array, + int64_t *values, + bool *inclusive, + size_t n_values); +bool omf_reader_array_regular_subblocks (struct OmfReader *reader, + const struct OmfArray *array, + uint32_t (*parents)[3], + uint32_t (*corners)[6], + size_t n_values); +bool omf_reader_array_freeform_subblocks64 (struct OmfReader *reader, + const struct OmfArray *array, + uint32_t (*parents)[3], + double (*corners)[6], + size_t n_values); +bool omf_reader_array_freeform_subblocks32 (struct OmfReader *reader, + const struct OmfArray *array, + uint32_t (*parents)[3], + float (*corners)[6], + size_t n_values); +bool omf_reader_array_numbers_float64 (struct OmfReader *reader, + const struct OmfArray *array, + double *values, + bool *mask, + size_t n_values); +bool omf_reader_array_numbers_float32 (struct OmfReader *reader, + const struct OmfArray *array, + float *values, + bool *mask, + size_t n_values); +bool omf_reader_array_indices (struct OmfReader *reader, + const struct OmfArray *array, + uint32_t *values, + bool *mask, + size_t n_values); +bool omf_reader_array_vectors32x2 (struct OmfReader *reader, + const struct OmfArray *array, + float (*values)[2], + bool *mask, + size_t n_values); +bool omf_reader_array_vectors64x2 (struct OmfReader *reader, + const struct OmfArray *array, + double (*values)[2], + bool *mask, + size_t n_values); +bool omf_reader_array_vectors32x3 (struct OmfReader *reader, + const struct OmfArray *array, + float (*values)[3], + bool *mask, + size_t n_values); +bool omf_reader_array_vectors64x3 (struct OmfReader *reader, + const struct OmfArray *array, + double (*values)[3], + bool *mask, + size_t n_values); +bool omf_reader_array_booleans (struct OmfReader *reader, + const struct OmfArray *array, + bool *values, + bool *mask, + size_t n_values); +bool omf_reader_array_colors (struct OmfReader *reader, + const struct OmfArray *array, + uint8_t (*values)[4], + bool *mask, + size_t n_values); +``` diff --git a/docs/c/reader_iterators.md b/docs/c/reader_iterators.md new file mode 100644 index 0000000..db8d98f --- /dev/null +++ b/docs/c/reader_iterators.md @@ -0,0 +1,126 @@ +# Reader Iterators + +## Reading Iterator Objects + +Each array type has a matching iterator type. +These can be used to iterate over the values of an array without allocating temporary buffers. + +Note that these objects are **not thread safe**. +You must create it, iterator over it, and free it all on the same thread. +Iterators may outlast the reader object; they have their own copy of the open file handle. + +### Methods + +#### omf_…_next + +```c +bool omf_scalars32_next(OmfScalars32 *iter, float *scalar); +bool omf_scalars64_next(OmfScalars64 *iter, double *scalar); +bool omf_vertices32_next(OmfVertices32 *iter, float vertex[3]); +bool omf_vertices64_next(OmfVertices64 *iter, double vertex[3]); +bool omf_segments_next(OmfSegments *iter, uint32_t segment[2]); +bool omf_triangles_next(OmfTriangles *iter, uint32_t triangles[3]); +bool omf_gradient_next(OmfGradient *iter, uint8_t rgba[4]); +bool omf_texcoords32_next(OmfTexcoords32 *iter, float uv[2]); +bool omf_texcoords64_next(OmfTexcoords64 *iter, double uv[2]); +bool omf_boundaries_float32_next(OmfBoundariesFloat32 *iter, float *value, bool *inclusive); +bool omf_boundaries_float64_next(OmfBoundariesFloat64 *iter, double *value, bool *inclusive); +bool omf_boundaries_int64_next(OmfBoundariesInt64 *iter, int64_t *value, bool *inclusive); +bool omf_numbers_float32_next(OmfNumbersFloat32 *iter, float *number, bool *is_null); +bool omf_numbers_float64_next(OmfNumbersFloat64 *iter, double *number, bool *is_null); +bool omf_numbers_int64_next(OmfNumbersInt64 *iter, int64_t *number, bool *is_null); +bool omf_indices_next(OmfIndices *iter, uint32_t *index, bool *is_null); +bool omf_booleans_next(OmfBooleans *iter, bool *boolean, bool *is_null); +bool omf_omf_colors_next(OmfColors *iter, uint8_t rgba[4], bool *is_null); +bool omf_names_next(OmfNames *iter, const char **string, size_t *len); +bool omf_text_next(OmfText *iter, const char **string, size_t *len); +``` + +These functions must be called from the same thread that created the iterator. + +Retrieve the next value from an iterator. +If another item is available, fills in the output arguments are returns true. +Returns false without changing the output arguments if no more items are available. + +Strings will always be nul-terminated but the length is also output for convenience. +The string buffer will be valid until `next` is called again. + +These iterators will end early if something fails. +Use `omf_error()` to distinguish normal termination for an error. + +#### omf_…_free + +```c +void omf_scalars32_free(OmfScalars32 *iter); +void omf_scalars64_free(OmfScalars64 *iter); +void omf_vertices32_free(OmfVertices32 *iter); +void omf_vertices64_free(OmfVertices64 *iter); +void omf_segments_free(OmfSegments *iter); +void omf_triangles_free(OmfTriangles *iter); +void omf_gradient_free(OmfGradient *iter); +void omf_texcoords32_free(OmfTexcoords32 *iter); +void omf_texcoords64_free(OmfTexcoords64 *iter); +void omf_boundaries_float64_free(OmfBoundariesFloat64 *iter); +void omf_boundaries_float32_free(OmfBoundariesFloat32 *iter); +void omf_boundaries_int64_free(OmfBoundariesInt64 *iter); +void omf_numbers_float32_free(OmfNumbersFloat32 *iter); +void omf_numbers_float64_free(OmfNumbersFloat64 *iter); +void omf_numbers_int64_free(OmfNumbersInt64 *iter); +void omf_indices_free(OmfIndices *iter); +void omf_booleans_free(OmfBooleans *iter); +void omf_colors_free(OmfColors *iter); +void omf_names_free(OmfNames *iter); +void omf_text_free(OmfText *iter); +``` + +Frees an iterator, releasing all resources it is holding. +Must be called from the same thread that created the iterator. + +## OmfReader + +### Methods + +#### omf_reader_array_…_iter + +```c +OmfScalars32 *omf_reader_array_scalars32_iter(OmfReader *reader, const OmfArray *array); +OmfScalars64 *omf_reader_array_scalars64_iter(OmfReader *reader, const OmfArray *array); +OmfVertices32 *omf_reader_array_vertices32_iter(OmfReader *reader, const OmfArray *array); +OmfVertices64 *omf_reader_array_vertices64_iter(OmfReader *reader, const OmfArray *array); +OmfSegments *omf_reader_array_segments_iter(OmfReader *reader, const OmfArray *array); +OmfTriangles *omf_reader_array_triangles_iter(OmfReader *reader, const OmfArray *array); +OmfNames *omf_reader_array_names_iter(OmfReader *reader, const OmfArray *array); +OmfGradient *omf_reader_array_gradient_iter(OmfReader *reader, const OmfArray *array); +OmfTexcoords32 *omf_reader_array_texcoords32_iter(OmfReader *reader, const OmfArray *array); +OmfTexcoords64 *omf_reader_array_texcoords64_iter(OmfReader *reader, const OmfArray *array); +OmfBoundariesFloat32 *omf_reader_array_boundaries_float32_iter(OmfReader *reader, const OmfArray *array); +OmfBoundariesFloat64 *omf_reader_array_boundaries_float64_iter(OmfReader *reader, const OmfArray *array); +OmfBoundariesInt64 *omf_reader_array_boundaries_int64_iter(OmfReader *reader, const OmfArray *array); +OmfRegularSubblocks *omf_reader_array_regular_subblocks_iter(OmfReader *reader, const OmfArray *array); +OmfFreeformSubblocks32 *omf_reader_array_freeform_subblocks32_iter(OmfReader *reader, const OmfArray *array); +OmfFreeformSubblocks64 *omf_reader_array_freeform_subblocks64_iter(OmfReader *reader, const OmfArray *array); +OmfNumbersFloat32 *omf_reader_array_numbers_float32_iter(OmfReader *reader, const OmfArray *array); +OmfNumbersFloat64 *omf_reader_array_numbers_float64_iter(OmfReader *reader, const OmfArray *array); +OmfNumbersInt64 *omf_reader_array_numbers_int64_iter(OmfReader *reader, const OmfArray *array); +OmfIndices *omf_reader_array_indices_iter(OmfReader *reader, const OmfArray *array); +OmfVectors32x2 *omf_reader_array_vectors32x2_iter(OmfReader *reader, const OmfArray *array); +OmfVectors64x2 *omf_reader_array_vectors64x2_iter(OmfReader *reader, const OmfArray *array); +OmfVectors32x3 *omf_reader_array_vectors32x3_iter(OmfReader *reader, const OmfArray *array); +OmfVectors64x3 *omf_reader_array_vectors64x3_iter(OmfReader *reader, const OmfArray *array); +OmfText *omf_reader_array_text_iter(OmfReader *reader, const OmfArray *array); +OmfBooleans *omf_reader_array_booleans_iter(OmfReader *reader, const OmfArray *array); +OmfColors *omf_reader_array_colors_iter(OmfReader *reader, const OmfArray *array); +``` + +Get an iterator for reading an array from the OMF file. +Returns null on error. +Pass the result to the matching free function when you're finished with it. + +All floating-point arrays will automatically cast from 32-bit to 64-bit, +but not the reverse because that would lose precision. + +For number arrays, +dates can be read into `int64_t` or `double` as the number of whole days since the epoch. +Date-time can be read into `int64_t` as the number of microseconds since the epoch, +or into `double` as the number of seconds since the epoch, including a fractional component, +with a small loss of precision. diff --git a/docs/c/writer.md b/docs/c/writer.md new file mode 100644 index 0000000..5ba03e5 --- /dev/null +++ b/docs/c/writer.md @@ -0,0 +1,435 @@ +# Writer + +## OmfHandle + +An opaque pointer used for creating nested structures in an OMF file. +Many [OmfWriter](#OmfWriter) functions return a handle that points to the newly created object, +while others take that handle and create their new object inside the existing object. + +Not all handle types are valid in all places. +You'll get an error at run-time if you pass the wrong type. +Handles belong to a specific writer and remain valid until [omf_writer_finish](#omf_writer_finish) is called, +you don't need to free them yourself and you shouldn't try to use them on a second writer. + + +## OmfWriter + +The class used for writing OMF files. + +Typical usage pattern is: + +1. Create the writer object using [omf_writer_open](#omf_writer_open). +1. Fill in an [OmfProject](project.md#OmfProject) struct then call [omf_writer_project](#omf_writer_project) +to add it to the writer. +1. For each element you want to store: + 1. Write the arrays and images. + 1. Fill in the required struct with the array pointers and other details then add it to the project. + 1. Repeat for the attributes, adding them to the newly created element. +1. Call [omf_writer_finish](#omf_writer_finish) to validate the data, +finish writing the file, and free the writer. + +You can also fill in the `OmfProject::elements` and `OmfElement::attributes` arrays all at once, +but that will often lead to messy code. + +### Methods + +#### omf_writer_open + +```c +OmfWriter *omf_writer_open(const char *path); +``` + +Creates a new writer. +The `path` string must be UTF-8 encoded. +The file will be created if it does not exist, or truncated and overwritten if it does. + +Returns the new writer, or null on error. +Pass the writer to `omf_writer_finish` or `omf_writer_cancel` when you're finished with it. + +#### omf_writer_finish + +```c +bool omf_writer_finish(OmfWriter *writer, OmfValidation **validation); +``` + +Validates the contents, writes the file index, closes the file, and frees the writer. + +Validation messages are stored in `validation` if it is non-null, +or written to `stdout` is it is null. Call `omf_validation_free` to free those messages. + +Returns false on error; the writer is still freed. + +#### omf_writer_compression + +```c +int32_t omf_writer_compression(OmfWriter *writer); +``` + +Returns the compression level that the writer will use, or -1 on error. + +#### omf_writer_set_compression + +```c +bool omf_writer_set_compression(OmfWriter *writer, int32_t compression); +``` + +Sets the compression level that the writer will use. +Pass an integer between 1 for fastest and 9 for most compressed, or -1 to use the default. + +Returns false on error. + +#### omf_writer_cancel + +```c +bool omf_writer_cancel(OmfWriter *writer); +``` + +Frees the writer without finishing it. +The partially written file will be deleted. + +Returns false if file deletion fails; the writer is still freed. + +#### omf_writer_project + +```c +OmfHandle *omf_writer_project(OmfWriter *writer, const OmfProject *project); +``` + +Copies the contents of `project` to the writer. +All `OmfArray` pointers inside `project` must have been previously written to this writer. + +Returns the project handle, or null on error. + +#### omf_writer_element + +```c +OmfHandle *omf_writer_element(OmfWriter *writer, + OmfHandle *handle, + const OmfElement *element); +``` + +Copies the contents of `element` into list elements in the object identified by `handle`, +which can be either the project or a composite element. + +Returns a handle to the new element, or null on error. + +#### omf_writer_attribute + +```c +OmfHandle *omf_writer_attribute(OmfWriter *writer, + OmfHandle *handle, + const OmfAttribute *attribute); +``` + +Copies the contents of `attribute` into the list the attributes of the object identified by `handle`, +which can be either an element or a category attribute. + +Returns a handle to the new attribute, or null on error. + +#### omf_writer_metadata_null + +```c +bool omf_writer_metadata_null(OmfWriter *writer, OmfHandle *handle, const char *name); +``` + +Adds a metadata item with null value. +The `handle` can be the project, an element, an attribute, or a metadata list or object. +`name` must be a UTF-8 encoded string, or null if adding to a metadata list. + +Returns false on error. + +#### omf_writer_metadata_boolean + +```c +bool omf_writer_metadata_boolean(OmfWriter *writer, + OmfHandle *handle, + const char *name, + bool value); +``` + +Adds a metadata item with boolean value. +The `handle` can be the project, an element, an attribute, or a metadata list or object. +`name` must be a UTF-8 encoded string, or null if adding to a metadata list. + +Returns false on error. + +#### omf_writer_metadata_number + +```c +bool omf_writer_metadata_number(OmfWriter *writer, + OmfHandle *handle, + const char *name, + double value); +``` + +Adds a metadata item with double value. +The `handle` can be the project, an element, an attribute, or a metadata list or object. +`name` must be a UTF-8 encoded string, or null if adding to a metadata list. + +Returns false on error. + +#### omf_writer_metadata_string + +```c +bool omf_writer_metadata_string(OmfWriter *writer, + OmfHandle *handle, + const char *name, + const char *value); +``` + +Adds a metadata item with string value. +The `handle` can be the project, an element, an attribute, or a metadata list or object. +`name` must be a UTF-8 encoded string, or null if adding to a metadata list. + +Returns false on error. + +#### omf_writer_metadata_list + +```c +OmfHandle *omf_writer_metadata_list(OmfWriter *writer, + OmfHandle *handle, + const char *name); +``` + +Adds a metadata item with empty list value. +The `handle` can be the project, an element, an attribute, or a metadata list or object. +`name` must be a UTF-8 encoded string, or null if adding to a metadata list. + +Returns a metadata object handle, or null on error. +When adding metadata inside this handle names will be ignored and order maintained. + +#### omf_writer_metadata_object + +```c +OmfHandle *omf_writer_metadata_object(OmfWriter *writer, + OmfHandle *handle, + const char *name); +``` + +Adds a metadata item with empty object value. +The `handle` can be the project, an element, an attribute, or a metadata list or object. +`name` must be a UTF-8 encoded string, or null if adding to a metadata list. + +Returns a metadata object handle, or null on error. +When adding further values inside this handle the value names will be used to create +a collection of key/value pairs. + +#### omf_writer_image_png8 + +```c +const OmfArray *omf_writer_image_png8(OmfWriter *writer, + uint32_t width, + uint32_t height, + OmfImageMode mode, + const uint8_t *pixels); +``` + +Writes an image in PNG encoding. This method supports 8 bits per channel, +use [omf_writer_image_png16](#omf_writer_image_png16) for 16-bit images. +The length of `pixels` must be `width * height * mode`. + +Pixel order is across the top row, then the second, and so on. +Channel order is ($r_0, g_0, b_0, a_0, r_1, g_1, b_1, a_1, ...$) +with the exact channels present depending `mode`. +Don't use any padding or row alignment. + +Returns the new `OmfArray*` pointer, or null on error. + +#### omf_writer_image_png16 + +```c +const OmfArray *omf_writer_image_png16(OmfWriter *writer, + uint32_t width, + uint32_t height, + OmfImageMode mode, + const uint16_t *pixels); +``` + +16-bit version of [omf_writer_image_png8](#omf_writer_image_png8). + +#### omf_writer_image_jpeg + +```c +const OmfArray *omf_writer_image_jpeg(OmfWriter *writer, + uint32_t width, + uint32_t height, + const uint8_t *pixels, + uint32_t quality); +``` + +Writes an image to the OMF file in JPEG encoding. JPEG only supports 8-bit RGB. +The length of `pixels` must be `width * height * 3`. + +Compared to PNG, JPEG encoding will give much smaller file sizes at the cost of reduced quality. +It is best used for photos or high resolution scans where the fine detail is mostly noise +and PNG encoding would be excessively large. +If your image comes from an existing JPEG file then you should use +[omf_writer_image_bytes](#omf_writer_image_bytes) or [omf_writer_image_file](#omf_writer_image_file) +instead; repeatedly encoding will further reduce the quality. + +Returns the new `OmfArray*` pointer, or null on error. + +#### omf_writer_image_bytes + +```c +const OmfArray *omf_writer_image_bytes(OmfWriter *writer, + const uint8_t *bytes, + size_t n_bytes); +``` + +Writes an image from bytes already in PNG or JPEG encoding. + +Returns the new `OmfArray*` pointer, or null on error. + +#### omf_writer_image_file + +```c +const OmfArray *omf_writer_image_file(OmfWriter *writer, const char *path); +``` + +Writes an existing PNG or JPEG image file. The `path` string must be UTF-8 encoded. + +Returns the new `OmfArray*` pointer, or null on error. + +#### omf_writer_array_bytes + +```c +const OmfArray *omf_writer_array_bytes(OmfWriter *writer, + OmfArrayType array_type, + uint64_t item_count, + const char *bytes, + size_t n_bytes); +``` + +Writes a previously compressed Parquet array. +Useful if you want to use an alternative Parquet implementation, +or if you're copying arrays from one file to another. +Check for the OMF file format documentation for the Parquet schema that each array type much match. + +#### omf_writer_array_… + +```c +const OmfArray *omf_writer_array_scalars64 (OmfWriter *writer, + const double *values, + size_t length); +const OmfArray *omf_writer_array_scalars32 (OmfWriter *writer, + const float *values, + size_t length); +const OmfArray *omf_writer_array_vertices64 (OmfWriter *writer, + const double (*values)[3], + size_t length); +const OmfArray *omf_writer_array_vertices32 (OmfWriter *writer, + const float (*values)[3], + size_t length); +const OmfArray *omf_writer_array_segments (OmfWriter *writer, + const uint32_t (*values)[2], + size_t length); +const OmfArray *omf_writer_array_triangles (OmfWriter *writer, + const uint32_t (*values)[3], + size_t length); +const OmfArray *omf_writer_array_names (OmfWriter *writer, + const char *const *values, + size_t length); +const OmfArray *omf_writer_array_gradient (OmfWriter *writer, + const uint8_t (*values)[4], + size_t length); +const OmfArray *omf_writer_array_texcoords64 (OmfWriter *writer, + const double (*values)[2], + size_t length); +const OmfArray *omf_writer_array_texcoords32 (OmfWriter *writer, + const float (*values)[2], + size_t length); +const OmfArray *omf_writer_array_boundaries_float32 (OmfWriter *writer, + const float *values, + const bool *inclusive, + size_t length); +const OmfArray *omf_writer_array_boundaries_float64 (OmfWriter *writer, + const double *values, + const bool *inclusive, + size_t length); +const OmfArray *omf_writer_array_boundaries_int64 (OmfWriter *writer, + const int64_t *values, + const bool *inclusive, + size_t length); +const OmfArray *omf_writer_array_boundaries_date (OmfWriter *writer, + const int64_t *values, + const bool *inclusive, + size_t length); +const OmfArray *omf_writer_array_boundaries_date_time (OmfWriter *writer, + const int64_t *values, + const bool *inclusive, + size_t length); +const OmfArray *omf_writer_array_regular_subblocks (OmfWriter *writer, + const uint32_t (*parents)[3], + const uint32_t (*corners)[6], + size_t length); +const OmfArray *omf_writer_array_freeform_subblocks32 (OmfWriter *writer, + const uint32_t (*parents)[3], + const float (*corners)[6], + size_t length); +const OmfArray *omf_writer_array_freeform_subblocks64 (OmfWriter *writer, + const uint32_t (*parents)[3], + const double (*corners)[6], + size_t length); +const OmfArray *omf_writer_array_numbers_float32 (OmfWriter *writer, + const float *values, + const bool *mask, + size_t length); +const OmfArray *omf_writer_array_numbers_float64 (OmfWriter *writer, + const double *values, + const bool *mask, + size_t length); +const OmfArray *omf_writer_array_numbers_int64 (OmfWriter *writer, + const int64_t *values, + const bool *mask, + size_t length); +const OmfArray *omf_writer_array_numbers_date (OmfWriter *writer, + const int32_t *values, + const bool *mask, + size_t length); +const OmfArray *omf_writer_array_numbers_date_time (OmfWriter *writer, + const int64_t *values, + const bool *mask, + size_t length); +const OmfArray *omf_writer_array_indices (OmfWriter *writer, + const uint32_t *values, + const bool *mask, + size_t length); +const OmfArray *omf_writer_array_vectors32x2 (OmfWriter *writer, + const float (*values)[2], + const bool *mask, + size_t length); +const OmfArray *omf_writer_array_vectors64x2 (OmfWriter *writer, + const double (*values)[2], + const bool *mask, + size_t length); +const OmfArray *omf_writer_array_vectors32x3 (OmfWriter *writer, + const float (*values)[3], + const bool *mask, + size_t length); +const OmfArray *omf_writer_array_vectors64x3 (OmfWriter *writer, + const double (*values)[3], + const bool *mask, + size_t length); +const OmfArray *omf_writer_array_text (OmfWriter *writer, + const char *const *values, + size_t length); +const OmfArray *omf_writer_array_booleans (OmfWriter *writer, + const bool *values, + const bool *mask, + size_t length); +const OmfArray *omf_writer_array_colors (OmfWriter *writer, + const uint8_t (*values)[4], + const bool *mask, + size_t length); +``` + +Functions for writing OMF arrays, pulling data from contiguous C arrays. + +For nullable types the `mask` array may be null if all values are valid. +Otherwise it should be the same length as the values array, +containing true where the value is null or false where it is valid. + +Strings here are always nul-terminated. +Only a limited set of input types is supported. +The iterator API below can provide more flexibility, and avoid copying in some cases. diff --git a/docs/c/writer_iterators.md b/docs/c/writer_iterators.md new file mode 100644 index 0000000..788c8d1 --- /dev/null +++ b/docs/c/writer_iterators.md @@ -0,0 +1,156 @@ +# Writer Iterators + +## Omf…Source + +```c +typedef bool (*OmfScalar32Source)(void *object, float *scalar); +typedef bool (*OmfScalar64Source)(void *object, double *scalar); +typedef bool (*OmfVertex32Source)(void *object, float vertex[3]); +typedef bool (*OmfVertex64Source)(void *object, double vertex[3]); +typedef bool (*OmfSegmentSource)(void *object, uint32_t segment[2]); +typedef bool (*OmfTriangleSource)(void *object, uint32_t triangle[3]); +typedef bool (*OmfNameSource)(void *object, const char **value); +typedef bool (*OmfGradientSource)(void *object, uint8_t rgba[4]); +typedef bool (*OmfTexcoord32Source)(void *object, float uv[2]); +typedef bool (*OmfTexcoord64Source)(void *object, double uv[2]); +typedef bool (*OmfBoundaryFloat32Source)(void *object, float *value, bool *inclusive); +typedef bool (*OmfBoundaryFloat64Source)(void *object, double *value, bool *inclusive); +typedef bool (*OmfBoundaryInt64Source)(void *object, int64_t *value, bool *inclusive); +typedef bool (*OmfRegularSubblockSource)(void *object, uint32_t parent_index[3], uint32_t corners[6]); +typedef bool (*OmfFreeformSubblock32Source)(void *object, uint32_t parent_index[3], float corners[6]); +typedef bool (*OmfFreeformSubblock64Source)(void *object, uint32_t parent_index[3], double corners[6]); +typedef bool (*OmfNumberFloat32Source)(void *object, float *number, bool *is_null); +typedef bool (*OmfNumberFloat64Source)(void *object, double *number, bool *is_null); +typedef bool (*OmfNumberInt64Source)(void *object, int64_t *number, bool *is_null); +typedef bool (*OmfIndexSource)(void *object, uint32_t *index, bool *is_null); +typedef bool (*OmfVector32x2Source)(void *object, float vector[2], bool *is_null); +typedef bool (*OmfVector32x3Source)(void *object, float vector[3], bool *is_null); +typedef bool (*OmfVector64x2Source)(void *object, double vector[2], bool *is_null); +typedef bool (*OmfVector64x3Source)(void *object, double vector[3], bool *is_null); +typedef bool (*OmfTextSource)(void *object, const char **string, size_t *len); +typedef bool (*OmfBooleanSource)(void *object, bool *boolean, bool *is_null); +typedef bool (*OmfColorSource)(void *object, uint8_t rgba[4], bool *is_null); +``` + +Callback function types for the various array sources. + +When called, the function should either fill in the output arguments with the next item and return true, +or return false if no more items are available. +The function will be called repeatedly until it returns false. +The common `object` argument provides the object that was passed to the writer function, +use it to store your iterator state. +For nullable types `*is_null` is initialized to false so you can ignore it if all your values are valid. + +For `OmfNameSource` and `OmfTextSource` set `len` if you know the string length, +or leave it untouched for nul-terminated strings. +`OmfNameSource` will treat a null string as empty, while `OmfTextSource` will store the null value. +For both, the string buffer only needs to be valid until the next call to your function. + +For dates you should use `OmfNumberInt64Source` and output the number of days since the epoch. +For date-time use the same but output the number of microseconds since the epoch in UTC. + +If you're implementing these functions in C++ then they **must not** raise exceptions. + +## OmfWriter + +### Methods + +#### omf_writer_array_…_iter + +```c +const OmfArray *omf_writer_array_scalars32_iter (OmfWriter *writer, + OmfScalar32Source source, + void *object); +const OmfArray *omf_writer_array_scalars64_iter (OmfWriter *writer, + OmfScalar64Source source, + void *object); +const OmfArray *omf_writer_array_vertices32_iter (OmfWriter *writer, + OmfVertex32Source source, + void *object); +const OmfArray *omf_writer_array_vertices64_iter (OmfWriter *writer, + OmfVertex64Source source, + void *object); +const OmfArray *omf_writer_array_segments_iter (OmfWriter *writer, + OmfSegmentSource source, + void *object); +const OmfArray *omf_writer_array_triangles_iter (OmfWriter *writer, + OmfTriangleSource source, + void *object); +const OmfArray *omf_writer_array_names_iter (OmfWriter *writer, + OmfNameSource source, + void *object); +const OmfArray *omf_writer_array_gradient_iter (OmfWriter *writer, + OmfGradientSource source, + void *object); +const OmfArray *omf_writer_array_texcoords32_iter (OmfWriter *writer, + OmfTexcoord32Source source, + void *object); +const OmfArray *omf_writer_array_texcoords64_iter (OmfWriter *writer, + OmfTexcoord64Source source, + void *object); +const OmfArray *omf_writer_array_boundaries_float32_iter (OmfWriter *writer, + OmfBoundaryFloat32Source source, + void *object); +const OmfArray *omf_writer_array_boundaries_float64_iter (OmfWriter *writer, + OmfBoundaryFloat64Source source, + void *object); +const OmfArray *omf_writer_array_boundaries_int64_iter (OmfWriter *writer, + OmfBoundaryInt64Source source, + void *object); +const OmfArray *omf_writer_array_boundaries_date_iter (OmfWriter *writer, + OmfBoundaryInt64Source source, + void *object); +const OmfArray *omf_writer_array_boundaries_date_time_iter (OmfWriter *writer, + OmfBoundaryInt64Source source, + void *object); +const OmfArray *omf_writer_array_regular_subblocks_iter (OmfWriter *writer, + OmfRegularSubblockSource source, + void *object); +const OmfArray *omf_writer_array_freeform_subblocks32_iter (OmfWriter *writer, + OmfFreeformSubblock32Source source, + void *object); +const OmfArray *omf_writer_array_freeform_subblocks64_iter (OmfWriter *writer, + OmfFreeformSubblock64Source source, + void *object); +const OmfArray *omf_writer_array_numbers_float32_iter (OmfWriter *writer, + OmfNumberFloat32Source source, + void *object); +const OmfArray *omf_writer_array_numbers_float64_iter (OmfWriter *writer, + OmfNumberFloat64Source source, + void *object); +const OmfArray *omf_writer_array_numbers_int64_iter (OmfWriter *writer, + OmfNumberInt64Source source, + void *object); +const OmfArray *omf_writer_array_numbers_date_iter (OmfWriter *writer, + OmfNumberInt64Source source, + void *object); +const OmfArray *omf_writer_array_numbers_date_time_iter (OmfWriter *writer, + OmfNumberInt64Source source, + void *object); +const OmfArray *omf_writer_array_indices_iter (OmfWriter *writer, + OmfIndexSource source, + void *object); +const OmfArray *omf_writer_array_vectors32x2_iter (OmfWriter *writer, + OmfVector32x2Source source, + void *object); +const OmfArray *omf_writer_array_vectors32x3_iter (OmfWriter *writer, + OmfVector32x3Source source, + void *object); +const OmfArray *omf_writer_array_vectors64x2_iter (OmfWriter *writer, + OmfVector64x2Source source, + void *object); +const OmfArray *omf_writer_array_vectors64x3_iter (OmfWriter *writer, + OmfVector64x3Source source, + void *object); +const OmfArray *omf_writer_array_text_iter (OmfWriter *writer, + OmfTextSource source, + void *object); +const OmfArray *omf_writer_array_booleans_iter (OmfWriter *writer, + OmfBooleanSource source, + void *object); +const OmfArray *omf_writer_array_colors_iter (OmfWriter *writer, + OmfColorSource source, + void *object); +``` + +These functions add an array by repeatedly calling of the callback functions defined above. diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 0000000..cb3c683 --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1,35 @@ +# Changelog + + + +## Version 2.0 + +- New Rust library and C API. +The new code focuses on security, ease of use, and performance. + +- Use ZIP as the container format, with an identifying comment. + +- Use Apache Parquet to compress array data. + +- The JSON data is now compressed. + +- The JSON structure has been reworked into something more predictable, +consistent, and flexible. A JSON [schema](schema_index.md) is provided as documentation +and specification. + +- Arbitrary JSON metadata is now supported on the project, elements, and attributes. + +- Added JPEG image support. + +- Added boolean-valued attributes. + +- Added UV mapped textures. + +- All attributes now have a standard representation for null values. +Going forward you should avoid using NaN or other flag values like -9999 for nulls. + +- Block models can now have sub-blocks. +Regular sub-blocks lie on a grid within the parent block, +while free-form sub-blocks can be anywhere. + +- Grid surfaces can now be regularly spaced. diff --git a/docs/format.md b/docs/format.md new file mode 100644 index 0000000..49795f1 --- /dev/null +++ b/docs/format.md @@ -0,0 +1,67 @@ +# OMF File Format + +The basic structure of an OMF 2 file is a [ZIP archive](https://en.wikipedia.org/wiki/ZIP_(file_format)), +with ZIP64 extensions. + +The files in the archive are stored without compression. +Each type of file uses a separate and data-specific compression separate from the ZIP archive, +and compressing them again would be a waste of time. + +The top-level zip comment will contain the format name and version, +such as `Open Mining Format 2.0`. +Due to the structure of ZIP files that comment will always be the final bytes in the file. +A ZIP created by another application won't have this comment so won't be recognized as an OMF file. + +The OMF file will contain three types of files: the JSON index, arrays, and images. + + +## JSON Index + +The index is a gzip-compressed JSON document called `index.json.gz`. +This file is required, and describes the project, elements, and attributes in the OMF file. + +See the [JSON schema](schema_index.md) documentation for a specification of the index. + + +## Arrays + +Arrays are stored in (Apache Parquet)[https://en.wikipedia.org/wiki/Apache_Parquet] format, +using a `.parquet` extension. +Each array of vertex locations, triangles, etc. will be in a separate file within the archive. +The JSON index will refer to array files by name. + +Several different array types are used, +see the [Parquet schema](parquet.md) documentation for a specification of each one. + + +## Images + +Images are encoded in either [PNG](https://en.wikipedia.org/wiki/PNG) +or [JPEG](https://en.wikipedia.org/wiki/JPEG) encoded files, +which should have `.png` and `.jpg` extensions respectively. +The JSON index will refer to image files by name. + +PNG encoding can store grayscale, grayscale-alpha, RGB or RGBA data with 8 or 16 bits per channel. +The compression is lossless. + +JPEG encoding can store only 8-bit per channel RGB. +They use lossy compression which won't preserve fine details but gives a smaller file size. +This is suitable for high-resolution maps and scans where it will give much smaller file sizes. + + +## Versions + +The current version is 2.0. + +When new features are added the **minor** version number will be incremented, +to 2.1, then 2.2, and so on. + +If features are ever removed, +or changed such that the format can't store something it used to store, +the **major** version number will be incremented. +This won't be done lightly, so it is unlikely to ever happen. + +When writing files the OMF library will use the oldest version possible, +down to 2.0, based on what is being stored. +Even if the library can write version 2.2 files, if you don't use any of the new features a +2.1 or 2.0 file may be written. diff --git a/docs/images/colormap_continuous.svg b/docs/images/colormap_continuous.svg new file mode 100644 index 0000000..b698c15 --- /dev/null +++ b/docs/images/colormap_continuous.svg @@ -0,0 +1,185 @@ + + + + + + + + + + + + + + + Continuous colormap + + Values + Gradient + + + + + max + min + + + + + diff --git a/docs/images/grid2_regular.svg b/docs/images/grid2_regular.svg new file mode 100644 index 0000000..105fd96 --- /dev/null +++ b/docs/images/grid2_regular.svg @@ -0,0 +1,431 @@ + + + + + + + + + + + + + + + u + + v + + A 2D regular grid with size [a, b] and count [5, 4] + + + + + + + + + + + + + + a + b + b + b + b + a + a + a + a + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/images/grid2_tensor.svg b/docs/images/grid2_tensor.svg new file mode 100644 index 0000000..71c21f8 --- /dev/null +++ b/docs/images/grid2_tensor.svg @@ -0,0 +1,426 @@ + + + + + + + + + + + + + + + u + + v + + A 2D tensor grid + + + + + + + + + + + + + + a₀ + b₃ + b₂ + b₁ + b₀ + a₁ + a₂ + a₃ + a₄ + v = [ b₀ b₁ b₃ b₃ ] + u = [ a₀ a₁ a₂ a₃ a₄ ] + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/images/grid3_regular.svg b/docs/images/grid3_regular.svg new file mode 100644 index 0000000..901513d --- /dev/null +++ b/docs/images/grid3_regular.svg @@ -0,0 +1,795 @@ + + + + + + + + + + + + + + + + + + c + c + c + c + c + c + + + + + + + + + + + + + + + + + a + u + b + b + b + b + a + a + a + a + + + + + + v + w + + + A 3D regular grid with size [a, b, c] and count [5, 4, 6] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/images/grid3_tensor.svg b/docs/images/grid3_tensor.svg new file mode 100644 index 0000000..720d0ff --- /dev/null +++ b/docs/images/grid3_tensor.svg @@ -0,0 +1,835 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + c₅ + c₄ + c₃ + c₂ + c₁ + c₀ + + + + + + + + + + + + + + + + + a₀ + u + b₀ + b₁ + b₂ + b₃ + a₁ + a₂ + a₃ + a₄ + + + + + + v + w + + + A 3D tensor grid + w = [ c₀ c₁ c₂ c₃ c₄ c₅ ] + v = [ b₀ b₁ b₃ b₃ ] + u = [ a₀ a₁ a₂ a₃ a₄ ] + + diff --git a/docs/images/subblocks_regular.svg b/docs/images/subblocks_regular.svg new file mode 100644 index 0000000..6bc4317 --- /dev/null +++ b/docs/images/subblocks_regular.svg @@ -0,0 +1,1161 @@ + + + + + + + + Example of regular sub-blocks + Parents + Corners + i + j + k + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 6 + 6 + 6 + 5 + 5 + 4 + 4 + 3 + 3 + 2 + 2 + 1 + 1 + 0 + 0 + 0 + 0 + … + 0 + 0 + … + 0 + 0 + … + 0 + 0 + … + 0 + 0 + … + 1 + 0 + … + 0 + 1 + 0 + i + j + k + 0 + 0 + … + 3 + 0 + … + 0 + 4 + … + 3 + 2 + … + 4 + 2 + … + 0 + 0 + … + i + j + k + 3 + 4 + … + 6 + 2 + … + 3 + 6 + … + 4 + 6 + … + 6 + 6 + … + 6 + 6 + … + + + + + + + + + + + diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..2f7a539 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,65 @@ +# Home + +Version 0.1.0-beta.1 + +Specification and library for Open Mining Format version 2, +a standard for mining data interchange backed by the +[Global Mining Guidelines Group](https://gmggroup.org). + +> WARNING: +> This is an alpha release of OMF 2. The storage format and libraries might be changed in +> backward-incompatible ways and are not subject to any SLA or deprecation policy. +> +> Further, this code is unfinished and may not be secure. +> Don't use it to open files you don't trust, and don't use it in production yet. + + +## What is OMF + +OMF is an open-source serialization format and library to support data interchange +across the entire mining community. +Its goal is to standardize file formats and promote collaboration. + +This repository provides a file format specification and a Rust library for reading and writing files, +plus a wrapper to use that library from C. + + +## What OMF Stores + +### Elements + +- Points. +- Line segments. +- Triangulated surfaces. +- Grid surfaces. + - Regular or tensor grid spacing. + - Any orientation. +- Block models, with optional sub-blocks. + - Regular or tensor grid spacing. + - Any orientation. + - Regular sub-blocks that lie on a grid within their parent, with octree or arbitrary layout. + - Free-form sub-blocks that don't lie on any grid. +- Composite elements made out of any of the above. + + +### Attributes + +- Floating-point or signed integer values. +- Date and date-time values. +- Category values, storing an index used to look up name, color, or other sub-attributes. +- Boolean or filter values. +- 2D and 3D vectors. +- Text values. +- Color values. +- Projected texture images. +- UV mapped texture images. + +Attributes values can be valid or null. +They can be attached to different parts of each element type, +such as the vertices vs. faces of a surface, +or the parent blocks vs. sub-blocks of a block model. + + +## Using OMF + +See the [getting started](start.md) page. diff --git a/docs/parquet.md b/docs/parquet.md new file mode 100644 index 0000000..ccfe751 --- /dev/null +++ b/docs/parquet.md @@ -0,0 +1,143 @@ +# Parquet Schemas + +OMF uses several different types of arrays. +Most types accept more than one Parquet schema, allowing flexibility on the types used. + +The sections below describe each array type and give the accepted schema for it. + +Take care when using 32-bit floating point values for vertex locations. +They will make file sizes smaller, but it's easy to lose precision. +When converting 64-bit floating-point to 32-bit you should calculate and subtract the +center point from all vertex locations, +then store that offset in the element origin field. + +Using 8-bit or 16-bit unsigned integers won't affect the file size, +because Parquet already compresses those efficiently. +It will still help applications loading the file, +letting them use the smaller type when loading the data without needing to check maximum values first. + +## Scalar + +Floating-point scalar values. +```text +--8<-- "docs/parquet/Scalar.txt" +``` + +## Vertex + +Vertex locations in 3D. Add the project and element origins. +```text +--8<-- "docs/parquet/Vertex.txt" +``` + +## Segment + +Line segments as indices into a vertex array. +```text +--8<-- "docs/parquet/Segment.txt" +``` + +## Triangle + +Triangles as indices into a vertex array. +Triangle winding should be counter-clockwise around an outward-pointing normal +```text +--8<-- "docs/parquet/Triangle.txt" +``` + +## Name + +Non-nullable category names. +These should be unique and non-empty. +```text +--8<-- "docs/parquet/Name.txt" +``` + +## Gradient + +Non-nullable colormap or category colors. +```text +--8<-- "docs/parquet/Gradient.txt" +``` + +## Texcoord + +UV texture coordinates. +Values outside [0, 1] should cause the texture to wrap. +```text +--8<-- "docs/parquet/Texcoord.txt" +``` + +## Boundary + +Discrete color-map boundaries. +If the `inclusive` column is true then the boundary is less than or equal to the value, +otherwise it is less the value. +```text +--8<-- "docs/parquet/Boundary.txt" +``` + +## RegularSubblock + +Parent indices and corners of regular sub-blocks. +These sub-blocks lie on a regular grid within their parent block and defined by integer indices on +that grid. +```text +--8<-- "docs/parquet/RegularSubblock.txt" +``` + +## FreeformSubblock + +Parent indices and corners of free-form sub-blocks. +These sub-blocks can be anywhere within their parent block and are defined relative to it. +```text +--8<-- "docs/parquet/FreeformSubblock.txt" +``` + +## Number + +Nullable number values, which can be floating-point, signed integer, date, or date-time. +Date-time must be in UTC with microsecond precision. +```text +--8<-- "docs/parquet/Number.txt" +``` + +## Index + +Nullable category index values. +```text +--8<-- "docs/parquet/Index.txt" +``` + +## Vector + +Nullable 2D or 3D vectors. +```text +--8<-- "docs/parquet/Vector.txt" +``` + +## Text + +Nullable text. +Some application may treat null as equivalent to an empty string. +```text +--8<-- "docs/parquet/Text.txt" +``` + +## Boolean + +Nullable booleans. +The values are optional, allowing them to be true, false, or null. +Applications that don't support +[three-valued logic](https://en.wikipedia.org/wiki/Three-valued_logic) may treat null as false. +```text +--8<-- "docs/parquet/Boolean.txt" +``` + +## Color + +Nullable colors, in 8-bit RGB or RGBA. +Omitting the alpha column out is equivalent to setting all alpha values to 255. +```text +--8<-- "docs/parquet/Color.txt" +``` diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..9dbabe2 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,4 @@ +mkdocs +markdown-callouts +python-markdown-math +pymdown-extensions diff --git a/docs/schema_index.md b/docs/schema_index.md new file mode 100644 index 0000000..8064514 --- /dev/null +++ b/docs/schema_index.md @@ -0,0 +1,26 @@ +# OMF JSON Schema + +The JSON index within the OMF file is specified using a [JSON Schema](http://json-schema.org/). +Below are links to all of the objects within the schema, with their documentation. + +- [Project](schema/Project.md) is the root object. + - [Element](schema/Element.md) stores the name, geometry, attributes, and other details of one object. + - [Geometry](schema/Geometry.md) picks from points, lines, surface, etc. + - [PointSet](schema/PointSet.md) describes a set of points. + - [LineSet](schema/LineSet.md) describes a set of straight line segments. + - [Surface](schema/Surface.md) describes a triangulated surface. + - [GridSurface](schema/GridSurface.md) describes a 2D grid surface. + - [Grid2](schema/Grid2.md) defines the grid, regular or tensor. + - [Orient2](schema/Orient2.md) gives the position and orientation. + - [BlockModel](schema/BlockModel.md) describes a block model. + - [Grid3](schema/Grid3.md) defines the grid, regular or tensor. + - [Orient3](schema/Orient3.md) gives the position and orientation. + - [Subblocks](schema/Subblocks.md) describes sub-blocks within each parent block. + - [SubblockMode](schema/SubblockMode.md) further restricts sub-blocks to octree or fully sub-blocked. + - [Composite](schema/Composite.md) contains other elements under a single name. + - [Attribute](schema/Attribute.md) stores data that is attached to an element. + - [Location](schema/Location.md) says where the attribute is attached. + - [AttributeData](schema/AttributeData.md) picks from several type of data. + - [NumberColormap](schema/NumberColormap.md) maps from numbers to colors. + - [NumberColormapRange](schema/NumberColormapRange.md) holds a number range for a colormap. +- [Array](schema/Array.md) points to an array or image file in the archive. diff --git a/docs/start.md b/docs/start.md new file mode 100644 index 0000000..ef70856 --- /dev/null +++ b/docs/start.md @@ -0,0 +1,60 @@ +# Getting Started + +## Overview of an OMF File + +An OMF file is a ZIP archive with an identifying comment. +It contains a JSON [index](format.md#json-index) document, +plus other files for [arrays](format.md#arrays) and [images](format.md#images) that the index refers to. + +The root object of the JSON document is a **project**. +The project contains one or more **elements** each of which describes a separate object, +like a set of points, a triangulated surface, or a block model. +Each element can have any number of **attributes** which describe things like assay measurements on points, +colors on triangles, and estimation outputs on blocks. + +Bulk data, like **images** or **arrays** of vertex locations, +are not stored as JSON but as separate files within the archive. +The JSON data refers to each data file by name, +and contains details for linking them together into rich objects. +Images may use PNG or JPEG encoding, while arrays use Apache Parquet encoding. + +> WARNING: +> When reading OMF files, beware of "zip bombs" where data is maliciously crafted to expand to an +> excessive size when decompressed, leading to a potential denial of service attack. +> Use the limits provided by the C and Rust APIs, and check sizes before allocating memory. + + +## Rust API + +See the [Rust API documentation](rust/omf/index.html). + + +## C API + +See the [C API documentation](c/index.md). + + +## Python API + +Not yet written. + + +## Write Your Own + +To create your own reading or writing code, +start with the [file format](format.md) specification, +[JSON schema](schema_index.md), +and [Parquet schema](parquet.md) documentation. + +How you proceed depends on the language you're working in. +You will probably want to start by finding good libraries for: + +- UTF-8 character encoding. +- JSON parsing. +- Deflate compression and decompression, a.k.a. Zlib. +- Apache Parquet compression and decompression. +- Reading and writing PNG and JPEG images. + +> WARNING: +> Make sure that these libraries are secure against malicious data, +> and keep track of any security updates for them. diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css new file mode 100644 index 0000000..760a420 --- /dev/null +++ b/docs/stylesheets/extra.css @@ -0,0 +1,3 @@ +dd { + margin-left: 2em; +} \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..b841a39 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,79 @@ +site_name: OMF + +use_directory_urls: false + +exclude_docs: | + /requirements.txt + /build.sh + +extra_javascript: + - https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-AMS-MML_HTMLorMML + +markdown_extensions: + - def_list + - mdx_math: + enable_dollar_delimiter: true + - callouts + - pymdownx.snippets + +extra_css: + - stylesheets/extra.css + +nav: + - 'index.md' + - 'start.md' + - 'changelog.md' + - 'Format': 'format.md' + - 'Parquet': 'parquet.md' + - 'JSON-Schema': + - 'schema_index.md' + - 'schema/Array.md' + - 'schema/Attribute.md' + - 'schema/AttributeData.md' + - 'schema/BlockModel.md' + - 'schema/Composite.md' + - 'schema/Element.md' + - 'schema/Geometry.md' + - 'schema/Grid2.md' + - 'schema/Grid3.md' + - 'schema/GridSurface.md' + - 'schema/LineSet.md' + - 'schema/Location.md' + - 'schema/NumberColormap.md' + - 'schema/Orient2.md' + - 'schema/Orient3.md' + - 'schema/PointSet.md' + - 'schema/Project.md' + - 'schema/SubblockMode.md' + - 'schema/Subblocks.md' + - 'schema/Surface.md' + - 'C': + - 'c/index.md' + - 'c/errors.md' + - 'c/metadata.md' + - 'c/arrays.md' + - 'c/images.md' + - 'c/project.md' + - 'c/element.md' + - 'Geometries': + - 'c/geometry/pointset.md' + - 'c/geometry/lineset.md' + - 'c/geometry/surface.md' + - 'c/geometry/gridsurface.md' + - 'c/geometry/composite.md' + - 'c/geometry/blockmodel.md' + - 'c/attribute.md' + - 'c/colormap.md' + - 'c/grids.md' + - 'c/reader.md' + - 'c/reader_iterators.md' + - 'c/writer.md' + - 'c/writer_iterators.md' + - 'c/omf1.md' + - 'Examples': + - 'c/examples/pyramid.md' + - 'c/examples/metadata.md' + - 'c/examples/geometries.md' + - 'c/examples/attributes.md' + - 'c/examples/textures.md' + - 'Rust': 'rust/omf/index.html' diff --git a/omf-c/Cargo.toml b/omf-c/Cargo.toml new file mode 100644 index 0000000..ff77c74 --- /dev/null +++ b/omf-c/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "omfc" +version = "0.1.0-beta.1" +description = "C bindings for `omf`." +authors = ["Tim Evans "] +license = "MIT" +edition = "2021" +publish = false + +[lib] +name = "omfc" +crate-type = ["cdylib"] + +[dependencies] +omf = { path = ".." } +chrono.workspace = true +image.workspace = true +serde_json.workspace = true +thiserror.workspace = true + +[build-dependencies] +omf = { path = ".." } +cbindgen.workspace = true diff --git a/omf-c/build.rs b/omf-c/build.rs new file mode 100644 index 0000000..2f4ae58 --- /dev/null +++ b/omf-c/build.rs @@ -0,0 +1,50 @@ +// Generates some functions then calls cbindgen to generate the header file. + +use std::fmt::Write; + +fn main() { + // cbindgen.toml contains the configuration. + let crate_dir = + std::env::var("CARGO_MANIFEST_DIR").expect("missing CARGO_MANIFEST_DIR env var"); + let profile = std::env::var("PROFILE").expect("missing PROFILE env var"); + let mut config = cbindgen::Config::from_root_or_default(&crate_dir); + config.after_includes = Some(defines()); + cbindgen::generate_with_config(&crate_dir, config) + .unwrap() + .write_to_file(format!("{crate_dir}/../target/{profile}/omf.h")); + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=cbindgen.toml"); + println!("cargo:rerun-if-changed=src/"); +} + +fn defines() -> String { + use omf::{ + format_full_name, CRATE_NAME, CRATE_VERSION, FORMAT_EXTENSION, FORMAT_NAME, + FORMAT_VERSION_MAJOR, FORMAT_VERSION_MINOR, FORMAT_VERSION_PRERELEASE, + }; + let mut defines = format!( + " +// File format name and version numbers. +#define OMF_FORMAT_EXTENSION \"{FORMAT_EXTENSION}\" +#define OMF_FORMAT_NAME \"{FORMAT_NAME}\" +#define OMF_FORMAT_VERSION_MAJOR {FORMAT_VERSION_MAJOR} +#define OMF_FORMAT_VERSION_MINOR {FORMAT_VERSION_MINOR} +#define OMF_CRATE_NAME \"{CRATE_NAME}\" +#define OMF_CRATE_VERSION \"{CRATE_VERSION}\" +" + ); + if let Some(pre) = FORMAT_VERSION_PRERELEASE { + writeln!( + &mut defines, + "#define OMF_FORMAT_VERSION_PRERELEASE \"{pre}\"" + ) + .unwrap(); + } + writeln!( + &mut defines, + "#define OMF_FORMAT_NAME_FULL \"{full}\"", + full = format_full_name() + ) + .unwrap(); + defines +} diff --git a/omf-c/cbindgen.toml b/omf-c/cbindgen.toml new file mode 100644 index 0000000..37bb66d --- /dev/null +++ b/omf-c/cbindgen.toml @@ -0,0 +1,27 @@ +language = "C" +pragma_once = true +documentation = false +usize_is_size_t = true +cpp_compat = true +line_length = 100 +style = "both" +tab_width = 4 + +[parse] +parse_deps = true +include = ["omf"] + +[export] +prefix = "Omf" +renaming_overrides_prefixing = true +include = ["Status"] + +[enum] +rename_variants = "ScreamingSnakeCase" +prefix_with_name = true + +[export.rename] +"CError" = "OmfError" +"FORMAT_VERSION_MAJOR" = "OMF_FORMAT_VERSION_MAJOR" +"FORMAT_VERSION_MINOR" = "OMF_FORMAT_VERSION_MINOR" +"FORMAT_VERSION_PRERELEASE" = "OMF_FORMAT_VERSION_PRERELEASE" diff --git a/omf-c/examples/CMakeLists.txt b/omf-c/examples/CMakeLists.txt new file mode 100644 index 0000000..add395d --- /dev/null +++ b/omf-c/examples/CMakeLists.txt @@ -0,0 +1,19 @@ +set(CMAKE_CONFIGURATION_TYPES Release) +cmake_minimum_required(VERSION 3.22) +project(omf_c_examples) + +if(MSVC) + set(CMAKE_C_FLAGS "/utf-8 /D_CRT_SECURE_NO_WARNINGS /W3 /WX /O2") + link_libraries(omfc.dll.lib) +else() + set(CMAKE_C_FLAGS "-g -Wall -Werror -O3") + link_libraries(libomfc.so) +endif() +include_directories(../../target/release) +link_directories(../../target/release) + +add_executable(pyramid pyramid.c) +add_executable(metadata metadata.c) +add_executable(geometries geometries.c) +add_executable(attributes attributes.c) +add_executable(textures textures.c) diff --git a/omf-c/examples/README.md b/omf-c/examples/README.md new file mode 100644 index 0000000..abc3c1a --- /dev/null +++ b/omf-c/examples/README.md @@ -0,0 +1,13 @@ +# Example uses of the C API + +Examples in this folder, best read in order: + +- [`pyramid.c`](./pyramid.c) writes a file containing a small surface and line-set of a square + pyramid, then reads it back and prints everything. +- [`metadata.c`](./metadata.c) stores and retrieves metadata, including nested structures. +- [`geometries.c`](./geometries.c) stores and retrieves the remaining element geometries: + point set, grid surface, block models, and composite. +- [`attributes.c`](./attributes.c) puts different types of attributes on a cube surface, then + reads back and prints a few of them. +- [`textures.c`](./textures.c) creates mapped and projected textures from a pre-existing image + file and reads them back as pixels. diff --git a/omf-c/examples/attributes.c b/omf-c/examples/attributes.c new file mode 100644 index 0000000..94853ac --- /dev/null +++ b/omf-c/examples/attributes.c @@ -0,0 +1,397 @@ +// Writes an OMF file containing all non-texture attributes on a cube surface, then reads +// back some of those attributes. + +#include +#include +#include +#include +#include + +static const double VERTICES[][3] = { + { 0.0, 0.0, 0.0 }, + { 1.0, 0.0, 0.0 }, + { 1.0, 1.0, 0.0 }, + { 0.0, 1.0, 0.0 }, + { 0.0, 0.0, 1.0 }, + { 1.0, 0.0, 1.0 }, + { 1.0, 1.0, 1.0 }, + { 0.0, 1.0, 1.0 }, +}; + +static const uint32_t TRIANGLES[][3] = { + { 0, 2, 1 }, + { 0, 3, 2 }, + { 0, 1, 5 }, + { 0, 5, 4 }, + { 1, 2, 6 }, + { 1, 6, 5 }, + { 2, 3, 7 }, + { 2, 7, 6 }, + { 3, 0, 4 }, + { 3, 4, 7 }, + { 4, 5, 6 }, + { 4, 6, 7 }, +}; + +static const double PATH_VECTORS_3D[][3] = { + { 1.0, 0.0, 0.0 }, + { 0.0, 1.0, 0.0 }, + { -1.0, 0.0, 0.0 }, + { 0.0, 0.0, 1.0 }, + { 0.0, 0.0, -1.0 }, + { -1.0, 0.0, 0.0 }, + { 0.0, -1.0, 0.0 }, + { 1.0, 0.0, 0.0 }, +}; + +static const double OUTWARD_VECTORS_2D[][2] = { + { 0.0, 0.0 }, + { 0.0, 0.0 }, + { 0.0, -1.0 }, + { 0.0, -1.0 }, + { 1.0, 0.0 }, + { 1.0, 0.0 }, + { 0.0, 1.0 }, + { 0.0, 1.0 }, + { -1.0, 0.0 }, + { -1.0, 0.0 }, + { 0.0, 0.0 }, + { 0.0, 0.0 }, +}; + +static const bool OUTWARD_VECTORS_2D_MASK[] = { + true, true, false, false, false, false, false, false, false, false, true, true, +}; + +static const bool FIRST_TRIANGLE[] = { + true, false, true, false, true, false, true, false, true, false, true, false, +}; + +static const uint8_t COLORS[][4] = { + { 0, 0, 0, 255 }, + { 255, 0, 0, 255 }, + { 255, 255, 0, 255 }, + { 0, 255, 0, 255 }, + { 0, 0, 255, 255 }, + { 255, 0, 255, 255 }, + { 255, 255, 255, 255 }, + { 0, 255, 255, 255 }, +}; + +static const char *FACE_STRINGS[] = { + "down", "down", + "south", "south", + "east", "east", + "north", "north", + "west", "west", + "up", "up", +}; + +static const char *VERTEX_STRINGS[] = { + "origin", NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, +}; + +static const uint32_t CATEGORY_VALUES[] = { + 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, +}; + +static const char *CATEGORY_NAMES[] = { + "ceiling", + "floor", + "wall", +}; + +static const int64_t CATEGORY_IDS[] = { + 1024, + 1025, + -1, +}; + +static const uint8_t CATEGORY_COLORS[][4] = { + { 255, 0, 0, 255 }, + { 0, 255, 0, 255 }, + { 0, 0, 255, 255 }, +}; + +static const float NUMBERS[] = { + 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, +}; + +static const uint8_t GRADIENT[][4] = { + { 255, 0, 0, 255 }, + { 255, 255, 0, 255 }, +}; + +static const float DISCRETE_BOUNDARIES[] = { + 1.0, 4.0, 5.5, 7.5, +}; +static const bool DISCRETE_INCLUSIVE[] = { + true, // Includes the 1.0 value. + false, // Excludes the 4.0 value. + false, + false, +}; +static const uint8_t DISCRETE_COLORS[][4] = { + { 255, 0, 0, 255 }, + { 255, 85, 0, 255 }, + { 255, 170, 0, 255 }, + { 255, 255, 0, 255 }, +}; + +static const int64_t DATETIMES_MS[] = { + -93706495806958LL, // -1000-07-24T01:49:53.042 + -1465596606958LL, // 1923-07-24T01:49:53.042 + 1690163393042LL, // 2023-07-24T01:49:53.042 + 4845836993042LL, // 2123-07-24T01:49:53.042 + 32521312193042LL, // 3000-07-24T01:49:53.042 + 253388396993042LL, // 9999-07-24T01:49:53.042 + 0LL, // 1970-01-01T00:00:00.000 (the epoch) + -2051264047219200000LL, // -65000000-01-01T00:00:00.000 (65 million years ago) +}; + +static bool write(const char *path) { + OmfError *error; + OmfWriter *writer; + OmfProject project; + OmfElement element; + OmfSurface surface; + OmfAttribute attribute; + OmfCategoryData cat_data; + OmfNumberData num_data; + OmfContinuousColormap c_cmap; + OmfDiscreteColormap d_cmap; + OmfHandle *proj_handle, *cube_handle, *attr_handle; + + // Open the file and create a project. + writer = omf_writer_open(path); + project = omf_project_init("attributes.omf"); + proj_handle = omf_writer_project(writer, &project); + + // Create the cube element and keep that handle too. + surface = omf_surface_init( + omf_writer_array_vertices64(writer, VERTICES, 8), + omf_writer_array_triangles(writer, TRIANGLES, 12) + ); + element = omf_element_init("Cube"); + element.surface = &surface; + cube_handle = omf_writer_element(writer, proj_handle, &element); + + // Masked 2D vectors on the faces. The attribute data is the array. + attribute = omf_attribute_init("Outward", OMF_LOCATION_PRIMITIVES); + attribute.description = "A vector on each face pointing outward in the XY plane, or null " + "if the face is parallel to the XY plane."; + attribute.vector_data = omf_writer_array_vectors64x2( + writer, OUTWARD_VECTORS_2D, OUTWARD_VECTORS_2D_MASK, 12); + omf_writer_attribute(writer, cube_handle, &attribute); + + // 3D vectors on the vertices. The attribute data is just the array for this type. + attribute = omf_attribute_init("Path", OMF_LOCATION_VERTICES); + attribute.description = "From each vertex, points toward the next vertex in a closed and " + "non-intersecting path around the cube"; + attribute.vector_data = omf_writer_array_vectors64x3(writer, PATH_VECTORS_3D, NULL, 8); + omf_writer_attribute(writer, cube_handle, &attribute); + + // Boolean values on faces. The attribute data is the array. + attribute = omf_attribute_init("First triangle", OMF_LOCATION_PRIMITIVES); + attribute.description = "Filter that selects the first triangle of each square face."; + attribute.boolean_data = omf_writer_array_booleans(writer, FIRST_TRIANGLE, NULL, 12); + omf_writer_attribute(writer, cube_handle, &attribute); + + // Color values on vertices. The attribute data is the array. + attribute = omf_attribute_init("Position", OMF_LOCATION_VERTICES); + attribute.description = "Transforms the vertex positions into RGB colors."; + attribute.color_data = omf_writer_array_colors(writer, COLORS, NULL, 8); + omf_writer_attribute(writer, cube_handle, &attribute); + + // Text values on faces. The attribute data is the string array. + attribute = omf_attribute_init("Directions", OMF_LOCATION_PRIMITIVES); + attribute.description = "Strings giving the direction of each face."; + attribute.text_data = omf_writer_array_text(writer, FACE_STRINGS, 12); + omf_writer_attribute(writer, cube_handle, &attribute); + + // Masked string values on vertices. The attribute data is the string array. + attribute = omf_attribute_init("Origin", OMF_LOCATION_PRIMITIVES); + attribute.description = "A string on just the origin vertex."; + attribute.text_data = omf_writer_array_text(writer, VERTEX_STRINGS, 12); + omf_writer_attribute(writer, cube_handle, &attribute); + + // Category values on faces. This is more complicated because we need to store the legend + // as well. + cat_data = omf_category_data_init(); + cat_data.values = omf_writer_array_indices(writer, CATEGORY_VALUES, NULL, 12); + cat_data.names = omf_writer_array_names(writer, CATEGORY_NAMES, 3); + cat_data.gradient = omf_writer_array_gradient(writer, CATEGORY_COLORS, 3); + attribute = omf_attribute_init("Face type", OMF_LOCATION_PRIMITIVES); + attribute.description = "The type of each face: wall, floor, or ceiling."; + attribute.category_data = &cat_data; + attr_handle = omf_writer_attribute(writer, cube_handle, &attribute); + + /// Add an integer sub-attribute to that category attribute. + num_data = omf_number_data_init(); + num_data.values = omf_writer_array_numbers_int64(writer, CATEGORY_IDS, NULL, 3); + attribute = omf_attribute_init("Discrete", OMF_LOCATION_CATEGORIES); + attribute.description = "Category ids."; + attribute.number_data = &num_data; + omf_writer_attribute(writer, attr_handle, &attribute); + + // Number values on vertices with a discrete colormap. + c_cmap = omf_continuous_colormap_init( + 0.0, 7.0, omf_writer_array_gradient(writer, GRADIENT, 2)); + num_data = omf_number_data_init(); + num_data.continuous_colormap = &c_cmap; + num_data.values = omf_writer_array_numbers_float32(writer, NUMBERS, NULL, 8); + attribute = omf_attribute_init("Continuous", OMF_LOCATION_VERTICES); + attribute.description = "Numbers with a continuous colormap, shading from red to yellow."; + attribute.number_data = &num_data; + omf_writer_attribute(writer, cube_handle, &attribute); + + // Number values on vertices with a discrete colormap. + d_cmap = omf_discrete_colormap_init(); + d_cmap.boundaries = omf_writer_array_boundaries_float32( + writer, DISCRETE_BOUNDARIES, DISCRETE_INCLUSIVE, 4); + d_cmap.gradient = omf_writer_array_gradient(writer, DISCRETE_COLORS, 5); + num_data = omf_number_data_init(); + num_data.discrete_colormap = &d_cmap; + num_data.values = omf_writer_array_numbers_float32(writer, NUMBERS, NULL, 8); + attribute = omf_attribute_init("Discrete", OMF_LOCATION_VERTICES); + attribute.description = "Numbers with a discrete colormap, shading from red to yellow with " + "each color applied to two vertices."; + attribute.number_data = &num_data; + omf_writer_attribute(writer, cube_handle, &attribute); + + // Datetime values on vertices with no colormap. + num_data = omf_number_data_init(); + num_data.values = omf_writer_array_numbers_date_time(writer, DATETIMES_MS, NULL, 8); + attribute = omf_attribute_init("Date-times", OMF_LOCATION_VERTICES); + attribute.description = "A scattering of date-time values as milliseconds since the epoch."; + attribute.units = "datetime[ms]"; + attribute.number_data = &num_data; + omf_writer_attribute(writer, cube_handle, &attribute); + + // Finish writing and close the file. + omf_writer_finish(writer, NULL); + + // Check for errors. + if ((error = omf_error()) != NULL) { + fprintf(stderr, "[write failed] %s (%d)\n", error->message, error->code); + omf_error_free(error); + return false; + } + return true; +} + +static void print_numbers_float32(OmfReader *reader, const OmfArray *array) { + OmfArrayInfo info; + OmfNumbersFloat32 *iter; + float data; + bool is_null; + + info = omf_reader_array_info(reader, array); + assert(info.array_type == OMF_ARRAY_TYPE_NUMBERS_FLOAT32); + iter = omf_reader_array_numbers_float32_iter(reader, array); + while (omf_numbers_float32_next(iter, &data, &is_null)) { + assert(!is_null); + printf(" %g\n", data); + } + omf_numbers_float32_free(iter); +} + +static void print_vectors64x2(OmfReader *reader, const OmfArray *array) { + OmfArrayInfo info; + OmfVectors64x2 *iter; + double data[2]; + bool is_null; + + info = omf_reader_array_info(reader, array); + assert(info.array_type == OMF_ARRAY_TYPE_VECTORS64X2); + iter = omf_reader_array_vectors64x2_iter(reader, array); + while (omf_vectors64x2_next(iter, data, &is_null)) { + if (is_null) { + printf(" null\n"); + } else { + printf(" { %g, %g }\n", data[0], data[1]); + } + } + omf_vectors64x2_free(iter); +} + +static void print_text(OmfReader *reader, const OmfArray *array) { + OmfArrayInfo info; + OmfText *iter; + const char *data; + size_t len; + + info = omf_reader_array_info(reader, array); + assert(info.array_type == OMF_ARRAY_TYPE_TEXT); + iter = omf_reader_array_text_iter(reader, array); + while (omf_text_next(iter, &data, &len)) { + if (data == NULL) { + assert(len == 0); + printf(" null\n"); + } else { + assert(len == strlen(data)); + printf(" \"%s\"\n", data); + } + } + omf_text_free(iter); +} + +static bool read(const char *path) { + OmfReader *reader; + const OmfProject *project; + OmfError *error; + const OmfAttribute *attribute; + + // Open the file and read the project. + reader = omf_reader_open(path); + project = omf_reader_project(reader, NULL); + if (!project) { + error = omf_error(); + fprintf(stderr, "[read failed] %s (%d)\n", error->message, error->code); + omf_error_free(error); + return false; + } + printf("name: %s\n", project->name); + + // Read a few of those attributes back and print them out. + + // Masked vector attribute. + attribute = &project->elements[0].attributes[0]; + printf("%s:\n", attribute->name); + print_vectors64x2(reader, attribute->vector_data); + + // String attribute. + attribute = &project->elements[0].attributes[4]; + printf("%s:\n", attribute->name); + print_text(reader, attribute->text_data); + + // Masked string attribute. + attribute = &project->elements[0].attributes[5]; + printf("%s:\n", attribute->name); + print_text(reader, attribute->text_data); + + // Number attribute. + attribute = &project->elements[0].attributes[7]; + printf("%s:\n", attribute->name); + print_numbers_float32(reader, attribute->number_data->values); + + // Close the reader only once we're done with `project`. + omf_reader_close(reader); + + // Check for errors. + if ((error = omf_error()) != NULL) { + fprintf(stderr, "[read failed] %s (%d)\n", error->message, + error->code); + omf_error_free(error); + return false; + } + return true; +} + +int main() { + if (!write("attributes.omf")) return 1; + if (!read("attributes.omf")) return 1; + return 0; +} diff --git a/omf-c/examples/attributes_output.txt b/omf-c/examples/attributes_output.txt new file mode 100644 index 0000000..196c058 --- /dev/null +++ b/omf-c/examples/attributes_output.txt @@ -0,0 +1,49 @@ +name: attributes.omf +Outward: + null + null + { 0, -1 } + { 0, -1 } + { 1, 0 } + { 1, 0 } + { 0, 1 } + { 0, 1 } + { -1, 0 } + { -1, 0 } + null + null +Directions: + "down" + "down" + "south" + "south" + "east" + "east" + "north" + "north" + "west" + "west" + "up" + "up" +Origin: + "origin" + null + null + null + null + null + null + null + null + null + null + null +Continuous: + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 diff --git a/omf-c/examples/build.bat b/omf-c/examples/build.bat new file mode 100644 index 0000000..aeb07ce --- /dev/null +++ b/omf-c/examples/build.bat @@ -0,0 +1,25 @@ +@REM Run 'cargo build --release --all' before running this script. +@echo off +if not exist "build" mkdir build +cd build || ( exit /b 1 ) +set CMAKE_BUILD_TYPE=Release +cmake .. || ( exit /b 1 ) +cmake --build . --config Release || ( exit /b 1 ) +set path=..\..\..\target\release;%path% + +.\Release\pyramid.exe > pyramid.txt || exit /b +git diff --ignore-cr-at-eol --no-index --exit-code pyramid.txt ..\pyramid_output.txt || exit /b + +.\Release\metadata.exe > metadata.txt || exit /b +git diff --ignore-cr-at-eol --no-index --exit-code metadata.txt ..\metadata_output.txt || exit /b + +.\Release\geometries.exe > geometries.txt || exit /b +git diff --ignore-cr-at-eol --no-index --exit-code geometries.txt ..\geometries_output.txt || exit /b + +.\Release\attributes.exe > attributes.txt || exit /b +git diff --ignore-cr-at-eol --no-index --exit-code attributes.txt ..\attributes_output.txt || exit /b + +.\Release\textures.exe > textures.txt || exit /b +git diff --ignore-cr-at-eol --no-index --exit-code textures.txt ..\textures_output.txt || exit /b + +echo All OK diff --git a/omf-c/examples/build.sh b/omf-c/examples/build.sh new file mode 100644 index 0000000..e3b7a76 --- /dev/null +++ b/omf-c/examples/build.sh @@ -0,0 +1,24 @@ +#!/usr/bin/sh +# Run 'cargo build --all --release' before running this script. +set -e +mkdir -p build +cd build +cmake .. +cmake --build . +export PATH="../../../target/release;$PATH" +./pyramid > pyramid.txt +echo "Run pyramid" +diff -u --strip-trailing-cr pyramid.txt ../pyramid_output.txt +./metadata > metadata.txt +echo "Run metadata" +diff -u --strip-trailing-cr metadata.txt ../metadata_output.txt +./geometries > geometries.txt +echo "Run geometries" +diff -u --strip-trailing-cr geometries.txt ../geometries_output.txt +./attributes > attributes.txt +echo "Run attributes" +diff -u --strip-trailing-cr attributes.txt ../attributes_output.txt +./textures > textures.txt +echo "Run textures" +diff -u --strip-trailing-cr textures.txt ../textures_output.txt +echo "All OK" diff --git a/omf-c/examples/example.png b/omf-c/examples/example.png new file mode 100644 index 0000000..17eb5aa Binary files /dev/null and b/omf-c/examples/example.png differ diff --git a/omf-c/examples/geometries.c b/omf-c/examples/geometries.c new file mode 100644 index 0000000..bdc9feb --- /dev/null +++ b/omf-c/examples/geometries.c @@ -0,0 +1,248 @@ +// Writes an OMF file containing one of each of the remaining element geometries. +// Surface and LineSet are covered in pyramid.c so won't be repeated here. + +#include +#include +#include +#include + +// 2D Tensor data. +static const double TENSOR_U[] = { 2.0, 1.0 }; +static const double TENSOR_V[] = { 1.0, 1.0 }; +static const double TENSOR_W[] = { 0.5 }; + +// 2D grid heights. +static const float HEIGHTS[] = { + -1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, +}; + +// 2D vertices. +static const float VERTICES[][3] = { + { 10.0, 0.0, -1.0 }, + { 12.0, 0.0, -1.0 }, + { 13.0, 0.0, -1.0 }, + { 10.0, 1.0, -1.0 }, + { 12.0, 1.0, 1.0 }, + { 13.0, 1.0, -1.0 }, + { 10.0, 2.0, -1.0 }, + { 12.0, 2.0, -1.0 }, + { 13.0, 2.0, -1.0 }, +}; + +static const uint32_t REGULAR_SUBBLOCK_PARENTS[][3] = { + { 0, 0, 0 }, + { 0, 0, 0 }, + { 0, 0, 0 }, + { 1, 0, 0 }, +}; + +static const uint32_t REGULAR_SUBBLOCK_CORNERS[][6] = { + { 0, 1, 0, 1, 2, 1 }, + { 1, 0, 0, 2, 1, 1 }, + { 1, 1, 0, 2, 2, 2 }, + { 0, 0, 0, 2, 2, 2 }, +}; + +static const uint32_t FREEFORM_SUBBLOCK_PARENTS[][3] = { + { 0, 0, 0 }, + { 0, 0, 0 }, + { 1, 0, 0 }, +}; + +static const float FREEFORM_SUBBLOCK_CORNERS[][6] = { + { 0.0, 0.0, 0.0, 0.5, 1.0, 0.17f }, + { 0.0, 0.0, 0.17f, 0.5, 1.0, 1.0 }, + { 0.0, 0.0, 0.0, 1.0, 1.0, 1.0 }, +}; + +typedef struct { + const float (*vertices)[3]; + size_t length; + size_t index; +} VertexIter; + +static bool next_vertex(void *object, double out[3]) { + VertexIter *iter = object; + if (iter->index >= iter->length) { + return false; + } else { + out[0] = iter->vertices[iter->index][0]; + out[1] = iter->vertices[iter->index][1]; + out[2] = iter->vertices[iter->index][2]; + ++iter->index; + return true; + } +} + +static bool write(const char *path) { + OmfError *error; + OmfWriter *writer; + OmfHandle *proj_handle, *comp_handle; + OmfProject project; + OmfElement element; + OmfPointSet point_set; + OmfComposite composite; + OmfGridSurface grid_surface; + OmfTensorGrid2 tensor_grid2; + OmfTensorGrid3 tensor_grid3; + OmfRegularGrid3 regular_grid3; + OmfBlockModel block_model; + OmfRegularSubblocks subblocks; + OmfFreeformSubblocks freeform_subblocks; + VertexIter iter; + + // Open the file and create a project. + writer = omf_writer_open(path); + project = omf_project_init("geometries.omf"); + proj_handle = omf_writer_project(writer, &project); + + // Composite element, sub-elements added below. + composite = omf_composite_init(); + element = omf_element_init("Container"); + element.composite = &composite; + element.description = "Contains a grid surface, plus a point set of the vertices of that grid."; + comp_handle = omf_writer_element(writer, proj_handle, &element); + + // GridSurface. + tensor_grid2 = omf_tensor_grid2_init( + omf_writer_array_scalars64(writer, TENSOR_U, 2), + omf_writer_array_scalars64(writer, TENSOR_V, 2) + ); + grid_surface = omf_grid_surface_init(); + grid_surface.orient.origin[0] = 10.0; + grid_surface.tensor_grid = &tensor_grid2; + grid_surface.heights = omf_writer_array_scalars32(writer, HEIGHTS, 9); + element = omf_element_init("GridSurface"); + element.description = "An example 2D grid surface."; + element.grid_surface = &grid_surface; + omf_writer_element(writer, comp_handle, &element); + + // PointSet. + // Write the vertices using the iterator API. + iter.vertices = VERTICES; + iter.index = 0; + iter.length = 9; + point_set = omf_point_set_init(omf_writer_array_vertices64_iter(writer, &next_vertex, &iter)); + element = omf_element_init("PointSet"); + element.description = "Points that should be in the same places as the grid vertices."; + element.point_set = &point_set; + omf_writer_element(writer, comp_handle, &element); + + // BlockModel with tensor grid and no sub-blocks. + tensor_grid3 = omf_tensor_grid3_init( + omf_writer_array_scalars64(writer, TENSOR_U, 2), + omf_writer_array_scalars64(writer, TENSOR_V, 2), + omf_writer_array_scalars64(writer, TENSOR_W, 1) + ); + block_model = omf_block_model_init(); + block_model.tensor_grid = &tensor_grid3; + element = omf_element_init("Tensor block model"); + element.block_model = &block_model; + omf_writer_element(writer, proj_handle, &element); + + // BlockModel with regular sub-blocks. + regular_grid3 = omf_regular_grid3_init(1.0, 1.0, 1.0, 2, 1, 1); + subblocks = omf_regular_subblocks_init( + 2, 2, 2, + omf_writer_array_regular_subblocks(writer, REGULAR_SUBBLOCK_PARENTS, REGULAR_SUBBLOCK_CORNERS, 4) + ); + block_model = omf_block_model_init(); + block_model.regular_grid = ®ular_grid3; + block_model.regular_subblocks = &subblocks; + element = omf_element_init("Regular block model with regular sub-blocks"); + element.block_model = &block_model; + omf_writer_element(writer, proj_handle, &element); + + // BlockModel with free-form sub-blocks. + regular_grid3 = omf_regular_grid3_init(1.0, 1.0, 1.0, 2, 1, 1); + freeform_subblocks = omf_freeform_subblocks_init( + omf_writer_array_freeform_subblocks32(writer, FREEFORM_SUBBLOCK_PARENTS, FREEFORM_SUBBLOCK_CORNERS, 3) + ); + block_model = omf_block_model_init(); + block_model.regular_grid = ®ular_grid3; + block_model.freeform_subblocks = &freeform_subblocks; + element = omf_element_init("Regular block model with free-form sub-blocks"); + element.block_model = &block_model; + omf_writer_element(writer, proj_handle, &element); + + // Finish writing and close the file. + omf_writer_finish(writer, NULL); + + // Check for errors. + if ((error = omf_error()) != NULL) { + fprintf(stderr, "[write failed] %s (%d)\n", error->message, error->code); + omf_error_free(error); + return false; + } + return true; +} + +static bool read(const char *path) { + OmfReader *reader; + const OmfProject *project; + const OmfElement *element; + OmfError *error; + double u[2], v[2], heights[9], x, y, z; + size_t i, j; + double vertices[9][3]; + + // Open the file and read the project. + reader = omf_reader_open(path); + project = omf_reader_project(reader, NULL); + if (!project) { + error = omf_error(); + fprintf(stderr, "[read failed] %s (%d)\n", error->message, error->code); + omf_error_free(error); + return false; + } + printf("name: %s\n", project->name); + + // Read and print the grid surface. + element = &project->elements[0].composite->elements[0]; + printf("element: %s\n", element->name); + omf_reader_array_scalars64(reader, element->grid_surface->tensor_grid->u, u, 2); + omf_reader_array_scalars64(reader, element->grid_surface->tensor_grid->v, v, 2); + // The heights were written as 'float' but can be read back as 'double'. Casting to larger + // types within the same category (floating point, unsigned int, and signed int) is allowed. + omf_reader_array_scalars64(reader, element->grid_surface->heights, heights, 9); + y = element->grid_surface->orient.origin[1]; + for (j = 0; j <= 2; j++) { + x = element->grid_surface->orient.origin[0]; + for (i = 0; i <= 2; i++) { + z = heights[j * 3 + i] + element->grid_surface->orient.origin[2]; + printf(" %g %g %g\n", x, y, z); + if (i < 2) { + x += u[i]; + } + } + if (j < 2) { + y += v[j]; + } + } + + // Read and print the points. + element = &project->elements[0].composite->elements[1]; + printf("element: %s\n", element->name); + omf_reader_array_vertices64(reader, element->point_set->vertices, vertices, 9); + for (i = 0; i < 9; i++) { + printf(" %g %g %g\n", vertices[i][0], vertices[i][1], vertices[i][2]); + } + + // Close the reader only once we're done with `project`. + omf_reader_close(reader); + + // Check for errors. + if ((error = omf_error()) != NULL) { + fprintf(stderr, "[read failed] %s (%d)\n", error->message, + error->code); + omf_error_free(error); + return false; + } + return true; +} + +int main() { + if (!write("geometries.omf")) return 1; + if (!read("geometries.omf")) return 1; + return 0; +} diff --git a/omf-c/examples/geometries_output.txt b/omf-c/examples/geometries_output.txt new file mode 100644 index 0000000..52da1da --- /dev/null +++ b/omf-c/examples/geometries_output.txt @@ -0,0 +1,21 @@ +name: geometries.omf +element: GridSurface + 10 0 -1 + 12 0 -1 + 13 0 -1 + 10 1 -1 + 12 1 1 + 13 1 -1 + 10 2 -1 + 12 2 -1 + 13 2 -1 +element: PointSet + 10 0 -1 + 12 0 -1 + 13 0 -1 + 10 1 -1 + 12 1 1 + 13 1 -1 + 10 2 -1 + 12 2 -1 + 13 2 -1 diff --git a/omf-c/examples/metadata.c b/omf-c/examples/metadata.c new file mode 100644 index 0000000..5a41305 --- /dev/null +++ b/omf-c/examples/metadata.c @@ -0,0 +1,172 @@ +// Demonstrates OMF metadata storage and retrieval. + +#include +#include +#include + +static bool write(const char *path) { + OmfError *error; + OmfWriter *writer; + OmfProject project; + OmfHandle *proj_handle, *array_handle, *object_handle; + + // Open file. + writer = omf_writer_open(path); + // Create project and keep the handle to it. + project = omf_project_init("metadata.omf"); + proj_handle = omf_writer_project(writer, &project); + + // Add a metadata value of each simple type. This is added directly to the project, but + // an element or attribute handle will work too. This all gets stored as a chunk of + // arbitrary JSON data in the file. Attaching too much meaning to metadata values may + // make your file less useful in other applications as they won't necessarily know what + // it means. + // + // Metadata keys, and values when they're strings, must be UTF-8 encoded. ASCII is also + // acceptable because it's a subset of UTF-8. + + // Null values store only the key. This can be used where just the presence of the key is + // useful or where a value isn't known. + omf_writer_metadata_null(writer, proj_handle, "version"); + // Boolean values store true or false. + omf_writer_metadata_boolean(writer, proj_handle, "is_draft", true); + // Number values store a double value. + omf_writer_metadata_number(writer, proj_handle, "importance", 2.6); + // String value. This could also be used to store date or date/time values, which should + // be in ISO 8601 format. + omf_writer_metadata_string(writer, proj_handle, "source", "omf example code"); + + // We can also store arrays of metadata values. Items in an array can have different types. + // The same `omf_writer_metadata_*` functions are used to append array items, but the `key` + // argument is ignored and should be null. + array_handle = omf_writer_metadata_list(writer, proj_handle, "list"); + omf_writer_metadata_string(writer, array_handle, NULL, "first value"); + omf_writer_metadata_string(writer, array_handle, NULL, "second value"); + omf_writer_metadata_number(writer, array_handle, NULL, 3); + + // Finally we have object values, which contain their own key/value pairs. This is a good + // way to group and label application-specific data for example. + object_handle = omf_writer_metadata_object(writer, proj_handle, "my-company"); + omf_writer_metadata_string( + writer, object_handle, "project-uuid", "550e8400-e29b-41d4-a716-446655440000"); + omf_writer_metadata_string( + writer, object_handle, "project-uri", "https://example.com/"); + omf_writer_metadata_string( + writer, object_handle, "project-revision", "1.4.2"); + + // Finish writing and close the file. + omf_writer_finish(writer, NULL); + + // Check for errors. The `omf_error` call will return the *first* error, even if several + // functions failed since after detecting an invalid argument. + if ((error = omf_error()) != NULL) { + fprintf(stderr, "[write failed] %s (%d)\n", error->message, error->code); + omf_error_free(error); + return false; + } + return true; +} + +static void print_indent(int indent) { + int i; + + for (i = 0; i < indent; i++) { + printf(" "); + } +} + +static void print_metadata_value(const OmfValue *value, int indent, bool is_array_item) { + size_t i; + + print_indent(indent); + if (!is_array_item) { + printf("\"%s\": ", value->name); + } + // First check the value type. + switch (value->type) { + case OMF_VALUE_TYPE_NULL: + // No valid fields for null values. + printf("null,\n"); + break; + case OMF_VALUE_TYPE_BOOLEAN: + // `boolean` field is valid. + printf("%s,\n", value->boolean ? "true" : "false"); + break; + case OMF_VALUE_TYPE_NUMBER: + // `number` field is valid. + printf("%g,\n", value->number); + break; + case OMF_VALUE_TYPE_STRING: + // `string` field is valid. + printf("\"%s\",\n", value->string); + break; + case OMF_VALUE_TYPE_LIST: + // `values` and `n_values` fields are valid and contain ordered values. + printf("[\n"); + for (i = 0; i < value->n_values; i++) { + print_metadata_value(&value->values[i], indent + 1, true); + } + print_indent(indent); + printf("],\n"); + break; + default: // OMF_VALUE_TYPE_OBJECT + // `values` and `n_values` fields are valid and contain named values. + printf("{\n"); + for (i = 0; i < value->n_values; i++) { + print_metadata_value(&value->values[i], indent + 1, false); + } + print_indent(indent); + printf("},\n"); + break; + } +} + +static bool read(const char *path) { + OmfReader *reader; + OmfError *error; + const OmfProject *project; + size_t i; + + // Open the file. + reader = omf_reader_open(path); + // Read the project. + project = omf_reader_project(reader, NULL); + if (!project) { + error = omf_error(); + fprintf(stderr, "[read failed] %s (%d)\n", error->message, error->code); + omf_error_free(error); + return false; + } + // Print project contents. + printf("name: %s\n", project->name); + + // Metadata is stored as a list of `OmfValue` structs in `project->metadata` with length + // `project->n_metadata`. The order that values were written in is not preserved. + // Inside `OmfValue` the `type` field stores the type of the value and defines which other + // fields are valid. Unused fields will be zeroed. + // + // The `OmfElement` and `OmfAttribute` fields have matching metadata fields. + printf("metadata: {\n"); + for (i = 0; i < project->n_metadata; i++) { + print_metadata_value(&project->metadata[i], 1, false); + } + printf("}\n"); + + // Close the reader only once we're done with `project`. + omf_reader_close(reader); + + // Check for errors. + if ((error = omf_error()) != NULL) { + fprintf(stderr, "[read failed] %s (%d)\n", error->message, + error->code); + omf_error_free(error); + return false; + } + return true; +} + +int main() { + if (!write("metadata.omf")) return 1; + if (!read("metadata.omf")) return 1; + return 0; +} diff --git a/omf-c/examples/metadata_output.txt b/omf-c/examples/metadata_output.txt new file mode 100644 index 0000000..874bc51 --- /dev/null +++ b/omf-c/examples/metadata_output.txt @@ -0,0 +1,17 @@ +name: metadata.omf +metadata: { + "importance": 2.6, + "is_draft": true, + "list": [ + "first value", + "second value", + 3, + ], + "my-company": { + "project-revision": "1.4.2", + "project-uri": "https://example.com/", + "project-uuid": "550e8400-e29b-41d4-a716-446655440000", + }, + "source": "omf example code", + "version": null, +} diff --git a/omf-c/examples/pyramid.c b/omf-c/examples/pyramid.c new file mode 100644 index 0000000..8a4a5ad --- /dev/null +++ b/omf-c/examples/pyramid.c @@ -0,0 +1,196 @@ +// Writes a small OMF file containing two elements: the surface and outline of a square pyramid. +// Then reads that file back and prints the data. + +#include +#include +#include + +static const float VERTICES[][3] = { + { -1.0, -1.0, 0.0 }, + { 1.0, -1.0, 0.0 }, + { 1.0, 1.0, 0.0 }, + { -1.0, 1.0, 0.0 }, + { 0.0, 0.0, 1.0 }, +}; + +static const uint32_t TRIANGLES[][3] = { + { 0, 1, 4 }, + { 1, 2, 4 }, + { 2, 3, 4 }, + { 3, 0, 4 }, + { 0, 2, 1 }, + { 0, 3, 2 }, +}; + +const uint32_t SEGMENTS[][2] = { + { 0, 1 }, + { 1, 2 }, + { 2, 3 }, + { 3, 0 }, + { 0, 4 }, + { 1, 4 }, + { 2, 4 }, + { 3, 4 }, +}; + +static bool write(const char *path) { + OmfError *error; + OmfWriter *writer; + OmfProject project; + OmfSurface surface; + OmfLineSet line_set; + OmfElement element; + const OmfArray *vertices; + OmfHandle *proj_handle, *ele_handle, *tags_handle; + + omf_reader_limits(NULL); + // Open file. + writer = omf_writer_open(path); + // Fill in `project` with the required name and optional description. + project = omf_project_init("pyramid.omf"); + project.description = "Contains a square pyramid."; + project.author = "Somebody"; + proj_handle = omf_writer_project(writer, &project); + + // First a surface element. Start writing the vertex and triangle arrays + // and putting them in `surface`. + vertices = omf_writer_array_vertices32(writer, VERTICES, 5); + surface = omf_surface_init( + vertices, omf_writer_array_triangles(writer, TRIANGLES, 6)); + // Fill in `element` with the surface and other fields. + element = omf_element_init("Pyramid surface"); + element.surface = &surface; + element.color_set = true; + element.color[0] = 255; + element.color[1] = 128; + element.color[2] = 0; + element.color[3] = 255; // Opaque + // Write the element. + ele_handle = omf_writer_element(writer, proj_handle, &element); + // Add metadata to that element. + omf_writer_metadata_string(writer, ele_handle, "revision", "1.2"); + tags_handle = omf_writer_metadata_list(writer, ele_handle, "tags"); + omf_writer_metadata_string(writer, tags_handle, NULL, "foo"); + omf_writer_metadata_string(writer, tags_handle, NULL, "bar"); + + // Second a line-set element. This uses the same vertices array as the + // surface. If we wrote it a second time the duplicate would be detected + // and removed but we can also pass it in to both geometries. + line_set = omf_line_set_init( + vertices, omf_writer_array_segments(writer, SEGMENTS, 8)); + // Clear and fill in `element` again. + element = omf_element_init("Pyramid outline"); + element.line_set = &line_set; + element.color_set = true; + element.color[0] = 0; + element.color[1] = 0; + element.color[2] = 0; + element.color[3] = 128; // 50% transparent + // And write it. + omf_writer_element(writer, proj_handle, &element); + + // Finish writing and close the file. + omf_writer_finish(writer, NULL); + + // Check for errors. The `omf_error` call will return the *first* error, + // even if several functions failed since after detecting an invalid + // argument. + if ((error = omf_error()) != NULL) { + fprintf(stderr, "[write failed] %s (%d)\n", error->message, error->code); + fflush(stderr); + omf_error_free(error); + return false; + } + return true; +} + +static bool read(const char *path) { + OmfReader *reader; + OmfError *error; + const OmfProject *project; + const OmfElement *e; + // Dynamically allocated buffer for vertices. + float (*vertices)[3]; + // For the triangles and segments we'll use fixed-size buffers for simplicity. Initialise + // these buffers to zero so that if the read fails we don't end up printing uninitialised + // memory. + uint32_t segments[8][2] = { 0 }; + size_t i; + OmfArrayInfo info; + OmfTriangles *tri_iter; + uint32_t tri[3]; + + // Open the file. + reader = omf_reader_open(path); + // Read the project. + project = omf_reader_project(reader, NULL); + if (!project) { + error = omf_error(); + fprintf(stderr, "[read failed] %s (%d)\n", error->message, error->code); + omf_error_free(error); + return false; + } + // Print project contents. + printf("name: %s\n", project->name); + printf("description: %s\n", project->description); + printf("coordinate_reference_system: %s\n", project->coordinate_reference_system); + printf("origin: %g, %g, %g\n", project->origin[0], project->origin[1], project->origin[2]); + printf("author: %s\n", project->author); + e = &project->elements[0]; + printf("surface:\n"); + printf(" name: %s\n", e->name); + printf(" description: %s\n", e->description); + printf(" color: #%02x%02x%02x%02x\n", e->color[0], e->color[1], e->color[2], e->color[3]); + printf(" origin: %g, %g, %g\n", + e->surface->origin[0], e->surface->origin[1], e->surface->origin[2]); + // Allocate a buffer for the vertices, pretending we don't know the required length already. + // Calloc Initializes the memory to zero for us. + info = omf_reader_array_info(reader, e->surface->vertices); + vertices = calloc(info.item_count, sizeof(float[3])); + if (vertices == NULL) { + fprintf(stderr, "memory allocation failed"); + return 1; + } + omf_reader_array_vertices32(reader, e->surface->vertices, vertices, info.item_count); + printf(" vertices:\n"); + for (i = 0; i < info.item_count; i++) { + printf(" % g, % g, % g\n", vertices[i][0], vertices[i][1], vertices[i][2]); + } + // Read the triangles using the iterator API. + printf(" triangles:\n"); + tri_iter = omf_reader_array_triangles_iter(reader, e->surface->triangles); + while (omf_triangles_next(tri_iter, tri)) { + printf(" %d, %d, %d\n", tri[0], tri[1], tri[2]); + } + e = &project->elements[1]; + printf("line-set:\n"); + printf(" name: %s\n", e->name); + printf(" description: %s\n", e->description); + printf(" color: #%02x%02x%02x%02x\n", e->color[0], e->color[1], e->color[2], e->color[3]); + printf(" origin: %g, %g, %g\n", + e->line_set->origin[0], e->line_set->origin[1], e->line_set->origin[2]); + // Read the segments into a fixed-size buffer. + omf_reader_array_segments(reader, e->line_set->segments, segments, 8); + printf(" segments:\n"); + for (i = 0; i < 8; i++) { + printf(" %d, %d\n", segments[i][0], segments[i][1]); + } + + // Close the reader only once we're done with `project`. + omf_reader_close(reader); + + // Check for errors. + if ((error = omf_error()) != NULL) { + fprintf(stderr, "[read failed] %s (%d)\n", error->message, + error->code); + omf_error_free(error); + return false; + } + return true; +} + +int main(void) { + if (!write("pyramid.omf")) return 1; + if (!read("pyramid.omf")) return 1; + return 0; +} diff --git a/omf-c/examples/pyramid_output.txt b/omf-c/examples/pyramid_output.txt new file mode 100644 index 0000000..81171da --- /dev/null +++ b/omf-c/examples/pyramid_output.txt @@ -0,0 +1,37 @@ +name: pyramid.omf +description: Contains a square pyramid. +coordinate_reference_system: (null) +origin: 0, 0, 0 +author: Somebody +surface: + name: Pyramid surface + description: (null) + color: #ff8000ff + origin: 0, 0, 0 + vertices: + -1, -1, 0 + 1, -1, 0 + 1, 1, 0 + -1, 1, 0 + 0, 0, 1 + triangles: + 0, 1, 4 + 1, 2, 4 + 2, 3, 4 + 3, 0, 4 + 0, 2, 1 + 0, 3, 2 +line-set: + name: Pyramid outline + description: (null) + color: #00000080 + origin: 0, 0, 0 + segments: + 0, 1 + 1, 2 + 2, 3 + 3, 0 + 0, 4 + 1, 4 + 2, 4 + 3, 4 diff --git a/omf-c/examples/textures.c b/omf-c/examples/textures.c new file mode 100644 index 0000000..8ea1755 --- /dev/null +++ b/omf-c/examples/textures.c @@ -0,0 +1,198 @@ +// Writes an OMF file containing a projected texture and a mapped texture. + +#include +#include +#include +#include + +static const double VERTICES[][3] = { + { -1.0, -1.0, 0.0 }, + { 1.0, -1.0, 0.0 }, + { 1.0, 1.0, 0.0 }, + { -1.0, 1.0, 0.0 }, + // The tip vertex is duplicated here so it can have different texture coordinates for + // each side face. + { 0.0, 0.0, 1.0 }, + { 0.0, 0.0, 1.0 }, + { 0.0, 0.0, 1.0 }, + { 0.0, 0.0, 1.0 }, +}; + +static const uint32_t TRIANGLES[][3] = { + { 0, 1, 4 }, + { 1, 2, 5 }, + { 2, 3, 6 }, + { 3, 0, 7 }, + { 0, 2, 1 }, + { 0, 3, 2 }, +}; + +static const float TEXCOORDS[][2] = { + { 0.5, 0.0 }, + { 1.0, 0.5 }, + { 0.5, 1.0 }, + { 0.0, 0.5 }, + { 1.0, 0.0 }, + { 1.0, 1.0 }, + { 0.0, 1.0 }, + { 0.0, 0.0 }, +}; + +static const char EXAMPLE_PNG[] = { + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, + 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x40, + 0x08, 0x02, 0x00, 0x00, 0x00, 0x25, 0x0b, 0xe6, 0x89, 0x00, 0x00, 0x00, + 0x09, 0x70, 0x48, 0x59, 0x73, 0x00, 0x00, 0x02, 0x80, 0x00, 0x00, 0x02, + 0x80, 0x01, 0x5b, 0x4d, 0x5f, 0x70, 0x00, 0x00, 0x00, 0x19, 0x74, 0x45, + 0x58, 0x74, 0x53, 0x6f, 0x66, 0x74, 0x77, 0x61, 0x72, 0x65, 0x00, 0x77, + 0x77, 0x77, 0x2e, 0x69, 0x6e, 0x6b, 0x73, 0x63, 0x61, 0x70, 0x65, 0x2e, + 0x6f, 0x72, 0x67, 0x9b, 0xee, 0x3c, 0x1a, 0x00, 0x00, 0x01, 0x9a, 0x49, + 0x44, 0x41, 0x54, 0x68, 0xde, 0xd5, 0xda, 0x3b, 0x72, 0xc3, 0x30, 0x0c, + 0x04, 0x50, 0x71, 0x6f, 0x1b, 0x17, 0xb9, 0x41, 0xba, 0x9c, 0x21, 0xb7, + 0xf4, 0xcc, 0xde, 0x40, 0x2e, 0x92, 0xc9, 0x78, 0xfc, 0x91, 0xf0, 0x25, + 0x01, 0x55, 0x2e, 0x48, 0x61, 0x5f, 0x21, 0x4b, 0x24, 0x31, 0xb6, 0x9f, + 0x7d, 0x4b, 0xbd, 0x3e, 0xbe, 0x53, 0x6f, 0x8f, 0xd4, 0xbb, 0xef, 0x97, + 0xb1, 0x8f, 0xaf, 0xae, 0x80, 0xfd, 0x32, 0xfe, 0x7e, 0x64, 0x1a, 0x90, + 0x9d, 0x3e, 0xdb, 0x80, 0x09, 0xe9, 0x53, 0x0d, 0x98, 0x93, 0x3e, 0xcf, + 0x80, 0x69, 0xe9, 0x93, 0x0c, 0x98, 0x99, 0x3e, 0xc3, 0x80, 0xc9, 0xe9, + 0xc3, 0x0d, 0x98, 0x9f, 0x3e, 0xd6, 0x80, 0x25, 0xe9, 0x03, 0x0d, 0x58, + 0x95, 0x3e, 0xca, 0x80, 0x85, 0xe9, 0x43, 0x0c, 0x58, 0x9b, 0xde, 0x6f, + 0xc0, 0xf2, 0xf4, 0x4e, 0x03, 0x2a, 0xa4, 0xf7, 0x18, 0x50, 0x24, 0xbd, + 0xd9, 0x80, 0x3a, 0xe9, 0x6d, 0x06, 0x94, 0x4a, 0x6f, 0x30, 0xa0, 0x5a, + 0x7a, 0xad, 0x01, 0x05, 0xd3, 0xab, 0x0c, 0xa8, 0x99, 0x5e, 0x6e, 0x40, + 0xd9, 0xf4, 0x42, 0x03, 0x2a, 0xa7, 0x97, 0x18, 0x50, 0x3c, 0xfd, 0xa9, + 0x01, 0xf5, 0xd3, 0x1f, 0x1b, 0xd0, 0x22, 0xfd, 0x81, 0x01, 0x5d, 0xd2, + 0xbf, 0x33, 0xa0, 0x51, 0xfa, 0x97, 0x06, 0xf4, 0x4a, 0xff, 0x6c, 0x40, + 0xbb, 0xf4, 0x0f, 0x06, 0x74, 0x4c, 0x7f, 0x6f, 0x18, 0xfb, 0xd6, 0xfb, + 0xc2, 0xf6, 0x79, 0x6d, 0x1c, 0x9f, 0xbf, 0xcf, 0x40, 0x53, 0x03, 0xef, + 0xff, 0x85, 0xda, 0x19, 0xf8, 0xfc, 0x1e, 0x68, 0x64, 0xe0, 0xbb, 0x37, + 0x71, 0x0b, 0x03, 0x8f, 0xbf, 0x85, 0x8a, 0x1b, 0x28, 0xf9, 0x1a, 0x2d, + 0x6b, 0xa0, 0x7c, 0x3d, 0x50, 0xd0, 0x40, 0xed, 0x8a, 0xac, 0x94, 0x81, + 0xb6, 0x35, 0x71, 0x11, 0x03, 0x3d, 0xbb, 0x12, 0xcb, 0x0d, 0xf4, 0xed, + 0x4a, 0x2c, 0x36, 0xf0, 0x7c, 0x88, 0x6c, 0x67, 0x6e, 0x89, 0x81, 0xa2, + 0x51, 0xe2, 0xbd, 0xd1, 0xc9, 0x06, 0x4a, 0x07, 0x6a, 0x76, 0xa7, 0xa7, + 0x19, 0xa8, 0x18, 0xab, 0x3c, 0x1f, 0x98, 0x60, 0xa0, 0x6e, 0xb8, 0xfe, + 0x84, 0x26, 0xd5, 0x40, 0xf5, 0x0c, 0xd3, 0x19, 0x59, 0x92, 0x81, 0x96, + 0x49, 0xd6, 0x53, 0xca, 0x70, 0x03, 0x8d, 0xf3, 0x1c, 0xe7, 0xc4, 0x81, + 0x06, 0xda, 0xa7, 0xfa, 0x4e, 0xea, 0x43, 0x0c, 0x74, 0xcd, 0x76, 0xf7, + 0x4a, 0x38, 0x0d, 0xf4, 0xd6, 0x8f, 0xe8, 0x56, 0x31, 0x1b, 0x18, 0x50, + 0x3c, 0xa8, 0x5f, 0xc8, 0x60, 0x60, 0x4c, 0xe5, 0xb8, 0x8e, 0x2d, 0x95, + 0x81, 0x61, 0x65, 0x43, 0x7b, 0xe6, 0x84, 0x06, 0x46, 0xd6, 0x8c, 0xee, + 0x5a, 0x3c, 0x35, 0x30, 0xb8, 0x60, 0x42, 0xdf, 0xe8, 0x81, 0x81, 0xf1, + 0xd5, 0x72, 0x3a, 0x77, 0x5f, 0x1a, 0x98, 0x52, 0x2a, 0xad, 0x77, 0xfa, + 0xc1, 0xc0, 0xac, 0x3a, 0x99, 0xdd, 0xeb, 0xff, 0x06, 0x26, 0x16, 0xb9, + 0x01, 0x89, 0x4f, 0xa5, 0x0e, 0xa3, 0x13, 0xdc, 0x24, 0x00, 0x00, 0x00, + 0x00, 0x49, 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82 +}; +static const size_t EXAMPLE_PNG_LEN = 525; + +static bool write(const char *path) { + OmfError *error; + OmfWriter *writer; + OmfProject project; + OmfElement element; + OmfSurface surface; + OmfAttribute attribute; + OmfMappedTexture mapped; + OmfProjectedTexture projected; + OmfHandle *proj_handle, *surf_handle; + const OmfArray *image; + + // Open the file and create a project. + writer = omf_writer_open(path); + project = omf_project_init("textures.omf"); + proj_handle = omf_writer_project(writer, &project); + + // Create a pyramid surface element and keep that handle too. + surface = omf_surface_init( + omf_writer_array_vertices64(writer, VERTICES, 8), + omf_writer_array_triangles(writer, TRIANGLES, 5) + ); + element = omf_element_init("Pyramid"); + element.surface = &surface; + surf_handle = omf_writer_element(writer, proj_handle, &element); + + // Add the image that both texture types will use. + image = omf_writer_image_bytes(writer, EXAMPLE_PNG, EXAMPLE_PNG_LEN); + + // Add mapped texture attribute. + mapped = omf_mapped_texture_init(image, omf_writer_array_texcoords32(writer, TEXCOORDS, 8)); + attribute = omf_attribute_init("Mapped", OMF_LOCATION_VERTICES); + attribute.mapped_texture_data = &mapped; + omf_writer_attribute(writer, surf_handle, &attribute); + + // Add projected texture attribute. + projected = omf_projected_texture_init(image); + projected.orient.origin[0] = -1.0; + projected.orient.origin[1] = -1.0; + projected.orient.origin[2] = 0.0; + projected.orient.u[0] = 1.0; + projected.orient.u[1] = 0.0; + projected.orient.u[2] = 0.0; + projected.orient.v[0] = 0.0; + projected.orient.v[1] = 0.0; + projected.orient.v[2] = 1.0; + projected.width = 2.0; + projected.height = 2.0; + attribute = omf_attribute_init("Projected", OMF_LOCATION_PROJECTED); + attribute.projected_texture_data = &projected; + omf_writer_attribute(writer, surf_handle, &attribute); + + // Finish writing and close the file. + omf_writer_finish(writer, NULL); + + // Check for errors. + if ((error = omf_error()) != NULL) { + fprintf(stderr, "[write failed] %s (%d)\n", error->message, error->code); + omf_error_free(error); + return false; + } + return true; +} + +static bool read(const char *path) { + OmfReader *reader; + const OmfProject *project; + OmfError *error; + OmfImageData *image_data; + + // Open the file and read the project. + reader = omf_reader_open(path); + project = omf_reader_project(reader, NULL); + if (!project) { + error = omf_error(); + fprintf(stderr, "[read failed] %s (%d)\n", error->message, error->code); + omf_error_free(error); + return false; + } + printf("name: %s\n", project->name); + + // Read the texture image back. + image_data = omf_reader_image( + reader, project->elements[0].attributes[0].mapped_texture_data->image); + assert(image_data); + assert(image_data->mode == OMF_IMAGE_MODE_RGB); + assert(image_data->width == 64); + assert(image_data->height == 64); + // This now contains the pixels in RGB order with no padding. + assert(image_data->uint8); + // Free the image data. + omf_image_data_free(image_data); + + // Close the reader only once we're done with `project`. + omf_reader_close(reader); + + // Check for errors. + if ((error = omf_error()) != NULL) { + fprintf(stderr, "[read failed] %s (%d)\n", error->message, error->code); + omf_error_free(error); + return false; + } + return true; +} + +int main() { + if (!write("textures.omf")) return 1; + if (!read("textures.omf")) return 1; + return 0; +} diff --git a/omf-c/examples/textures_output.txt b/omf-c/examples/textures_output.txt new file mode 100644 index 0000000..2053074 --- /dev/null +++ b/omf-c/examples/textures_output.txt @@ -0,0 +1 @@ +name: textures.omf diff --git a/omf-c/src/arrays.rs b/omf-c/src/arrays.rs new file mode 100644 index 0000000..5b010af --- /dev/null +++ b/omf-c/src/arrays.rs @@ -0,0 +1,179 @@ +use crate::{ + error::Error, + ffi_tools::{FfiConvert, FfiStorage}, +}; + +pub enum Array { + Image(omf::Array), + Scalar(omf::Array), + Vertex(omf::Array), + Segment(omf::Array), + Triangle(omf::Array), + Name(omf::Array), + Gradient(omf::Array), + Texcoord(omf::Array), + Boundary(omf::Array), + RegularSubblock(omf::Array), + FreeformSubblock(omf::Array), + Number(omf::Array), + Index(omf::Array), + Vector(omf::Array), + Text(omf::Array), + Boolean(omf::Array), + Color(omf::Array), +} + +macro_rules! define_array { + ($( $name:ident, )*) => { + impl Array { + pub(crate) fn data_type(&self) -> omf::DataType { + match self { $( + Self::$name(_) => omf::DataType::$name, + )* } + } + } + + $( + impl TryFrom<&Array> for omf::Array { + type Error = Error; + + fn try_from(value: &Array) -> Result { + use omf::ArrayType; + match value { + Array::$name(a) => Ok(a.clone()), + _ => Err(Error::ArrayTypeWrong { + src: "", + found: value.data_type(), + expected: omf::array_type::$name::DATA_TYPE, + }), + } + } + } + + impl FfiConvert> for Array { + fn convert(omf_array: omf::Array, _st: &mut FfiStorage) -> Self { + Self::$name(omf_array) + } + } + )* + }; +} + +define_array! { + Image, + Scalar, + Vertex, + Segment, + Triangle, + Name, + Gradient, + Texcoord, + Boundary, + RegularSubblock, + FreeformSubblock, + Number, + Index, + Vector, + Text, + Boolean, + Color, +} + +macro_rules! array_action { + ($input:expr, |$name:ident| $( $action:tt )*) => { + match $input { + Array::Image($name) => $($action)*, + Array::Scalar($name) => $($action)*, + Array::Vertex($name) => $($action)*, + Array::Segment($name) => $($action)*, + Array::Triangle($name) => $($action)*, + Array::Name($name) => $($action)*, + Array::Gradient($name) => $($action)*, + Array::Texcoord($name) => $($action)*, + Array::Boundary($name) => $($action)*, + Array::RegularSubblock($name) => $($action)*, + Array::FreeformSubblock($name) => $($action)*, + Array::Number($name) => $($action)*, + Array::Index($name) => $($action)*, + Array::Vector($name) => $($action)*, + Array::Text($name) => $($action)*, + Array::Boolean($name) => $($action)*, + Array::Color($name) => $($action)*, + } + }; +} + +pub(crate) fn array_from_ptr<'a, T>( + ptr: *const Array, + src: &'static str, +) -> Result, Error> +where + T: omf::ArrayType, + omf::Array: TryFrom<&'a Array, Error = Error>, +{ + let array = unsafe { crate::ffi_tools::arg::ref_from_ptr(src, ptr) }?; + omf::Array::::try_from(array) +} + +pub(crate) fn array_from_ptr_opt<'a, T>( + ptr: *const Array, + src: &'static str, +) -> Result>, Error> +where + T: omf::ArrayType, + omf::Array: TryFrom<&'a Array, Error = Error>, +{ + if ptr.is_null() { + Ok(None) + } else { + array_from_ptr(ptr, src).map(Some) + } +} + +macro_rules! array { + ($arg:ident) => { + crate::arrays::array_from_ptr($arg, stringify!($arg)) + }; +} + +pub(crate) use {array, array_action}; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +#[repr(i32)] +pub enum ArrayType { + #[default] + Invalid = -1, + Image, + Scalars32, + Scalars64, + Vertices32, + Vertices64, + Segments, + Triangles, + Names, + Gradient, + Texcoords32, + Texcoords64, + BoundariesFloat32, + BoundariesFloat64, + BoundariesInt64, + BoundariesDate, + BoundariesDateTime, + RegularSubblocks, + FreeformSubblocks32, + FreeformSubblocks64, + NumbersFloat32, + NumbersFloat64, + NumbersInt64, + NumbersDate, // accessed as i64 days since the epoch + NumbersDateTime, // accessed as i64 microseconds since the epoch + Indices, + Vectors32x2, + Vectors64x2, + Vectors32x3, + Vectors64x3, + Text, + Booleans, + Colors, +} diff --git a/omf-c/src/attributes.rs b/omf-c/src/attributes.rs new file mode 100644 index 0000000..4e7e6ed --- /dev/null +++ b/omf-c/src/attributes.rs @@ -0,0 +1,160 @@ +use std::{ffi::c_char, ptr::null}; + +use crate::{arrays::Array, metadata::Value}; + +#[derive(Debug)] +#[repr(C)] +pub struct Attribute { + pub name: *const c_char, + pub description: *const c_char, + pub units: *const c_char, + pub n_metadata: usize, + pub metadata: *const Value, + pub location: omf::Location, + pub boolean_data: *const Array, + pub vector_data: *const Array, + pub text_data: *const Array, + pub color_data: *const Array, + pub number_data: *const NumberData, + pub category_data: *const CategoryData, + pub mapped_texture_data: *const MappedTexture, + pub projected_texture_data: *const ProjectedTexture, +} + +impl Default for Attribute { + fn default() -> Self { + Self { + name: null(), + description: null(), + units: null(), + n_metadata: 0, + metadata: null(), + location: Default::default(), + number_data: null(), + boolean_data: null(), + vector_data: null(), + text_data: null(), + color_data: null(), + category_data: null(), + mapped_texture_data: null(), + projected_texture_data: null(), + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct NumberData { + pub values: *const Array, + pub continuous_colormap: *const ContinuousColormap, + pub discrete_colormap: *const DiscreteColormap, +} + +impl Default for NumberData { + fn default() -> Self { + Self { + values: null(), + continuous_colormap: null(), + discrete_colormap: null(), + } + } +} + +#[derive(Debug, Default)] +#[repr(i32)] +pub enum RangeType { + #[default] + Float, + Integer, + Date, + DateTime, +} + +#[derive(Debug)] +#[repr(C)] +pub struct ContinuousColormap { + pub range_type: RangeType, + pub min: f64, + pub max: f64, + pub min_int: i64, + pub max_int: i64, + pub gradient: *const Array, +} + +impl Default for ContinuousColormap { + fn default() -> Self { + Self { + range_type: RangeType::Float, + min: 0.0, + max: 1.0, + min_int: 0, + max_int: 100, + gradient: null(), + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct DiscreteColormap { + pub boundaries: *const Array, + pub gradient: *const Array, +} + +impl Default for DiscreteColormap { + fn default() -> Self { + Self { + boundaries: null(), + gradient: null(), + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct CategoryData { + pub values: *const Array, + pub names: *const Array, + pub gradient: *const Array, + pub attributes: *const Attribute, + pub n_attributes: usize, +} + +impl Default for CategoryData { + fn default() -> Self { + Self { + values: null(), + names: null(), + gradient: null(), + attributes: null(), + n_attributes: 0, + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct MappedTexture { + pub image: *const Array, + pub texcoords: *const Array, +} + +#[derive(Debug)] +#[repr(C)] +pub struct ProjectedTexture { + pub image: *const Array, + pub orient: omf::Orient2, + pub width: f64, + pub height: f64, +} + +impl Default for ProjectedTexture { + fn default() -> Self { + Self { + image: null(), + orient: Default::default(), + width: 100.0, + height: 100.0, + } + } +} diff --git a/omf-c/src/elements.rs b/omf-c/src/elements.rs new file mode 100644 index 0000000..99e43ef --- /dev/null +++ b/omf-c/src/elements.rs @@ -0,0 +1,349 @@ +use std::{ffi::c_char, ptr::null}; + +use crate::{arrays::Array, attributes::Attribute, metadata::Value}; + +#[derive(Debug, Default)] +#[repr(C)] +pub struct FileVersion { + pub major: u32, + pub minor: u32, +} + +impl From<[u32; 2]> for FileVersion { + fn from([major, minor]: [u32; 2]) -> Self { + Self { major, minor } + } +} + +#[derive(Debug, Default, Clone, Copy)] +#[repr(C)] +pub struct Limits { + pub json_bytes: u64, + pub image_bytes: u64, + pub image_dim: u32, + pub validation: u32, +} + +impl From for Limits { + fn from(value: omf::file::Limits) -> Self { + Limits { + json_bytes: value.json_bytes.unwrap_or(0), + image_bytes: value.image_bytes.unwrap_or(0), + image_dim: value.image_dim.unwrap_or(0), + validation: value.validation.unwrap_or(0), + } + } +} + +impl From for omf::file::Limits { + fn from(value: Limits) -> Self { + fn none_if_default(x: T) -> Option { + if x == Default::default() { + Some(x) + } else { + None + } + } + Self { + json_bytes: none_if_default(value.json_bytes), + image_bytes: none_if_default(value.image_bytes), + image_dim: none_if_default(value.image_dim), + validation: none_if_default(value.validation), + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct Project { + pub name: *const c_char, + pub description: *const c_char, + pub coordinate_reference_system: *const c_char, + pub units: *const c_char, + pub author: *const c_char, + pub application: *const c_char, + pub date: i64, + pub origin: [f64; 3], + pub n_metadata: usize, + pub metadata: *const Value, + pub n_elements: usize, + pub elements: *const Element, +} + +impl Default for Project { + fn default() -> Self { + Self { + name: null(), + description: null(), + coordinate_reference_system: null(), + units: null(), + author: null(), + application: null(), + date: omf::date_time::utc_now().timestamp_micros(), + origin: [0.0, 0.0, 0.0], + n_metadata: 0, + metadata: null(), + n_elements: 0, + elements: null(), + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct Element { + pub name: *const c_char, + pub description: *const c_char, + pub color_set: bool, + pub color: [u8; 4], + pub n_metadata: usize, + pub metadata: *const Value, + pub n_attributes: usize, + pub attributes: *const Attribute, + pub point_set: *const PointSet, + pub line_set: *const LineSet, + pub surface: *const Surface, + pub grid_surface: *const GridSurface, + pub block_model: *const BlockModel, + pub composite: *const Composite, +} + +impl Default for Element { + fn default() -> Self { + Self { + name: null(), + description: null(), + color_set: false, + color: [0, 0, 0, u8::MAX], + n_metadata: 0, + metadata: null(), + n_attributes: 0, + attributes: null(), + point_set: null(), + line_set: null(), + surface: null(), + grid_surface: null(), + block_model: null(), + composite: null(), + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct PointSet { + pub origin: [f64; 3], + pub vertices: *const Array, +} + +impl Default for PointSet { + fn default() -> Self { + Self { + origin: [0.0, 0.0, 0.0], + vertices: null(), + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct LineSet { + pub origin: [f64; 3], + pub vertices: *const Array, + pub segments: *const Array, +} + +impl Default for LineSet { + fn default() -> Self { + Self { + origin: [0.0, 0.0, 0.0], + vertices: null(), + segments: null(), + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct Surface { + pub origin: [f64; 3], + pub vertices: *const Array, + pub triangles: *const Array, +} + +impl Default for Surface { + fn default() -> Self { + Self { + origin: [0.0, 0.0, 0.0], + vertices: null(), + triangles: null(), + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct GridSurface { + pub orient: omf::Orient2, + pub regular_grid: *const RegularGrid2, + pub tensor_grid: *const TensorGrid2, + pub heights: *const Array, +} + +impl Default for GridSurface { + fn default() -> Self { + Self { + orient: Default::default(), + regular_grid: null(), + tensor_grid: null(), + heights: null(), + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct BlockModel { + pub orient: omf::Orient3, + pub regular_grid: *const RegularGrid3, + pub tensor_grid: *const TensorGrid3, + pub regular_subblocks: *const RegularSubblocks, + pub freeform_subblocks: *const FreeformSubblocks, +} + +impl Default for BlockModel { + fn default() -> Self { + Self { + orient: Default::default(), + regular_grid: null(), + tensor_grid: null(), + regular_subblocks: null(), + freeform_subblocks: null(), + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct Composite { + pub n_elements: usize, + pub elements: *const Element, +} + +impl Default for Composite { + fn default() -> Self { + Self { + n_elements: 0, + elements: null(), + } + } +} + +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +#[repr(i32)] +pub enum SubblockMode { + None = 0, + Octree, + Full, +} + +impl From> for SubblockMode { + fn from(value: Option) -> Self { + match value { + Some(omf::SubblockMode::Full) => Self::Full, + Some(omf::SubblockMode::Octree) => Self::Octree, + None => Self::None, + } + } +} + +impl From for Option { + fn from(value: SubblockMode) -> Self { + match value { + SubblockMode::Octree => Some(omf::SubblockMode::Octree), + SubblockMode::Full => Some(omf::SubblockMode::Full), + _ => None, + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct RegularSubblocks { + pub count: [u32; 3], + pub subblocks: *const Array, + pub mode: SubblockMode, +} + +#[derive(Debug)] +#[repr(C)] +pub struct FreeformSubblocks { + pub subblocks: *const Array, +} + +#[derive(Debug)] +#[repr(C)] +pub struct RegularGrid2 { + pub size: [f64; 2], + pub count: [u32; 2], +} + +impl Default for RegularGrid2 { + fn default() -> Self { + Self { + size: [1.0, 1.0], + count: [10, 10], + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct TensorGrid2 { + pub u: *const Array, + pub v: *const Array, +} + +impl Default for TensorGrid2 { + fn default() -> Self { + Self { + u: null(), + v: null(), + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct RegularGrid3 { + pub size: [f64; 3], + pub count: [u32; 3], +} + +impl Default for RegularGrid3 { + fn default() -> Self { + Self { + size: [1.0, 1.0, 1.0], + count: [10, 10, 10], + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct TensorGrid3 { + pub u: *const Array, + pub v: *const Array, + pub w: *const Array, +} + +impl Default for TensorGrid3 { + fn default() -> Self { + Self { + u: null(), + v: null(), + w: null(), + } + } +} diff --git a/omf-c/src/error/mod.rs b/omf-c/src/error/mod.rs new file mode 100644 index 0000000..bfbf37c --- /dev/null +++ b/omf-c/src/error/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod state; +mod types; + +pub use state::{set_error, set_panic}; +pub use types::{Error, InvalidArg}; diff --git a/omf-c/src/error/state.rs b/omf-c/src/error/state.rs new file mode 100644 index 0000000..473c97d --- /dev/null +++ b/omf-c/src/error/state.rs @@ -0,0 +1,102 @@ +use std::{any::Any, cell::RefCell, ffi::c_char, ptr::null_mut}; + +use crate::ffi_tools::{into_ffi_free, FfiConvert, FfiStorage, IntoFfi}; + +use super::Error; + +thread_local! { + static ERROR_STATE: RefCell> = Default::default(); +} + +/// Sets the error state for the current thread, if it is not already set. +pub fn set_error(error: Error) { + ERROR_STATE.with(|cell| { + cell.borrow_mut().get_or_insert(error); + }) +} + +/// Sets the error state from a panic payload, if it is not already set. +pub fn set_panic(payload: Box) { + let message = if let Some(s) = payload.downcast_ref::() { + s.clone() + } else if let Some(s) = payload.downcast_ref::<&str>() { + (*s).to_owned() + } else { + "unknown payload".to_owned() + }; + set_error(Error::Panic(message)); +} + +#[derive(Debug)] +#[repr(C)] +pub struct CError { + /// Status code. + pub code: i32, + /// Error detail. The meaning depends on the status code. + pub detail: i32, + /// A nul-terminated string containing a human-readable error message in English. + pub message: *const c_char, +} + +impl FfiConvert for CError { + fn convert(error: Error, storage: &mut FfiStorage) -> Self { + Self { + code: error.code(), + detail: error.detail(), + message: storage.keep_string(error.to_string()), + } + } +} + +#[no_mangle] +pub extern "C" fn omf_error() -> *mut CError { + match ERROR_STATE.with(|cell| cell.take()) { + Some(error) => error.into_ffi(), + None => null_mut(), + } +} + +#[no_mangle] +pub extern "C" fn omf_error_clear() { + ERROR_STATE.with(|cell| cell.take()); +} + +#[no_mangle] +pub extern "C" fn omf_error_peek() -> i32 { + ERROR_STATE + .with(|cell| cell.borrow().as_ref().map(Error::code)) + .unwrap_or(0) +} + +#[no_mangle] +pub extern "C" fn omf_error_free(error: *mut CError) { + unsafe { into_ffi_free(error) }; +} + +#[cfg(test)] +mod tests { + use std::ffi::CStr; + + use crate::ffi_tools::catch; + + use super::*; + + #[test] + fn test_io_error() { + catch::error(|| -> Result<(), Error> { + let err = std::fs::read_to_string("does_not_exist.txt").unwrap_err(); + Err(omf::error::Error::IoError(err).into()) + }); + let c_err = omf_error(); + let message = unsafe { CStr::from_ptr((*c_err).message) } + .to_str() + .unwrap(); + // Different OS will give different error messages, only check start and end. + assert!( + message.starts_with("File IO error: ") && message.ends_with(" (os error 2)"), + "Got: {message}\nExpected: File IO error: .* (os error 2)" + ); + assert!(omf_error().is_null()); + omf_error_free(c_err); + } +} diff --git a/omf-c/src/error/types.rs b/omf-c/src/error/types.rs new file mode 100644 index 0000000..966bde8 --- /dev/null +++ b/omf-c/src/error/types.rs @@ -0,0 +1,125 @@ +/// Describes how an argument can be invalid. +#[derive(Debug, thiserror::Error)] +pub enum InvalidArg { + #[error("{0} must not be null")] + Null(&'static str), + #[error("{0} must not be null unless {1} is zero")] + NullArray(&'static str, &'static str), + #[error("{0} must be UTF-8 encoded")] + NotUtf8(&'static str), + #[error("one of the {0} options must be non-null")] + NoOptionSet(&'static str), + #[error("invalid handle")] + Handle, + #[error("{0}")] + HandleType(&'static str), + #[error("invalid enum value")] + Enum, +} + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("Panic: {0}")] + Panic(String), + #[error("Invalid argument: {0}")] + InvalidArgument(#[from] InvalidArg), + #[error("Invalid call: {0}")] + InvalidCall(String), + #[error("Error: buffer length {found} does not match expected length {expected}")] + BufferLengthWrong { found: u64, expected: u64 }, + #[error("Error: '{src}' data type {found:?} does not match expected type {expected:?}")] + ArrayTypeWrong { + src: &'static str, + found: omf::DataType, + expected: omf::DataType, + }, + #[error("{message}")] + External { + code: i32, + detail: i32, + message: String, + }, +} + +impl Error { + pub fn code(&self) -> i32 { + match self { + Error::Panic(_) => Status::Panic as i32, + Error::InvalidArgument(_) => Status::InvalidArgument as i32, + Error::InvalidCall(_) => Status::InvalidCall as i32, + Error::BufferLengthWrong { .. } => Status::BufferLengthWrong as i32, + Error::ArrayTypeWrong { .. } => Status::ArrayTypeWrong as i32, + Error::External { code, .. } => *code, + } + } + + pub fn detail(&self) -> i32 { + match self { + Error::External { detail, .. } => *detail, + _ => 0, + } + } +} + +impl From for Error { + fn from(error: omf::error::Error) -> Self { + use omf::error::Error::*; + let message = error.to_string(); + let (status, detail) = match error { + OutOfMemory => (Status::OutOfMemory, 0), + IoError(e) => (Status::IoError, e.raw_os_error().unwrap_or(0)), + NotOmf(_) => (Status::NotOmf, 0), + NewerVersion(_, _) => (Status::NewerVersion, 0), + PreReleaseVersion(_, _, _) => (Status::PreRelease, 0), + DeserializationFailed(_) => (Status::DeserializationFailed, 0), + SerializationFailed(_) => (Status::SerializationFailed, 0), + ValidationFailed(_) => (Status::ValidationFailed, 0), + LimitExceeded(_) => (Status::LimitExceeded, 0), + NotImageData => (Status::NotImageData, 0), + InvalidData(_) => (Status::InvalidData, 0), + ImageError(_) => (Status::ImageError, 0), + UnsafeCast(_, _) => (Status::UnsafeCast, 0), + NotParquetData => (Status::NotParquetData, 0), + ZipMemberMissing(_) => (Status::ZipMemberMissing, 0), + ZipError(_) => (Status::ZipError, 0), + ParquetSchemaMismatch(_, _) => (Status::ParquetSchemaMismatch, 0), + ParquetError(_) => (Status::ParquetError, 0), + _ => (Status::Panic, 0), + }; + Self::External { + code: status as i32, + detail, + message, + } + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +#[repr(i32)] +pub enum Status { + #[default] + Success = 0, + Panic, + InvalidArgument, + InvalidCall, + OutOfMemory, + IoError, + NotOmf, + NewerVersion, + PreRelease, + DeserializationFailed, + SerializationFailed, + ValidationFailed, + LimitExceeded, + NotImageData, + NotParquetData, + ArrayTypeWrong, + BufferLengthWrong, + InvalidData, + UnsafeCast, + ZipMemberMissing, + ZipError, + ParquetSchemaMismatch, + ParquetError, + ImageError, +} diff --git a/omf-c/src/ffi_tools/arg.rs b/omf-c/src/ffi_tools/arg.rs new file mode 100644 index 0000000..1fbb13f --- /dev/null +++ b/omf-c/src/ffi_tools/arg.rs @@ -0,0 +1,206 @@ +use std::ffi::{c_char, CStr}; + +use crate::error::{Error, InvalidArg}; + +/// Get a reference from a pointer or an error on null. +/// +/// # Safety +/// +/// Pointer must be null or valid. See `core::ptr::const_ptr::as_ref` for details. +pub unsafe fn ref_from_ptr(arg_name: &'static str, ptr: *const T) -> Result<&T, Error> { + unsafe { ptr.as_ref() }.ok_or(InvalidArg::Null(arg_name).into()) +} + +/// Get a mutable reference from a pointer or an error on null. +/// +/// # Safety +/// +/// Pointer must be null or valid. See `core::ptr::const_ptr::as_mut` for details. +pub unsafe fn mut_from_ptr(arg_name: &'static str, ptr: *mut T) -> Result<&mut T, Error> { + unsafe { ptr.as_mut() }.ok_or(InvalidArg::Null(arg_name).into()) +} + +/// Consumes a pointer, returning the contained value. +/// +/// # Safety +/// +/// Pointer must be null or valid. +pub unsafe fn consume_ptr(arg_name: &'static str, ptr: *mut T) -> Result { + if ptr.is_null() { + Err(InvalidArg::Null(arg_name).into()) + } else { + Ok(*unsafe { Box::from_raw(ptr) }) + } +} + +/// Copies from a C `const char*` returning an error if the pointer is null, or an error +/// if the string is null or not UTF-8 encoded. +/// +/// # Safety +/// +/// `ptr` must point to a valid nul-terminated string. See `CStr::from_ptr` for details. +pub unsafe fn string_from_ptr(src: &'static str, ptr: *const c_char) -> Result { + if ptr.is_null() { + Err(InvalidArg::Null(src).into()) + } else { + Ok(unsafe { CStr::from_ptr(ptr) } + .to_str() + .map_err(|_| InvalidArg::NotUtf8(src))? + .to_owned()) + } +} + +/// Copies from a C `const char*` returning an error if the pointer is null, or if the +/// string is not UTF-8 encoded. +/// +/// If `ptr` is null then an empty string is returned. +/// +/// # Safety +/// +/// `ptr` must be a valid string. See `CStr::from_ptr` for details. +pub unsafe fn string_from_ptr_or_null( + src: &'static str, + ptr: *const c_char, +) -> Result { + if ptr.is_null() { + Ok(String::new()) + } else { + Ok(unsafe { CStr::from_ptr(ptr) } + .to_str() + .map_err(|_| InvalidArg::NotUtf8(src))? + .to_owned()) + } +} + +/// Creates a slice from a pointer without copying, returning an error if the pointer is +/// null and the size isn't zero. +/// +/// # Safety +/// +/// `ptr` must point to at least `n` elements. See `std::slice::from_raw_parts` for details. +pub unsafe fn slice_from_ptr( + src: &'static str, + src_n: &'static str, + ptr: *const T, + n: usize, +) -> Result<&'static [T], Error> { + if !ptr.is_null() { + Ok(unsafe { std::slice::from_raw_parts(ptr, n) }) + } else if n == 0 { + Ok(&[]) + } else { + Err(InvalidArg::NullArray(src, src_n).into()) + } +} + +/// Creates a slice from a pointer without copying, returning an error if the pointer is +/// null and the size isn't zero. +/// +/// # Safety +/// +/// `ptr` must point to at least `n` elements. See `std::slice::from_raw_parts` for details. +pub unsafe fn slice_mut_from_ptr( + src: &'static str, + src_n: &'static str, + ptr: *mut T, + n: usize, +) -> Result<&'static mut [T], Error> { + if !ptr.is_null() { + Ok(unsafe { std::slice::from_raw_parts_mut(ptr, n) }) + } else if n == 0 { + Ok(&mut []) + } else { + Err(InvalidArg::NullArray(src, src_n).into()) + } +} + +/// Like `slice_mut_from_ptr` but also checks the length. +pub unsafe fn slice_mut_from_ptr_len( + src: &'static str, + src_n: &'static str, + ptr: *mut T, + n: usize, + len: u64, +) -> Result<&'static mut [T], Error> { + if n as u64 != len { + Err(Error::BufferLengthWrong { + found: n as u64, + expected: len, + }) + } else if !ptr.is_null() { + Ok(unsafe { std::slice::from_raw_parts_mut(ptr, n) }) + } else if n == 0 { + Ok(&mut []) + } else { + Err(InvalidArg::NullArray(src, src_n).into()) + } +} + +/// Call `ref_from_ptr` using the name of the argument. +macro_rules! not_null ( + ($name:expr) => { unsafe { crate::ffi_tools::arg::ref_from_ptr(stringify!($name), $name) } }; +); + +/// Call `ref_from_ptr` using the name of the argument. +macro_rules! not_null_mut ( + ($name:expr) => { unsafe { crate::ffi_tools::arg::mut_from_ptr(stringify!($name), $name) } }; +); + +/// Call `consume_ptr` using the name of the argument. +macro_rules! not_null_consume ( + ($name:expr) => { unsafe { crate::ffi_tools::arg::consume_ptr(stringify!($name), $name) } }; +); + +/// Call `string_from_ptr` using the name of the argument. +macro_rules! string_not_null { + ($name:expr) => { + unsafe { crate::ffi_tools::arg::string_from_ptr(stringify!($name), $name) } + }; +} + +/// Call `slice_from_ptr` using the names of the arguments. +macro_rules! slice { + ($name:expr, $count:expr) => { + unsafe { + crate::ffi_tools::arg::slice_from_ptr( + stringify!($name), + stringify!($count), + $name, + $count, + ) + } + }; +} + +/// Call `slice_mut_from_ptr` using the names of the arguments. +macro_rules! slice_mut { + ($name:expr, $count:expr) => { + unsafe { + crate::ffi_tools::arg::slice_mut_from_ptr( + stringify!($name), + stringify!($count), + $name, + $count, + ) + } + }; +} + +/// Call `slice_mut_from_ptr` using the names of the arguments. +macro_rules! slice_mut_len { + ($name:expr, $count:expr, $len:expr) => { + unsafe { + crate::ffi_tools::arg::slice_mut_from_ptr_len( + stringify!($name), + stringify!($count), + $name, + $count, + $len, + ) + } + }; +} + +pub(crate) use { + not_null, not_null_consume, not_null_mut, slice, slice_mut, slice_mut_len, string_not_null, +}; diff --git a/omf-c/src/ffi_tools/catch.rs b/omf-c/src/ffi_tools/catch.rs new file mode 100644 index 0000000..54b9b94 --- /dev/null +++ b/omf-c/src/ffi_tools/catch.rs @@ -0,0 +1,66 @@ +//! Panic and result error catching code. + +use std::panic::UnwindSafe; + +use crate::error::{set_error, set_panic, Error}; + +/// Catches both panics and `Result` results in a function. +/// +/// Returns an `Option` that is `None` when an error occurs, calling code should use +/// `unwrap_or` or similar for the error return value. +pub fn error(func: F) -> Option +where + F: FnOnce() -> Result + UnwindSafe, +{ + match std::panic::catch_unwind(func) { + Ok(Ok(value)) => Some(value), + Ok(Err(error)) => { + set_error(error); + None + } + Err(payload) => { + set_panic(payload); + None + } + } +} + +/// Catches just panics in a function. +/// +/// Returns an `Option` that is `None` when an error occurs, calling code should use +/// `unwrap_or` or similar for the error return value. +pub fn panic(func: impl FnOnce() -> T + UnwindSafe) -> Option { + match std::panic::catch_unwind(func) { + Ok(value) => Some(value), + Err(payload) => { + set_panic(payload); + None + } + } +} + +/// Catches just panics in a function, returning a boolean. +pub fn panic_bool(func: impl FnOnce() + UnwindSafe) -> bool { + panic(|| { + func(); + true + }) + .unwrap_or(false) +} + +/// Catches `Result` results in a function. +/// +/// Returns an `Option` that is `None` when an error occurs, calling code should use +/// `unwrap_or` or similar for the error return value. +pub fn error_only(func: F) -> Option +where + F: FnOnce() -> Result, +{ + match func() { + Ok(value) => Some(value), + Err(error) => { + set_error(error); + None + } + } +} diff --git a/omf-c/src/ffi_tools/into_ffi.rs b/omf-c/src/ffi_tools/into_ffi.rs new file mode 100644 index 0000000..329f3ed --- /dev/null +++ b/omf-c/src/ffi_tools/into_ffi.rs @@ -0,0 +1,258 @@ +use std::{ + ffi::c_char, + panic::{RefUnwindSafe, UnwindSafe}, + ptr::{null, null_mut}, +}; + +use super::{catch, typeless::Typeless}; + +/// A trait that defines how an object can be moved into a pointer and then freed later. +/// +/// This is similar to the built-in `From` with a matching `IntoFfi` trait. +pub trait FfiConvert +where + Self: Sized + RefUnwindSafe + 'static, + T: UnwindSafe + 'static, +{ + /// Turns an object into a pointer, putting any required allocations into `storage`. + fn convert(value: T, storage: &mut FfiStorage) -> Self; +} + +/// Provides the `into_ffi` method on the wrapped type, retuning a pointer to the new +/// wrapper. Call `ffi_from_free` to free the returned pointer. +/// +/// Don't implement this directly, implement `FfiConvert` or `FfiWrapper` instead. +pub trait IntoFfi: Sized + UnwindSafe + 'static +where + T: FfiConvert, +{ + fn into_ffi(self) -> *mut T; +} + +impl IntoFfi for T +where + U: FfiConvert, + T: UnwindSafe + 'static, +{ + /// Calls `FfiFrom::convert` on the matching implementation. + #[inline] + fn into_ffi(self) -> *mut U { + catch::panic(|| { + let mut storage = FfiStorage::new(); + let wrapper: U = FfiConvert::convert(self, &mut storage); + Box::into_raw(Box::new(WrapperAndStorage { wrapper, storage })).cast() + }) + .unwrap_or_else(null_mut) + } +} + +/// This trait declares that a type is an FFI wrapper for `T`, adding an easy way to +/// implement `FfiFrom` for simple wrappers. +pub trait FfiWrapper +where + Self: Sized + RefUnwindSafe + 'static, + T: UnwindSafe + 'static, +{ + /// Create the wrapper object, which may safely contain pointers into boxes and vecs + /// inside `value`. + fn wrap(value: &T) -> Self; +} + +impl, T> FfiConvert for U +where + T: UnwindSafe + 'static, +{ + fn convert(value: T, storage: &mut FfiStorage) -> Self { + let wrapper = Self::wrap(&value); + storage.keep(value); + wrapper + } +} + +/// Stores arbitrary data needed by an FFI wrapper. +#[derive(Default, Debug)] +pub struct FfiStorage { + data: Vec, +} + +impl FfiStorage { + pub fn new() -> Self { + Default::default() + } + + /// Keep an object alive, returing a pointer to it. + pub fn keep(&mut self, value: T) -> *const T { + let (ptr, typeless) = Typeless::new(value); + self.data.push(typeless); + ptr.cast_const() + } + + /// Keep an object alive, returing a pointer to it. + pub fn keep_mut(&mut self, value: T) -> *mut T { + let (ptr, typeless) = Typeless::new(value); + self.data.push(typeless); + ptr + } + + /// Keep a vec alive, returing a pointer to the contents. + pub fn keep_vec(&mut self, values: Vec) -> *const T { + let ptr = values.as_ptr(); + self.keep(values); + ptr + } + + /// Nul-terminate a string, keep it alive, and return the `const char*` pointer. + /// + /// Returns null if the string is empty. + pub fn keep_string(&mut self, input: impl Into) -> *const c_char { + let mut bytes: Vec = input.into().into_bytes(); + if bytes.is_empty() { + null() + } else { + bytes.push(0); + self.keep_vec(bytes).cast() + } + } + + /// Convert an object, keeping its depdendent data in here as well. + pub fn convert(&mut self, value: T) -> U + where + U: FfiConvert + UnwindSafe + 'static, + T: UnwindSafe + 'static, + { + U::convert(value, self) + } + + pub fn convert_ptr(&mut self, value: T) -> *const U + where + U: FfiConvert + UnwindSafe + 'static, + T: UnwindSafe + 'static, + { + let wrapper = self.convert(value); + self.keep(wrapper) + } + + pub fn convert_ptr_mut(&mut self, value: T) -> *mut U + where + U: FfiConvert + UnwindSafe + 'static, + T: UnwindSafe + 'static, + { + let wrapper = self.convert(value); + self.keep_mut(wrapper) + } + + /// Convert an optional object, keeping its depdendent data in here as well, + /// returning a pointer or null. + pub fn convert_option(&mut self, value: Option) -> *const U + where + U: FfiConvert + UnwindSafe + 'static, + T: UnwindSafe + 'static, + { + if let Some(val) = value { + self.convert_ptr(val) + } else { + null() + } + } + + pub fn convert_iter_term( + &mut self, + values: impl IntoIterator, + ) -> (*const U, usize) + where + U: FfiConvert + Default + UnwindSafe + 'static, + T: UnwindSafe + 'static, + { + let vec: Vec<_> = values + .into_iter() + .map(|x| self.convert(x)) + .chain(Some(Default::default())) + .collect(); + let len = vec.len() - 1; + let ptr = self.keep_vec(vec); + (ptr, len) + } +} + +/// Frees a pointer created by `IntoFfi::into_ffi`. +/// +/// Does nothing if the pointer is null. +/// +/// # Safety +/// +/// `ptr` must have been returned by `IntoFfi::into_ffi` and have not been modified. +pub unsafe fn into_ffi_free(ptr: *mut T) -> bool { + catch::panic_bool(|| { + if !ptr.is_null() { + let ws_ptr: *mut WrapperAndStorage = ptr.cast(); + unsafe { + _ = Box::from_raw(ws_ptr); + } + } + }) +} + +#[allow(dead_code)] +#[repr(C)] +struct WrapperAndStorage { + wrapper: W, + storage: FfiStorage, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ffi_storage_keep() { + let mut s = FfiStorage::new(); + let x = 42; + let y = s.keep(x); + assert_eq!(unsafe { *y }, 42); + drop(s); + } + + #[test] + fn ffi_storage_keep_vec() { + let mut s = FfiStorage::new(); + let x = vec![42, 43, 44]; + let y = s.keep_vec(x); + assert_eq!(unsafe { *y }, 42); + assert_eq!(unsafe { *y.add(1) }, 43); + assert_eq!(unsafe { *y.add(2) }, 44); + drop(s); + } + + #[test] + fn ffi_storage_keep_string() { + let mut s = FfiStorage::new(); + let ptr = s.keep_string("test string"); + let slice = unsafe { std::slice::from_raw_parts(ptr.cast::(), 12) }; + assert_eq!(slice, b"test string\0"); + drop(s); + } + + #[test] + #[allow(dead_code)] + fn ffi_into_and_free() { + struct Test { + inner: String, + } + struct TestWrapper { + inner: *const c_char, + } + impl FfiConvert for TestWrapper { + fn convert(value: Test, storage: &mut FfiStorage) -> Self { + Self { + inner: storage.keep_string(value.inner), + } + } + } + let test = Test { + inner: "Test string".to_owned(), + }; + let wrapper: *mut TestWrapper = test.into_ffi(); + assert!(!wrapper.is_null()); + unsafe { into_ffi_free(wrapper) }; + } +} diff --git a/omf-c/src/ffi_tools/mod.rs b/omf-c/src/ffi_tools/mod.rs new file mode 100644 index 0000000..7ef1405 --- /dev/null +++ b/omf-c/src/ffi_tools/mod.rs @@ -0,0 +1,6 @@ +pub mod arg; +pub mod catch; +mod into_ffi; +mod typeless; + +pub use into_ffi::{into_ffi_free, FfiConvert, FfiStorage, FfiWrapper, IntoFfi}; diff --git a/omf-c/src/ffi_tools/typeless.rs b/omf-c/src/ffi_tools/typeless.rs new file mode 100644 index 0000000..db2d648 --- /dev/null +++ b/omf-c/src/ffi_tools/typeless.rs @@ -0,0 +1,39 @@ +use std::ffi::c_void; + +/// An typeless container that holds a typed value. +/// +/// The only action it can perform is dropping the contained value. Used to store the data +/// behind FFI pointers. +#[derive(Debug)] +pub struct Typeless { + ptr: *mut c_void, + drop: fn(*mut c_void), +} + +impl Typeless { + /// Moves `value` into the typeless container, returning a safe pointer to it along + /// with the container. + pub fn new(value: T) -> (*mut T, Self) { + let ptr: *mut T = Box::into_raw(Box::new(value)); + ( + ptr, + Self { + ptr: ptr.cast(), + drop: Self::drop_func::, + }, + ) + } + + fn drop_func(ptr: *mut c_void) { + // Safety: we know where this came from so the type cast is safe. + unsafe { + _ = Box::from_raw(ptr.cast::()); + }; + } +} + +impl Drop for Typeless { + fn drop(&mut self) { + (self.drop)(self.ptr); + } +} diff --git a/omf-c/src/from_omf.rs b/omf-c/src/from_omf.rs new file mode 100644 index 0000000..8896dc3 --- /dev/null +++ b/omf-c/src/from_omf.rs @@ -0,0 +1,320 @@ +use std::ptr::null; + +use omf::date_time::{date_time_to_i64, date_to_i64}; + +use crate::{ + attributes::*, + elements::*, + ffi_tools::{FfiConvert, FfiStorage}, + metadata::*, +}; + +fn convert_metadata( + input: impl IntoIterator, + st: &mut FfiStorage, +) -> (*const Value, usize) { + st.convert_iter_term(input) +} + +impl FfiConvert for Value { + fn convert(value: serde_json::Value, st: &mut FfiStorage) -> Self { + let mut wrap = Value::default(); + match value { + serde_json::Value::Null => wrap.r#type = ValueType::Null, + serde_json::Value::Bool(b) => { + wrap.r#type = ValueType::Boolean; + wrap.boolean = b; + } + serde_json::Value::Number(n) => { + wrap.r#type = ValueType::Number; + wrap.number = n.as_f64().unwrap_or(f64::NAN); + } + serde_json::Value::String(s) => { + wrap.r#type = ValueType::String; + wrap.string = st.keep_string(s); + } + serde_json::Value::Array(a) => { + wrap.r#type = ValueType::List; + (wrap.values, wrap.n_values) = st.convert_iter_term(a); + } + serde_json::Value::Object(map) => { + wrap.r#type = ValueType::Object; + (wrap.values, wrap.n_values) = convert_metadata(map, st); + } + } + wrap + } +} + +impl FfiConvert<(String, serde_json::Value)> for Value { + fn convert((name, value): (String, serde_json::Value), st: &mut FfiStorage) -> Self { + let mut out: Self = Value::convert(value, st); + out.name = st.keep_string(name); + out + } +} + +impl FfiConvert for Project { + fn convert(project: omf::Project, st: &mut FfiStorage) -> Self { + let (metadata, n_metadata) = convert_metadata(project.metadata, st); + let (elements, n_elements) = st.convert_iter_term(project.elements); + Project { + name: st.keep_string(project.name), + description: st.keep_string(project.description), + coordinate_reference_system: st.keep_string(project.coordinate_reference_system), + units: st.keep_string(project.units), + origin: project.origin, + author: st.keep_string(project.author), + application: st.keep_string(project.application), + date: project.date.timestamp_micros(), + n_metadata, + metadata, + n_elements, + elements, + } + } +} + +impl FfiConvert for Element { + fn convert(element: omf::Element, st: &mut FfiStorage) -> Self { + let mut wrap = Self { + name: st.keep_string(element.name), + description: st.keep_string(element.description), + color_set: element.color.is_some(), + color: element.color.unwrap_or([0; 4]), + ..Default::default() + }; + (wrap.metadata, wrap.n_metadata) = convert_metadata(element.metadata, st); + (wrap.attributes, wrap.n_attributes) = st.convert_iter_term(element.attributes); + match element.geometry { + omf::Geometry::PointSet(p) => wrap.point_set = st.convert_ptr(p), + omf::Geometry::LineSet(l) => wrap.line_set = st.convert_ptr(l), + omf::Geometry::Surface(s) => wrap.surface = st.convert_ptr(s), + omf::Geometry::GridSurface(g) => wrap.grid_surface = st.convert_ptr(g), + omf::Geometry::BlockModel(b) => wrap.block_model = st.convert_ptr(b), + omf::Geometry::Composite(c) => wrap.composite = st.convert_ptr(c), + } + wrap + } +} + +impl FfiConvert for PointSet { + fn convert(point_set: omf::PointSet, st: &mut FfiStorage) -> Self { + Self { + origin: point_set.origin, + vertices: st.convert_ptr(point_set.vertices), + } + } +} + +impl FfiConvert for LineSet { + fn convert(line_set: omf::LineSet, st: &mut FfiStorage) -> Self { + Self { + origin: line_set.origin, + vertices: st.convert_ptr(line_set.vertices), + segments: st.convert_ptr(line_set.segments), + } + } +} + +impl FfiConvert for Surface { + fn convert(surface: omf::Surface, st: &mut FfiStorage) -> Self { + Self { + origin: surface.origin, + vertices: st.convert_ptr(surface.vertices), + triangles: st.convert_ptr(surface.triangles), + } + } +} + +impl FfiConvert for GridSurface { + fn convert(grid_surface: omf::GridSurface, st: &mut FfiStorage) -> Self { + let mut regular_grid = null(); + let mut tensor_grid = null(); + match grid_surface.grid { + omf::Grid2::Regular { size, count } => { + regular_grid = st.keep(RegularGrid2 { size, count }) + } + omf::Grid2::Tensor { u, v } => { + let u = st.convert_ptr(u); + let v = st.convert_ptr(v); + tensor_grid = st.keep(TensorGrid2 { u, v }) + } + } + Self { + orient: grid_surface.orient, + regular_grid, + tensor_grid, + heights: st.convert_option(grid_surface.heights), + } + } +} + +impl FfiConvert for BlockModel { + fn convert(block_model: omf::BlockModel, st: &mut FfiStorage) -> Self { + let mut regular_grid = null(); + let mut tensor_grid = null(); + let mut regular_subblocks = null(); + let mut freeform_subblocks = null(); + // Grid. + match block_model.grid { + omf::Grid3::Regular { size, count } => { + regular_grid = st.keep(RegularGrid3 { size, count }); + } + omf::Grid3::Tensor { u, v, w } => { + let u = st.convert_ptr(u); + let v = st.convert_ptr(v); + let w = st.convert_ptr(w); + tensor_grid = st.keep(TensorGrid3 { u, v, w }) + } + } + // Sub-blocks. + match block_model.subblocks { + Some(omf::Subblocks::Regular { + count, + subblocks, + mode, + }) => { + let subblocks = st.convert_ptr(subblocks); + regular_subblocks = st.keep(RegularSubblocks { + count, + subblocks, + mode: mode.into(), + }); + } + Some(omf::Subblocks::Freeform { subblocks }) => { + let subblocks = st.convert_ptr(subblocks); + freeform_subblocks = st.keep(FreeformSubblocks { subblocks }); + } + None => {} + } + Self { + orient: block_model.orient, + regular_grid, + tensor_grid, + regular_subblocks, + freeform_subblocks, + } + } +} + +impl FfiConvert for Composite { + fn convert(composite: omf::Composite, st: &mut FfiStorage) -> Self { + let (elements, n_elements) = st.convert_iter_term(composite.elements); + Self { + n_elements, + elements, + } + } +} + +fn convert_colormap(cmap: Option, st: &mut FfiStorage) -> NumberData { + match cmap { + Some(omf::NumberColormap::Continuous { range, gradient }) => { + let mut colormap = ContinuousColormap::default(); + match range { + omf::NumberRange::Float { min, max } => { + colormap.range_type = RangeType::Float; + colormap.min = min; + colormap.max = max; + } + omf::NumberRange::Integer { min, max } => { + colormap.range_type = RangeType::Integer; + colormap.min_int = min; + colormap.max_int = max; + } + omf::NumberRange::Date { min, max } => { + colormap.range_type = RangeType::Date; + colormap.min_int = date_to_i64(min); + colormap.max_int = date_to_i64(max); + } + omf::NumberRange::DateTime { min, max } => { + colormap.range_type = RangeType::DateTime; + colormap.min_int = date_time_to_i64(min); + colormap.max_int = date_time_to_i64(max); + } + } + colormap.gradient = st.convert_ptr(gradient); + NumberData { + continuous_colormap: st.keep(colormap), + ..Default::default() + } + } + Some(omf::NumberColormap::Discrete { + boundaries, + gradient, + }) => { + let boundaries = st.convert_ptr(boundaries); + let gradient = st.convert_ptr(gradient); + NumberData { + discrete_colormap: st.keep(DiscreteColormap { + boundaries, + gradient, + }), + ..Default::default() + } + } + None => Default::default(), + } +} + +impl FfiConvert for Attribute { + fn convert(attribute: omf::Attribute, st: &mut FfiStorage) -> Self { + let mut wrap = Self { + name: st.keep_string(attribute.name), + description: st.keep_string(attribute.description), + location: attribute.location, + ..Default::default() + }; + (wrap.metadata, wrap.n_metadata) = convert_metadata(attribute.metadata, st); + match attribute.data { + omf::AttributeData::Number { values, colormap } => { + let mut number_data = convert_colormap(colormap, st); + number_data.values = st.convert_ptr(values); + wrap.number_data = st.keep(number_data); + } + omf::AttributeData::Vector { values } => wrap.vector_data = st.convert_ptr(values), + omf::AttributeData::Text { values } => wrap.text_data = st.convert_ptr(values), + omf::AttributeData::Category { + values, + names, + gradient, + attributes, + } => { + let values = st.convert_ptr(values); + let names = st.convert_ptr(names); + let gradient = st.convert_option(gradient); + let (attributes, n_attributes) = st.convert_iter_term(attributes); + wrap.category_data = st.keep(CategoryData { + values, + names, + gradient, + attributes, + n_attributes, + }); + } + omf::AttributeData::Boolean { values } => wrap.boolean_data = st.convert_ptr(values), + omf::AttributeData::Color { values } => wrap.color_data = st.convert_ptr(values), + omf::AttributeData::MappedTexture { image, texcoords } => { + let image = st.convert_ptr(image); + let texcoords = st.convert_ptr(texcoords); + wrap.mapped_texture_data = st.keep(MappedTexture { image, texcoords }); + } + omf::AttributeData::ProjectedTexture { + image, + orient, + width, + height, + } => { + let image = st.convert_ptr(image); + wrap.projected_texture_data = st.keep(ProjectedTexture { + image, + orient, + width, + height, + }); + } + } + wrap + } +} diff --git a/omf-c/src/image_data.rs b/omf-c/src/image_data.rs new file mode 100644 index 0000000..609d41b --- /dev/null +++ b/omf-c/src/image_data.rs @@ -0,0 +1,65 @@ +use std::{fmt::Debug, ptr::null}; + +use crate::ffi_tools::{into_ffi_free, FfiWrapper}; + +#[repr(C)] +pub struct ImageData { + pub width: u32, + pub height: u32, + pub mode: ImageMode, + pub uint8: *const u8, + pub uint16: *const u16, +} + +impl FfiWrapper for ImageData { + fn wrap(value: &image::DynamicImage) -> Self { + let mut uint8 = null(); + let mut uint16 = null(); + match value { + image::DynamicImage::ImageLuma8(img) => uint8 = img.as_ptr(), + image::DynamicImage::ImageLumaA8(img) => uint8 = img.as_ptr(), + image::DynamicImage::ImageRgb8(img) => uint8 = img.as_ptr(), + image::DynamicImage::ImageRgba8(img) => uint8 = img.as_ptr(), + image::DynamicImage::ImageLuma16(img) => uint16 = img.as_ptr(), + image::DynamicImage::ImageLumaA16(img) => uint16 = img.as_ptr(), + image::DynamicImage::ImageRgb16(img) => uint16 = img.as_ptr(), + image::DynamicImage::ImageRgba16(img) => uint16 = img.as_ptr(), + _ => panic!("unexpected image type"), + } + Self { + width: value.width(), + height: value.height(), + mode: value.color().into(), + uint8, + uint16, + } + } +} + +#[no_mangle] +pub extern "C" fn omf_image_data_free(data: *mut ImageData) -> bool { + unsafe { into_ffi_free(data) } +} + +#[derive(Debug)] +#[repr(i32)] +#[non_exhaustive] +pub enum ImageMode { + Gray = 1, + GrayAlpha = 2, + Rgb = 3, + Rgba = 4, +} + +impl From for ImageMode { + fn from(value: image::ColorType) -> Self { + use image::ColorType::*; + match value { + L8 | L16 => Self::Gray, + La8 | La16 => Self::GrayAlpha, + Rgb8 | Rgb16 | Rgb32F => Self::Rgb, + Rgba8 | Rgba16 | Rgba32F => Self::Rgba, + _ => panic!("unexpected image type"), + } + } +} diff --git a/omf-c/src/init_functions.rs b/omf-c/src/init_functions.rs new file mode 100644 index 0000000..b94180f --- /dev/null +++ b/omf-c/src/init_functions.rs @@ -0,0 +1,154 @@ +use std::ffi::c_char; + +use crate::{arrays::Array, attributes::*, elements::*}; + +macro_rules! create ( + ($ty:ty) => { + <$ty>::default() + }; + ($ty:ty $( , $field:ident )* ) => { + { + let mut obj = <$ty>::default(); + $( obj.$field = $field; )* + obj + } + }; +); + +#[no_mangle] +pub extern "C" fn omf_project_init(name: *const c_char) -> Project { + create!(Project, name) +} + +#[no_mangle] +pub extern "C" fn omf_element_init(name: *const c_char) -> Element { + create!(Element, name) +} + +#[no_mangle] +pub extern "C" fn omf_attribute_init(name: *const c_char, location: omf::Location) -> Attribute { + create!(Attribute, name, location) +} + +#[no_mangle] +pub extern "C" fn omf_point_set_init(vertices: *const Array) -> PointSet { + create!(PointSet, vertices) +} + +#[no_mangle] +pub extern "C" fn omf_line_set_init(vertices: *const Array, segments: *const Array) -> LineSet { + create!(LineSet, vertices, segments) +} + +#[no_mangle] +pub extern "C" fn omf_surface_init(vertices: *const Array, triangles: *const Array) -> Surface { + create!(Surface, vertices, triangles) +} + +#[no_mangle] +pub extern "C" fn omf_grid_surface_init() -> GridSurface { + create!(GridSurface) +} + +#[no_mangle] +pub extern "C" fn omf_block_model_init() -> BlockModel { + create!(BlockModel) +} + +#[no_mangle] +pub extern "C" fn omf_composite_init() -> Composite { + create!(Composite) +} + +#[no_mangle] +pub extern "C" fn omf_number_data_init() -> NumberData { + create!(NumberData) +} + +#[no_mangle] +pub extern "C" fn omf_category_data_init() -> CategoryData { + create!(CategoryData) +} + +#[no_mangle] +pub extern "C" fn omf_discrete_colormap_init() -> DiscreteColormap { + create!(DiscreteColormap) +} + +#[no_mangle] +pub extern "C" fn omf_continuous_colormap_init( + min: f64, + max: f64, + gradient: *const Array, +) -> ContinuousColormap { + create!(ContinuousColormap, min, max, gradient) +} + +#[no_mangle] +pub extern "C" fn omf_tensor_grid2_init(u: *const Array, v: *const Array) -> TensorGrid2 { + create!(TensorGrid2, u, v) +} + +#[no_mangle] +pub extern "C" fn omf_tensor_grid3_init( + u: *const Array, + v: *const Array, + w: *const Array, +) -> TensorGrid3 { + create!(TensorGrid3, u, v, w) +} + +#[no_mangle] +pub extern "C" fn omf_regular_grid2_init(du: f64, dv: f64, nu: u32, nv: u32) -> RegularGrid2 { + RegularGrid2 { + size: [du, dv], + count: [nu, nv], + } +} + +#[no_mangle] +pub extern "C" fn omf_regular_grid3_init( + du: f64, + dv: f64, + dw: f64, + nu: u32, + nv: u32, + nw: u32, +) -> RegularGrid3 { + RegularGrid3 { + size: [du, dv, dw], + count: [nu, nv, nw], + } +} + +#[no_mangle] +pub extern "C" fn omf_regular_subblocks_init( + nu: u32, + nv: u32, + nw: u32, + subblocks: *const Array, +) -> RegularSubblocks { + RegularSubblocks { + count: [nu, nv, nw], + subblocks, + mode: SubblockMode::None, + } +} + +#[no_mangle] +pub extern "C" fn omf_freeform_subblocks_init(subblocks: *const Array) -> FreeformSubblocks { + FreeformSubblocks { subblocks } +} + +#[no_mangle] +pub extern "C" fn omf_mapped_texture_init( + image: *const Array, + texcoords: *const Array, +) -> MappedTexture { + MappedTexture { image, texcoords } +} + +#[no_mangle] +pub extern "C" fn omf_projected_texture_init(image: *const Array) -> ProjectedTexture { + create!(ProjectedTexture, image) +} diff --git a/omf-c/src/lib.rs b/omf-c/src/lib.rs new file mode 100644 index 0000000..9e9be4f --- /dev/null +++ b/omf-c/src/lib.rs @@ -0,0 +1,131 @@ +//! C wrapper for the [omf](omf) crate. +//! +//! Doesn't export any new Rust APIs. The C API is documented in the core OMF docs. + +#![deny(unsafe_op_in_unsafe_fn)] + +mod arrays; +mod attributes; +mod elements; +mod error; +mod ffi_tools; +mod from_omf; +mod image_data; +mod init_functions; +mod metadata; +mod omf1; +mod read_iterators; +mod reader; +mod to_omf; +mod validation; +mod writer; +mod writer_handle; + +#[cfg(test)] +mod tests { + use std::ffi::CStr; + use std::ptr::{null, null_mut}; + + use crate::error::state::{omf_error, omf_error_free}; + use crate::init_functions::*; + use crate::read_iterators::*; + use crate::reader::*; + use crate::writer::*; + + /// Same as the "pyramid" C example but written in Rust, to test the C API without + /// any C code being involved. + #[test] + fn pyramid_rust() { + const VERTICES: &[[f32; 3]] = &[ + [-1.0, -1.0, 0.0], + [1.0, -1.0, 0.0], + [1.0, 1.0, 0.0], + [-1.0, 1.0, 0.0], + [0.0, 0.0, 1.0], + ]; + const TRIANGLES: &[[u32; 3]] = &[ + [0, 1, 4], + [1, 2, 4], + [2, 3, 4], + [3, 0, 4], + [0, 2, 1], + [0, 3, 2], + ]; + const SEGMENTS: &[[u32; 2]] = &[ + [0, 1], + [1, 2], + [2, 3], + [3, 0], + [0, 4], + [1, 4], + [2, 4], + [3, 4], + ]; + const NAME: &str = "pyramid.omf\0"; + const PATH: &str = "../target/tmp/pyramid-rust.omf\0"; + const SURFACE_NAME: &str = "Pyramid surface\0"; + const LINE_SET_NAME: &str = "Pyramid edges\0"; + + // Open. + let writer = omf_writer_open(PATH.as_ptr().cast()); + // Init project. + let mut project = omf_project_init(NAME.as_ptr().cast()); + project.name = "Test Project".as_ptr().cast(); + let proj_handle = omf_writer_project(writer, &project); + // Add surface. + let vertices = omf_writer_array_vertices32(writer, VERTICES.as_ptr(), VERTICES.len()); + let surface = omf_surface_init( + vertices, + omf_writer_array_triangles(writer, TRIANGLES.as_ptr(), TRIANGLES.len()), + ); + let mut element = omf_element_init(SURFACE_NAME.as_ptr().cast()); + element.surface = &surface; + element.color_set = true; + element.color = [255, 128, 0, 255]; + let ele_handle = omf_writer_element(writer, proj_handle, &element); + // Add metadata to that element. + omf_writer_metadata_string( + writer, + ele_handle, + "revision\0".as_ptr().cast(), + "1.2\0".as_ptr().cast(), + ); + let tags_handle = omf_writer_metadata_list(writer, ele_handle, "tags\0".as_ptr().cast()); + omf_writer_metadata_string(writer, tags_handle, null(), "foo\0".as_ptr().cast()); + omf_writer_metadata_string(writer, tags_handle, null(), "bar\0".as_ptr().cast()); + // Line-set element. + let line_set = omf_line_set_init( + vertices, + omf_writer_array_segments(writer, SEGMENTS.as_ptr(), SEGMENTS.len()), + ); + element = omf_element_init(LINE_SET_NAME.as_ptr().cast()); + element.line_set = &line_set; + element.color_set = true; + element.color = [0, 0, 0, 0]; + omf_writer_element(writer, proj_handle, &element); + // Finish and close. + omf_writer_finish(writer, null_mut()); + let error = omf_error(); + if !error.is_null() { + let s = unsafe { CStr::from_ptr(error.read().message) }; + assert!(false, "Error: {s:?}"); + omf_error_free(error); + } + + // Re-open to read. + let reader = omf_reader_open(PATH.as_ptr().cast()); + let project = unsafe { omf_reader_project(reader, null_mut()).as_ref() }.unwrap(); + let name = unsafe { CStr::from_ptr(project.name) }.to_str().unwrap(); + assert_eq!(name, "Test Project"); + assert_eq!(project.n_elements, 2); + let surface = unsafe { project.elements.as_ref() }.unwrap(); + let vertices_array = unsafe { surface.surface.as_ref().unwrap().vertices }; + let iter = omf_reader_array_vertices32_iter(reader, vertices_array); + let mut vertices = Vec::new(); + let mut vertex = [0.0_f32; 3]; + while omf_vertices32_next(iter, vertex.as_mut_ptr()) { + vertices.push(vertex); + } + assert_eq!(vertices, VERTICES); + } +} diff --git a/omf-c/src/metadata.rs b/omf-c/src/metadata.rs new file mode 100644 index 0000000..dfdc8c0 --- /dev/null +++ b/omf-c/src/metadata.rs @@ -0,0 +1,95 @@ +//! FFI types for OMF metadata. +//! +//! These wrap `serde_json::Value` into something usable from C. I've avoided using wrapping +//! a Rust enum here because tagged enums in C are a bit messy and easy to use incorrectly +//! by treating an f64 as a pointer and crashing. + +use std::{ffi::c_char, ptr::null}; + +use crate::{ + error::Error, + ffi_tools::arg::{slice, string_from_ptr, string_from_ptr_or_null}, +}; + +#[allow(clippy::enum_variant_names)] +#[derive(Debug, Default)] +#[repr(i32)] +pub enum ValueType { + #[default] + Null, + Boolean, + Number, + String, + List, + Object, +} + +#[derive(Debug)] +#[repr(C)] +pub struct Value { + pub name: *const c_char, + pub r#type: ValueType, + pub boolean: bool, + pub number: f64, + pub string: *const c_char, + pub values: *const Value, + pub n_values: usize, +} + +impl Value { + fn as_json_value(&self) -> Result { + match self.r#type { + ValueType::Null => Ok(serde_json::Value::Null), + ValueType::Boolean => Ok(self.boolean.into()), + ValueType::Number => Ok(self.number.into()), + ValueType::String => { + Ok(unsafe { string_from_ptr_or_null("metadata value", self.string) }?.into()) + } + ValueType::Object => { + Self::values_as_json_map(self.values, self.n_values).map(serde_json::Value::Object) + } + ValueType::List => { + Self::values_as_json_vec(self.values, self.n_values).map(serde_json::Value::Array) + } + } + } + + fn as_json_pair(&self) -> Result<(String, serde_json::Value), Error> { + let name = unsafe { string_from_ptr("value.name", self.name) }?; + Ok((name, self.as_json_value()?)) + } + + fn values_as_json_vec( + values: *const Value, + n_values: usize, + ) -> Result, Error> { + slice!(values, n_values)? + .iter() + .map(Self::as_json_value) + .collect() + } + + pub fn values_as_json_map( + values: *const Value, + n_values: usize, + ) -> Result, Error> { + slice!(values, n_values)? + .iter() + .map(Self::as_json_pair) + .collect() + } +} + +impl Default for Value { + fn default() -> Self { + Self { + name: null(), + r#type: Default::default(), + boolean: false, + number: 0.0, + string: null(), + n_values: 0, + values: null(), + } + } +} diff --git a/omf-c/src/omf1.rs b/omf-c/src/omf1.rs new file mode 100644 index 0000000..2ba0a2e --- /dev/null +++ b/omf-c/src/omf1.rs @@ -0,0 +1,120 @@ +use std::{ffi::c_char, path::PathBuf, ptr::null_mut, sync::Mutex}; + +use crate::{ + elements::Limits, + ffi_tools::{ + arg::{not_null, not_null_mut, string_not_null}, + catch, + }, + validation::{handle_validation, Validation}, +}; + +#[derive(Debug, Default)] +pub struct Omf1Converter(Mutex); + +macro_rules! inner { + ($converter:ident) => { + not_null_mut!($converter)?.0.lock().expect("intact lock") + }; +} + +#[no_mangle] +pub extern "C" fn omf_omf1_detect(path: *const c_char) -> bool { + catch::error(|| { + let path = PathBuf::from(string_not_null!(path)?); + omf::omf1::detect_open(&path)?; + Ok(()) + }) + .is_some() +} + +#[no_mangle] +pub extern "C" fn omf_omf1_converter_new() -> *mut Omf1Converter { + catch::error(|| Ok(Box::into_raw(Default::default()))).unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_omf1_converter_free(converter: *mut Omf1Converter) -> bool { + catch::error(|| { + if !converter.is_null() { + unsafe { + _ = Box::from_raw(converter); + } + } + Ok(()) + }) + .is_some() +} + +#[no_mangle] +pub extern "C" fn omf_omf1_converter_compression(converter: *mut Omf1Converter) -> i32 { + catch::error(|| Ok(inner!(converter).compression().level() as i32)).unwrap_or(-1) +} + +#[no_mangle] +pub extern "C" fn omf_omf1_converter_set_compression( + converter: *mut Omf1Converter, + compression: i32, +) -> bool { + catch::error(|| { + inner!(converter).set_compression(if compression == -1 { + omf::file::Compression::default() + } else { + omf::file::Compression::new(compression.clamp(0, 9) as u32) + }); + Ok(()) + }) + .is_some() +} + +#[no_mangle] +pub extern "C" fn omf_omf1_converter_limits(converter: *mut Omf1Converter) -> Limits { + catch::error(|| { + let limits = if converter.is_null() { + Default::default() + } else { + inner!(converter).limits() + }; + Ok(limits.into()) + }) + .unwrap_or_default() +} + +#[no_mangle] +pub extern "C" fn omf_omf1_converter_set_limits( + converter: *mut Omf1Converter, + limits: *const Limits, +) -> bool { + catch::error(|| { + let limits = not_null!(limits)?; + inner!(converter).set_limits((*limits).into()); + Ok(()) + }) + .is_some() +} + +#[no_mangle] +pub extern "C" fn omf_omf1_converter_convert( + converter: *mut Omf1Converter, + input_path: *const c_char, + output_path: *const c_char, + validation: *mut *mut Validation, +) -> bool { + catch::error(|| { + let converter = inner!(converter); + let input_path = PathBuf::from(string_not_null!(input_path)?); + let output_path = PathBuf::from(string_not_null!(output_path)?); + match converter.convert_open(input_path, output_path) { + Ok(warnings) => { + handle_validation(&warnings, validation); + Ok(()) + } + Err(omf::error::Error::ValidationFailed(problems)) => { + handle_validation(&problems, validation); + Err(omf::error::Error::ValidationFailed(problems).into()) + } + Err(e) => Err(e.into()), + } + }) + .is_some() +} diff --git a/omf-c/src/read_iterators.rs b/omf-c/src/read_iterators.rs new file mode 100644 index 0000000..8e6d404 --- /dev/null +++ b/omf-c/src/read_iterators.rs @@ -0,0 +1,837 @@ +use std::{ffi::c_char, ptr::null}; + +use crate::{ + error::{set_error, Error}, + ffi_tools::{arg::not_null_mut, catch}, +}; + +// Utility functions. + +macro_rules! inner { + ($arg:ident) => { + &mut not_null_mut!($arg)?.0 + }; +} + +fn alloc(t: T) -> *mut T { + Box::into_raw(Box::new(t)) +} + +fn free(obj: *mut T) { + if !obj.is_null() { + unsafe { + drop(Box::from_raw(obj)); + } + } +} + +fn width_cast(input: *mut T) -> *mut [T; N] { + input.cast() +} + +fn bytes_to_chars(input: *const u8) -> *const c_char { + input.cast() +} + +fn write_to_ptr(ptr: *mut T, value: T) { + if let Some(m) = unsafe { ptr.as_mut() } { + *m = value; + } +} + +fn write_default_to_ptr(ptr: *mut T) { + write_to_ptr(ptr, Default::default()) +} + +#[derive(Debug, Default)] +struct BytesCache(Vec); + +impl BytesCache { + fn set_string(&mut self, string: &str, value: *mut *const c_char, len: *mut usize) { + self.0.clear(); + self.0.extend(string.as_bytes()); + self.0.push(0); + write_to_ptr(value, bytes_to_chars(self.0.as_ptr())); + write_to_ptr(len, self.0.len() - 1); + } +} + +pub(crate) fn next_simple( + iter: &mut impl Iterator>, + value: *mut T, +) -> Result { + match iter.next() { + Some(Ok(v)) => { + write_to_ptr(value, v); + Ok(true) + } + Some(Err(err)) => Err(err.into()), + None => Ok(false), + } +} + +pub(crate) fn next_option( + iter: &mut impl Iterator, omf::error::Error>>, + value: *mut T, + is_null: *mut bool, +) -> Result { + match iter.next() { + Some(Ok(Some(v))) => { + write_to_ptr(value, v); + write_to_ptr(is_null, false); + Ok(true) + } + Some(Ok(None)) => { + write_default_to_ptr(value); + write_to_ptr(is_null, true); + Ok(true) + } + Some(Err(err)) => Err(err.into()), + None => Ok(false), + } +} + +pub(crate) fn next_option_convert( + iter: &mut impl Iterator, omf::error::Error>>, + value: *mut U, + is_null: *mut bool, + convert: impl FnOnce(T) -> U, +) -> Result { + match iter.next() { + Some(Ok(Some(v))) => { + write_to_ptr(value, convert(v)); + write_to_ptr(is_null, false); + Ok(true) + } + Some(Ok(None)) => { + write_default_to_ptr(value); + write_to_ptr(is_null, true); + Ok(true) + } + Some(Err(err)) => Err(err.into()), + None => Ok(false), + } +} + +pub(crate) fn next_wide( + iter: &mut impl Iterator>, + value: *mut T, +) -> Result { + next_simple(iter, width_cast(value)) +} + +pub(crate) fn next_wide_option( + iter: &mut impl Iterator, omf::error::Error>>, + value: *mut T, + is_null: *mut bool, +) -> Result +where + [T; N]: Default, +{ + next_option(iter, width_cast(value), is_null) +} + +pub(crate) fn next_boundary( + iter: &mut impl Iterator, omf::error::Error>>, + value: *mut T, + inclusive: *mut bool, +) -> Result { + match iter.next() { + Some(Ok(boundary)) => { + write_to_ptr(value, boundary.value()); + write_to_ptr(inclusive, boundary.is_inclusive()); + Ok(true) + } + Some(Err(err)) => Err(err.into()), + None => Ok(false), + } +} + +// f32 scalars. + +pub struct Scalars32(omf::data::GenericScalars); +pub(crate) fn scalars32_new(iter: omf::data::GenericScalars) -> *mut Scalars32 { + alloc(Scalars32(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_scalars32_free(iter: *mut Scalars32) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_scalars32_next(iter: *mut Scalars32, value: *mut f32) -> bool { + catch::error_only(|| next_simple(inner!(iter), value)).unwrap_or(false) +} + +// f64 scalars, can cast from f32. + +pub struct Scalars64(omf::data::Scalars); + +pub(crate) fn scalars64_new(iter: omf::data::Scalars) -> *mut Scalars64 { + alloc(Scalars64(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_scalars64_free(iter: *mut Scalars64) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_scalars64_next(iter: *mut Scalars64, value: *mut f64) -> bool { + catch::error_only(|| next_simple(inner!(iter), value)).unwrap_or(false) +} + +// f32 vertices. + +pub struct Vertices32(omf::data::GenericArrays); + +pub(crate) fn vertices32_new(iter: omf::data::GenericArrays) -> *mut Vertices32 { + alloc(Vertices32(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_vertices32_free(iter: *mut Vertices32) { + free(iter); +} +#[no_mangle] +pub extern "C" fn omf_vertices32_next(iter: *mut Vertices32, value: *mut f32) -> bool { + catch::error_only(|| next_wide(inner!(iter), value)).unwrap_or(false) +} + +// f64 vertices, can cast from f32. + +pub struct Vertices64(omf::data::Vertices); + +pub(crate) fn vertices64_new(iter: omf::data::Vertices) -> *mut Vertices64 { + alloc(Vertices64(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_vertices64_free(iter: *mut Vertices64) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_vertices64_next(iter: *mut Vertices64, value: *mut f64) -> bool { + catch::error_only(|| next_wide(inner!(iter), value)).unwrap_or(false) +} + +// Segments. + +pub struct Segments(omf::data::GenericPrimitives<2>); + +pub(crate) fn segments_new(iter: omf::data::GenericPrimitives<2>) -> *mut Segments { + alloc(Segments(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_segments_free(iter: *mut Segments) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_segments_next(iter: *mut Segments, value: *mut u32) -> bool { + catch::error_only(|| next_wide(inner!(iter), value)).unwrap_or(false) +} + +// Triangles. + +pub struct Triangles(omf::data::GenericPrimitives<3>); +pub(crate) fn triangles_new(iter: omf::data::GenericPrimitives<3>) -> *mut Triangles { + alloc(Triangles(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_triangles_free(iter: *mut Triangles) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_triangles_next(iter: *mut Triangles, value: *mut u32) -> bool { + catch::error_only(|| next_wide(inner!(iter), value)).unwrap_or(false) +} + +// Gradient. + +pub struct Gradient(omf::data::Gradient); +pub(crate) fn gradient_new(iter: omf::data::Gradient) -> *mut Gradient { + alloc(Gradient(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_gradient_free(iter: *mut Gradient) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_gradient_next(iter: *mut Gradient, value: *mut u8) -> bool { + catch::error_only(|| next_wide(inner!(iter), value)).unwrap_or(false) +} + +// f32 texture coordinates. + +pub struct Texcoords32(omf::data::GenericArrays); + +pub(crate) fn texcoords32_new(iter: omf::data::GenericArrays) -> *mut Texcoords32 { + alloc(Texcoords32(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_texcoords32_free(iter: *mut Texcoords32) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_texcoords32_next(iter: *mut Texcoords32, value: *mut f32) -> bool { + catch::error_only(|| next_wide(inner!(iter), value)).unwrap_or(false) +} + +// f64 texcoords, can cast from f32. + +pub struct Texcoords64(omf::data::Texcoords); + +pub(crate) fn texcoords64_new(iter: omf::data::Texcoords) -> *mut Texcoords64 { + alloc(Texcoords64(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_texcoords64_free(iter: *mut Texcoords64) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_texcoords64_next(iter: *mut Texcoords64, value: *mut f64) -> bool { + catch::error_only(|| next_wide(inner!(iter), value)).unwrap_or(false) +} + +// f64 discrete colormap boundaries. + +pub struct BoundariesFloat64(omf::data::BoundariesF64); + +pub(crate) fn boundaries_float64_new(iter: omf::data::BoundariesF64) -> *mut BoundariesFloat64 { + alloc(BoundariesFloat64(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_boundaries_float64_free(iter: *mut BoundariesFloat64) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_boundaries_float64_next( + iter: *mut BoundariesFloat64, + value: *mut f64, + inclusive: *mut bool, +) -> bool { + catch::error_only(|| next_boundary(inner!(iter), value, inclusive)).unwrap_or(false) +} + +// f32 discrete colormap boundaries. + +pub struct BoundariesFloat32(omf::data::GenericBoundaries); + +pub(crate) fn boundaries_float32_new( + iter: omf::data::GenericBoundaries, +) -> *mut BoundariesFloat32 { + alloc(BoundariesFloat32(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_boundaries_float32_free(iter: *mut BoundariesFloat32) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_boundaries_float32_next( + iter: *mut BoundariesFloat32, + value: *mut f32, + inclusive: *mut bool, +) -> bool { + catch::error_only(|| next_boundary(inner!(iter), value, inclusive)).unwrap_or(false) +} + +// i64 discrete colormap boundaries. + +pub struct BoundariesInt64(omf::data::BoundariesI64); + +pub(crate) fn boundaries_int64_new(iter: omf::data::BoundariesI64) -> *mut BoundariesInt64 { + alloc(BoundariesInt64(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_boundaries_int64_free(iter: *mut BoundariesInt64) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_boundaries_int64_next( + iter: *mut BoundariesInt64, + value: *mut i64, + inclusive: *mut bool, +) -> bool { + catch::error_only(|| next_boundary(inner!(iter), value, inclusive)).unwrap_or(false) +} + +// f32 numbers. + +pub struct NumbersFloat32(omf::data::GenericNumbers); + +pub(crate) fn numbers_float32_new(iter: omf::data::GenericNumbers) -> *mut NumbersFloat32 { + alloc(NumbersFloat32(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_numbers_float32_free(iter: *mut NumbersFloat32) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_numbers_float32_next( + iter: *mut NumbersFloat32, + value: *mut f32, + is_null: *mut bool, +) -> bool { + catch::error_only(|| next_option(inner!(iter), value, is_null)).unwrap_or(false) +} + +// f64 numbers, casting from date and date-time as well. + +pub struct NumbersFloat64(omf::data::NumbersF64); + +pub(crate) fn numbers_float64_new(iter: omf::data::NumbersF64) -> *mut NumbersFloat64 { + alloc(NumbersFloat64(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_numbers_float64_free(iter: *mut NumbersFloat64) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_numbers_float64_next( + iter: *mut NumbersFloat64, + value: *mut f64, + is_null: *mut bool, +) -> bool { + catch::error_only(|| next_option(inner!(iter), value, is_null)).unwrap_or(false) +} + +// i64 numbers, casting from date and date-time as well. + +pub struct NumbersInt64(omf::data::NumbersI64); + +pub(crate) fn numbers_int64_new(iter: omf::data::NumbersI64) -> *mut NumbersInt64 { + alloc(NumbersInt64(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_numbers_int64_free(iter: *mut NumbersInt64) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_numbers_int64_next( + iter: *mut NumbersInt64, + value: *mut i64, + is_null: *mut bool, +) -> bool { + catch::error_only(|| next_option(inner!(iter), value, is_null)).unwrap_or(false) +} + +// Nullable indices. + +pub struct Indices(omf::data::Indices); + +pub(crate) fn indices_new(iter: omf::data::Indices) -> *mut Indices { + alloc(Indices(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_indices_free(iter: *mut Indices) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_indices_next( + iter: *mut Indices, + value: *mut u32, + is_null: *mut bool, +) -> bool { + catch::error_only(|| next_option(inner!(iter), value, is_null)).unwrap_or(false) +} + +// Nullable booleans. + +pub struct Booleans(omf::data::Booleans); + +pub(crate) fn booleans_new(iter: omf::data::Booleans) -> *mut Booleans { + alloc(Booleans(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_booleans_free(iter: *mut Booleans) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_booleans_next( + iter: *mut Booleans, + value: *mut bool, + is_null: *mut bool, +) -> bool { + catch::error_only(|| next_option(inner!(iter), value, is_null)).unwrap_or(false) +} + +// Nullable colors. + +pub struct Colors(omf::data::Colors); +pub(crate) fn colors_new(iter: omf::data::Colors) -> *mut Colors { + alloc(Colors(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_colors_free(iter: *mut Colors) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_omf_colors_next( + iter: *mut Colors, + value: *mut u8, + is_null: *mut bool, +) -> bool { + catch::error_only(|| next_wide_option(inner!(iter), value, is_null)).unwrap_or(false) +} + +// Non-nullable name strings. + +pub struct Names { + iter: omf::data::Names, + bytes: BytesCache, +} + +pub(crate) fn names_new(iter: omf::data::Names) -> *mut Names { + Box::into_raw(Box::new(Names { + iter, + bytes: Default::default(), + })) +} + +#[no_mangle] +pub extern "C" fn omf_names_free(iter: *mut Names) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_names_next( + iter: *mut Names, + value: *mut *const c_char, + len: *mut usize, +) -> bool { + catch::error_only(|| { + let this = not_null_mut!(iter)?; + match this.iter.next() { + Some(Ok(s)) => { + this.bytes.set_string(&s, value, len); + Ok(true) + } + Some(Err(err)) => { + set_error(err.into()); + Ok(false) + } + None => Ok(false), + } + }) + .unwrap_or(false) +} + +// Nullable text strings. + +pub struct Text { + iter: omf::data::Text, + bytes: BytesCache, +} + +pub(crate) fn text_new(iter: omf::data::Text) -> *mut Text { + Box::into_raw(Box::new(Text { + iter, + bytes: Default::default(), + })) +} + +#[no_mangle] +pub extern "C" fn omf_text_free(iter: *mut Text) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_text_next( + iter: *mut Text, + value: *mut *const c_char, + len: *mut usize, +) -> bool { + catch::error_only(|| { + let this = not_null_mut!(iter)?; + match this.iter.next() { + Some(Ok(Some(s))) => { + this.bytes.set_string(&s, value, len); + Ok(true) + } + Some(Ok(None)) => { + write_to_ptr(value, null()); + write_to_ptr(len, 0); + Ok(true) + } + Some(Err(err)) => { + set_error(err.into()); + Ok(false) + } + None => Ok(false), + } + }) + .unwrap_or(false) +} + +// Regular sub-blocks + +pub struct RegularSubblocks(omf::data::RegularSubblocks); + +pub(crate) fn regular_subblocks_new(iter: omf::data::RegularSubblocks) -> *mut RegularSubblocks { + alloc(RegularSubblocks(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_regular_subblocks_free(iter: *mut RegularSubblocks) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_regular_subblocks_next( + iter: *mut RegularSubblocks, + parent_index: *mut u32, + corners: *mut u32, +) -> bool { + catch::error_only(|| { + let this = not_null_mut!(iter)?; + match this.0.next() { + Some(Ok((p, c))) => { + write_to_ptr(width_cast(parent_index), p); + write_to_ptr(width_cast(corners), c); + Ok(true) + } + Some(Err(err)) => { + set_error(err.into()); + Ok(false) + } + None => Ok(false), + } + }) + .unwrap_or(false) +} + +// Free-form sub-blocks with f64 corners, casts from f64 + +pub struct FreeformSubblocks32(omf::data::GenericFreeformSubblocks); + +pub(crate) fn freeform_subblocks32_new( + iter: omf::data::GenericFreeformSubblocks, +) -> *mut FreeformSubblocks32 { + alloc(FreeformSubblocks32(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_freeform_subblocks32_free(iter: *mut FreeformSubblocks32) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_freeform_subblocks32_next( + iter: *mut FreeformSubblocks32, + parent_index: *mut u32, + corners: *mut f32, +) -> bool { + catch::error_only(|| { + let this = not_null_mut!(iter)?; + match this.0.next() { + Some(Ok((p, c))) => { + write_to_ptr(width_cast(parent_index), p); + write_to_ptr(width_cast(corners), c); + Ok(true) + } + Some(Err(err)) => { + set_error(err.into()); + Ok(false) + } + None => Ok(false), + } + }) + .unwrap_or(false) +} + +// Free-form sub-blocks with f32 corners + +pub struct FreeformSubblocks64(omf::data::FreeformSubblocks); + +pub(crate) fn freeform_subblocks64_new( + iter: omf::data::FreeformSubblocks, +) -> *mut FreeformSubblocks64 { + alloc(FreeformSubblocks64(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_freeform_subblocks64_free(iter: *mut FreeformSubblocks64) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_freeform_subblocks64_next( + iter: *mut FreeformSubblocks64, + parent_index: *mut u32, + corners: *mut f64, +) -> bool { + catch::error_only(|| { + let this = not_null_mut!(iter)?; + match this.0.next() { + Some(Ok((p, c))) => { + write_to_ptr(width_cast(parent_index), p); + write_to_ptr(width_cast(corners), c); + Ok(true) + } + Some(Err(err)) => { + set_error(err.into()); + Ok(false) + } + None => Ok(false), + } + }) + .unwrap_or(false) +} + +// 3D vectors with type f64, casts from anything + +pub struct Vectors64x3(omf::data::Vectors); + +pub(crate) fn vectors64x3_new(iter: omf::data::Vectors) -> *mut Vectors64x3 { + alloc(Vectors64x3(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_vectors64x3_free(iter: *mut Vectors64x3) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_vectors64x3_next( + iter: *mut Vectors64x3, + value: *mut f64, + is_null: *mut bool, +) -> bool { + catch::error_only(|| next_wide_option(inner!(iter), value, is_null)).unwrap_or(false) +} + +// 3D vectors with type f32, casts from 2D f32 + +pub struct Vectors32x3(Vec32Iter); + +enum Vec32Iter { + X2(omf::data::GenericOptionalArrays), + X3(omf::data::GenericOptionalArrays), +} + +pub(crate) fn vectors32x3_new(iter: omf::data::Vectors) -> *mut Vectors32x3 { + let vec32_iter = match iter { + omf::data::Vectors::F32x2(i) => Vec32Iter::X2(i), + omf::data::Vectors::F32x3(i) => Vec32Iter::X3(i), + _ => panic!("wrong vector type"), + }; + alloc(Vectors32x3(vec32_iter)) +} + +#[no_mangle] +pub extern "C" fn omf_vectors32x3_free(iter: *mut Vectors32x3) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_vectors32x3_next( + iter: *mut Vectors32x3, + value: *mut f32, + is_null: *mut bool, +) -> bool { + catch::error_only(|| { + let iter = inner!(iter); + match iter { + Vec32Iter::X2(i) => next_option_convert(i, width_cast(value), is_null, vec_2d_to_3d), + Vec32Iter::X3(i) => next_wide_option(i, value, is_null), + } + }) + .unwrap_or(false) +} + +fn vec_2d_to_3d([x, y]: [T; 2]) -> [T; 3] { + [x, y, T::ZERO] +} + +// 2D vectors with type f64, casts from 2D f32 + +pub struct Vectors64x2(Vec2DIter); + +enum Vec2DIter { + F32(omf::data::GenericOptionalArrays), + F64(omf::data::GenericOptionalArrays), +} + +pub(crate) fn vectors64x2_new(iter: omf::data::Vectors) -> *mut Vectors64x2 { + let vec32_iter = match iter { + omf::data::Vectors::F32x2(i) => Vec2DIter::F32(i), + omf::data::Vectors::F64x2(i) => Vec2DIter::F64(i), + _ => panic!("wrong vector type"), + }; + alloc(Vectors64x2(vec32_iter)) +} + +#[no_mangle] +pub extern "C" fn omf_vectors64x2_free(iter: *mut Vectors64x2) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_vectors64x2_next( + iter: *mut Vectors64x2, + value: *mut f64, + is_null: *mut bool, +) -> bool { + catch::error_only(|| { + let iter = inner!(iter); + match iter { + Vec2DIter::F32(i) => { + next_option_convert(i, width_cast(value), is_null, |[x, y]| [x.into(), y.into()]) + } + Vec2DIter::F64(i) => next_wide_option(i, value, is_null), + } + }) + .unwrap_or(false) +} + +// 2D vectors with type f32, no casting + +pub struct Vectors32x2(omf::data::GenericOptionalArrays); + +pub(crate) fn vectors32x2_new(iter: omf::data::GenericOptionalArrays) -> *mut Vectors32x2 { + alloc(Vectors32x2(iter)) +} + +#[no_mangle] +pub extern "C" fn omf_vectors32x2_free(iter: *mut Vectors32x2) { + free(iter); +} + +#[no_mangle] +pub extern "C" fn omf_vectors32x2_next( + iter: *mut Vectors32x2, + value: *mut f32, + is_null: *mut bool, +) -> bool { + catch::error_only(|| next_wide_option(inner!(iter), value, is_null)).unwrap_or(false) +} diff --git a/omf-c/src/reader.rs b/omf-c/src/reader.rs new file mode 100644 index 0000000..5350ab9 --- /dev/null +++ b/omf-c/src/reader.rs @@ -0,0 +1,1177 @@ +use std::{ffi::c_char, io::Read, path::PathBuf, ptr::null_mut, sync::Mutex}; + +use crate::{ + arrays::{array, array_action, Array, ArrayType}, + elements::{FileVersion, Limits, Project}, + error::Error, + ffi_tools::{ + arg::{not_null, not_null_consume, slice_mut, slice_mut_len, string_not_null}, + catch, FfiStorage, IntoFfi, + }, + image_data::ImageData, + read_iterators::*, + validation::{handle_validation, Validation}, +}; + +struct ReaderWrapper { + pub inner: omf::file::Reader, + pub storage: FfiStorage, + pub project_loaded: bool, +} + +macro_rules! unsafe_cast { + ($from:literal -> $to:literal) => { + Err(omf::error::Error::UnsafeCast($from, $to).into()) + }; +} + +pub struct Reader(Mutex); + +impl ReaderWrapper { + fn array_type(&self, array: &Array) -> Result { + Ok(match array { + Array::Image(_) => ArrayType::Image, + Array::Name(_) => ArrayType::Names, + Array::Text(_) => ArrayType::Text, + Array::Boolean(_) => ArrayType::Booleans, + Array::Segment(_) => ArrayType::Segments, + Array::Triangle(_) => ArrayType::Triangles, + Array::Gradient(_) => ArrayType::Gradient, + Array::RegularSubblock(_) => ArrayType::RegularSubblocks, + Array::Index(_) => ArrayType::Indices, + Array::Color(_) => ArrayType::Colors, + Array::Scalar(a) => match self.inner.array_scalars(a)? { + omf::data::Scalars::F32(_) => ArrayType::Scalars32, + omf::data::Scalars::F64(_) => ArrayType::Scalars64, + }, + Array::Vertex(a) => match self.inner.array_vertices(a)? { + omf::data::Vertices::F32(_) => ArrayType::Vertices32, + omf::data::Vertices::F64(_) => ArrayType::Vertices64, + }, + Array::Texcoord(a) => match self.inner.array_texcoords(a)? { + omf::data::Texcoords::F32(_) => ArrayType::Texcoords32, + omf::data::Texcoords::F64(_) => ArrayType::Texcoords64, + }, + Array::Boundary(a) => match self.inner.array_boundaries(a)? { + omf::data::Boundaries::F32(_) => ArrayType::BoundariesFloat32, + omf::data::Boundaries::F64(_) => ArrayType::BoundariesFloat64, + omf::data::Boundaries::I64(_) => ArrayType::BoundariesInt64, + omf::data::Boundaries::Date(_) => ArrayType::BoundariesDate, + omf::data::Boundaries::DateTime(_) => ArrayType::BoundariesDateTime, + }, + Array::FreeformSubblock(a) => match self.inner.array_freeform_subblocks(a)? { + omf::data::FreeformSubblocks::F32(_) => ArrayType::FreeformSubblocks32, + omf::data::FreeformSubblocks::F64(_) => ArrayType::FreeformSubblocks64, + }, + Array::Number(a) => match self.inner.array_numbers(a)? { + omf::data::Numbers::F32(_) => ArrayType::NumbersFloat32, + omf::data::Numbers::F64(_) => ArrayType::NumbersFloat64, + omf::data::Numbers::I64(_) => ArrayType::NumbersInt64, + omf::data::Numbers::Date(_) => ArrayType::NumbersDate, + omf::data::Numbers::DateTime(_) => ArrayType::NumbersDateTime, + }, + Array::Vector(a) => match self.inner.array_vectors(a)? { + omf::data::Vectors::F32x2(_) => ArrayType::Vectors32x2, + omf::data::Vectors::F64x2(_) => ArrayType::Vectors64x2, + omf::data::Vectors::F32x3(_) => ArrayType::Vectors32x3, + omf::data::Vectors::F64x3(_) => ArrayType::Vectors64x3, + }, + }) + } + + fn array_info(&self, array: &Array) -> Result { + let array_type = self.array_type(array)?; + array_action!(array, |a| { + Ok(ArrayInfo { + array_type, + item_count: a.item_count(), + compressed_size: self.inner.array_compressed_size(a)?, + }) + }) + } +} + +#[no_mangle] +pub extern "C" fn omf_reader_open(path: *const c_char) -> *mut Reader { + catch::error(|| { + let path = PathBuf::from(string_not_null!(path)?); + let wrapper = ReaderWrapper { + inner: omf::file::Reader::open(path)?, + storage: FfiStorage::new(), + project_loaded: false, + }; + Ok(Box::into_raw(Box::new(Reader(Mutex::new(wrapper))))) + }) + .unwrap_or(null_mut()) +} + +#[no_mangle] +pub extern "C" fn omf_reader_close(reader: *mut Reader) -> bool { + catch::panic_bool(|| { + _ = not_null_consume!(reader); + }) +} + +macro_rules! wrapper { + ($reader:ident) => { + not_null!($reader)?.0.lock().expect("intact lock") + }; +} + +#[no_mangle] +pub extern "C" fn omf_reader_project( + reader: *mut Reader, + validation: *mut *mut Validation, +) -> *const Project { + catch::error(|| { + let mut wrapper = wrapper!(reader); + if wrapper.project_loaded { + return Err(Error::InvalidCall( + "second call to 'omf_reader_project' on this reader".to_owned(), + )); + } + let result = wrapper.inner.project(); + match &result { + Ok((_, warnings)) => handle_validation(warnings, validation), + Err(omf::error::Error::ValidationFailed(errors)) => { + handle_validation(errors, validation) + } + _ => {} + } + let (project, _) = result?; + wrapper.project_loaded = true; + Ok(wrapper.storage.convert_ptr_mut(project)) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_version(reader: *mut Reader) -> FileVersion { + catch::error(|| { + let wrapper = wrapper!(reader); + Ok(wrapper.inner.version().into()) + }) + .unwrap_or_default() +} + +#[no_mangle] +pub extern "C" fn omf_reader_limits(reader: *mut Reader) -> Limits { + catch::error(|| { + let limits = if reader.is_null() { + Default::default() + } else { + wrapper!(reader).inner.limits() + }; + Ok(limits.into()) + }) + .unwrap_or_default() +} + +#[no_mangle] +pub extern "C" fn omf_reader_set_limits(reader: *mut Reader, limits: *const Limits) -> bool { + catch::error(|| { + let limits = not_null!(limits)?; + wrapper!(reader).inner.set_limits((*limits).into()); + Ok(true) + }) + .unwrap_or(false) +} + +#[derive(Debug, Default)] +#[repr(C)] +pub struct ArrayInfo { + pub array_type: ArrayType, + pub item_count: u64, + pub compressed_size: u64, +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_info(reader: *mut Reader, array: *const Array) -> ArrayInfo { + catch::error(|| { + let array = not_null!(array)?; + wrapper!(reader).array_info(array) + }) + .unwrap_or_default() +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_bytes( + reader: *mut Reader, + array: *const Array, + output: *mut c_char, + n_output: usize, +) -> bool { + catch::error(|| { + let wrapper = wrapper!(reader); + let mut read = array_action!(not_null!(array)?, |a| wrapper.inner.array_bytes_reader(a))?; + let found = u64::try_from(n_output).expect("usize fits in u64"); + if read.len() != found { + return Err(Error::BufferLengthWrong { + found, + expected: read.len(), + }); + } + read.read_exact(slice_mut!(output.cast(), n_output)?) + .map_err(omf::error::Error::from)?; + Ok(true) + }) + .unwrap_or(false) +} + +// Image + +#[no_mangle] +pub extern "C" fn omf_reader_image(reader: *mut Reader, array: *const Array) -> *mut ImageData { + catch::error(|| Ok(wrapper!(reader).inner.image(&array!(array)?)?.into_ffi())) + .unwrap_or_else(null_mut) +} + +// Scalars + +#[no_mangle] +pub extern "C" fn omf_reader_array_scalars32_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Scalars32 { + catch::error( + || match wrapper!(reader).inner.array_scalars(&array!(array)?)? { + omf::data::Scalars::F32(i) => Ok(scalars32_new(i)), + omf::data::Scalars::F64(_) => unsafe_cast!("64-bit float" -> "32-bit float"), + }, + ) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_scalars64_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Scalars64 { + catch::error(|| { + Ok(scalars64_new( + wrapper!(reader).inner.array_scalars(&array!(array)?)?, + )) + }) + .unwrap_or_else(null_mut) +} + +fn check_slice(n_values: usize, expected: u64) -> Result<(), Error> { + let found = u64::try_from(n_values).expect("usize fits in u64"); + if found != expected { + return Err(Error::BufferLengthWrong { found, expected }); + } + Ok(()) +} + +fn into_slice( + values: *mut T, + n_values: usize, + expected: u64, + iter: impl IntoIterator>, +) -> Result<(), Error> { + check_slice(n_values, expected)?; + for (out, val) in slice_mut!(values, n_values)?.iter_mut().zip(iter) { + *out = val?; + } + Ok(()) +} + +fn into_slice_nullable( + values: *mut T, + mask: *mut bool, + n_values: usize, + expected: u64, + iter: impl IntoIterator, omf::error::Error>>, +) -> Result<(), Error> { + check_slice(n_values, expected)?; + for ((out, is_null), val) in slice_mut!(values, n_values)? + .iter_mut() + .zip(slice_mut!(mask, n_values)?) + .zip(iter) + { + let value = val?; + *is_null = value.is_none(); + *out = value.unwrap_or_default(); + } + Ok(()) +} + +fn into_slice_nullable_convert( + values: *mut U, + mask: *mut bool, + n_values: usize, + expected: u64, + iter: impl IntoIterator, omf::error::Error>>, + f: impl Fn(T) -> U + Copy, +) -> Result<(), Error> { + check_slice(n_values, expected)?; + for ((out, is_null), val) in slice_mut!(values, n_values)? + .iter_mut() + .zip(slice_mut!(mask, n_values)?) + .zip(iter) + { + let value = val?; + *is_null = value.is_none(); + *out = value.map(f).unwrap_or_default(); + } + Ok(()) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_scalars64( + reader: *mut Reader, + array: *const Array, + values: *mut f64, + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = wrapper!(reader).inner.array_scalars(&array)?; + into_slice(values, n_values, array.item_count(), iter) + }) + .is_some() +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_scalars32( + reader: *mut Reader, + array: *const Array, + values: *mut f32, + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = match wrapper!(reader).inner.array_scalars(&array)? { + omf::data::Scalars::F32(i) => i, + omf::data::Scalars::F64(_) => return unsafe_cast!("64-bit float" -> "32-bit float"), + }; + into_slice(values, n_values, array.item_count(), iter) + }) + .is_some() +} + +// Vertices + +#[no_mangle] +pub extern "C" fn omf_reader_array_vertices32_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Vertices32 { + catch::error( + || match wrapper!(reader).inner.array_vertices(&array!(array)?)? { + omf::data::Vertices::F32(i) => Ok(vertices32_new(i)), + omf::data::Vertices::F64(_) => unsafe_cast!("64-bit float" -> "32-bit float"), + }, + ) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_vertices64_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Vertices64 { + catch::error(|| { + Ok(vertices64_new( + wrapper!(reader).inner.array_vertices(&array!(array)?)?, + )) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_vertices64( + reader: *mut Reader, + array: *const Array, + values: *mut [f64; 3], + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = wrapper!(reader).inner.array_vertices(&array)?; + into_slice(values, n_values, array.item_count(), iter) + }) + .is_some() +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_vertices32( + reader: *mut Reader, + array: *const Array, + values: *mut [f32; 3], + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = match wrapper!(reader).inner.array_vertices(&array)? { + omf::data::Vertices::F32(i) => i, + omf::data::Vertices::F64(_) => return unsafe_cast!("64-bit float" -> "32-bit float"), + }; + into_slice(values, n_values, array.item_count(), iter) + }) + .is_some() +} + +// Segments + +#[no_mangle] +pub extern "C" fn omf_reader_array_segments_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Segments { + catch::error(|| { + Ok(segments_new( + wrapper!(reader).inner.array_segments(&array!(array)?)?, + )) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_segments( + reader: *mut Reader, + array: *const Array, + values: *mut [u32; 2], + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = wrapper!(reader).inner.array_segments(&array)?; + into_slice(values, n_values, array.item_count(), iter) + }) + .is_some() +} + +// Triangles + +#[no_mangle] +pub extern "C" fn omf_reader_array_triangles_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Triangles { + catch::error(|| { + Ok(triangles_new( + wrapper!(reader).inner.array_triangles(&array!(array)?)?, + )) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_triangles( + reader: *mut Reader, + array: *const Array, + values: *mut [u32; 3], + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = wrapper!(reader).inner.array_triangles(&array)?; + into_slice(values, n_values, array.item_count(), iter) + }) + .is_some() +} + +// Names + +#[no_mangle] +pub extern "C" fn omf_reader_array_names_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Names { + catch::error(|| { + Ok(names_new( + wrapper!(reader).inner.array_names(&array!(array)?)?, + )) + }) + .unwrap_or_else(null_mut) +} + +// Gradient + +#[no_mangle] +pub extern "C" fn omf_reader_array_gradient_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Gradient { + catch::error(|| { + Ok(gradient_new( + wrapper!(reader).inner.array_gradient(&array!(array)?)?, + )) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_gradient( + reader: *mut Reader, + array: *const Array, + values: *mut [u8; 4], + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = wrapper!(reader).inner.array_gradient(&array)?; + into_slice(values, n_values, array.item_count(), iter) + }) + .is_some() +} + +// Texture coordinates + +#[no_mangle] +pub extern "C" fn omf_reader_array_texcoords32_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Texcoords32 { + catch::error( + || match wrapper!(reader).inner.array_texcoords(&array!(array)?)? { + omf::data::Texcoords::F32(i) => Ok(texcoords32_new(i)), + omf::data::Texcoords::F64(_) => unsafe_cast!("64-bit float" -> "32-bit float"), + }, + ) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_texcoords64_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Texcoords64 { + catch::error(|| { + Ok(texcoords64_new( + wrapper!(reader).inner.array_texcoords(&array!(array)?)?, + )) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_texcoords64( + reader: *mut Reader, + array: *const Array, + values: *mut [f64; 2], + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = wrapper!(reader).inner.array_texcoords(&array)?; + into_slice(values, n_values, array.item_count(), iter) + }) + .is_some() +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_texcoords32( + reader: *mut Reader, + array: *const Array, + values: *mut [f32; 2], + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = match wrapper!(reader).inner.array_texcoords(&array)? { + omf::data::Texcoords::F32(i) => i, + omf::data::Texcoords::F64(_) => return unsafe_cast!("64-bit float" -> "32-bit float"), + }; + into_slice(values, n_values, array.item_count(), iter) + }) + .is_some() +} + +// Boundaries + +#[no_mangle] +pub extern "C" fn omf_reader_array_boundaries_float32_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut BoundariesFloat32 { + catch::error( + || match wrapper!(reader).inner.array_boundaries(&array!(array)?)? { + omf::data::Boundaries::F32(i) => Ok(boundaries_float32_new(i)), + omf::data::Boundaries::F64(_) => unsafe_cast!("64-bit float" -> "32-bit float"), + omf::data::Boundaries::I64(_) => unsafe_cast!("64-bit int" -> "32-bit float"), + omf::data::Boundaries::Date(_) => unsafe_cast!("date" -> "32-bit float"), + omf::data::Boundaries::DateTime(_) => unsafe_cast!("date-time" -> "32-bit float"), + }, + ) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_boundaries_float64_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut BoundariesFloat64 { + catch::error(|| { + Ok(boundaries_float64_new( + wrapper!(reader) + .inner + .array_boundaries(&array!(array)?)? + .try_into_f64()?, + )) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_boundaries_int64_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut BoundariesInt64 { + catch::error(|| { + Ok(boundaries_int64_new( + wrapper!(reader) + .inner + .array_boundaries(&array!(array)?)? + .try_into_i64()?, + )) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_boundaries_float64( + reader: *mut Reader, + array: *const Array, + values: *mut f64, + inclusive: *mut bool, + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = wrapper!(reader) + .inner + .array_boundaries(&array)? + .try_into_f64()?; + let values = slice_mut_len!(values, n_values, array.item_count())?; + let inclusive = slice_mut_len!(inclusive, n_values, array.item_count())?; + for ((out, inc), val) in values.iter_mut().zip(inclusive.iter_mut()).zip(iter) { + let bound = val?; + *out = bound.value(); + *inc = bound.is_inclusive(); + } + Ok(()) + }) + .is_some() +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_boundaries_int64( + reader: *mut Reader, + array: *const Array, + values: *mut i64, + inclusive: *mut bool, + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = wrapper!(reader) + .inner + .array_boundaries(&array)? + .try_into_i64()?; + let values = slice_mut_len!(values, n_values, array.item_count())?; + let inclusive = slice_mut_len!(inclusive, n_values, array.item_count())?; + for ((out, inc), val) in values.iter_mut().zip(inclusive.iter_mut()).zip(iter) { + let bound = val?; + *out = bound.value(); + *inc = bound.is_inclusive(); + } + Ok(()) + }) + .is_some() +} + +// Regular sub-blocks + +#[no_mangle] +pub extern "C" fn omf_reader_array_regular_subblocks_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut RegularSubblocks { + catch::error(|| { + Ok(regular_subblocks_new( + wrapper!(reader) + .inner + .array_regular_subblocks(&array!(array)?)?, + )) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_regular_subblocks( + reader: *mut Reader, + array: *const Array, + parents: *mut [u32; 3], + corners: *mut [u32; 6], + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = wrapper!(reader).inner.array_regular_subblocks(&array)?; + let parents = slice_mut_len!(parents, n_values, array.item_count())?; + let corners = slice_mut_len!(corners, n_values, array.item_count())?; + for ((p, c), block) in parents.iter_mut().zip(corners.iter_mut()).zip(iter) { + (*p, *c) = block?; + } + Ok(()) + }) + .is_some() +} + +// Freeform sub-blocks + +#[no_mangle] +pub extern "C" fn omf_reader_array_freeform_subblocks32_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut FreeformSubblocks32 { + catch::error(|| { + let wrapper = wrapper!(reader); + match wrapper.inner.array_freeform_subblocks(&array!(array)?)? { + omf::data::FreeformSubblocks::F32(i) => Ok(freeform_subblocks32_new(i)), + omf::data::FreeformSubblocks::F64(_) => unsafe_cast!("64-bit float" -> "32-bit float"), + } + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_freeform_subblocks64_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut FreeformSubblocks64 { + catch::error(|| { + Ok(freeform_subblocks64_new( + wrapper!(reader) + .inner + .array_freeform_subblocks(&array!(array)?)?, + )) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_freeform_subblocks64( + reader: *mut Reader, + array: *const Array, + parents: *mut [u32; 3], + corners: *mut [f64; 6], + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = wrapper!(reader).inner.array_freeform_subblocks(&array)?; + let parents = slice_mut_len!(parents, n_values, array.item_count())?; + let corners = slice_mut_len!(corners, n_values, array.item_count())?; + for ((p, c), block) in parents.iter_mut().zip(corners.iter_mut()).zip(iter) { + (*p, *c) = block?; + } + Ok(()) + }) + .is_some() +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_freeform_subblocks32( + reader: *mut Reader, + array: *const Array, + parents: *mut [u32; 3], + corners: *mut [f32; 6], + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = match wrapper!(reader).inner.array_freeform_subblocks(&array)? { + omf::data::FreeformSubblocks::F32(i) => i, + omf::data::FreeformSubblocks::F64(_) => { + return unsafe_cast!("64-bit float" -> "32-bit float") + } + }; + let parents = slice_mut_len!(parents, n_values, array.item_count())?; + let corners = slice_mut_len!(corners, n_values, array.item_count())?; + for ((p, c), block) in parents.iter_mut().zip(corners.iter_mut()).zip(iter) { + (*p, *c) = block?; + } + Ok(()) + }) + .is_some() +} + +// Numbers + +#[no_mangle] +pub extern "C" fn omf_reader_array_numbers_float32_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut NumbersFloat32 { + catch::error( + || match wrapper!(reader).inner.array_numbers(&array!(array)?)? { + omf::data::Numbers::F32(i) => Ok(numbers_float32_new(i)), + omf::data::Numbers::F64(_) => unsafe_cast!("64-bit float" -> "32-bit float"), + omf::data::Numbers::I64(_) => unsafe_cast!("64-bit int" -> "32-bit float"), + omf::data::Numbers::Date(_) => unsafe_cast!("date" -> "32-bit float"), + omf::data::Numbers::DateTime(_) => unsafe_cast!("date-time" -> "32-bit float"), + }, + ) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_numbers_float64_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut NumbersFloat64 { + catch::error(|| { + Ok(numbers_float64_new( + wrapper!(reader) + .inner + .array_numbers(&array!(array)?)? + .try_into_f64()?, + )) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_numbers_int64_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut NumbersInt64 { + catch::error(|| { + Ok(numbers_int64_new( + wrapper!(reader) + .inner + .array_numbers(&array!(array)?)? + .try_into_i64()?, + )) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_numbers_float64( + reader: *mut Reader, + array: *const Array, + values: *mut f64, + mask: *mut bool, + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = wrapper!(reader) + .inner + .array_numbers(&array)? + .try_into_f64()?; + into_slice_nullable(values, mask, n_values, array.item_count(), iter) + }) + .is_some() +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_numbers_float32( + reader: *mut Reader, + array: *const Array, + values: *mut f32, + mask: *mut bool, + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + match wrapper!(reader).inner.array_numbers(&array)? { + omf::data::Numbers::F32(i) => { + into_slice_nullable(values, mask, n_values, array.item_count(), i) + } + omf::data::Numbers::F64(_) => unsafe_cast!("64-bit float" -> "32-bit float"), + omf::data::Numbers::I64(_) => unsafe_cast!("64-bit int" -> "32-bit float"), + omf::data::Numbers::Date(_) => unsafe_cast!("date" -> "32-bit float"), + omf::data::Numbers::DateTime(_) => unsafe_cast!("date-time" -> "32-bit float"), + } + }) + .is_some() +} + +// Indices + +#[no_mangle] +pub extern "C" fn omf_reader_array_indices_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Indices { + catch::error(|| { + Ok(indices_new( + wrapper!(reader).inner.array_indices(&array!(array)?)?, + )) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_indices( + reader: *mut Reader, + array: *const Array, + values: *mut u32, + mask: *mut bool, + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = wrapper!(reader).inner.array_indices(&array)?; + into_slice_nullable(values, mask, n_values, array.item_count(), iter) + }) + .is_some() +} + +// Vectors + +#[no_mangle] +pub extern "C" fn omf_reader_array_vectors32x2_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Vectors32x2 { + catch::error( + || match wrapper!(reader).inner.array_vectors(&array!(array)?)? { + omf::data::Vectors::F32x2(i) => Ok(vectors32x2_new(i)), + omf::data::Vectors::F64x2(_) => unsafe_cast!("64-bit float" -> "32-bit float"), + omf::data::Vectors::F32x3(_) | omf::data::Vectors::F64x3(_) => { + unsafe_cast!("3D" -> "2D") + } + }, + ) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_vectors64x2_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Vectors64x2 { + catch::error(|| { + let iter = wrapper!(reader).inner.array_vectors(&array!(array)?)?; + match &iter { + omf::data::Vectors::F32x2(_) | omf::data::Vectors::F64x2(_) => { + Ok(vectors64x2_new(iter)) + } + omf::data::Vectors::F32x3(_) | omf::data::Vectors::F64x3(_) => { + unsafe_cast!("3D" -> "2D") + } + } + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_vectors32x3_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Vectors32x3 { + catch::error(|| { + let iter = wrapper!(reader).inner.array_vectors(&array!(array)?)?; + match &iter { + omf::data::Vectors::F32x3(_) | omf::data::Vectors::F32x2(_) => { + Ok(vectors32x3_new(iter)) + } + omf::data::Vectors::F64x2(_) | omf::data::Vectors::F64x3(_) => { + unsafe_cast!("64-bit float" -> "32-bit float") + } + } + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_vectors64x3_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Vectors64x3 { + catch::error(|| { + Ok(vectors64x3_new( + wrapper!(reader).inner.array_vectors(&array!(array)?)?, + )) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_vectors32x2( + reader: *mut Reader, + array: *const Array, + values: *mut [f32; 2], + mask: *mut bool, + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + match wrapper!(reader).inner.array_vectors(&array)? { + omf::data::Vectors::F32x2(i) => { + into_slice_nullable(values, mask, n_values, array.item_count(), i) + } + omf::data::Vectors::F64x2(_) => unsafe_cast!("64-bit float" -> "32-bit float"), + omf::data::Vectors::F32x3(_) | omf::data::Vectors::F64x3(_) => { + unsafe_cast!("3D" -> "2D") + } + } + }) + .is_some() +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_vectors64x2( + reader: *mut Reader, + array: *const Array, + values: *mut [f64; 2], + mask: *mut bool, + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let c = array.item_count(); + match wrapper!(reader).inner.array_vectors(&array)? { + omf::data::Vectors::F32x2(i) => { + into_slice_nullable_convert(values, mask, n_values, c, i, |[x, y]| { + [x as f64, y as f64] + }) + } + omf::data::Vectors::F64x2(i) => { + into_slice_nullable(values, mask, n_values, array.item_count(), i) + } + omf::data::Vectors::F32x3(_) | omf::data::Vectors::F64x3(_) => { + unsafe_cast!("3D" -> "2D") + } + } + }) + .is_some() +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_vectors32x3( + reader: *mut Reader, + array: *const Array, + values: *mut [f32; 3], + mask: *mut bool, + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let c = array.item_count(); + match wrapper!(reader).inner.array_vectors(&array)? { + omf::data::Vectors::F32x2(i) => { + into_slice_nullable_convert(values, mask, n_values, c, i, |[x, y]| [x, y, 0.0]) + } + omf::data::Vectors::F32x3(i) => into_slice_nullable(values, mask, n_values, c, i), + omf::data::Vectors::F64x2(_) | omf::data::Vectors::F64x3(_) => { + unsafe_cast!("64-bit float" -> "32-bit float") + } + } + }) + .is_some() +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_vectors64x3( + reader: *mut Reader, + array: *const Array, + values: *mut [f64; 3], + mask: *mut bool, + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let c = array.item_count(); + match wrapper!(reader).inner.array_vectors(&array)? { + omf::data::Vectors::F32x2(i) => { + into_slice_nullable_convert(values, mask, n_values, c, i, |[x, y]| { + [x as f64, y as f64, 0.0] + }) + } + omf::data::Vectors::F64x2(i) => { + into_slice_nullable_convert(values, mask, n_values, c, i, |[x, y]| [x, y, 0.0]) + } + omf::data::Vectors::F32x3(i) => { + into_slice_nullable_convert(values, mask, n_values, c, i, |[x, y, z]| { + [x as f64, y as f64, z as f64] + }) + } + omf::data::Vectors::F64x3(i) => into_slice_nullable(values, mask, n_values, c, i), + } + }) + .is_some() +} + +// Text + +#[no_mangle] +pub extern "C" fn omf_reader_array_text_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Text { + catch::error(|| { + Ok(text_new( + wrapper!(reader).inner.array_text(&array!(array)?)?, + )) + }) + .unwrap_or_else(null_mut) +} + +// Booleans + +#[no_mangle] +pub extern "C" fn omf_reader_array_booleans_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Booleans { + catch::error(|| { + Ok(booleans_new( + wrapper!(reader).inner.array_booleans(&array!(array)?)?, + )) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_booleans( + reader: *mut Reader, + array: *const Array, + values: *mut bool, + mask: *mut bool, + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = wrapper!(reader).inner.array_booleans(&array)?; + into_slice_nullable(values, mask, n_values, array.item_count(), iter) + }) + .is_some() +} + +// Colors + +#[no_mangle] +pub extern "C" fn omf_reader_array_colors_iter( + reader: *mut Reader, + array: *const Array, +) -> *mut Colors { + catch::error(|| { + Ok(colors_new( + wrapper!(reader).inner.array_colors(&array!(array)?)?, + )) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_reader_array_colors( + reader: *mut Reader, + array: *const Array, + values: *mut [u8; 4], + mask: *mut bool, + n_values: usize, +) -> bool { + catch::error(|| { + let array = array!(array)?; + let iter = wrapper!(reader).inner.array_colors(&array)?; + into_slice_nullable(values, mask, n_values, array.item_count(), iter) + }) + .is_some() +} diff --git a/omf-c/src/to_omf.rs b/omf-c/src/to_omf.rs new file mode 100644 index 0000000..8cd29f9 --- /dev/null +++ b/omf-c/src/to_omf.rs @@ -0,0 +1,347 @@ +use omf::date_time::{i64_to_date, i64_to_date_time}; + +use crate::{ + arrays::{array_from_ptr, array_from_ptr_opt}, + attributes::*, + elements::*, + error::{Error, InvalidArg}, + ffi_tools::arg::{slice_from_ptr, string_from_ptr_or_null}, + metadata::Value, +}; + +fn option(is_set: bool, value: T) -> Option { + if is_set { + Some(value) + } else { + None + } +} + +impl Project { + pub fn to_omf(&self) -> Result { + let slice = unsafe { + slice_from_ptr( + "element.attributes", + "element.n_attributes", + self.elements, + self.n_elements, + )? + }; + let elements = slice + .iter() + .map(|attr| attr.to_omf()) + .collect::>()?; + Ok(omf::Project { + name: unsafe { string_from_ptr_or_null("project.name", self.name) }?, + description: unsafe { + string_from_ptr_or_null("project.description", self.description) + }?, + coordinate_reference_system: unsafe { + string_from_ptr_or_null( + "project.coordinate_reference_system", + self.coordinate_reference_system, + ) + }?, + units: unsafe { string_from_ptr_or_null("project.units", self.units) }?, + origin: self.origin, + author: unsafe { string_from_ptr_or_null("project.author", self.author) }?, + application: unsafe { + string_from_ptr_or_null("project.application", self.application) + }?, + date: i64_to_date_time(self.date), + metadata: Value::values_as_json_map(self.metadata, self.n_metadata)?, + elements, + }) + } +} + +impl Element { + pub fn to_omf(&self) -> Result { + let geometry = if let Some(p) = unsafe { self.point_set.as_ref() } { + p.to_omf()? + } else if let Some(l) = unsafe { self.line_set.as_ref() } { + l.to_omf()? + } else if let Some(s) = unsafe { self.surface.as_ref() } { + s.to_omf()? + } else if let Some(g) = unsafe { self.grid_surface.as_ref() } { + g.to_omf()? + } else if let Some(b) = unsafe { self.block_model.as_ref() } { + b.to_omf()? + } else if let Some(c) = unsafe { self.composite.as_ref() } { + c.to_omf()? + } else { + return Err(InvalidArg::NoOptionSet("OmfElement geometry").into()); + }; + let slice = unsafe { + slice_from_ptr( + "element.attributes", + "element.n_attributes", + self.attributes, + self.n_attributes, + )? + }; + let attributes = slice + .iter() + .map(|attr| attr.to_omf()) + .collect::>()?; + Ok(omf::Element { + name: unsafe { string_from_ptr_or_null("element.name", self.name) }?, + description: unsafe { + string_from_ptr_or_null("element.description", self.description) + }?, + color: option(self.color_set, self.color), + metadata: Value::values_as_json_map(self.metadata, self.n_metadata)?, + geometry, + attributes, + }) + } +} + +impl RegularGrid2 { + pub fn to_omf(&self) -> omf::Grid2 { + omf::Grid2::Regular { + size: self.size, + count: self.count, + } + } +} + +impl RegularGrid3 { + pub fn to_omf(&self) -> omf::Grid3 { + omf::Grid3::Regular { + size: self.size, + count: self.count, + } + } +} + +impl TensorGrid2 { + pub fn to_omf(&self) -> Result { + Ok(omf::Grid2::Tensor { + u: array_from_ptr(self.u, "OmfGrid2.u")?, + v: array_from_ptr(self.v, "OmfGrid2.u")?, + }) + } +} + +impl TensorGrid3 { + pub fn to_omf(&self) -> Result { + Ok(omf::Grid3::Tensor { + u: array_from_ptr(self.u, "OmfGrid3.u")?, + v: array_from_ptr(self.v, "OmfGrid3.u")?, + w: array_from_ptr(self.w, "OmfGrid3.u")?, + }) + } +} + +impl PointSet { + pub fn to_omf(&self) -> Result { + Ok(omf::PointSet { + origin: self.origin, + vertices: array_from_ptr(self.vertices, "OmfPointSet.vertices")?, + } + .into()) + } +} + +impl LineSet { + pub fn to_omf(&self) -> Result { + Ok(omf::LineSet { + origin: self.origin, + vertices: array_from_ptr(self.vertices, "OmfLineSet.vertices")?, + segments: array_from_ptr(self.segments, "OmfLineSet.segments")?, + } + .into()) + } +} + +impl Surface { + pub fn to_omf(&self) -> Result { + Ok(omf::Surface { + origin: self.origin, + vertices: array_from_ptr(self.vertices, "OmfSurface.vertices")?, + triangles: array_from_ptr(self.triangles, "OmfSurface.triangles")?, + } + .into()) + } +} + +impl GridSurface { + pub fn to_omf(&self) -> Result { + let grid = if let Some(reg) = unsafe { self.regular_grid.as_ref() } { + reg.to_omf() + } else if let Some(ten) = unsafe { self.tensor_grid.as_ref() } { + ten.to_omf()? + } else { + return Err(InvalidArg::NoOptionSet("OmfGridSurface.*_grid").into()); + }; + Ok(omf::GridSurface { + orient: self.orient, + grid, + heights: array_from_ptr_opt(self.heights, "GridSurface.heights")?, + } + .into()) + } +} + +impl Composite { + pub fn to_omf(&self) -> Result { + let slice = unsafe { + slice_from_ptr( + "composite.elements", + "composite.n_elements", + self.elements, + self.n_elements, + )? + }; + let elements = slice.iter().map(|e| e.to_omf()).collect::>()?; + Ok(omf::Composite { elements }.into()) + } +} + +impl BlockModel { + pub fn to_omf(&self) -> Result { + let grid = if let Some(reg) = unsafe { self.regular_grid.as_ref() } { + reg.to_omf() + } else if let Some(ten) = unsafe { self.tensor_grid.as_ref() } { + ten.to_omf()? + } else { + return Err(InvalidArg::NoOptionSet("OmfBlockModel.*_grid").into()); + }; + let subblocks = if let Some(reg) = unsafe { self.regular_subblocks.as_ref() } { + Some(reg.to_omf()?) + } else if let Some(free) = unsafe { self.freeform_subblocks.as_ref() } { + Some(free.to_omf()?) + } else { + None + }; + Ok(omf::BlockModel { + orient: self.orient, + grid, + subblocks, + } + .into()) + } +} + +impl RegularSubblocks { + pub fn to_omf(&self) -> Result { + Ok(omf::Subblocks::Regular { + count: self.count, + subblocks: array_from_ptr(self.subblocks, "OmfRegularSubblocks.subblocks")?, + mode: self.mode.into(), + }) + } +} + +impl FreeformSubblocks { + pub fn to_omf(&self) -> Result { + Ok(omf::Subblocks::Freeform { + subblocks: array_from_ptr(self.subblocks, "OmfFreeformSubblocks.subblocks")?, + }) + } +} + +impl Attribute { + pub fn to_omf(&self) -> Result { + let data = if let Some(arr) = unsafe { self.boolean_data.as_ref() } { + omf::AttributeData::Boolean { + values: array_from_ptr(arr, "OmfAttribute.boolean_data")?, + } + } else if let Some(arr) = unsafe { self.vector_data.as_ref() } { + omf::AttributeData::Vector { + values: array_from_ptr(arr, "OmfAttribute.vector_data")?, + } + } else if let Some(arr) = unsafe { self.text_data.as_ref() } { + omf::AttributeData::Text { + values: array_from_ptr(arr, "OmfAttribute.text_data")?, + } + } else if let Some(arr) = unsafe { self.color_data.as_ref() } { + omf::AttributeData::Color { + values: array_from_ptr(arr, "OmfAttribute.color_data")?, + } + } else if let Some(n) = unsafe { self.number_data.as_ref() } { + let colormap = if let Some(c) = unsafe { n.continuous_colormap.as_ref() } { + Some(c.to_omf()?) + } else if let Some(d) = unsafe { n.discrete_colormap.as_ref() } { + Some(d.to_omf()?) + } else { + None + }; + omf::AttributeData::Number { + values: array_from_ptr(n.values, "OmfNumberData.values")?, + colormap, + } + } else if let Some(cat) = unsafe { self.category_data.as_ref() } { + let attributes = unsafe { + slice_from_ptr( + "CategoryData.attributes", + "CategoryData.n_attributes", + cat.attributes, + cat.n_attributes, + ) + }?; + omf::AttributeData::Category { + names: array_from_ptr(cat.names, "OmfCategoryData.names")?, + values: array_from_ptr(cat.values, "OmfCategoryData.names")?, + gradient: array_from_ptr_opt(cat.gradient, "OmfCategoryData.gradient")?, + attributes: attributes + .iter() + .map(|a| a.to_omf()) + .collect::>()?, + } + } else if let Some(map) = unsafe { self.mapped_texture_data.as_ref() } { + omf::AttributeData::MappedTexture { + image: array_from_ptr(map.image, "OmfMappedTexture.image")?, + texcoords: array_from_ptr(map.texcoords, "OmfMappedTexture.texcoords")?, + } + } else if let Some(proj) = unsafe { self.projected_texture_data.as_ref() } { + omf::AttributeData::ProjectedTexture { + image: array_from_ptr(proj.image, "OmfProjectedTexture.image")?, + orient: proj.orient, + width: proj.width, + height: proj.height, + } + } else { + return Err(InvalidArg::NoOptionSet("OmfAttribute.*_data").into()); + }; + Ok(omf::Attribute { + name: unsafe { string_from_ptr_or_null("attribute name", self.name) }?, + description: unsafe { + string_from_ptr_or_null("attribute description", self.description) + }?, + units: unsafe { string_from_ptr_or_null("attribute units", self.units) }?, + metadata: Value::values_as_json_map(self.metadata, self.n_metadata)?, + location: self.location, + data, + }) + } +} + +impl ContinuousColormap { + pub fn to_omf(&self) -> Result { + let range = match self.range_type { + RangeType::Integer => (self.min_int, self.max_int).into(), + RangeType::Date => (i64_to_date(self.min_int), i64_to_date(self.max_int)).into(), + RangeType::DateTime => ( + i64_to_date_time(self.min_int), + i64_to_date_time(self.max_int), + ) + .into(), + _ => (self.min, self.max).into(), + }; + Ok(omf::NumberColormap::Continuous { + range, + gradient: array_from_ptr(self.gradient, "ContinuousColormap.gradient")?, + }) + } +} + +impl DiscreteColormap { + pub fn to_omf(&self) -> Result { + Ok(omf::NumberColormap::Discrete { + boundaries: array_from_ptr(self.boundaries, "OmfDiscreteColormap.boundaries")?, + gradient: array_from_ptr(self.gradient, "OmfDiscreteColormap.gradient")?, + }) + } +} diff --git a/omf-c/src/validation.rs b/omf-c/src/validation.rs new file mode 100644 index 0000000..0bd958b --- /dev/null +++ b/omf-c/src/validation.rs @@ -0,0 +1,43 @@ +use std::{ffi::c_char, io::Write, ptr::null}; + +use crate::ffi_tools::{into_ffi_free, FfiConvert, FfiStorage, IntoFfi}; + +pub fn handle_validation(problems: &omf::validate::Problems, validation: *mut *mut Validation) { + if !problems.is_empty() { + if validation.is_null() { + let mut stdout = std::io::stdout().lock(); + _ = writeln!(stdout, "{problems}"); + _ = stdout.flush(); + } else { + let strings: Vec<_> = problems.iter().map(|p| p.to_string()).collect(); + unsafe { validation.write(strings.into_ffi()) } + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct Validation { + pub n_messages: usize, + pub messages: *const *const c_char, +} + +impl FfiConvert> for Validation { + fn convert(strings: Vec, storage: &mut FfiStorage) -> Self { + let n_messages = strings.len(); + let ptrs = strings + .into_iter() + .map(|s| storage.keep_string(s)) + .chain(Some(null())) + .collect(); + Self { + n_messages, + messages: storage.keep_vec(ptrs), + } + } +} + +#[no_mangle] +pub extern "C" fn omf_validation_free(ptr: *mut Validation) -> bool { + unsafe { into_ffi_free(ptr) } +} diff --git a/omf-c/src/writer.rs b/omf-c/src/writer.rs new file mode 100644 index 0000000..8473e34 --- /dev/null +++ b/omf-c/src/writer.rs @@ -0,0 +1,1599 @@ +use std::{ + ffi::{c_char, CStr}, + fs::File, + panic::RefUnwindSafe, + path::PathBuf, + ptr::{null, null_mut}, + sync::Mutex, +}; + +use omf::date_time::{i64_to_date, i64_to_date_time}; + +use crate::{ + arrays::{Array, ArrayType}, + error::{Error, InvalidArg}, + ffi_tools::{ + arg::{not_null, not_null_consume, slice, string_not_null}, + catch, FfiStorage, + }, + image_data::ImageMode, + validation::handle_validation, + writer_handle::{Handle, HandleComponent}, +}; + +use crate::{ + attributes::Attribute, + elements::{Element, Project}, + validation::Validation, +}; + +pub(crate) struct WriterWrapper { + pub path: PathBuf, + pub inner: omf::file::Writer, + pub project: Option, + pub storage: FfiStorage, +} + +impl WriterWrapper { + fn project_mut(&mut self) -> Result<&mut omf::Project, Error> { + self.project + .as_mut() + .ok_or_else(|| Error::InvalidCall("you must call omf_writer_project first".to_owned())) + } +} + +pub struct Writer(pub(crate) Mutex); + +macro_rules! wrapper { + ($writer:ident) => { + not_null!($writer)?.0.lock().expect("intact lock") + }; +} + +#[no_mangle] +pub extern "C" fn omf_writer_open(path: *const c_char) -> *mut Writer { + catch::error(|| { + let path = PathBuf::from(string_not_null!(path)?); + let wrapper = WriterWrapper { + inner: omf::file::Writer::open(&path)?, + path, + project: None, + storage: Default::default(), + }; + Ok(Box::into_raw(Box::new(Writer(Mutex::new(wrapper))))) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_writer_compression(writer: *mut Writer) -> i32 { + catch::error(|| Ok(wrapper!(writer).inner.compression().level() as i32)).unwrap_or(-1) +} + +#[no_mangle] +pub extern "C" fn omf_writer_set_compression(writer: *mut Writer, compression: i32) -> bool { + catch::error(|| { + wrapper!(writer) + .inner + .set_compression(omf::file::Compression::new(compression.clamp(0, 9) as u32)); + Ok(true) + }) + .unwrap_or(false) +} + +#[no_mangle] +pub extern "C" fn omf_writer_finish(writer: *mut Writer, validation: *mut *mut Validation) -> bool { + catch::error(|| { + let writer = not_null_consume!(writer)?; + // Clear validation if not null. + if !validation.is_null() { + unsafe { validation.write(null_mut()) }; + } + // Get the omf::Project and omf::file::Writer. + let wrapper = writer.0.into_inner().expect("intact lock"); + let project = wrapper.project.unwrap_or_default(); + // Finish writing file. + let result = wrapper.inner.finish(project); + match &result { + Ok((_file, warnings)) => handle_validation(warnings, validation), + Err(omf::error::Error::ValidationFailed(errors)) => { + handle_validation(errors, validation) + } + _ => {} + } + result?; + Ok(true) + }) + .unwrap_or(false) +} + +#[no_mangle] +pub extern "C" fn omf_writer_cancel(writer: *mut Writer) -> bool { + catch::error(|| { + let writer = not_null_consume!(writer)?; + let state = writer.0.into_inner().expect("intact lock"); + std::fs::remove_file(state.path).map_err(omf::error::Error::IoError)?; + Ok(true) + }) + .unwrap_or(false) +} + +fn omf_writer_metadata( + writer: *mut Writer, + handle: *mut Handle, + key: *const c_char, + value: impl Into, +) -> Result<*mut Handle, Error> { + let mut wrapper = not_null!(writer)?.0.lock().expect("intact lock"); + let handle = Handle::from_ptr(handle)?; + let json_value = value.into(); + let is_recursive = matches!( + json_value, + serde_json::Value::Array(_) | serde_json::Value::Object(_) + ); + let new_comp = match handle.metadata(wrapper.project_mut()?)? { + crate::writer_handle::HandleMetadata::Map(m) => { + let key = string_not_null!(key)?; + m.insert(key.clone(), json_value); + HandleComponent::Nested { key } + } + crate::writer_handle::HandleMetadata::Vec(v) => { + let index = v.len(); + v.push(json_value); + HandleComponent::Array { index } + } + }; + if is_recursive { + Ok(wrapper.storage.keep_mut(handle.join(new_comp))) + } else { + Ok(null_mut()) + } +} + +#[no_mangle] +pub extern "C" fn omf_writer_metadata_null( + writer: *mut Writer, + handle: *mut Handle, + name: *const c_char, +) -> bool { + catch::error(|| omf_writer_metadata(writer, handle, name, ())).is_some() +} + +#[no_mangle] +pub extern "C" fn omf_writer_metadata_boolean( + writer: *mut Writer, + handle: *mut Handle, + name: *const c_char, + value: bool, +) -> bool { + catch::error(|| omf_writer_metadata(writer, handle, name, value)).is_some() +} + +#[no_mangle] +pub extern "C" fn omf_writer_metadata_number( + writer: *mut Writer, + handle: *mut Handle, + name: *const c_char, + value: f64, +) -> bool { + catch::error(|| omf_writer_metadata(writer, handle, name, value)).is_some() +} + +#[no_mangle] +pub extern "C" fn omf_writer_metadata_string( + writer: *mut Writer, + handle: *mut Handle, + name: *const c_char, + value: *const c_char, +) -> bool { + catch::error(|| omf_writer_metadata(writer, handle, name, string_not_null!(value)?)).is_some() +} + +#[no_mangle] +pub extern "C" fn omf_writer_metadata_list( + writer: *mut Writer, + handle: *mut Handle, + name: *const c_char, +) -> *mut Handle { + let value = serde_json::Value::Array(Default::default()); + catch::error(|| omf_writer_metadata(writer, handle, name, value)).unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_writer_metadata_object( + writer: *mut Writer, + handle: *mut Handle, + name: *const c_char, +) -> *mut Handle { + let value = serde_json::Value::Object(Default::default()); + catch::error(|| omf_writer_metadata(writer, handle, name, value)).unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_writer_project(writer: *mut Writer, project: *const Project) -> *mut Handle { + catch::error(|| { + let mut wrapper = not_null!(writer)?.0.lock().expect("intact lock"); + if wrapper.project.is_some() { + return Err(Error::InvalidCall( + "second call to 'omf_writer_project' on this writer".to_owned(), + )); + } + wrapper.project = Some(not_null!(project)?.to_omf()?); + Ok(wrapper.storage.keep_mut(Handle::default())) + }) + .unwrap_or_else(null_mut) +} + +fn push_with_index(vec: &mut Vec, item: T) -> usize { + let index = vec.len(); + vec.push(item); + index +} + +#[no_mangle] +pub extern "C" fn omf_writer_element( + writer: *mut Writer, + handle: *mut Handle, + element: *const Element, +) -> *mut Handle { + catch::error(|| { + let mut wrapper = not_null!(writer)?.0.lock().expect("intact lock"); + let handle = Handle::from_ptr(handle)?; + let index = push_with_index( + handle.elements(wrapper.project_mut()?)?, + not_null!(element)?.to_omf()?, + ); + Ok(wrapper + .storage + .keep_mut(handle.join(HandleComponent::Element { index }))) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_writer_attribute( + writer: *mut Writer, + handle: *mut Handle, + attribute: *const Attribute, +) -> *mut Handle { + catch::error(|| { + let mut wrapper = not_null!(writer)?.0.lock().expect("intact lock"); + let handle = Handle::from_ptr(handle)?; + let index = push_with_index( + handle.attributes(wrapper.project_mut()?)?, + not_null!(attribute)?.to_omf()?, + ); + Ok(wrapper + .storage + .keep_mut(handle.join(HandleComponent::Attribute { index }))) + }) + .unwrap_or_else(null_mut) +} + +#[no_mangle] +pub extern "C" fn omf_writer_image_bytes( + writer: *mut Writer, + bytes: *const c_char, + n_bytes: usize, +) -> *const Array { + catch::error(|| { + let bytes_slice: &[u8] = slice!(bytes.cast(), n_bytes)?; + let mut wrapper = not_null!(writer)?.0.lock().expect("intact lock"); + let image = wrapper.inner.image_bytes(bytes_slice)?; + Ok(wrapper.storage.convert_ptr(image)) + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_bytes( + writer: *mut Writer, + array_type: ArrayType, + item_count: u64, + bytes: *const c_char, + n_bytes: usize, +) -> *const Array { + catch::error(|| { + let mut wrapper = not_null!(writer)?.0.lock().expect("intact lock"); + let bytes_slice: &[u8] = slice!(bytes.cast(), n_bytes)?; + macro_rules! arr { + ($name:ident) => { + Array::$name(wrapper.inner.array_bytes(item_count, bytes_slice)?) + }; + } + let array = match array_type { + ArrayType::Image => arr!(Image), + ArrayType::Scalars32 => arr!(Scalar), + ArrayType::Scalars64 => arr!(Scalar), + ArrayType::Vertices32 => arr!(Vertex), + ArrayType::Vertices64 => arr!(Vertex), + ArrayType::Segments => arr!(Segment), + ArrayType::Triangles => arr!(Triangle), + ArrayType::Names => arr!(Name), + ArrayType::Gradient => arr!(Gradient), + ArrayType::Texcoords32 => arr!(Texcoord), + ArrayType::Texcoords64 => arr!(Texcoord), + ArrayType::BoundariesFloat32 => arr!(Boundary), + ArrayType::BoundariesFloat64 => arr!(Boundary), + ArrayType::BoundariesInt64 => arr!(Boundary), + ArrayType::BoundariesDate => arr!(Boundary), + ArrayType::BoundariesDateTime => arr!(Boundary), + ArrayType::RegularSubblocks => arr!(RegularSubblock), + ArrayType::FreeformSubblocks32 => arr!(FreeformSubblock), + ArrayType::FreeformSubblocks64 => arr!(FreeformSubblock), + ArrayType::NumbersFloat32 => arr!(Number), + ArrayType::NumbersFloat64 => arr!(Number), + ArrayType::NumbersInt64 => arr!(Number), + ArrayType::NumbersDate => arr!(Number), + ArrayType::NumbersDateTime => arr!(Number), + ArrayType::Indices => arr!(Index), + ArrayType::Vectors32x2 => arr!(Vector), + ArrayType::Vectors64x2 => arr!(Vector), + ArrayType::Vectors32x3 => arr!(Vector), + ArrayType::Vectors64x3 => arr!(Vector), + ArrayType::Text => arr!(Text), + ArrayType::Booleans => arr!(Boolean), + ArrayType::Colors => arr!(Color), + _ => return Err(InvalidArg::Enum.into()), + }; + Ok(wrapper.storage.keep(array)) + }) + .unwrap_or_else(null) +} + +fn copy_pixels( + width: u32, + height: u32, + n_channels: usize, + pixels: *const T, +) -> Result, Error> { + let n_bytes = usize::try_from(width) + .expect("u32 fits in usize") + .saturating_mul(usize::try_from(height).expect("u32 fits in usize")) + .saturating_mul(n_channels); + let slice = slice!(pixels, n_bytes)?; + let mut vec = Vec::new(); + vec.try_reserve_exact(n_bytes) + .map_err(|_| omf::error::Error::OutOfMemory)?; + vec.extend_from_slice(slice); + Ok(vec) +} + +#[no_mangle] +pub extern "C" fn omf_writer_image_file(writer: *mut Writer, path: *const c_char) -> *const Array { + catch::error(|| { + let mut wrapper = not_null!(writer)?.0.lock().expect("intact lock"); + let f = File::open(string_not_null!(path)?).map_err(omf::error::Error::from)?; + let image = wrapper.inner.image_bytes_from(f)?; + Ok(wrapper.storage.convert_ptr(image)) + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_image_jpeg( + writer: *mut Writer, + width: u32, + height: u32, + pixels: *const u8, + quality: u32, +) -> *const Array { + catch::error(|| { + let mut wrapper = not_null!(writer)?.0.lock().expect("intact lock"); + let buffer = copy_pixels(width, height, 3, pixels)?; + let rgb = image::RgbImage::from_vec(width, height, buffer).expect("correct buffer size"); + let image = wrapper + .inner + .image_jpeg(&rgb, quality.clamp(1, 100) as u8)?; + Ok(wrapper.storage.convert_ptr(image)) + }) + .unwrap_or_else(null) +} + +fn write_png( + writer: *mut Writer, + width: u32, + height: u32, + pixels: *const P::Subpixel, +) -> Result<*const Array, Error> +where + P::Subpixel: RefUnwindSafe + 'static, + image::DynamicImage: From>>, +{ + let mut wrapper = not_null!(writer)?.0.lock().expect("intact lock"); + let buffer = copy_pixels(width, height, P::CHANNEL_COUNT.into(), pixels)?; + let image_obj = image::ImageBuffer::>::from_vec(width, height, buffer) + .expect("correct buffer size") + .into(); + let image = wrapper.inner.image_png(&image_obj)?; + Ok(wrapper.storage.convert_ptr(image)) +} + +#[no_mangle] +pub extern "C" fn omf_writer_image_png8( + writer: *mut Writer, + width: u32, + height: u32, + mode: ImageMode, + pixels: *const u8, +) -> *const Array { + #[allow(unreachable_patterns)] + catch::error(|| match mode { + ImageMode::Gray => write_png::>(writer, width, height, pixels), + ImageMode::GrayAlpha => write_png::>(writer, width, height, pixels), + ImageMode::Rgb => write_png::>(writer, width, height, pixels), + ImageMode::Rgba => write_png::>(writer, width, height, pixels), + _ => Err(Error::InvalidArgument(InvalidArg::Enum)), + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_image_png16( + writer: *mut Writer, + width: u32, + height: u32, + mode: ImageMode, + pixels: *const u16, +) -> *const Array { + #[allow(unreachable_patterns)] + catch::error(|| match mode { + ImageMode::Gray => write_png::>(writer, width, height, pixels), + ImageMode::GrayAlpha => write_png::>(writer, width, height, pixels), + ImageMode::Rgb => write_png::>(writer, width, height, pixels), + ImageMode::Rgba => write_png::>(writer, width, height, pixels), + _ => Err(Error::InvalidArgument(InvalidArg::Enum)), + }) + .unwrap_or_else(null) +} + +macro_rules! write_array_from { + ($writer:ident, $values:ident, $length:ident, $method:ident) => { + catch::error(|| { + let mut wrapper = wrapper!($writer); + let slice = slice!($values, $length)?; + let array = wrapper.inner.$method(slice.iter().copied())?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) + }; +} + +macro_rules! write_nullable_array_from { + ($writer:ident, $values:ident, $mask:ident, $length:ident, $method:ident) => { + catch::error(|| { + let mut wrapper = wrapper!($writer); + let array = if $mask.is_null() { + wrapper + .inner + .$method(slice!($values, $length)?.iter().copied().map(Some)) + } else { + wrapper.inner.$method( + slice!($values, $length)? + .iter() + .zip(slice!($mask, $length)?) + .map(|(v, m)| if *m { None } else { Some(*v) }), + ) + }?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) + }; +} + +macro_rules! write_nullable_array_from_convert { + ($writer:ident, $values:ident, $mask:ident, $length:ident, $method:ident, $convert:expr) => { + catch::error(|| { + let mut wrapper = wrapper!($writer); + let array = if $mask.is_null() { + wrapper + .inner + .$method(slice!($values, $length)?.iter().map(|v| Some($convert(*v)))) + } else { + wrapper.inner.$method( + slice!($values, $length)? + .iter() + .zip(slice!($mask, $length)?) + .map(|(v, m)| if *m { None } else { Some($convert(*v)) }), + ) + }?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) + }; +} + +macro_rules! write_array { + ($writer:ident . $method:ident ( $iter:expr )) => { + catch::error(|| { + let mut wrapper = wrapper!($writer); + let array = wrapper.inner.$method($iter)?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) + }; +} + +fn func_iter( + source: extern "C" fn(*mut (), *mut T) -> bool, + object: *mut (), +) -> impl Iterator { + std::iter::from_fn(move || { + let mut value = T::default(); + if source(object, &mut value) { + Some(value) + } else { + None + } + }) +} + +fn nullable_func_iter( + source: extern "C" fn(*mut (), *mut T, *mut bool) -> bool, + object: *mut (), +) -> impl Iterator> { + std::iter::from_fn(move || { + let mut value = T::default(); + let mut is_null = false; + if source(object, &mut value, &mut is_null) { + Some(if is_null { None } else { Some(value) }) + } else { + None + } + }) +} + +fn nullable_func_iter_convert( + source: extern "C" fn(*mut (), *mut T, *mut bool) -> bool, + object: *mut (), + convert: impl Fn(T) -> U, +) -> impl Iterator> { + std::iter::from_fn(move || { + let mut value = T::default(); + let mut is_null = false; + if source(object, &mut value, &mut is_null) { + Some(if is_null { None } else { Some(convert(value)) }) + } else { + None + } + }) +} + +fn wide_func_iter( + source: extern "C" fn(*mut (), *mut T) -> bool, + object: *mut (), +) -> impl Iterator +where + T: 'static, + [T; N]: Copy + Default, +{ + std::iter::from_fn(move || { + let mut value = <[T; N]>::default(); + if source(object, value.as_mut_ptr()) { + Some(value) + } else { + None + } + }) +} + +fn nullable_wide_func_iter( + source: extern "C" fn(*mut (), *mut T, *mut bool) -> bool, + object: *mut (), +) -> impl Iterator> +where + T: 'static, + [T; N]: Copy + Default, +{ + std::iter::from_fn(move || { + let mut value = <[T; N]>::default(); + let mut is_null = false; + if source(object, value.as_mut_ptr(), &mut is_null) { + Some(if is_null { None } else { Some(value) }) + } else { + None + } + }) +} + +pub type Scalar32Source = extern "C" fn(*mut (), *mut f32) -> bool; +pub type Scalar64Source = extern "C" fn(*mut (), *mut f64) -> bool; +pub type Vertex32Source = extern "C" fn(*mut (), *mut f32) -> bool; +pub type Vertex64Source = extern "C" fn(*mut (), *mut f64) -> bool; +pub type SegmentSource = extern "C" fn(*mut (), *mut u32) -> bool; +pub type TriangleSource = extern "C" fn(*mut (), *mut u32) -> bool; +pub type NameSource = extern "C" fn(*mut (), *mut *const c_char, *mut usize) -> bool; +pub type GradientSource = extern "C" fn(*mut (), *mut u8) -> bool; +pub type Texcoord32Source = extern "C" fn(*mut (), *mut f32) -> bool; +pub type Texcoord64Source = extern "C" fn(*mut (), *mut f64) -> bool; +pub type BoundaryFloat32Source = extern "C" fn(*mut (), *mut f32, *mut bool) -> bool; +pub type BoundaryFloat64Source = extern "C" fn(*mut (), *mut f64, *mut bool) -> bool; +pub type BoundaryInt64Source = extern "C" fn(*mut (), *mut i64, *mut bool) -> bool; +pub type RegularSubblockSource = extern "C" fn(*mut (), *mut u32, *mut u32) -> bool; +pub type FreeformSubblock32Source = extern "C" fn(*mut (), *mut u32, *mut f32) -> bool; +pub type FreeformSubblock64Source = extern "C" fn(*mut (), *mut u32, *mut f64) -> bool; +pub type NumberFloat32Source = extern "C" fn(*mut (), *mut f32, *mut bool) -> bool; +pub type NumberFloat64Source = extern "C" fn(*mut (), *mut f64, *mut bool) -> bool; +pub type NumberInt64Source = extern "C" fn(*mut (), *mut i64, *mut bool) -> bool; +pub type IndexSource = extern "C" fn(*mut (), *mut u32, *mut bool) -> bool; +pub type Vector32x2Source = extern "C" fn(*mut (), *mut f32, *mut bool) -> bool; +pub type Vector64x2Source = extern "C" fn(*mut (), *mut f64, *mut bool) -> bool; +pub type Vector32x3Source = extern "C" fn(*mut (), *mut f32, *mut bool) -> bool; +pub type Vector64x3Source = extern "C" fn(*mut (), *mut f64, *mut bool) -> bool; +pub type TextSource = extern "C" fn(*mut (), *mut *const c_char, *mut usize) -> bool; +pub type BooleanSource = extern "C" fn(*mut (), *mut bool, *mut bool) -> bool; +pub type ColorSource = extern "C" fn(*mut (), *mut u8, *mut bool) -> bool; + +// Scalars + +#[no_mangle] +pub extern "C" fn omf_writer_array_scalars32_iter( + writer: *mut Writer, + source: Scalar32Source, + object: *mut (), +) -> *const Array { + write_array!(writer.array_scalars(func_iter(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_scalars64_iter( + writer: *mut Writer, + source: Scalar64Source, + object: *mut (), +) -> *const Array { + write_array!(writer.array_scalars(func_iter(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_scalars64( + writer: *mut Writer, + values: *const f64, + length: usize, +) -> *const Array { + write_array_from!(writer, values, length, array_scalars) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_scalars32( + writer: *mut Writer, + values: *const f32, + length: usize, +) -> *const Array { + write_array_from!(writer, values, length, array_scalars) +} + +// Vertices + +#[no_mangle] +pub extern "C" fn omf_writer_array_vertices32_iter( + writer: *mut Writer, + source: Vertex32Source, + object: *mut (), +) -> *const Array { + write_array!(writer.array_vertices(wide_func_iter(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_vertices64_iter( + writer: *mut Writer, + source: Vertex64Source, + object: *mut (), +) -> *const Array { + write_array!(writer.array_vertices(wide_func_iter(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_vertices64( + writer: *mut Writer, + values: *const [f64; 3], + length: usize, +) -> *const Array { + write_array_from!(writer, values, length, array_vertices) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_vertices32( + writer: *mut Writer, + values: *const [f32; 3], + length: usize, +) -> *const Array { + write_array_from!(writer, values, length, array_vertices) +} + +// Segments + +#[no_mangle] +pub extern "C" fn omf_writer_array_segments_iter( + writer: *mut Writer, + source: SegmentSource, + object: *mut (), +) -> *const Array { + write_array!(writer.array_segments(wide_func_iter(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_segments( + writer: *mut Writer, + values: *const [u32; 2], + length: usize, +) -> *const Array { + write_array_from!(writer, values, length, array_segments) +} + +// Triangles + +#[no_mangle] +pub extern "C" fn omf_writer_array_triangles_iter( + writer: *mut Writer, + source: TriangleSource, + object: *mut (), +) -> *const Array { + write_array!(writer.array_triangles(wide_func_iter(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_triangles( + writer: *mut Writer, + values: *const [u32; 3], + length: usize, +) -> *const Array { + write_array_from!(writer, values, length, array_triangles) +} + +// Names + +#[no_mangle] +pub extern "C" fn omf_writer_array_names_iter( + writer: *mut Writer, + source: NameSource, + object: *mut (), +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let mut non_utf8 = false; + let array = wrapper.inner.array_names(std::iter::from_fn(|| { + let mut ptr = null(); + let mut len = usize::MAX; + if source(object, &mut ptr, &mut len) { + name_from_ptr(ptr, len, &mut non_utf8) + } else { + None + } + }))?; + if non_utf8 { + Err(Error::InvalidArgument(InvalidArg::NotUtf8("names"))) + } else { + Ok(wrapper.storage.convert_ptr(array)) + } + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_names( + writer: *mut Writer, + values: *const *const c_char, + length: usize, +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let slice = slice!(values, length)?; + let mut non_utf8 = false; + let array = wrapper.inner.array_names( + slice + .iter() + .map_while(|ptr| name_from_ptr(*ptr, usize::MAX, &mut non_utf8)), + )?; + if non_utf8 { + Err(Error::InvalidArgument(InvalidArg::NotUtf8("names"))) + } else { + Ok(wrapper.storage.convert_ptr(array)) + } + }) + .unwrap_or_else(null) +} + +fn name_from_ptr(ptr: *const c_char, len: usize, non_utf8: &mut bool) -> Option { + if ptr.is_null() { + Some(String::new()) + } else if len == usize::MAX { + // Read string as nul-terminated. + if let Ok(s) = unsafe { CStr::from_ptr(ptr) }.to_str() { + Some(s.to_string()) + } else { + *non_utf8 = true; + None + } + } else { + // Use size. + let slice = unsafe { std::slice::from_raw_parts(ptr.cast::(), len) }; + if let Ok(s) = std::str::from_utf8(slice) { + Some(s.to_owned()) + } else { + *non_utf8 = true; + None + } + } +} + +// Gradient + +#[no_mangle] +pub extern "C" fn omf_writer_array_gradient_iter( + writer: *mut Writer, + source: GradientSource, + object: *mut (), +) -> *const Array { + write_array!(writer.array_gradient(wide_func_iter(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_gradient( + writer: *mut Writer, + values: *const [u8; 4], + length: usize, +) -> *const Array { + write_array_from!(writer, values, length, array_gradient) +} + +// Texcoords + +#[no_mangle] +pub extern "C" fn omf_writer_array_texcoords32_iter( + writer: *mut Writer, + source: Texcoord32Source, + object: *mut (), +) -> *const Array { + write_array!(writer.array_texcoords(wide_func_iter(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_texcoords64_iter( + writer: *mut Writer, + source: Texcoord64Source, + object: *mut (), +) -> *const Array { + write_array!(writer.array_texcoords(wide_func_iter(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_texcoords64( + writer: *mut Writer, + values: *const [f64; 2], + length: usize, +) -> *const Array { + write_array_from!(writer, values, length, array_texcoords) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_texcoords32( + writer: *mut Writer, + values: *const [f32; 2], + length: usize, +) -> *const Array { + write_array_from!(writer, values, length, array_texcoords) +} + +// Boundaries + +fn boundary_iter( + source: extern "C" fn(*mut (), *mut T, *mut bool) -> bool, + object: *mut (), + convert: impl Fn(T) -> U, +) -> impl Iterator> { + std::iter::from_fn(move || { + let mut value = T::default(); + let mut inclusive = false; + if source(object, &mut value, &mut inclusive) { + if inclusive { + Some(omf::data::Boundary::LessEqual(convert(value))) + } else { + Some(omf::data::Boundary::Less(convert(value))) + } + } else { + None + } + }) +} + +fn boundary_iter_from( + values: &'static [T], + convert: impl Fn(T) -> U + 'static, +) -> impl Iterator> { + values + .iter() + .map(move |v| omf::data::Boundary::from_value(convert(*v), false)) +} + +fn boundary_iter_from_inc( + values: &'static [T], + inclusive: &'static [bool], + convert: impl Fn(T) -> U + 'static, +) -> impl Iterator> { + values + .iter() + .zip(inclusive.iter()) + .map(move |(v, i)| omf::data::Boundary::from_value(convert(*v), *i)) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_boundaries_float32_iter( + writer: *mut Writer, + source: BoundaryFloat32Source, + object: *mut (), +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let array = wrapper + .inner + .array_boundaries(boundary_iter(source, object, |x| x))?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_boundaries_float64_iter( + writer: *mut Writer, + source: BoundaryFloat64Source, + object: *mut (), +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let array = wrapper + .inner + .array_boundaries(boundary_iter(source, object, |x| x))?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_boundaries_int64_iter( + writer: *mut Writer, + source: BoundaryInt64Source, + object: *mut (), +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let array = wrapper + .inner + .array_boundaries(boundary_iter(source, object, |x| x))?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_boundaries_date_iter( + writer: *mut Writer, + source: BoundaryInt64Source, + object: *mut (), +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let array = wrapper + .inner + .array_boundaries(boundary_iter(source, object, i64_to_date))?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_boundaries_date_time_iter( + writer: *mut Writer, + source: BoundaryInt64Source, + object: *mut (), +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let array = + wrapper + .inner + .array_boundaries(boundary_iter(source, object, i64_to_date_time))?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_boundaries_float32( + writer: *mut Writer, + values: *const f32, + inclusive: *const bool, + length: usize, +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let values = slice!(values, length)?; + let array = if inclusive.is_null() { + wrapper + .inner + .array_boundaries(boundary_iter_from(values, |x| x)) + } else { + wrapper.inner.array_boundaries(boundary_iter_from_inc( + values, + slice!(inclusive, length)?, + |x| x, + )) + }?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_boundaries_float64( + writer: *mut Writer, + values: *const f64, + inclusive: *const bool, + length: usize, +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let values = slice!(values, length)?; + let array = if inclusive.is_null() { + wrapper + .inner + .array_boundaries(boundary_iter_from(values, |x| x)) + } else { + wrapper.inner.array_boundaries(boundary_iter_from_inc( + values, + slice!(inclusive, length)?, + |x| x, + )) + }?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_boundaries_int64( + writer: *mut Writer, + values: *const i64, + inclusive: *const bool, + length: usize, +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let values = slice!(values, length)?; + let array = if inclusive.is_null() { + wrapper + .inner + .array_boundaries(boundary_iter_from(values, |x| x)) + } else { + wrapper.inner.array_boundaries(boundary_iter_from_inc( + values, + slice!(inclusive, length)?, + |x| x, + )) + }?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_boundaries_date( + writer: *mut Writer, + values: *const i64, + inclusive: *const bool, + length: usize, +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let values = slice!(values, length)?; + let array = if inclusive.is_null() { + wrapper + .inner + .array_boundaries(boundary_iter_from(values, i64_to_date)) + } else { + let inc = slice!(inclusive, length)?; + wrapper + .inner + .array_boundaries(boundary_iter_from_inc(values, inc, i64_to_date)) + }?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_boundaries_date_time( + writer: *mut Writer, + values: *const i64, + inclusive: *const bool, + length: usize, +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let values = slice!(values, length)?; + let array = if inclusive.is_null() { + wrapper + .inner + .array_boundaries(boundary_iter_from(values, i64_to_date_time)) + } else { + let inc = slice!(inclusive, length)?; + wrapper + .inner + .array_boundaries(boundary_iter_from_inc(values, inc, i64_to_date_time)) + }?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +// Regular sub-blocks + +#[no_mangle] +pub extern "C" fn omf_writer_array_regular_subblocks_iter( + writer: *mut Writer, + source: RegularSubblockSource, + object: *mut (), +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let array = wrapper + .inner + .array_regular_subblocks(std::iter::from_fn(|| { + let mut parent = [0; 3]; + let mut corners = [0; 6]; + if source(object, parent.as_mut_ptr(), corners.as_mut_ptr()) { + Some((parent, corners)) + } else { + None + } + }))?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_regular_subblocks( + writer: *mut Writer, + parents: *const [u32; 3], + corners: *const [u32; 6], + length: usize, +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let parents = slice!(parents, length)?; + let corners = slice!(corners, length)?; + let array = wrapper + .inner + .array_regular_subblocks(parents.iter().copied().zip(corners.iter().copied()))?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +// Free-form sub-blocks + +#[no_mangle] +pub extern "C" fn omf_writer_array_freeform_subblocks32_iter( + writer: *mut Writer, + source: FreeformSubblock32Source, + object: *mut (), +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let array = wrapper + .inner + .array_freeform_subblocks(std::iter::from_fn(|| { + let mut parent = [0; 3]; + let mut corners = [0.0; 6]; + if source(object, parent.as_mut_ptr(), corners.as_mut_ptr()) { + Some((parent, corners)) + } else { + None + } + }))?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_freeform_subblocks64_iter( + writer: *mut Writer, + source: FreeformSubblock64Source, + object: *mut (), +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let array = wrapper + .inner + .array_freeform_subblocks(std::iter::from_fn(|| { + let mut parent = [0; 3]; + let mut corners = [0.0; 6]; + if source(object, parent.as_mut_ptr(), corners.as_mut_ptr()) { + Some((parent, corners)) + } else { + None + } + }))?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_freeform_subblocks32( + writer: *mut Writer, + parents: *const [u32; 3], + corners: *const [f32; 6], + length: usize, +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let parents = slice!(parents, length)?; + let corners = slice!(corners, length)?; + let array = wrapper + .inner + .array_freeform_subblocks(parents.iter().copied().zip(corners.iter().copied()))?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_freeform_subblocks64( + writer: *mut Writer, + parents: *const [u32; 3], + corners: *const [f64; 6], + length: usize, +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let parents = slice!(parents, length)?; + let corners = slice!(corners, length)?; + let array = wrapper + .inner + .array_freeform_subblocks(parents.iter().copied().zip(corners.iter().copied()))?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +// Numbers + +#[no_mangle] +pub extern "C" fn omf_writer_array_numbers_float32_iter( + writer: *mut Writer, + source: NumberFloat32Source, + object: *mut (), +) -> *const Array { + write_array!(writer.array_numbers(nullable_func_iter(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_numbers_float64_iter( + writer: *mut Writer, + source: NumberFloat64Source, + object: *mut (), +) -> *const Array { + write_array!(writer.array_numbers(nullable_func_iter(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_numbers_int64_iter( + writer: *mut Writer, + source: NumberInt64Source, + object: *mut (), +) -> *const Array { + write_array!(writer.array_numbers(nullable_func_iter(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_numbers_date_iter( + writer: *mut Writer, + source: NumberInt64Source, + object: *mut (), +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let array = + wrapper + .inner + .array_numbers(nullable_func_iter_convert(source, object, i64_to_date))?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_numbers_date_time_iter( + writer: *mut Writer, + source: NumberInt64Source, + object: *mut (), +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let array = wrapper.inner.array_numbers(nullable_func_iter_convert( + source, + object, + i64_to_date_time, + ))?; + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_numbers_float32( + writer: *mut Writer, + values: *const f32, + mask: *const bool, + length: usize, +) -> *const Array { + write_nullable_array_from!(writer, values, mask, length, array_numbers) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_numbers_float64( + writer: *mut Writer, + values: *const f64, + mask: *const bool, + length: usize, +) -> *const Array { + write_nullable_array_from!(writer, values, mask, length, array_numbers) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_numbers_int64( + writer: *mut Writer, + values: *const i64, + mask: *const bool, + length: usize, +) -> *const Array { + write_nullable_array_from!(writer, values, mask, length, array_numbers) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_numbers_date( + writer: *mut Writer, + values: *const i32, + mask: *const bool, + length: usize, +) -> *const Array { + let convert = |n: i32| i64_to_date(n.into()); + write_nullable_array_from_convert!(writer, values, mask, length, array_numbers, convert) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_numbers_date_time( + writer: *mut Writer, + values: *const i64, + mask: *const bool, + length: usize, +) -> *const Array { + write_nullable_array_from_convert!( + writer, + values, + mask, + length, + array_numbers, + i64_to_date_time + ) +} + +// Indices + +#[no_mangle] +pub extern "C" fn omf_writer_array_indices_iter( + writer: *mut Writer, + source: IndexSource, + object: *mut (), +) -> *const Array { + write_array!(writer.array_indices(nullable_func_iter(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_indices( + writer: *mut Writer, + values: *const u32, + mask: *const bool, + length: usize, +) -> *const Array { + write_nullable_array_from!(writer, values, mask, length, array_indices) +} + +// Vectors + +#[no_mangle] +pub extern "C" fn omf_writer_array_vectors32x2_iter( + writer: *mut Writer, + source: Vector32x2Source, + object: *mut (), +) -> *const Array { + write_array!(writer.array_vectors(nullable_wide_func_iter::<_, 2>(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_vectors32x3_iter( + writer: *mut Writer, + source: Vector32x3Source, + object: *mut (), +) -> *const Array { + write_array!(writer.array_vectors(nullable_wide_func_iter::<_, 3>(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_vectors64x2_iter( + writer: *mut Writer, + source: Vector64x2Source, + object: *mut (), +) -> *const Array { + write_array!(writer.array_vectors(nullable_wide_func_iter::<_, 2>(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_vectors64x3_iter( + writer: *mut Writer, + source: Vector64x3Source, + object: *mut (), +) -> *const Array { + write_array!(writer.array_vectors(nullable_wide_func_iter::<_, 3>(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_vectors32x2( + writer: *mut Writer, + values: *const [f32; 2], + mask: *const bool, + length: usize, +) -> *const Array { + write_nullable_array_from!(writer, values, mask, length, array_vectors) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_vectors64x2( + writer: *mut Writer, + values: *const [f64; 2], + mask: *const bool, + length: usize, +) -> *const Array { + write_nullable_array_from!(writer, values, mask, length, array_vectors) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_vectors32x3( + writer: *mut Writer, + values: *const [f32; 3], + mask: *const bool, + length: usize, +) -> *const Array { + write_nullable_array_from!(writer, values, mask, length, array_vectors) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_vectors64x3( + writer: *mut Writer, + values: *const [f64; 3], + mask: *const bool, + length: usize, +) -> *const Array { + write_nullable_array_from!(writer, values, mask, length, array_vectors) +} + +// Text + +#[no_mangle] +pub extern "C" fn omf_writer_array_text_iter( + writer: *mut Writer, + source: TextSource, + object: *mut (), +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let mut non_utf8 = false; + let array = wrapper.inner.array_text(std::iter::from_fn(|| { + let mut ptr = null(); + let mut len = usize::MAX; + if source(object, &mut ptr, &mut len) { + text_from_ptr(ptr, len, &mut non_utf8) + } else { + None + } + }))?; + if non_utf8 { + Err(Error::InvalidArgument(InvalidArg::NotUtf8("names"))) + } else { + Ok(wrapper.storage.convert_ptr(array)) + } + }) + .unwrap_or_else(null) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_text( + writer: *mut Writer, + values: *const *const c_char, + length: usize, +) -> *const Array { + catch::error(|| { + let mut wrapper = wrapper!(writer); + let slice = slice!(values, length)?; + let mut non_utf8 = false; + let array = wrapper.inner.array_text( + slice + .iter() + .map_while(|ptr| text_from_ptr(*ptr, usize::MAX, &mut non_utf8)), + )?; + if non_utf8 { + return Err(Error::InvalidArgument(InvalidArg::NotUtf8( + "array of names", + ))); + } + Ok(wrapper.storage.convert_ptr(array)) + }) + .unwrap_or_else(null) +} + +fn text_from_ptr(ptr: *const c_char, len: usize, non_utf8: &mut bool) -> Option> { + if ptr.is_null() { + Some(None) + } else if len == usize::MAX { + // Read string as nul-terminated. + if let Ok(s) = unsafe { CStr::from_ptr(ptr) }.to_str() { + Some(Some(s.to_string())) + } else { + *non_utf8 = true; + None + } + } else { + // Use size. + let slice = unsafe { std::slice::from_raw_parts(ptr.cast::(), len) }; + if let Ok(s) = std::str::from_utf8(slice) { + Some(Some(s.to_owned())) + } else { + *non_utf8 = true; + None + } + } +} + +// Booleans + +#[no_mangle] +pub extern "C" fn omf_writer_array_booleans_iter( + writer: *mut Writer, + source: BooleanSource, + object: *mut (), +) -> *const Array { + write_array!(writer.array_booleans(nullable_func_iter(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_booleans( + writer: *mut Writer, + values: *const bool, + mask: *const bool, + length: usize, +) -> *const Array { + write_nullable_array_from!(writer, values, mask, length, array_booleans) +} + +// Colors + +#[no_mangle] +pub extern "C" fn omf_writer_array_colors_iter( + writer: *mut Writer, + source: ColorSource, + object: *mut (), +) -> *const Array { + write_array!(writer.array_colors(nullable_wide_func_iter(source, object))) +} + +#[no_mangle] +pub extern "C" fn omf_writer_array_colors( + writer: *mut Writer, + values: *const [u8; 4], + mask: *const bool, + length: usize, +) -> *const Array { + write_nullable_array_from!(writer, values, mask, length, array_colors) +} diff --git a/omf-c/src/writer_handle.rs b/omf-c/src/writer_handle.rs new file mode 100644 index 0000000..796e3f6 --- /dev/null +++ b/omf-c/src/writer_handle.rs @@ -0,0 +1,144 @@ +use crate::{ + error::{Error, InvalidArg}, + ffi_tools::arg::not_null, +}; + +#[derive(Debug, Clone)] +pub enum HandleComponent { + Nested { key: String }, + Array { index: usize }, + Element { index: usize }, + Attribute { index: usize }, +} + +pub enum HandleMetadata<'a> { + Map(&'a mut serde_json::Map), + Vec(&'a mut Vec), +} + +#[derive(Debug, Clone, Default)] +pub struct Handle(Vec); + +impl Handle { + pub fn from_ptr(handle: *mut Handle) -> Result<&'static Self, Error> { + not_null!(handle) + } + + pub fn join(&self, comp: HandleComponent) -> Self { + let mut new_handle = self.clone(); + new_handle.0.push(comp); + new_handle + } + + pub fn metadata<'a>(&self, project: &'a mut omf::Project) -> Result, Error> { + match HandleObject::new(project, self)? { + HandleObject::Project(omf::Project { metadata, .. }) + | HandleObject::Element(omf::Element { metadata, .. }) + | HandleObject::Attribute(omf::Attribute { metadata, .. }) + | HandleObject::MetadataMap(metadata) => Ok(HandleMetadata::Map(metadata)), + HandleObject::MetadataVec(vec) => Ok(HandleMetadata::Vec(vec)), + } + } + + pub fn elements<'a>( + &self, + project: &'a mut omf::Project, + ) -> Result<&'a mut Vec, Error> { + match HandleObject::new(project, self)? { + HandleObject::Project(omf::Project { elements, .. }) + | HandleObject::Element(omf::Element { + geometry: omf::Geometry::Composite(omf::Composite { elements, .. }), + .. + }) => Ok(elements), + _ => Err( + InvalidArg::HandleType("a project or composite element handle is required").into(), + ), + } + } + + pub fn attributes<'a>( + &self, + project: &'a mut omf::Project, + ) -> Result<&'a mut Vec, Error> { + match HandleObject::new(project, self)? { + HandleObject::Element(omf::Element { attributes, .. }) => Ok(attributes), + HandleObject::Attribute(omf::Attribute { + data: omf::AttributeData::Category { attributes, .. }, + .. + }) => Ok(attributes), + _ => Err( + InvalidArg::HandleType("an element or category attribute handle is required") + .into(), + ), + } + } +} + +enum HandleObject<'a> { + Project(&'a mut omf::Project), + MetadataMap(&'a mut serde_json::Map), + MetadataVec(&'a mut Vec), + Element(&'a mut omf::Element), + Attribute(&'a mut omf::Attribute), +} + +impl<'a> HandleObject<'a> { + fn new(project: &'a mut omf::Project, handle: &Handle) -> Result { + let mut obj = Self::Project(project); + for comp in &handle.0 { + obj = obj.next(comp).ok_or(InvalidArg::Handle)?; + } + Ok(obj) + } + + fn next(self, comp: &HandleComponent) -> Option { + match (&comp, self) { + // nested metadata + ( + HandleComponent::Nested { key }, + Self::MetadataMap(metadata) + | Self::Project(omf::Project { metadata, .. }) + | Self::Element(omf::Element { metadata, .. }) + | Self::Attribute(omf::Attribute { metadata, .. }), + ) => match metadata.get_mut(key)? { + serde_json::Value::Array(v) => Some(Self::MetadataVec(v)), + serde_json::Value::Object(m) => Some(Self::MetadataMap(m)), + _ => None, + }, + // nested metadata within array + (HandleComponent::Array { index }, Self::MetadataVec(vec)) => { + match vec.get_mut(*index)? { + serde_json::Value::Array(v) => Some(Self::MetadataVec(v)), + serde_json::Value::Object(m) => Some(Self::MetadataMap(m)), + _ => None, + } + } + // element within project + (HandleComponent::Element { index }, Self::Project(p)) => { + Some(Self::Element(p.elements.get_mut(*index)?)) + } + // element within composite element + ( + HandleComponent::Element { index }, + Self::Element(omf::Element { + geometry: omf::Geometry::Composite(c), + .. + }), + ) => Some(Self::Element(c.elements.get_mut(*index)?)), + // attribute within element + (HandleComponent::Attribute { index }, Self::Element(e)) => { + Some(Self::Attribute(e.attributes.get_mut(*index)?)) + } + // attribute within category attribute + ( + HandleComponent::Attribute { index }, + Self::Attribute(omf::Attribute { + data: omf::AttributeData::Category { attributes, .. }, + .. + }), + ) => Some(Self::Attribute(attributes.get_mut(*index)?)), + // otherwise invalid + _ => None, + } + } +} diff --git a/omf.schema.json b/omf.schema.json new file mode 100644 index 0000000..1697e12 --- /dev/null +++ b/omf.schema.json @@ -0,0 +1,946 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "https://github.com/gmggroup/omf-rust/blob/main/omf.schema.json", + "title": "Open Mining Format 2.0-beta.1", + "type": "object", + "required": [ + "date" + ], + "properties": { + "application": { + "type": "string" + }, + "author": { + "type": "string" + }, + "coordinate_reference_system": { + "type": "string" + }, + "date": { + "type": "string", + "format": "date-time" + }, + "description": { + "type": "string" + }, + "elements": { + "default": [], + "type": "array", + "items": { + "$ref": "#/definitions/Element" + } + }, + "metadata": { + "type": "object", + "additionalProperties": true + }, + "name": { + "type": "string" + }, + "origin": { + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "maxItems": 3, + "minItems": 3 + }, + "units": { + "type": "string" + } + }, + "definitions": { + "Array": { + "type": "object", + "required": [ + "filename", + "item_count" + ], + "properties": { + "filename": { + "type": "string" + }, + "item_count": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + } + }, + "Attribute": { + "type": "object", + "required": [ + "data", + "location", + "name" + ], + "properties": { + "data": { + "$ref": "#/definitions/AttributeData" + }, + "description": { + "type": "string" + }, + "location": { + "$ref": "#/definitions/Location" + }, + "metadata": { + "type": "object", + "additionalProperties": true + }, + "name": { + "type": "string" + }, + "units": { + "type": "string" + } + } + }, + "AttributeData": { + "oneOf": [ + { + "type": "object", + "required": [ + "type", + "values" + ], + "properties": { + "colormap": { + "anyOf": [ + { + "$ref": "#/definitions/NumberColormap" + }, + { + "type": "null" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "Number" + ] + }, + "values": { + "$ref": "#/definitions/Array" + } + } + }, + { + "type": "object", + "required": [ + "type", + "values" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "Vector" + ] + }, + "values": { + "$ref": "#/definitions/Array" + } + } + }, + { + "type": "object", + "required": [ + "type", + "values" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "Text" + ] + }, + "values": { + "$ref": "#/definitions/Array" + } + } + }, + { + "type": "object", + "required": [ + "names", + "type", + "values" + ], + "properties": { + "attributes": { + "type": "array", + "items": { + "$ref": "#/definitions/Attribute" + } + }, + "gradient": { + "anyOf": [ + { + "$ref": "#/definitions/Array" + }, + { + "type": "null" + } + ] + }, + "names": { + "$ref": "#/definitions/Array" + }, + "type": { + "type": "string", + "enum": [ + "Category" + ] + }, + "values": { + "$ref": "#/definitions/Array" + } + } + }, + { + "type": "object", + "required": [ + "type", + "values" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "Boolean" + ] + }, + "values": { + "$ref": "#/definitions/Array" + } + } + }, + { + "type": "object", + "required": [ + "type", + "values" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "Color" + ] + }, + "values": { + "$ref": "#/definitions/Array" + } + } + }, + { + "type": "object", + "required": [ + "image", + "texcoords", + "type" + ], + "properties": { + "image": { + "$ref": "#/definitions/Array" + }, + "texcoords": { + "$ref": "#/definitions/Array" + }, + "type": { + "type": "string", + "enum": [ + "MappedTexture" + ] + } + } + }, + { + "type": "object", + "required": [ + "height", + "image", + "orient", + "type", + "width" + ], + "properties": { + "height": { + "type": "number", + "format": "double" + }, + "image": { + "$ref": "#/definitions/Array" + }, + "orient": { + "$ref": "#/definitions/Orient2" + }, + "type": { + "type": "string", + "enum": [ + "ProjectedTexture" + ] + }, + "width": { + "type": "number", + "format": "double" + } + } + } + ] + }, + "Element": { + "type": "object", + "required": [ + "geometry", + "name" + ], + "properties": { + "attributes": { + "type": "array", + "items": { + "$ref": "#/definitions/Attribute" + } + }, + "color": { + "type": [ + "array", + "null" + ], + "items": { + "type": "integer", + "format": "uint8", + "maximum": 255.0, + "minimum": 0.0 + }, + "maxItems": 4, + "minItems": 4 + }, + "description": { + "type": "string" + }, + "geometry": { + "$ref": "#/definitions/Geometry" + }, + "metadata": { + "type": "object", + "additionalProperties": true + }, + "name": { + "type": "string" + } + } + }, + "Geometry": { + "oneOf": [ + { + "type": "object", + "required": [ + "type", + "vertices" + ], + "properties": { + "origin": { + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "maxItems": 3, + "minItems": 3 + }, + "type": { + "type": "string", + "enum": [ + "PointSet" + ] + }, + "vertices": { + "$ref": "#/definitions/Array" + } + } + }, + { + "type": "object", + "required": [ + "segments", + "type", + "vertices" + ], + "properties": { + "origin": { + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "maxItems": 3, + "minItems": 3 + }, + "segments": { + "$ref": "#/definitions/Array" + }, + "type": { + "type": "string", + "enum": [ + "LineSet" + ] + }, + "vertices": { + "$ref": "#/definitions/Array" + } + } + }, + { + "type": "object", + "required": [ + "triangles", + "type", + "vertices" + ], + "properties": { + "origin": { + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "maxItems": 3, + "minItems": 3 + }, + "triangles": { + "$ref": "#/definitions/Array" + }, + "type": { + "type": "string", + "enum": [ + "Surface" + ] + }, + "vertices": { + "$ref": "#/definitions/Array" + } + } + }, + { + "type": "object", + "required": [ + "grid", + "orient", + "type" + ], + "properties": { + "grid": { + "$ref": "#/definitions/Grid2" + }, + "heights": { + "anyOf": [ + { + "$ref": "#/definitions/Array" + }, + { + "type": "null" + } + ] + }, + "orient": { + "$ref": "#/definitions/Orient2" + }, + "type": { + "type": "string", + "enum": [ + "GridSurface" + ] + } + } + }, + { + "type": "object", + "required": [ + "grid", + "orient", + "type" + ], + "properties": { + "grid": { + "$ref": "#/definitions/Grid3" + }, + "orient": { + "$ref": "#/definitions/Orient3" + }, + "subblocks": { + "anyOf": [ + { + "$ref": "#/definitions/Subblocks" + }, + { + "type": "null" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "BlockModel" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "elements": { + "default": [], + "type": "array", + "items": { + "$ref": "#/definitions/Element" + } + }, + "type": { + "type": "string", + "enum": [ + "Composite" + ] + } + } + } + ] + }, + "Grid2": { + "oneOf": [ + { + "type": "object", + "required": [ + "count", + "size", + "type" + ], + "properties": { + "count": { + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "maxItems": 2, + "minItems": 2 + }, + "size": { + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "maxItems": 2, + "minItems": 2 + }, + "type": { + "type": "string", + "enum": [ + "Regular" + ] + } + } + }, + { + "type": "object", + "required": [ + "type", + "u", + "v" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "Tensor" + ] + }, + "u": { + "$ref": "#/definitions/Array" + }, + "v": { + "$ref": "#/definitions/Array" + } + } + } + ] + }, + "Grid3": { + "oneOf": [ + { + "type": "object", + "required": [ + "count", + "size", + "type" + ], + "properties": { + "count": { + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "maxItems": 3, + "minItems": 3 + }, + "size": { + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "maxItems": 3, + "minItems": 3 + }, + "type": { + "type": "string", + "enum": [ + "Regular" + ] + } + } + }, + { + "type": "object", + "required": [ + "type", + "u", + "v", + "w" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "Tensor" + ] + }, + "u": { + "$ref": "#/definitions/Array" + }, + "v": { + "$ref": "#/definitions/Array" + }, + "w": { + "$ref": "#/definitions/Array" + } + } + } + ] + }, + "Location": { + "type": "string", + "enum": [ + "Vertices", + "Primitives", + "Subblocks", + "Elements", + "Projected", + "Categories" + ] + }, + "NumberColormap": { + "oneOf": [ + { + "type": "object", + "required": [ + "gradient", + "range", + "type" + ], + "properties": { + "gradient": { + "$ref": "#/definitions/Array" + }, + "range": { + "$ref": "#/definitions/NumberRange" + }, + "type": { + "type": "string", + "enum": [ + "Continuous" + ] + } + } + }, + { + "type": "object", + "required": [ + "boundaries", + "gradient", + "type" + ], + "properties": { + "boundaries": { + "$ref": "#/definitions/Array" + }, + "gradient": { + "$ref": "#/definitions/Array" + }, + "type": { + "type": "string", + "enum": [ + "Discrete" + ] + } + } + } + ] + }, + "NumberRange": { + "anyOf": [ + { + "type": "object", + "required": [ + "max", + "min" + ], + "properties": { + "max": { + "type": "number", + "format": "double" + }, + "min": { + "type": "number", + "format": "double" + } + } + }, + { + "type": "object", + "required": [ + "max", + "min" + ], + "properties": { + "max": { + "type": "integer", + "format": "int64" + }, + "min": { + "type": "integer", + "format": "int64" + } + } + }, + { + "type": "object", + "required": [ + "max", + "min" + ], + "properties": { + "max": { + "type": "string", + "format": "date" + }, + "min": { + "type": "string", + "format": "date" + } + } + }, + { + "type": "object", + "required": [ + "max", + "min" + ], + "properties": { + "max": { + "type": "string", + "format": "date-time" + }, + "min": { + "type": "string", + "format": "date-time" + } + } + } + ] + }, + "Orient2": { + "type": "object", + "required": [ + "origin" + ], + "properties": { + "origin": { + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "maxItems": 3, + "minItems": 3 + }, + "u": { + "default": [ + 1.0, + 0.0, + 0.0 + ], + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "maxItems": 3, + "minItems": 3 + }, + "v": { + "default": [ + 0.0, + 1.0, + 0.0 + ], + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "maxItems": 3, + "minItems": 3 + } + } + }, + "Orient3": { + "type": "object", + "required": [ + "origin" + ], + "properties": { + "origin": { + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "maxItems": 3, + "minItems": 3 + }, + "u": { + "default": [ + 1.0, + 0.0, + 0.0 + ], + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "maxItems": 3, + "minItems": 3 + }, + "v": { + "default": [ + 0.0, + 1.0, + 0.0 + ], + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "maxItems": 3, + "minItems": 3 + }, + "w": { + "default": [ + 0.0, + 0.0, + 1.0 + ], + "type": "array", + "items": { + "type": "number", + "format": "double" + }, + "maxItems": 3, + "minItems": 3 + } + } + }, + "SubblockMode": { + "type": "string", + "enum": [ + "Octree", + "Full" + ] + }, + "Subblocks": { + "oneOf": [ + { + "type": "object", + "required": [ + "count", + "subblocks", + "type" + ], + "properties": { + "count": { + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "maxItems": 3, + "minItems": 3 + }, + "mode": { + "anyOf": [ + { + "$ref": "#/definitions/SubblockMode" + }, + { + "type": "null" + } + ] + }, + "subblocks": { + "$ref": "#/definitions/Array" + }, + "type": { + "type": "string", + "enum": [ + "Regular" + ] + } + } + }, + { + "type": "object", + "required": [ + "subblocks", + "type" + ], + "properties": { + "subblocks": { + "$ref": "#/definitions/Array" + }, + "type": { + "type": "string", + "enum": [ + "Freeform" + ] + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/src/array.rs b/src/array.rs new file mode 100644 index 0000000..d0723e5 --- /dev/null +++ b/src/array.rs @@ -0,0 +1,214 @@ +use std::marker::PhantomData; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[cfg(feature = "parquet")] +use crate::data::write_checks::ArrayWriteCheck; +use crate::{validate::Reason, SubblockMode}; + +pub trait ArrayType { + const DATA_TYPE: DataType; +} + +macro_rules! array_types { + ($(#[doc = $doc:literal] $name:ident,)*) => { + #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, JsonSchema)] + pub enum DataType { + $($name,)* + } + + pub mod array_type { + use super::*; + $( + #[derive(Debug, Default, Clone, Copy, PartialEq, Serialize, Deserialize, JsonSchema)] + pub struct $name {} + impl ArrayType for $name { + const DATA_TYPE: DataType = DataType::$name; + } + )* + } + }; +} + +array_types! { + /// An image in PNG or JPEG encoding. + Image, + /// Floating-point scalar values. + Scalar, + /// Vertex locations in 3D. Add the project and element origins. + Vertex, + /// Line segments as indices into a vertex array. + Segment, + /// Triangles as indices into a vertex array. + Triangle, + /// Non-nullable category names. + Name, + /// Non-nullable colormap or category colors. + Gradient, + /// UV texture coordinates. + Texcoord, + /// Discrete color-map boundaries. + Boundary, + /// Parent indices and corners of regular sub-blocks. + RegularSubblock, + /// Parent indices and corners of free-form sub-blocks. + FreeformSubblock, + /// Nullable number values, floating-point or signed integer. + Number, + /// Nullable category index values. + Index, + /// Nullable 2D or 3D vectors. + Vector, + /// Nullable text. + Text, + /// Nullable booleans. + Boolean, + /// Nullable colors. + Color, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Array { + filename: String, + item_count: u64, + #[serde(default, skip_serializing, skip_deserializing)] + private: ArrayPrivate, + #[serde(default, skip_serializing, skip_deserializing)] + _marker: PhantomData, +} + +impl Array { + pub(crate) fn new(filename: String, item_count: u64) -> Self { + Self { + filename, + item_count, + private: Default::default(), + _marker: Default::default(), + } + } + + pub(crate) fn constrain(&mut self, constraint: Constraint) -> Result<(), Reason> { + assert_eq!( + constraint.data_type(), + self.data_type(), + "invalid constraint {constraint:?} for {:?} array", + self.data_type() + ); + self.private.constraint = Some(constraint); + Ok(()) + } + + pub(crate) fn constraint(&self) -> &Constraint { + self.private + .constraint + .as_ref() + .expect("array should have been validated") + } + + /// The filename of the array data within the zip file. + pub(crate) fn filename(&self) -> &str { + &self.filename + } + + /// Number of items in the decompressed array. Zero for images. + pub fn item_count(&self) -> u64 { + self.item_count + } + + /// The type of the array, based on `T`. + pub fn data_type(&self) -> DataType { + T::DATA_TYPE + } +} + +#[cfg(feature = "parquet")] +impl Array { + pub(crate) fn add_write_checks(mut self, checks: Vec) -> Self { + self.private.checks.extend(checks); + self + } + + pub(crate) fn run_write_checks(&self) -> Vec { + let mut reasons = Vec::new(); + for check in &self.private.checks { + if let Err(r) = check.check(self) { + reasons.push(r); + } + } + reasons + } +} + +#[cfg(not(feature = "parquet"))] +impl Array { + pub(crate) fn run_write_checks(&self) -> Vec { + Vec::new() + } +} + +#[derive(Debug, Default, Clone)] +struct ArrayPrivate { + constraint: Option, + #[cfg(feature = "parquet")] + checks: Vec, +} + +impl PartialEq for ArrayPrivate { + fn eq(&self, _other: &Self) -> bool { + // Don't let this private data interfere with tests. + true + } +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum Constraint { + Image, + Scalar, + Size, + Vertex, + Segment(u64), + Triangle(u64), + Name, + Gradient, + Texcoord, + Boundary, + RegularSubblock { + block_count: [u32; 3], + subblock_count: [u32; 3], + mode: Option, + }, + FreeformSubblock { + block_count: [u32; 3], + }, + Number, + Index(u64), + Vector, + String, + Boolean, + Color, +} + +impl Constraint { + pub fn data_type(&self) -> DataType { + match self { + Self::Image => DataType::Image, + Self::Scalar | Self::Size => DataType::Scalar, + Self::Vertex => DataType::Vertex, + Self::Segment(_) => DataType::Segment, + Self::Triangle(_) => DataType::Triangle, + Self::Name => DataType::Name, + Self::Gradient => DataType::Gradient, + Self::Texcoord => DataType::Texcoord, + Self::Boundary => DataType::Boundary, + Self::RegularSubblock { .. } => DataType::RegularSubblock, + Self::FreeformSubblock { .. } => DataType::FreeformSubblock, + Self::Number => DataType::Number, + Self::Index(_) => DataType::Index, + Self::Vector => DataType::Vector, + Self::String => DataType::Text, + Self::Boolean => DataType::Boolean, + Self::Color => DataType::Color, + } + } +} diff --git a/src/attribute.rs b/src/attribute.rs new file mode 100644 index 0000000..04da96f --- /dev/null +++ b/src/attribute.rs @@ -0,0 +1,500 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::{ + array::Constraint, + array_type, + colormap::NumberRange, + validate::{Validate, Validator}, + Array, NumberColormap, Orient2, +}; + +/// The various types of data that can be attached to an [`Attribute`](crate::Attribute). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type")] +pub enum AttributeData { + /// Number data with flexible type. + /// + /// Values can be stored as 32 or 64-bit signed integers, 32 or 64-bit floating point, + /// date, or date-time. Valid dates are approximately ±262,000 years + /// from the common era. Date-time values are written with microsecond accuracy, + /// and times are always in UTC. + Number { + /// Array with `Number` type storing the attribute values. + values: Array, + /// Optional colormap. If absent then the importing application should invent one. + /// + /// Make sure the colormap uses the same number type as `values`. + #[serde(default, skip_serializing_if = "Option::is_none")] + colormap: Option, + }, + /// 2D or 3D vector data. + Vector { + /// Array with `Vector` type storing the attribute values. + values: Array, + }, + /// Text data. + Text { + /// Array with `Text` type storing the attribute values. + values: Array, + }, + /// Category data. + /// + /// A name is required for each category, a color is optional, and other values can be attached + /// as sub-attributes. + Category { + /// Array with `Index` type storing the category indices. + /// + /// Values are indices into the `names` array, `colors` array, and any sub-attributes, + /// and must be within range for them. + values: Array, + /// Array with `Name` type storing category names. + names: Array, + /// Optional array with `Gradient` type storing category colors. + /// + /// If present, must be the same length as `names`. If absent then the importing + /// application should invent colors. + #[serde(default, skip_serializing_if = "Option::is_none")] + gradient: Option>, + /// Additional attributes that use the same indices. + /// + /// This could be used to store the density of rock types in a lithology attribute for + /// example. The location field of these attributes must be + /// `Categories`[crate::Location::Categories]. They must have the same length as `names`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + attributes: Vec, + }, + /// Boolean or filter data. + Boolean { + /// Array with `Boolean` type storing the attribute values. + /// + /// These values may be true, false, or null. Applications that don't support + /// [three-valued logic](https://en.wikipedia.org/wiki/Three-valued_logic) may treat + /// null as false. + values: Array, + }, + /// Color data. + Color { + /// Array with `Color` type storing the attribute values. + /// + /// Null values may be replaced by the element color, or a default color as the + /// application prefers. + values: Array, + }, + /// A texture applied with [UV mapping](https://en.wikipedia.org/wiki/UV_mapping). + /// + /// Typically applied to surface vertices. Applications may ignore other locations. + MappedTexture { + /// Array with `Image` type storing the texture image. + image: Array, + /// Array with `Texcoord` type storing the UV texture coordinates. + /// + /// Each item is a normalized (U, V) pair. For values outside [0, 1] the texture wraps. + texcoords: Array, + }, + /// A texture defined as a rectangle in space projected along its normal. + /// + /// Behavior of the texture outside the projected rectangle is not defined. The texture + /// might repeat, clip the element, or itself be clipped to reveal the flat color of the + /// element. + /// + /// The attribute location must be [`Projected`](crate::Location::Projected). + ProjectedTexture { + /// Array with `Image` type storing the texture image. + image: Array, + /// Orientation of the image. + orient: Orient2, + /// Width of the image projection in space. + width: f64, + /// Height of the image projection in space. + height: f64, + }, +} + +impl AttributeData { + /// True if the attribute data length is zero. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Length of the attribute data; zero for projected textures. + pub fn len(&self) -> u64 { + match self { + Self::Number { values, .. } => values.item_count(), + Self::Vector { values } => values.item_count(), + Self::Text { values } => values.item_count(), + Self::Category { values, .. } => values.item_count(), + Self::Boolean { values } => values.item_count(), + Self::Color { values, .. } => values.item_count(), + Self::MappedTexture { texcoords, .. } => texcoords.item_count(), + Self::ProjectedTexture { .. } => 0, + } + } + + pub(crate) fn type_name(&self) -> &'static str { + match self { + Self::Number { .. } => "Number", + Self::Vector { .. } => "Vector", + Self::Text { .. } => "String", + Self::Category { .. } => "Category", + Self::Boolean { .. } => "Boolean", + Self::Color { .. } => "Color", + Self::MappedTexture { .. } => "MappedTexture", + Self::ProjectedTexture { .. } => "ProjectedTexture", + } + } +} + +/// Describes data attached to an [`Element`](crate::Element). +/// +/// Each [`Element`](crate::Element) can have zero or more attributes, +/// each attached to different parts of the element and each containing different types of data. +/// On a set of points, one attribute might contain gold assay results and another rock-type classifications. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Attribute { + /// Attribute name. Should be unique within the containing element. + pub name: String, + /// Optional attribute description. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub description: String, + /// Optional unit of the attribute data, default empty. + /// + /// OMF does not currently attempt to standardize the strings you can use here, but our + /// recommendations are: + /// + /// - Use full names, so "kilometers" rather than "km". The abbreviations for non-metric units + /// aren't consistent and complex units can be confusing. + /// + /// - Use plurals, so "feet" rather than "foot". + /// + /// - Avoid ambiguity, so "long tons" rather than just "tons". + /// + /// - Accept American and British spellings, so "meter" and "metre" are the same. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub units: String, + /// Attribute metadata. + #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] + pub metadata: serde_json::Map, + /// Selects which part of the element the attribute is attached to. + /// + /// See the documentation for each [`Geometry`](crate::Geometry) variant for a list of what + /// locations are valid. + pub location: Location, + /// The attribute data. + pub data: AttributeData, +} + +impl Attribute { + pub(crate) fn new(name: impl Into, location: Location, data: AttributeData) -> Self { + Self { + name: name.into(), + description: Default::default(), + units: Default::default(), + metadata: Default::default(), + location, + data, + } + } + + /// Convenience function to create a number attribute. + pub fn from_numbers( + name: impl Into, + location: Location, + values: Array, + ) -> Self { + Self::new( + name, + location, + AttributeData::Number { + values, + colormap: None, + }, + ) + } + + /// Convenience function to create a number attribute with a continuous colormap. + pub fn from_numbers_continuous_colormap( + name: impl Into, + location: Location, + values: Array, + range: impl Into, + gradient: Array, + ) -> Self { + Self::new( + name, + location, + AttributeData::Number { + values, + colormap: Some(NumberColormap::Continuous { + range: range.into(), + gradient, + }), + }, + ) + } + + /// Convenience function to create a number attribute with a discrete colormap. + pub fn from_numbers_discrete_colormap( + name: impl Into, + location: Location, + values: Array, + boundaries: Array, + gradient: Array, + ) -> Self { + Self::new( + name, + location, + AttributeData::Number { + values, + colormap: Some(NumberColormap::Discrete { + boundaries, + gradient, + }), + }, + ) + } + + /// Convenience function to create a vector attribute. + pub fn from_vectors( + name: impl Into, + location: Location, + values: Array, + ) -> Self { + Self::new(name, location, AttributeData::Vector { values }) + } + + /// Convenience function to create a string attribute. + pub fn from_strings( + name: impl Into, + location: Location, + values: Array, + ) -> Self { + Self::new(name, location, AttributeData::Text { values }) + } + + /// Convenience function to create a category attribute. + pub fn from_categories( + name: impl Into, + location: Location, + values: Array, + names: Array, + gradient: Option>, + attributes: impl IntoIterator, + ) -> Self { + Self::new( + name, + location, + AttributeData::Category { + values, + names, + gradient, + attributes: attributes.into_iter().collect(), + }, + ) + } + + /// Convenience function to create a number attribute. + pub fn from_booleans( + name: impl Into, + location: Location, + values: Array, + ) -> Self { + Self::new(name, location, AttributeData::Boolean { values }) + } + + /// Convenience function to create a color attribute. + pub fn from_colors( + name: impl Into, + location: Location, + values: Array, + ) -> Self { + Self::new(name, location, AttributeData::Color { values }) + } + + /// Convenience function to create a mapped texture attribute. + pub fn from_texture_map( + name: impl Into, + image: Array, + location: Location, + texcoords: Array, + ) -> Self { + Self::new( + name, + location, + AttributeData::MappedTexture { image, texcoords }, + ) + } + + /// Convenience function to create a projected texture attribute. + pub fn from_texture_project( + name: impl Into, + image: Array, + orient: Orient2, + width: f64, + height: f64, + ) -> Self { + Self::new( + name, + Location::Projected, + AttributeData::ProjectedTexture { + image, + orient, + width, + height, + }, + ) + } + + /// Returns the length of the attribute, or zero for a projected texture. + pub fn len(&self) -> u64 { + self.data.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// Describes what part of the geometry an attribute attaches to. +/// +/// See the documentation for each [`Geometry`](crate::Geometry) variant for a list of what +/// locations are valid. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +#[repr(i32)] +pub enum Location { + /// The attribute contains one value for each point, vertex, or block corner. + #[default] + Vertices, + /// The attribute contains one value for each line segment, triangle, or block. + /// For sub-blocked block models that means parent blocks. + Primitives, + /// The attribute contains one value for each sub-block in a block model. + Subblocks, + /// The attribute contains one value for each sub-element in a + /// [`Composite`](crate::Geometry::Composite). + Elements, + /// Used by [projected texture](crate::AttributeData::ProjectedTexture) attributes. + /// The texture is projected onto the element + Projected, + /// Used for category sub-attributes. + /// The attribute contains one value for each category. + Categories, +} + +impl Validate for Attribute { + fn validate_inner(&mut self, val: &mut Validator) { + val.enter("Attribute").name(&self.name).obj(&mut self.data); + } +} + +impl Validate for AttributeData { + fn validate_inner(&mut self, val: &mut Validator) { + match self { + Self::Number { values, colormap } => { + val.enter("AttributeData::Number") + .array(values, Constraint::Number, "values") + .obj(colormap); + } + Self::Vector { values } => { + val.enter("AttributeData::Vector") + .array(values, Constraint::Vector, "values"); + } + Self::Text { values } => { + val.enter("AttributeData::String") + .array(values, Constraint::String, "values"); + } + Self::Category { + names, + gradient, + values, + attributes, + } => { + val.enter("AttributeData::Category") + .array(values, Constraint::Index(names.item_count()), "values") + .array(names, Constraint::Name, "names") + .array_opt(gradient.as_mut(), Constraint::Gradient, "colors") + .array_size_opt( + gradient.as_ref().map(|a| a.item_count()), + names.item_count(), + "colors", + ) + .objs(attributes.iter_mut()) + .attrs_on_attribute(attributes, names.item_count()); + } + Self::Boolean { values } => { + val.enter("AttributeData::Boolean") + .array(values, Constraint::Boolean, "values"); + } + Self::Color { values } => { + val.enter("AttributeData::Color") + .array(values, Constraint::Color, "values"); + } + Self::ProjectedTexture { + orient, + width, + height, + image, + } => { + val.enter("AttributeData::ProjectedTexture") + .obj(orient) + .finite(*width, "width") + .finite(*height, "height") + .above_zero(*width, "width") + .above_zero(*height, "height") + .array(image, Constraint::Image, "image"); + } + Self::MappedTexture { texcoords, image } => { + val.enter("AttributeData::MappedTexture") + .array(texcoords, Constraint::Texcoord, "texcoords") + .array(image, Constraint::Image, "image"); + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::{colormap::NumberRange, Array}; + + use super::*; + + #[test] + fn serialize_attribute() { + let mut attributes = vec![ + Attribute { + name: "filter".to_owned(), + description: "description of filter".to_owned(), + units: Default::default(), + metadata: Default::default(), + location: Location::Vertices, + data: AttributeData::Boolean { + values: Array::new("1.parquet".to_owned(), 100).into(), + }, + }, + Attribute { + name: "assay".to_owned(), + description: "description of assay".to_owned(), + units: "parts per million".to_owned(), + metadata: Default::default(), + location: Location::Primitives, + data: AttributeData::Number { + values: Array::new("2.parquet".to_owned(), 100).into(), + colormap: Some(NumberColormap::Continuous { + range: NumberRange::Float { + min: 0.0, + max: 100.0, + }, + gradient: Array::new("3.parquet".to_owned(), 128), + }), + }, + }, + ]; + for a in &mut attributes { + a.validate().unwrap(); + let s = serde_json::to_string(a).unwrap(); + let b = serde_json::from_str(&s).unwrap(); + assert_eq!(a, &b); + } + } +} diff --git a/src/block_model.rs b/src/block_model.rs new file mode 100644 index 0000000..237c1cc --- /dev/null +++ b/src/block_model.rs @@ -0,0 +1,255 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::{ + array::Constraint, + array_type, + validate::{Validate, Validator}, + Array, Grid3, Location, Orient3, +}; + +/// Block model geometry with optional sub-blocks. +/// +/// First, the `orient` field defines the position and orientation of a (U, V, W) space relative +/// to the project, which could be just an offset or a full rotation as well. Then the `grid` +/// field defines the size and number of parent blocks aligned with that space and starting at +/// (0, 0, 0). [Sub-blocks](crate::Subblocks) can then optionally be added inside those parent +/// blocks using a variety of layouts. +/// +/// While sub-blocks are supported on tensor grids it isn't a common arrangement and many +/// applications won't load them. +/// +/// ### Attribute Locations +/// +/// - [`Vertices`](crate::Location::Vertices) puts attribute values on the corners of the +/// parent blocks. If the block count is $(N_0, N_1, N_2)$ then there must be +/// $(N_0 + 1) · (N_1 + 1) · (N_2 + 1)$ values. Ordering increases U first, then V, then W. +/// +/// - [`Blocks`](crate::Location::Primitives) puts attribute values on the centroids of the +/// parent block. If the block count is $(N_0, N_1, N_2)$ then there must be +/// $N_0 · N_1 · N_2$ values. Ordering increases U first, then V, then W. +/// +/// - [`Subblocks`](crate::Location::Subblocks) puts attribute values on sub-block centroids. +/// The number and values and their ordering matches the `parents` and `corners` arrays. +/// +/// To have attribute values on undivided parent blocks in this mode there must be a sub-block +/// that covers the whole parent block. +#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct BlockModel { + /// Orientation of the block model. + pub orient: Orient3, + /// Block sizes. + pub grid: Grid3, + /// Optional sub-blocks, which can be regular or free-form divisions of the parent blocks. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subblocks: Option, +} + +impl BlockModel { + pub fn new(orient: Orient3, grid: Grid3) -> Self { + Self { + orient, + grid, + subblocks: None, + } + } + + pub fn with_subblocks(orient: Orient3, grid: Grid3, subblocks: Subblocks) -> Self { + Self { + orient, + grid, + subblocks: Some(subblocks), + } + } + + pub fn with_regular_subblocks( + orient: Orient3, + grid: Grid3, + subblock_count: [u32; 3], + subblocks: Array, + mode: Option, + ) -> Self { + Self { + orient, + grid, + subblocks: Some(Subblocks::Regular { + count: subblock_count, + subblocks, + mode, + }), + } + } + + pub fn with_freeform_subblocks( + orient: Orient3, + grid: Grid3, + subblocks: Array, + ) -> Self { + Self { + orient, + grid, + subblocks: Some(Subblocks::Freeform { subblocks }), + } + } + + /// Returns true if the model has sub-blocks. + pub fn has_subblocks(&self) -> bool { + self.subblocks.is_some() + } + + pub fn location_len(&self, location: Location) -> Option { + match (&self.subblocks, location) { + (_, Location::Vertices) => Some(self.grid.flat_corner_count()), + (_, Location::Primitives) => Some(self.grid.flat_count()), + (Some(s), Location::Subblocks) => Some(s.len()), + _ => None, + } + } +} + +/// Stores sub-blocks of a block model. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type")] +pub enum Subblocks { + /// Divide each parent block into a regular grid of `count` cells. Sub-blocks each covers + /// a non-overlapping cuboid subset of that grid. + /// + /// Sub-blocks are described by the `parents` and `corners` arrays. Those arrays must be the + /// same length and matching rows in each describe the same sub-block. Each row in `parents` + /// is an IJK index on the block model grid. Each row of + /// `corners` is $(i_{min}, j_{min}, k_{min}, i_{max}, j_{max}, k_{max})$, all integers, that + /// refer to the *vertices* of the sub-block grid within the parent block. For example: + /// + /// - A block with minimum size in the corner of the parent block would be (0, 0, 0, 1, 1, 1). + /// + /// - If the `subblock_count` is (5, 5, 3) then a sub-block covering the whole parent would + /// be (0, 0, 0, 5, 5, 3). + /// + /// Sub-blocks must stay within their parent, must have a non-zero size in all directions, and + /// should not overlap. Further restrictions can be applied by the `mode` field, see + /// [`SubblockMode`](crate::SubblockMode) for details. + /// + /// ![Example of regular sub-blocks](../images/subblocks_regular.svg "Regular sub-bloks") + Regular { + /// The sub-block grid size. + /// + /// Must be greater than zero in all directions. If `mode` is octree then these must also + /// be powers of two but they don't have to be equal. + count: [u32; 3], + /// Array with `RegularSubblock` type storing the sub-block parent indices and corners + /// relative to the sub-block grid within the parent. + subblocks: Array, + /// If present this further restricts the sub-block layout. + mode: Option, + }, + /// Divide each parent block into any number and arrangement of non-overlapping cuboid regions. + /// + /// Sub-blocks are described by the `parents` and `corners` arrays. Each row in `parents` is + /// an IJK index on the block model grid. Each row of `corners` is + /// $(i_{min}, j_{min}, k_{min}, i_{max}, j_{max}, k_{max})$ in floating-point and relative + /// to the parent block, running from 0.0 to 1.0 across the parent. For example: + /// + /// - A sub-block covering the whole parent will be (0.0, 0.0, 0.0, 1.0, 1.0, 1.0) + /// no matter the size of the parent. + /// + /// - A sub-block covering the bottom third of the parent block would be + /// (0.0, 0.0, 0.0, 1.0, 1.0, 0.3333) and one covering the top two-thirds would be + /// (0.0, 0.0, 0.3333, 1.0, 1.0, 1.0), again no matter the size of the parent. + /// + /// Sub-blocks must stay within their parent, must have a non-zero size in all directions, + /// and shouldn't overlap. + Freeform { + /// Array with `FreeformSubblock` type storing the sub-block parent indices and corners + /// relative to the parent. + subblocks: Array, + }, +} + +impl Subblocks { + /// The number of sub-blocks. + pub fn len(&self) -> u64 { + match self { + Self::Regular { subblocks, .. } => subblocks.item_count(), + Self::Freeform { subblocks, .. } => subblocks.item_count(), + } + } + + /// True if there are no sub-blocks. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns the optional sub-block mode. + /// + /// Currently this will always be `None` for free-form sub-blocks. + pub fn mode(&self) -> Option { + match self { + Subblocks::Regular { mode, .. } => *mode, + _ => None, + } + } + + fn validate(&mut self, block_count: [u32; 3], val: &mut Validator) { + match self { + Subblocks::Regular { + count, + subblocks, + mode, + } => { + val.enter("Subblocks::Regular") + .above_zero_seq(*count, "count") + .subblock_mode_and_count(*mode, *count) + .array( + subblocks, + Constraint::RegularSubblock { + block_count, + subblock_count: *count, + mode: *mode, + }, + "subblocks", + ); + } + Subblocks::Freeform { subblocks } => { + val.enter("Subblocks::Freeform").array( + subblocks, + Constraint::FreeformSubblock { block_count }, + "subblocks", + ); + } + } + } +} + +/// A optional mode for regular sub-blocks. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +pub enum SubblockMode { + /// Sub-blocks form a octree-like inside the parent block. + /// + /// To form this structure, cut the parent block in half in all directions to create + /// eight child blocks. Repeat that cut for some or all of those children, and continue + /// doing that until the limit on sub-block count is reached or until the sub-blocks + /// accurately model the inputs. + /// + /// The sub-block count must be a power of two in each direction. This isn't strictly an + /// octree because the sub-block count doesn't have to be the *same* in all directions. + /// For example you can have count (16, 16, 2) and blocks will stop dividing the the W + /// direction after the first split. + Octree, + /// Parent blocks are fully divided or not divided at all. + /// + /// Applications reading this mode may choose to merge sub-blocks with matching attributes + /// to reduce the overall number of them. + Full, +} + +impl Validate for BlockModel { + fn validate_inner(&mut self, val: &mut Validator) { + let mut v = val + .enter("BlockModel") + .obj(&mut self.orient) + .obj(&mut self.grid); + if let Some(subblocks) = &mut self.subblocks { + subblocks.validate(self.grid.count(), &mut v); + } + } +} diff --git a/src/colormap.rs b/src/colormap.rs new file mode 100644 index 0000000..a086316 --- /dev/null +++ b/src/colormap.rs @@ -0,0 +1,142 @@ +use std::fmt::Display; + +use chrono::{DateTime, NaiveDate, Utc}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::{ + array::Constraint, + array_type, + validate::{Validate, Validator}, + Array, +}; + +/// Specifies the minimum and maximum values of a number colormap. +/// +/// Values outside this range will use the color at the ends of the gradient. +/// The variant used should match the type of the number array. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(untagged)] +pub enum NumberRange { + Float { + min: f64, + max: f64, + }, + Integer { + min: i64, + max: i64, + }, + Date { + min: NaiveDate, + max: NaiveDate, + }, + DateTime { + min: DateTime, + max: DateTime, + }, +} + +impl Display for NumberRange { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + NumberRange::Float { min, max } => write!(f, "[{min}, {max}]"), + NumberRange::Integer { min, max } => write!(f, "[{min}, {max}]"), + NumberRange::Date { min, max } => write!(f, "[{min}, {max}]"), + NumberRange::DateTime { min, max } => write!(f, "[{min}, {max}]"), + } + } +} + +impl From<(f64, f64)> for NumberRange { + fn from((min, max): (f64, f64)) -> Self { + Self::Float { min, max } + } +} + +impl From<(i64, i64)> for NumberRange { + fn from((min, max): (i64, i64)) -> Self { + Self::Integer { min, max } + } +} + +impl From<(i32, i32)> for NumberRange { + fn from((min, max): (i32, i32)) -> Self { + Self::Integer { + min: min.into(), + max: max.into(), + } + } +} + +impl From<(NaiveDate, NaiveDate)> for NumberRange { + fn from((min, max): (NaiveDate, NaiveDate)) -> Self { + Self::Date { min, max } + } +} + +impl From<(DateTime, DateTime)> for NumberRange { + fn from((min, max): (DateTime, DateTime)) -> Self { + Self::DateTime { min, max } + } +} + +/// Describes a mapping of floating-point value to colors. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type")] +pub enum NumberColormap { + /// A continuous colormap linearly samples a color gradient within a defined range. + /// + /// A value X% of way between `min` and `max` should use the color from X% way down + /// gradient. When that X doesn't land directly on a color use the average of + /// the colors on either side, inverse-weighted by the distance to each. + /// + /// Values below the minimum use the first color in the gradient array. Values above + /// the maximum use the last. + /// + /// ![Diagram of a continuous colormap](../images/colormap_continuous.svg "Continuous colormap") + Continuous { + /// Value range. + range: NumberRange, + /// Array with `Gradient` type storing the smooth color gradient. + gradient: Array, + }, + /// A discrete colormap divides the number line into adjacent but non-overlapping + /// ranges and gives a flat color to each range. + /// + /// Values above the last boundary use `end_color`. + Discrete { + /// Array with `Boundary` type storing the smooth color gradient, containing the value + /// and inclusiveness of each boundary. Values must increase along the array. + /// Boundary values type should match the type of the number array. + boundaries: Array, + /// Array with `Gradient` type storing the colors of the discrete ranges. + /// Length must be one more than `boundaries`, with the extra color used for values above + /// the last boundary. + gradient: Array, + }, +} + +impl Validate for NumberColormap { + fn validate_inner(&mut self, val: &mut Validator) { + match self { + NumberColormap::Continuous { range, gradient } => { + val.enter("NumberColormap::Continuous") + .min_max(*range) + .array(gradient, Constraint::Gradient, "gradient"); + } + NumberColormap::Discrete { + boundaries, + gradient, + } => { + val.enter("NumberColormap::Discrete") + .array(boundaries, Constraint::Boundary, "boundaries") + .array(gradient, Constraint::Gradient, "gradient") + .array_size( + gradient.item_count(), + boundaries.item_count() + 1, + "gradient", + ); + } + } + } +} diff --git a/src/data/iterators.rs b/src/data/iterators.rs new file mode 100644 index 0000000..3d83c40 --- /dev/null +++ b/src/data/iterators.rs @@ -0,0 +1,372 @@ +use std::{fmt::Display, fs::File}; + +use chrono::{DateTime, NaiveDate, Utc}; + +use crate::{ + date_time::{date_time_to_f64, date_time_to_i64, date_to_f64, date_to_i64}, + error::Error, + file::SubFile, + pqarray::read::SimpleIter, +}; + +use super::{ + BoundaryValues, GenericArrays, GenericFreeformSubblocks, GenericNumbers, GenericOptionalArrays, + GenericScalars, NumberType, +}; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Boundary { + Less(T), + LessEqual(T), +} + +impl Boundary { + pub fn from>(other: Boundary) -> Self { + match other { + Boundary::Less(value) => Self::Less(value.into()), + Boundary::LessEqual(value) => Self::LessEqual(value.into()), + } + } + + pub fn from_value(value: T, is_inclusive: bool) -> Self { + if is_inclusive { + Self::LessEqual(value) + } else { + Self::Less(value) + } + } + + pub fn value(self) -> T { + match self { + Boundary::Less(value) | Boundary::LessEqual(value) => value, + } + } + + pub fn is_inclusive(self) -> bool { + match self { + Boundary::Less(_) => false, + Boundary::LessEqual(_) => true, + } + } + + pub fn map(self, func: impl FnOnce(T) -> U) -> Boundary { + match self { + Boundary::Less(t) => Boundary::Less(func(t)), + Boundary::LessEqual(t) => Boundary::LessEqual(func(t)), + } + } +} + +impl Display for Boundary { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Less(value) => write!(f, "< {value}"), + Self::LessEqual(value) => write!(f, "≤ {value}"), + } + } +} + +/// Iterator for reading scalar data. +/// +/// Casts to `f64` by default or you can access the variants directly. +#[derive(Debug)] +pub enum Scalars { + F32(GenericScalars), + F64(GenericScalars), +} + +impl Iterator for Scalars { + type Item = Result; + + fn next(&mut self) -> Option { + match self { + Self::F64(iter) => iter.next(), + Self::F32(iter) => iter.next().map(|r| r.map(Into::into)), + } + } +} + +/// Iterator for reading vertex data of various types. +/// +/// Can be used as an iterator that casts to `[f64; 3]` or you can access the variants directly. +#[derive(Debug)] +pub enum Vertices { + F32(GenericArrays), + F64(GenericArrays), +} + +impl Iterator for Vertices { + type Item = Result<[f64; 3], Error>; + + fn next(&mut self) -> Option { + match self { + Self::F64(iter) => iter.next(), + Self::F32(iter) => array_item_cast(iter.next()), + } + } +} + +fn array_item_cast, const N: usize>( + input: Option>, +) -> Option> { + input.map(|r| r.map(|a| a.map(Into::into))) +} + +/// Iterator for reading texture coordinate data of various types. +/// +/// Can be used as an iterator that casts to `[f64; 2]` or you can access the variants directly. +#[derive(Debug)] +pub enum Texcoords { + F32(GenericArrays), + F64(GenericArrays), +} + +impl Iterator for Texcoords { + type Item = Result<[f64; 2], Error>; + + fn next(&mut self) -> Option { + match self { + Self::F64(iter) => iter.next(), + Self::F32(iter) => array_item_cast(iter.next()), + } + } +} + +/// Iterator for reading number data of various types. +/// +/// You can access the variants directly or use the `try_into_f64` and `try_into_i64` methods. +/// These methods can both fail so aren't automatic. +#[derive(Debug)] +pub enum Numbers { + F32(GenericNumbers), + F64(GenericNumbers), + I64(GenericNumbers), + Date(GenericNumbers), + DateTime(GenericNumbers>), +} + +impl Numbers { + /// Turns this into an `f64` iterator, casting values. + /// + /// If the numbers use type `i64` this will fail with `Error::UnsafeCast`. Dates will become + /// days since the '1970-01-01' epoch. Date-times will become seconds since the + /// '1970-01-01T00:00:00Z' epoch with a small loss of precision. + /// + /// Currently can't fail but future number types might yield `Error::UnsafeCast`. + pub fn try_into_f64(self) -> Result { + match &self { + Numbers::I64(_) => Err(Error::UnsafeCast("64-bit integer", "64-bit float")), + Numbers::F32(_) | Numbers::F64(_) | Numbers::Date(_) | Numbers::DateTime(_) => { + Ok(NumbersF64(self)) + } + } + } + + /// Turns this into an `i64` iterator, casting values. + /// + /// Floating-point types will be rejected with `Error::UnsafeCast`. Dates will become + /// days since the '1970-01-01' epoch. Date-times will become microseconds since the + /// '1970-01-01T00:00:00Z' epoch. + pub fn try_into_i64(self) -> Result { + match self { + Numbers::F32(_) => Err(Error::UnsafeCast("32-bit float", "64-bit integer")), + Numbers::F64(_) => Err(Error::UnsafeCast("64-bit float", "64-bit integer")), + Numbers::I64(_) | Numbers::Date(_) | Numbers::DateTime(_) => Ok(NumbersI64(self)), + } + } +} + +pub struct NumbersF64(Numbers); + +impl Iterator for NumbersF64 { + type Item = Result, Error>; + + fn next(&mut self) -> Option { + match &mut self.0 { + Numbers::F32(i) => i.next().map(|r| r.map(|o| o.map(Into::into))), + Numbers::F64(i) => i.next(), + Numbers::Date(i) => i.next().map(|r| r.map(|o| o.map(date_to_f64))), + Numbers::DateTime(i) => i.next().map(|r| r.map(|o| o.map(date_time_to_f64))), + Numbers::I64(_) => None, + } + } +} + +pub struct NumbersI64(Numbers); + +impl Iterator for NumbersI64 { + type Item = Result, Error>; + + fn next(&mut self) -> Option { + match &mut self.0 { + Numbers::F32(_) | Numbers::F64(_) => None, + Numbers::I64(i) => i.next(), + Numbers::Date(i) => i.next().map(|r| r.map(|o| o.map(date_to_i64))), + Numbers::DateTime(i) => i.next().map(|r| r.map(|o| o.map(date_time_to_i64))), + } + } +} + +/// Iterator for reading vector data. +/// +/// Casts to `Option<[f64; 3]>` by default or you can access the variants directly. +/// 2D vectors are cast to a 3D vector with zero in the Z component. +#[derive(Debug)] +pub enum Vectors { + F32x2(GenericOptionalArrays), + F64x2(GenericOptionalArrays), + F32x3(GenericOptionalArrays), + F64x3(GenericOptionalArrays), +} + +impl Iterator for Vectors { + type Item = Result, Error>; + + fn next(&mut self) -> Option { + match self { + Self::F32x2(iter) => iter + .next() + .map(|r| r.map(|o| o.map(|[x, y]| [x.into(), y.into(), 0.0]))), + Self::F64x2(iter) => iter.next().map(|r| r.map(|o| o.map(|[x, y]| [x, y, 0.0]))), + Self::F32x3(iter) => { + let input = iter.next(); + input.map(|r| r.map(|o| o.map(|a| a.map(Into::into)))) + } + Self::F64x3(iter) => iter.next(), + } + } +} + +/// Generic iterator for boundary data. +#[derive(Debug)] +pub struct GenericBoundaries { + value: BoundaryValues, + inclusive: SimpleIter>, +} + +impl GenericBoundaries { + pub fn new( + value: SimpleIter>, + inclusive: SimpleIter>, + ) -> Self { + Self { + value: BoundaryValues::new(value), + inclusive, + } + } +} + +impl Iterator for GenericBoundaries { + type Item = Result, Error>; + + fn next(&mut self) -> Option { + match (self.value.next(), self.inclusive.next()) { + (Some(Err(e)), _) | (_, Some(Err(e))) => Some(Err(e)), + (None, _) | (_, None) => None, + (Some(Ok(value)), Some(Ok(false))) => Some(Ok(Boundary::Less(value))), + (Some(Ok(value)), Some(Ok(true))) => Some(Ok(Boundary::LessEqual(value))), + } + } +} + +/// Iterator for reading color data. +/// +/// Casting is the same as [`Numbers`](Numbers). +#[derive(Debug)] +pub enum Boundaries { + F32(GenericBoundaries), + F64(GenericBoundaries), + I64(GenericBoundaries), + Date(GenericBoundaries), + DateTime(GenericBoundaries>), +} + +impl Boundaries { + /// Turns this into an `f64` boundary iterator, casting values. + /// + /// If the numbers use type `i64` this will fail with `Error::UnsafeCast`. Dates will become + /// days since the '1970-01-01' epoch. Date-times will become seconds since the + /// '1970-01-01T00:00:00Z' epoch with a small loss of precision. + /// + /// Currently can't fail but future number types might yield `Error::UnsafeCast`. + pub fn try_into_f64(self) -> Result { + match &self { + Boundaries::I64(_) => Err(Error::UnsafeCast("64-bit integer", "64-bit float")), + Boundaries::F32(_) + | Boundaries::F64(_) + | Boundaries::Date(_) + | Boundaries::DateTime(_) => Ok(BoundariesF64(self)), + } + } + + /// Turns this into an `i64` boundary iterator, casting values. + /// + /// Floating-point types will be rejected with `Error::UnsafeCast`. Dates will become + /// days since the '1970-01-01' epoch. Date-times will become microseconds since the + /// '1970-01-01T00:00:00Z' epoch. + pub fn try_into_i64(self) -> Result { + match self { + Boundaries::F32(_) => Err(Error::UnsafeCast("32-bit float", "64-bit integer")), + Boundaries::F64(_) => Err(Error::UnsafeCast("64-bit float", "64-bit integer")), + Boundaries::I64(_) | Boundaries::Date(_) | Boundaries::DateTime(_) => { + Ok(BoundariesI64(self)) + } + } + } +} + +pub struct BoundariesF64(Boundaries); + +impl Iterator for BoundariesF64 { + type Item = Result, Error>; + + fn next(&mut self) -> Option { + match &mut self.0 { + Boundaries::F32(i) => i.next().map(|r| r.map(|o| o.map(Into::into))), + Boundaries::F64(i) => i.next(), + Boundaries::Date(i) => i.next().map(|r| r.map(|o| o.map(date_to_f64))), + Boundaries::DateTime(i) => i.next().map(|r| r.map(|o| o.map(date_time_to_f64))), + Boundaries::I64(_) => None, + } + } +} + +pub struct BoundariesI64(Boundaries); + +impl Iterator for BoundariesI64 { + type Item = Result, Error>; + + fn next(&mut self) -> Option { + match &mut self.0 { + Boundaries::F32(_) | Boundaries::F64(_) => None, + Boundaries::I64(i) => i.next(), + Boundaries::Date(i) => i.next().map(|r| r.map(|o| o.map(date_to_i64))), + Boundaries::DateTime(i) => i.next().map(|r| r.map(|o| o.map(date_time_to_i64))), + } + } +} + +/// Iterator for reading regular sub-block corner min/max data. +/// +/// Casts to `[f64; 6]` by default or you can access the variants directly. +/// Each item is `[min_x, min_y, min_z, max_x, max_y, max_z]`. +#[derive(Debug)] +pub enum FreeformSubblocks { + F32(GenericFreeformSubblocks), + F64(GenericFreeformSubblocks), +} + +impl Iterator for FreeformSubblocks { + type Item = Result<([u32; 3], [f64; 6]), Error>; + + fn next(&mut self) -> Option { + match self { + Self::F64(iter) => iter.next(), + Self::F32(iter) => match iter.next() { + Some(Ok((parent, corners))) => Some(Ok((parent, corners.map(Into::into)))), + Some(Err(e)) => Some(Err(e)), + None => None, + }, + } + } +} diff --git a/src/data/mod.rs b/src/data/mod.rs new file mode 100644 index 0000000..f622f77 --- /dev/null +++ b/src/data/mod.rs @@ -0,0 +1,8 @@ +mod iterators; +mod read_checks; +pub mod traits; +pub(crate) mod write_checks; + +pub use iterators::*; +pub use read_checks::*; +pub use traits::*; diff --git a/src/data/read_checks.rs b/src/data/read_checks.rs new file mode 100644 index 0000000..875fdf8 --- /dev/null +++ b/src/data/read_checks.rs @@ -0,0 +1,447 @@ +use std::{collections::HashSet, fs::File}; + +use crate::{ + array::Constraint, + error::{Error, InvalidData}, + file::SubFile, + pqarray::read::{MultiIter, NullableGroupIter, NullableIter, SimpleIter}, + SubblockMode, +}; + +use super::{ + write_checks::{subblock_is_octree_compat, valid_subblock_sizes}, + FloatType, NumberType, +}; + +/// Iterator for reading scalar data, supporting `f32` and `f64` types generically. +/// +/// When used for tensor block model sizes this checks that all sizes are greater than zero. +#[derive(Debug)] +pub struct GenericScalars { + inner: SimpleIter>, + is_size: bool, +} + +impl GenericScalars { + pub(crate) fn new(inner: SimpleIter>, constraint: &Constraint) -> Self { + Self { + inner, + is_size: matches!(constraint, Constraint::Size), + } + } +} + +impl Iterator for GenericScalars { + type Item = Result; + + fn next(&mut self) -> Option { + let item = self.inner.next()?; + if self.is_size { + if let Ok(value) = item { + if value <= T::default() { + return Some(Err(InvalidData::SizeZeroOrLess { + value: value.into(), + } + .into())); + } + } + } + Some(item) + } +} + +/// Iterator for reading nullable indices. +/// +/// Checks that all indices are within range. +#[derive(Debug)] +pub struct Indices { + inner: NullableIter>, + category_count: u64, +} + +impl Indices { + pub(crate) fn new(inner: NullableIter>, constraint: &Constraint) -> Self { + let &Constraint::Index(category_count) = constraint else { + panic!("invalid constraint"); + }; + Self { + inner, + category_count, + } + } +} + +impl Iterator for Indices { + type Item = Result, Error>; + + fn next(&mut self) -> Option { + let item = self.inner.next()?; + if let Ok(Some(i)) = &item { + let index: u64 = (*i).into(); + if index >= self.category_count { + return Some(Err(InvalidData::IndexOutOfRange { + value: index, + maximum: self.category_count.saturating_add(1), + } + .into())); + } + } + Some(item) + } +} + +/// Iterator for reading segments or triangles. +/// +/// Checks that all vertex indices are within range. +#[derive(Debug)] +pub struct GenericPrimitives { + inner: MultiIter, N>, + vertex_count: u64, +} + +impl GenericPrimitives { + pub(crate) fn new(inner: MultiIter, N>, constraint: &Constraint) -> Self { + let vertex_count = match constraint { + Constraint::Segment(n) | Constraint::Triangle(n) => *n, + _ => panic!("invalid constraint"), + }; + Self { + inner, + vertex_count, + } + } +} + +impl Iterator for GenericPrimitives { + type Item = Result<[u32; N], Error>; + + fn next(&mut self) -> Option { + let item = self.inner.next()?; + if let Ok(value) = &item { + for i in value { + let index: u64 = (*i).into(); + if index >= self.vertex_count { + return Some(Err(InvalidData::IndexOutOfRange { + value: index, + maximum: self.vertex_count.saturating_add(1), + } + .into())); + } + } + } + Some(item) + } +} + +/// Iterator for reading boundary values. +/// +/// Checks that the values are increasing. +#[derive(Debug)] +pub(super) struct BoundaryValues { + inner: SimpleIter>, + previous: Option, +} + +impl BoundaryValues { + pub(crate) fn new(inner: SimpleIter>) -> Self { + Self { + inner, + previous: None, + } + } +} + +impl Iterator for BoundaryValues { + type Item = Result; + + fn next(&mut self) -> Option { + let item = self.inner.next()?; + if let (Ok(v), Some(p)) = (&item, &self.previous) { + if v < p { + return Some(Err(InvalidData::BoundaryDecreases.into())); + } + } + Some(item) + } +} + +/// Iterator for reading free-form sub-blocks, supporting `f32` or `f64` generically. +/// +/// Checks that the parent index and corners are all valid. +#[derive(Debug)] +pub struct GenericFreeformSubblocks { + parents: MultiIter, 3>, + corners: MultiIter, 6>, + block_count: [u32; 3], +} + +impl GenericFreeformSubblocks { + pub(crate) fn new( + parents: MultiIter, 3>, + corners: MultiIter, 6>, + constraint: &Constraint, + ) -> Self { + let block_count = match constraint { + &Constraint::RegularSubblock { block_count, .. } + | &Constraint::FreeformSubblock { block_count } => block_count, + _ => panic!("invalid constraint"), + }; + Self { + parents, + corners, + block_count, + } + } +} + +impl Iterator for GenericFreeformSubblocks { + type Item = Result<([u32; 3], [T; 6]), Error>; + + fn next(&mut self) -> Option { + match (self.parents.next(), self.corners.next()) { + (None, _) | (_, None) => None, + (Some(Err(e)), _) | (_, Some(Err(e))) => Some(Err(e)), + (Some(Ok(parent)), Some(Ok(corners))) => { + if let Err(e) = check_freeform_corners(corners) { + Some(Err(e)) + } else if let Err(e) = check_parent(self.block_count, parent) { + Some(Err(e)) + } else { + Some(Ok((parent, corners))) + } + } + } + } +} + +/// Iterator for reading regular sub-blocks. +/// +/// Checks that the parent index and corners are all valid. +#[derive(Debug)] +pub struct RegularSubblocks { + parents: MultiIter, 3>, + corners: MultiIter, 6>, + block_count: [u32; 3], + subblock_count: [u32; 3], + mode: Option<(SubblockMode, HashSet<[u32; 3]>)>, +} + +impl RegularSubblocks { + pub(crate) fn new( + parents: MultiIter, 3>, + corners: MultiIter, 6>, + constraint: &Constraint, + ) -> Self { + let &Constraint::RegularSubblock { + block_count, + subblock_count, + mode, + } = constraint + else { + panic!("invalid constraint"); + }; + Self { + parents, + corners, + block_count, + subblock_count, + mode: mode.map(|m| (m, valid_subblock_sizes(m, subblock_count))), + } + } +} + +impl Iterator for RegularSubblocks { + type Item = Result<([u32; 3], [u32; 6]), Error>; + + fn next(&mut self) -> Option { + match (self.parents.next(), self.corners.next()) { + (None, _) | (_, None) => None, + (Some(Err(e)), _) | (_, Some(Err(e))) => Some(Err(e)), + (Some(Ok(parent)), Some(Ok(corners))) => { + if let Err(e) = check_regular_corners(self.subblock_count, &self.mode, corners) { + Some(Err(e)) + } else if let Err(e) = check_parent(self.block_count, parent) { + Some(Err(e)) + } else { + Some(Ok((parent, corners))) + } + } + } + } +} + +#[inline] +fn check_parent(block_count: [u32; 3], parent: [u32; 3]) -> Result<(), Error> { + for (count, index) in block_count.into_iter().zip(parent) { + if index >= count { + return Err(InvalidData::BlockIndexOutOfRange { + value: parent.map(Into::into), + maximum: block_count, + } + .into()); + } + } + Ok(()) +} + +#[inline] +fn check_regular_corners( + subblock_count: [u32; 3], + mode_and_sizes: &Option<(SubblockMode, HashSet<[u32; 3]>)>, + corners: [u32; 6], +) -> Result<(), Error> { + let corners: [u32; 6] = corners.map(Into::into); + for i in 0..3 { + if corners[i] >= corners[i + 3] { + return Err(InvalidData::RegularSubblockZeroSize { corners }.into()); + } + if corners[i + 3] > subblock_count[i] { + return Err(InvalidData::RegularSubblockOutsideParent { + corners, + maximum: subblock_count, + } + .into()); + } + } + if let Some((mode, valid_sizes)) = &mode_and_sizes { + let size = std::array::from_fn(|i| corners[i + 3] - corners[i]); + if !valid_sizes.contains(&size) { + return Err(InvalidData::RegularSubblockNotInMode { + corners, + mode: *mode, + } + .into()); + } + if *mode == SubblockMode::Octree && !subblock_is_octree_compat(&corners) { + return Err(InvalidData::RegularSubblockNotInMode { + corners, + mode: *mode, + } + .into()); + } + } + Ok(()) +} + +#[inline] +fn check_freeform_corners(corners: [T; 6]) -> Result<(), Error> { + for i in 0..3 { + if corners[i] < T::ZERO { + return Err(InvalidData::FreeformSubblockOutsideParent { + corners: corners.map(Into::into), + } + .into()); + } + if corners[i + 3] > T::ONE { + return Err(InvalidData::FreeformSubblockOutsideParent { + corners: corners.map(Into::into), + } + .into()); + } + if corners[i] >= corners[i + 3] { + return Err(InvalidData::FreeformSubblockZeroSize { + corners: corners.map(Into::into), + } + .into()); + } + } + Ok(()) +} + +/// Iterator for reading numbers, supporting `f32`, `f64`, `i64`, date, and date-time generically. +#[derive(Debug)] +pub struct GenericNumbers(pub(crate) NullableIter>); + +impl Iterator for GenericNumbers { + type Item = Result, Error>; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +/// Iterator for reading small fixed-size arrays, like vertices and texture coordinates. +#[derive(Debug)] +pub struct GenericArrays(pub(crate) MultiIter, N>); + +impl Iterator for GenericArrays { + type Item = Result<[T; N], Error>; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +/// Iterator for reading non-nullable colors, such as category legends. +#[derive(Debug)] +pub struct Gradient(pub(crate) MultiIter, 4>); + +impl Iterator for Gradient { + type Item = Result<[u8; 4], Error>; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +/// Iterator for reading small optional fixed-size arrays, like 2D and 3D vectors. +#[derive(Debug)] +pub struct GenericOptionalArrays( + pub(crate) NullableGroupIter, N>, +); + +impl Iterator for GenericOptionalArrays { + type Item = Result, Error>; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +/// Iterator for reading nullable colors. +#[derive(Debug)] +pub struct Colors(pub(crate) NullableGroupIter, 4>); + +impl Iterator for Colors { + type Item = Result, Error>; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +/// Iterator for reading nullable booleans. +#[derive(Debug)] +pub struct Booleans(pub(crate) NullableIter>); + +impl Iterator for Booleans { + type Item = Result, Error>; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +/// Iterator for reading non-nullable strings, such as category names. +#[derive(Debug)] +pub struct Names(pub(crate) SimpleIter>); + +impl Iterator for Names { + type Item = Result; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +/// Iterator for reading nullable strings. +#[derive(Debug)] +pub struct Text(pub(crate) NullableIter>); + +impl Iterator for Text { + type Item = Result, Error>; + + fn next(&mut self) -> Option { + self.0.next() + } +} diff --git a/src/data/traits.rs b/src/data/traits.rs new file mode 100644 index 0000000..7031e0a --- /dev/null +++ b/src/data/traits.rs @@ -0,0 +1,56 @@ +use chrono::{DateTime, NaiveDate, Utc}; + +use crate::pqarray::PqArrayType; + +pub trait FloatType: PqArrayType + Copy + Into + PartialOrd + Default + 'static { + const ZERO: Self; + const ONE: Self; +} + +impl FloatType for f32 { + const ZERO: Self = 0.0; + const ONE: Self = 1.0; +} + +impl FloatType for f64 { + const ZERO: Self = 0.0; + const ONE: Self = 1.0; +} + +pub trait NumberType: PqArrayType + Copy + PartialOrd + Default + 'static {} + +impl NumberType for f32 {} +impl NumberType for f64 {} +impl NumberType for i64 {} +impl NumberType for NaiveDate {} +impl NumberType for DateTime {} + +pub trait VectorSource: 'static { + const IS_3D: bool; + fn into_2d(self) -> [T; 2]; + fn into_3d(self) -> [T; 3]; +} + +impl VectorSource for [T; 2] { + const IS_3D: bool = false; + + fn into_2d(self) -> [T; 2] { + [self[0], self[1]] + } + + fn into_3d(self) -> [T; 3] { + [self[0], self[1], T::default()] + } +} + +impl VectorSource for [T; 3] { + const IS_3D: bool = true; + + fn into_2d(self) -> [T; 2] { + [self[0], self[1]] + } + + fn into_3d(self) -> [T; 3] { + [self[0], self[1], self[2]] + } +} diff --git a/src/data/write_checks.rs b/src/data/write_checks.rs new file mode 100644 index 0000000..6f53cb0 --- /dev/null +++ b/src/data/write_checks.rs @@ -0,0 +1,311 @@ +use std::collections::HashSet; + +use crate::{ + array::{Array, ArrayType, Constraint}, + error::InvalidData, + validate::Reason, + SubblockMode, +}; + +use super::NumberType; + +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum ArrayWriteCheck { + MinimumScalar(f64), + MaximumIndex(u32), + Subblocks([[u32; 3]; 3]), + RegularSubblocksMaximum([[u32; 6]; 3]), + RegularSubblockCorners(HashSet<[u32; 6]>), + Invalid(InvalidData), +} + +impl ArrayWriteCheck { + pub fn check(&self, array: &Array) -> Result<(), Reason> { + match (self, array.constraint()) { + (Self::MinimumScalar(min), Constraint::Size) => { + // Check that minimum scalar value is not zero or less. + if *min <= 0.0 { + return Err(Reason::InvalidData(InvalidData::SizeZeroOrLess { + value: *min, + })); + } + } + ( + Self::MaximumIndex(max_index), + Constraint::Segment(count) | Constraint::Triangle(count) | Constraint::Index(count), + ) => { + // Check the maximum index against the vertex count. + if u64::from(*max_index) >= *count { + return Err(Reason::InvalidData(InvalidData::IndexOutOfRange { + value: u64::from(*max_index), + maximum: (*count).saturating_sub(1), + })); + } + } + ( + Self::Subblocks(maximum_parents), + Constraint::RegularSubblock { block_count, .. } + | Constraint::FreeformSubblock { block_count, .. }, + ) => { + // Check parent block indices against the block count. + for (i, parent) in maximum_parents.iter().enumerate() { + if parent[i] >= block_count[i] { + return Err(Reason::InvalidData(InvalidData::BlockIndexOutOfRange { + value: *parent, + maximum: block_count.map(|n| n.saturating_sub(1)), + })); + } + } + } + ( + Self::RegularSubblocksMaximum(maximums), + Constraint::RegularSubblock { subblock_count, .. }, + ) => { + // Check regular sub-block indices against the sub-block count. + for (i, corners) in maximums.iter().enumerate() { + if corners[i + 3] > subblock_count[i] { + return Err(Reason::InvalidData( + InvalidData::RegularSubblockOutsideParent { + corners: *corners, + maximum: *subblock_count, + }, + )); + } + } + } + ( + Self::RegularSubblockCorners(all_corners), + Constraint::RegularSubblock { + subblock_count, + mode: Some(mode), + .. + }, + ) => { + // Check regular sub-block indices against the sub-block count. + let valid_sizes = valid_subblock_sizes(*mode, *subblock_count); + for corners in all_corners { + let size = std::array::from_fn(|i| corners[i + 3] - corners[i]); + if !valid_sizes.contains(&size) + || (*mode == SubblockMode::Octree && !subblock_is_octree_compat(corners)) + { + return Err(Reason::InvalidData(InvalidData::RegularSubblockNotInMode { + corners: *corners, + mode: *mode, + })); + } + } + } + (Self::Invalid(inv), _) => { + // Pass error on. + return Err(Reason::InvalidData(inv.clone())); + } + _ => (), + } + Ok(()) + } +} + +pub(crate) fn subblock_is_octree_compat(corners: &[u32; 6]) -> bool { + (0..3).all(|i| corners[i] % (corners[i + 3] - corners[i]) == 0) +} + +pub(crate) fn valid_subblock_sizes( + mode: SubblockMode, + subblock_count: [u32; 3], +) -> HashSet<[u32; 3]> { + let mut valid_sizes = HashSet::new(); + match mode { + SubblockMode::Octree => { + let mut size = subblock_count; + loop { + valid_sizes.insert(size); + if size == [1, 1, 1] { + break; + } + size = size.map(|n| (n / 2).max(1)); + } + } + SubblockMode::Full => { + valid_sizes.insert([1, 1, 1]); + valid_sizes.insert(subblock_count); + } + } + valid_sizes +} + +pub(crate) struct MaximumIndex(u32); + +impl MaximumIndex { + pub fn new() -> Self { + Self(0) + } + + pub fn visit>(&mut self, value: T) -> T { + self.0 = self.0.max(value.into()); + value + } + + pub fn visit_opt>(&mut self, value: Option) -> Option { + value.map(|v| self.visit(v)) + } + + pub fn visit_array, const N: usize>(&mut self, value: [T; N]) -> [T; N] { + for v in value { + self.visit(v); + } + value + } + + pub fn get(self) -> Vec { + vec![ArrayWriteCheck::MaximumIndex(self.0)] + } +} + +pub(crate) struct MinimumScalar(f64); + +impl MinimumScalar { + pub fn new() -> Self { + Self(f64::INFINITY) + } + + pub fn visit>(&mut self, value: T) -> T { + self.0 = self.0.min(value.into()); + value + } + + pub fn get(self) -> Vec { + vec![ArrayWriteCheck::MinimumScalar(self.0)] + } +} + +pub(crate) struct IncreasingBoundary { + previous: Option, + ok: bool, +} + +impl IncreasingBoundary { + pub fn new() -> Self { + Self { + previous: None, + ok: true, + } + } + + pub fn visit(&mut self, value: T) -> T { + if let Some(previous) = self.previous { + if value < previous { + self.ok = false; + } + } + self.previous = Some(value); + value + } + + pub fn get(self) -> Vec { + if self.ok { + vec![] + } else { + vec![ArrayWriteCheck::Invalid(InvalidData::BoundaryDecreases)] + } + } +} + +pub(crate) struct ParentIndices { + maximums: [[u32; 3]; 3], +} + +impl ParentIndices { + pub fn new() -> Self { + Self { + maximums: [[0; 3]; 3], + } + } + + pub fn visit>(&mut self, value: [T; 3]) { + let parent = value.map(Into::into); + for i in 0..3 { + if parent[i] > self.maximums[i][i] { + self.maximums[i] = parent; + } + } + } + + pub fn get(self) -> Vec { + vec![ArrayWriteCheck::Subblocks(self.maximums)] + } +} + +pub(crate) struct FreeformCorners(Option); + +impl FreeformCorners { + pub fn new() -> Self { + Self(None) + } + + pub fn visit>(&mut self, corners: [T; 6]) { + if self.0.is_none() { + let corners: [f64; 6] = corners.map(Into::into); + for i in 0..3 { + if corners[i] >= corners[i + 3] { + self.0 = Some(InvalidData::FreeformSubblockZeroSize { corners }); + return; + } + if corners[i] < 0.0 || corners[i + 3] > 1.0 { + self.0 = Some(InvalidData::FreeformSubblockOutsideParent { corners }); + return; + } + } + } + } + + pub fn get(self) -> Vec { + self.0.map(ArrayWriteCheck::Invalid).into_iter().collect() + } +} + +pub(crate) struct RegularCorners { + maximum_corners: [[u32; 6]; 3], + corners: HashSet<[u32; 6]>, + invalid_size: Option<[u32; 6]>, +} + +impl RegularCorners { + pub fn new() -> Self { + Self { + maximum_corners: [[0; 6]; 3], + corners: HashSet::new(), + invalid_size: None, + } + } + + pub fn visit>(&mut self, corners: [T; 6]) { + let corners: [u32; 6] = corners.map(Into::into); + self.corners.insert(corners); + for i in 0..3 { + if corners[i + 3] > self.maximum_corners[i][i + 3] { + self.maximum_corners[i] = corners; + } + } + if self.invalid_size.is_none() { + for i in 0..3 { + if corners[i] >= corners[i + 3] { + self.invalid_size = Some(corners); + break; + } + } + } + } + + pub fn get(self) -> Vec { + let mut out = vec![ + ArrayWriteCheck::RegularSubblocksMaximum(self.maximum_corners), + ArrayWriteCheck::RegularSubblockCorners(self.corners), + ]; + if let Some(corners) = self.invalid_size { + out.push(ArrayWriteCheck::Invalid( + InvalidData::RegularSubblockZeroSize { corners }, + )); + } + out + } +} diff --git a/src/date_time.rs b/src/date_time.rs new file mode 100644 index 0000000..1a12ba5 --- /dev/null +++ b/src/date_time.rs @@ -0,0 +1,83 @@ +//! Utility functions for date and date-time conversion. + +use chrono::{DateTime, Duration, NaiveDate, TimeZone, Utc}; + +/// Convert a date to the number of days since the epoch. +pub fn date_to_f64(date: NaiveDate) -> f64 { + date_to_i64(date) as f64 +} + +/// Convert a date-time the number of seconds since the epoch, including fractional seconds, +/// at a bit less than microsecond precision. +pub fn date_time_to_f64(date_time: DateTime) -> f64 { + const MICRO: i64 = 1_000_000; + let us = date_time_to_i64(date_time); + let s = us / MICRO; + let f = us % MICRO; + (s as f64) + (f as f64) / (MICRO as f64) +} + +/// Convert a date to the number of days since the epoch. +pub fn date_to_i64(date: NaiveDate) -> i64 { + date.signed_duration_since(NaiveDate::default()).num_days() +} + +/// Convert a date-time to the number of microseconds since the epoch. +pub fn date_time_to_i64(date_time: DateTime) -> i64 { + date_time.timestamp_micros() +} + +/// Convert a number of days since the epoch back to a date. +pub fn i64_to_date(value: i64) -> NaiveDate { + Duration::try_days(value) + .and_then(|d| NaiveDate::default().checked_add_signed(d)) + .unwrap_or(if value < 0 { + NaiveDate::MIN + } else { + NaiveDate::MAX + }) +} + +/// Convert a number of microseconds since the epoch back to a date. +pub fn i64_to_date_time(value: i64) -> DateTime { + DateTime::::default() + .checked_add_signed(Duration::microseconds(value)) + .unwrap_or(if value < 0 { + DateTime::::MIN_UTC + } else { + DateTime::::MAX_UTC + }) +} + +/// Convert a number of milliseconds since the epoch back to a date. +pub fn i64_milli_to_date_time(value: i64) -> DateTime { + Duration::try_milliseconds(value) + .and_then(|d| DateTime::::default().checked_add_signed(d)) + .unwrap_or(if value < 0 { + DateTime::::MIN_UTC + } else { + DateTime::::MAX_UTC + }) +} + +/// Convert a number of nanoseconds since the epoch back to a date. +pub fn i64_nano_to_date_time(value: i64) -> DateTime { + DateTime::::default() + .checked_add_signed(Duration::nanoseconds(value)) + .unwrap_or(if value < 0 { + DateTime::::MIN_UTC + } else { + DateTime::::MAX_UTC + }) +} + +/// Returns the current date and time in UTC. +pub fn utc_now() -> DateTime { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("valid system time"); + let naive = DateTime::from_timestamp(now.as_secs() as i64, now.subsec_nanos()) + .expect("valid timestamp") + .naive_utc(); + Utc.from_utc_datetime(&naive) +} diff --git a/src/element.rs b/src/element.rs new file mode 100644 index 0000000..fd5495b --- /dev/null +++ b/src/element.rs @@ -0,0 +1,73 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::{ + validate::{Validate, Validator}, + Attribute, Color, Geometry, Location, +}; + +/// Defines a single "object" or "shape" within the OMF file. +/// +/// Each shape has a name plus other optional metadata, a "geometry" that describes +/// a point-set, surface, etc., and a list of attributes that that exist on that geometry. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Element { + /// The element name. Names should be non-empty and unique. + pub name: String, + /// Optional element description. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub description: String, + /// Optional solid color. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub color: Option, + /// Arbitrary metadata. + #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] + pub metadata: serde_json::Map, + /// List of attributes, if any. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub attributes: Vec, + /// The geometry of the element. + pub geometry: Geometry, +} + +impl Element { + /// Create a new element with the given name and geometry. + /// + /// Geometries will be automatically converted from their objects into enum variants. + pub fn new(name: impl Into, geometry: impl Into) -> Self { + Self { + name: name.into(), + description: Default::default(), + metadata: Default::default(), + attributes: Default::default(), + geometry: geometry.into(), + color: None, + } + } + + /// Returns the valid locations for attributes on this element. + pub fn valid_locations(&self) -> &'static [Location] { + self.geometry.valid_locations() + } + + /// Returns the number of values needed for the given location. + pub fn location_len(&self, location: Location) -> Option { + self.geometry.location_len(location) + } +} + +impl Validate for Element { + fn validate_inner(&mut self, val: &mut Validator) { + val.enter("Element") + .name(&self.name) + .obj(&mut self.geometry) + .objs(&mut self.attributes) + .unique( + self.attributes.iter().map(|a| &a.name), + "attributes[..]::name", + false, + ) + .attrs_on_geometry(&self.attributes, &self.geometry); + } +} diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..6ba8cd0 --- /dev/null +++ b/src/error.rs @@ -0,0 +1,227 @@ +//! Error codes and details. + +use std::{collections::TryReserveError, fmt::Display}; + +use zip::result::ZipError; + +use crate::{validate, SubblockMode}; + +/// The types of limit that may be exceeded. +/// +/// Variants name the field in [`Limits`](crate::file::Limits) that was exceeded. +#[derive(Debug, thiserror::Error)] +pub enum Limit { + #[error("uncompressed JSON too big")] + JsonBytes, + #[error("uncompressed array or image too big")] + ArrayBytes, + #[error("image width or height is too large")] + ImageDim, +} + +fn format_corners(corners: &[T; 6]) -> String { + format!( + "[{}, {}, {}] to [{}, {}, {}]", + corners[0], corners[1], corners[2], corners[3], corners[4], corners[5], + ) +} + +/// Ways that data, either in a file or being written to one, can be invalid. +#[derive(Debug, Clone, PartialEq, thiserror::Error)] +pub enum InvalidData { + /// Data length does not match the Array. + #[error("Error: array length {found} does not the declared length {expected}")] + LengthMismatch { found: u64, expected: u64 }, + /// A size is <= 0. + #[error("size value {value} is zero or less")] + SizeZeroOrLess { value: f64 }, + /// A discrete colormap boundary is less than the previous boundary. + #[error("discrete colormap boundary decreases")] + BoundaryDecreases, + /// A segment, triangle, or category index is out of range. + #[error("index value {value} exceeds the maximum index {maximum}")] + IndexOutOfRange { value: u64, maximum: u64 }, + /// A block index is out of range. + #[error("block index {value:?} exceeds the maximum index {maximum:?}")] + BlockIndexOutOfRange { value: [u32; 3], maximum: [u32; 3] }, + /// A regular sub-block has zero or negative size. + #[error("sub-block {} has zero or negative size", format_corners(corners))] + RegularSubblockZeroSize { corners: [u32; 6] }, + /// A regular sub-block extends outside the parent. + #[error( + "sub-block {} exceeds the maximum {maximum:?}", + format_corners(corners) + )] + RegularSubblockOutsideParent { + corners: [u32; 6], + maximum: [u32; 3], + }, + /// A regular sub-block doesn't match the octree or full sub-block mode. + #[error("sub-block {} is invalid for {mode:?} mode", format_corners(corners))] + RegularSubblockNotInMode { + corners: [u32; 6], + mode: SubblockMode, + }, + /// A free-form sub-block has zero or negative size. + #[error("sub-block {} has zero or negative size", format_corners(corners))] + FreeformSubblockZeroSize { corners: [f64; 6] }, + /// A free-form sub-block is outside the [0.0, 1.0] parent range. + #[error( + "sub-block {} is outside the valid range of 0.0 to 1.0", + format_corners(corners) + )] + FreeformSubblockOutsideParent { corners: [f64; 6] }, +} + +/// Errors generated by this crate. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// Used when memory allocation fails. + #[error("Memory allocation failed")] + OutOfMemory, + /// Forward errors from file operations. + #[error("File IO error: {0}")] + IoError(std::io::Error), + /// When the correct file header is not detected. + #[error("Error: the zip comment does not identify this as an OMF file: '{0}'")] + NotOmf(String), + /// When the file version is newer than the library. + #[error("Version error: the file uses OMF v{0}.{1} but this version library can only read 0.9 and 2.0")] + NewerVersion(u32, u32), + /// The file version is pre-release and can't be loaded by release versions. + #[error("Version error: the file uses pre-release OMF v{0}.{1}-{2} and can't be loaded")] + PreReleaseVersion(u32, u32, String), + /// Forwards `serde_json` errors when deserializing. + #[error("JSON deserialization error: {0}")] + DeserializationFailed(#[from] serde_json::Error), + /// Forwards `serde_json` errors when serializing. + #[error("JSON serialization error: {0}")] + SerializationFailed(serde_json::Error), + /// Passes out errors detected during OMF validation. + #[error("Validation failed")] + ValidationFailed(#[from] validate::Problems), + /// When trying to cast `f64` to `f32` for example, as that would lose precision. + #[error("Error: can't cast from {0} to {1} without losing data")] + UnsafeCast(&'static str, &'static str), + /// Writing an image that isn't in PNG or JPEG format. + #[error("Error: image is not in PNG or JPEG encoding")] + NotImageData, + /// Writing an array that isn't in Parquet format. + #[error("Error: image is not in Parquet encoding")] + NotParquetData, + /// Tried to read something that exceeds the provided limits. + #[error("Error: safety limit exceeded")] + LimitExceeded(Limit), + /// Array data errors, when reading or writing. + #[error("Data error: {0}")] + InvalidData(Box), + /// A data file or index is missing from the zip. + #[error("Error: missing archive member '{0}'")] + ZipMemberMissing(String), + /// Zip read or write failed. + #[error("Zip error: {0}")] + ZipError(String), + /// When a Parquet schema doesn't match. + #[cfg(feature = "parquet")] + #[error("{}", mismatch_string(.0, .1))] + ParquetSchemaMismatch( + std::sync::Arc, + std::sync::Arc>, + ), + /// Forward errors from array operations. + #[cfg(feature = "parquet")] + #[error("{0}")] + ParquetError(Box), + /// Forward errors from image operations. + #[cfg(feature = "image")] + #[error("Image processing error: {0}")] + ImageError(Box), + /// Errors from OMF1 conversion. + #[cfg(feature = "omf1")] + #[error("OMF1 conversion failed: {0}")] + Omf1Error(#[from] crate::omf1::Omf1Error), +} + +impl From for Error { + fn from(value: InvalidData) -> Self { + Self::InvalidData(value.into()) + } +} + +impl From for Error { + fn from(error: std::io::Error) -> Self { + match error.kind() { + std::io::ErrorKind::OutOfMemory => Error::OutOfMemory, + std::io::ErrorKind::Other if error.get_ref().is_some_and(|r| r.is::()) => { + *error.into_inner().unwrap().downcast().unwrap() + } + _ => Error::IoError(error), + } + } +} + +impl From for Error { + fn from(_: TryReserveError) -> Self { + Error::OutOfMemory + } +} + +impl From for Error { + fn from(value: ZipError) -> Self { + match value { + ZipError::Io(e) => Self::IoError(e), + other => Self::ZipError(other.to_string()), + } + } +} + +#[cfg(feature = "image")] +impl From for Error { + fn from(value: image::ImageError) -> Self { + use image::error::LimitErrorKind; + match &value { + image::ImageError::Limits(err @ image::error::LimitError { .. }) => match err.kind() { + LimitErrorKind::DimensionError => Error::LimitExceeded(Limit::ImageDim), + LimitErrorKind::InsufficientMemory => Error::LimitExceeded(Limit::ArrayBytes), + _ => Error::ImageError(value.into()), + }, + _ => Error::ImageError(value.into()), + } + } +} + +#[cfg(feature = "parquet")] +impl From for Error { + fn from(value: parquet::errors::ParquetError) -> Self { + Self::ParquetError(value.into()) + } +} + +#[cfg(feature = "parquet")] +fn mismatch_string( + found: &parquet::schema::types::Type, + expected: &[parquet::schema::types::Type], +) -> String { + use parquet::schema::{printer::print_schema, types::Type}; + + fn schema_string(ty: &Type) -> String { + let mut buf = Vec::new(); + print_schema(&mut buf, ty); + String::from_utf8_lossy(&buf).trim_end().to_owned() + } + let mut out = format!( + "Parquet schema mismatch, found:\n{found}\n\n{expected_label}:\n", + found = schema_string(found), + expected_label = if expected.len() == 1 { + "Expected" + } else { + "Expected one of" + } + ); + for ty in expected { + out.push_str(&schema_string(ty)); + out.push('\n'); + } + out +} diff --git a/src/file/image.rs b/src/file/image.rs new file mode 100644 index 0000000..e773168 --- /dev/null +++ b/src/file/image.rs @@ -0,0 +1,62 @@ +use std::io::{BufReader, Cursor}; + +use crate::{array_type, error::Error, Array}; + +use super::{Limits, Reader, Writer}; + +impl From for image::io::Limits { + fn from(value: Limits) -> Self { + let mut out = Self::no_limits(); + out.max_alloc = value.image_bytes; + out.max_image_width = value.image_dim; + out.max_image_height = value.image_dim; + out + } +} + +impl Reader { + /// Read and decode an image. + pub fn image(&self, image: &Array) -> Result { + let f = BufReader::new(self.array_bytes_reader(image)?); + let mut reader = image::io::Reader::new(f).with_guessed_format()?; + reader.limits(self.limits().into()); + Ok(reader.decode()?) + } +} + +impl Writer { + /// Write an image in PNG encoding. + /// + /// This supports grayscale, grayscale + alpha, RGB, and RGBA, in 8 or 16 bits per channel. + pub fn image_png( + &mut self, + image: &image::DynamicImage, + ) -> Result, Error> { + let mut bytes = Vec::new(); + image.write_to(&mut Cursor::new(&mut bytes), image::ImageFormat::Png)?; + self.image_bytes(&bytes) + } + + /// Write an image in JPEG encoding. + /// + /// Unlike PNG this is limited to 8-bit RGB and compression is lossy, but it will give + /// much better compression ratios. The JPEG compression level is set by the `quality` + /// argument, from 1 to 100. 90 is a reasonable level for preserving fine detail in the image, + /// while lower values will give a smaller file. + /// + /// If you have an existing image in JPEG encoding you shouldn't be using this method, + /// instead add the raw bytes of the file with `writer.image_bytes(&bytes)` to avoid recompressing + /// the image and losing more detail. + pub fn image_jpeg( + &mut self, + image: &image::RgbImage, + quality: u8, + ) -> Result, Error> { + let mut bytes = Vec::new(); + image.write_with_encoder(image::codecs::jpeg::JpegEncoder::new_with_quality( + &mut Cursor::new(&mut bytes), + quality.clamp(1, 100), + ))?; + self.image_bytes(&bytes) + } +} diff --git a/src/file/mod.rs b/src/file/mod.rs new file mode 100644 index 0000000..fa39de6 --- /dev/null +++ b/src/file/mod.rs @@ -0,0 +1,14 @@ +//! Contains the [`Reader`] and [`Writer`] objects. + +#[cfg(feature = "image")] +mod image; +#[cfg(feature = "parquet")] +pub(crate) mod parquet; +mod reader; +mod sub_file; +mod writer; +mod zip_container; + +pub use reader::{Limits, Reader}; +pub use sub_file::SubFile; +pub use writer::{Compression, Writer}; diff --git a/src/file/parquet/mod.rs b/src/file/parquet/mod.rs new file mode 100644 index 0000000..d185063 --- /dev/null +++ b/src/file/parquet/mod.rs @@ -0,0 +1,3 @@ +mod reader; +pub(crate) mod schemas; +mod writer; diff --git a/src/file/parquet/reader.rs b/src/file/parquet/reader.rs new file mode 100644 index 0000000..da80ee0 --- /dev/null +++ b/src/file/parquet/reader.rs @@ -0,0 +1,280 @@ +use std::fs::File; + +use crate::{ + array_type, + data::*, + error::{Error, InvalidData}, + file::SubFile, + pqarray::PqArrayReader, + Array, ArrayType, +}; + +use super::{super::Reader, schemas}; + +impl Reader { + fn array_reader( + &self, + array: &Array, + ) -> Result>, Error> { + let f = self.array_bytes_reader(array)?; + let reader = PqArrayReader::new(f)?; + if array.item_count() != reader.len() { + Err(InvalidData::LengthMismatch { + found: array.item_count(), + expected: reader.len(), + } + .into()) + } else { + Ok(reader) + } + } + + /// Read an [`array_type::Scalar`](crate::array_type::Scalar) array. + pub fn array_scalars(&self, array: &Array) -> Result { + let reader = self.array_reader(array)?; + Ok(match schemas::Scalar::check(&reader)? { + schemas::Scalar::F32 => { + let inner = reader.iter_column::("scalar")?; + Scalars::F32(GenericScalars::new(inner, array.constraint())) + } + schemas::Scalar::F64 => { + let inner = reader.iter_column::("scalar")?; + Scalars::F64(GenericScalars::new(inner, array.constraint())) + } + }) + } + + /// Read an [`array_type::Vertex`](crate::array_type::Vertex) array. + pub fn array_vertices(&self, array: &Array) -> Result { + let reader = self.array_reader(array)?; + Ok(match schemas::Vertex::check(&reader)? { + schemas::Vertex::F32 => { + Vertices::F32(GenericArrays(reader.iter_multi_column(["x", "y", "z"])?)) + } + schemas::Vertex::F64 => { + Vertices::F64(GenericArrays(reader.iter_multi_column(["x", "y", "z"])?)) + } + }) + } + + /// Read an [`array_type::Segment`](crate::array_type::Segment) array. + pub fn array_segments( + &self, + array: &Array, + ) -> Result, Error> { + let reader = self.array_reader(array)?; + Ok(match schemas::Segment::check(&reader)? { + schemas::Segment::U32 => { + GenericPrimitives::new(reader.iter_multi_column(["a", "b"])?, array.constraint()) + } + }) + } + + /// Read an [`array_type::Triangle`](crate::array_type::Triangle) array. + pub fn array_triangles( + &self, + array: &Array, + ) -> Result, Error> { + let reader = self.array_reader(array)?; + Ok(match schemas::Triangle::check(&reader)? { + schemas::Triangle::U32 => GenericPrimitives::new( + reader.iter_multi_column(["a", "b", "c"])?, + array.constraint(), + ), + }) + } + + /// Read an [`array_type::Name`](crate::array_type::Name) array. + pub fn array_names(&self, array: &Array) -> Result { + let reader = self.array_reader(array)?; + schemas::Name::check(&reader)?; + reader.iter_column("name").map(Names) + } + + /// Read an [`array_type::Gradient`](crate::array_type::Gradient) array. + pub fn array_gradient(&self, array: &Array) -> Result { + let reader = self.array_reader(array)?; + Ok(match schemas::Gradient::check(&reader)? { + schemas::Gradient::Rgba8 => Gradient(reader.iter_multi_column(["r", "g", "b", "a"])?), + }) + } + + /// Read an [`array_type::Texcoord`](crate::array_type::Texcoord) array. + pub fn array_texcoords(&self, array: &Array) -> Result { + let reader = self.array_reader(array)?; + Ok(match schemas::Texcoord::check(&reader)? { + schemas::Texcoord::F32 => { + Texcoords::F32(GenericArrays(reader.iter_multi_column(["u", "v"])?)) + } + schemas::Texcoord::F64 => { + Texcoords::F64(GenericArrays(reader.iter_multi_column(["u", "v"])?)) + } + }) + } + + /// Read an [`array_type::Boundary`](crate::array_type::Boundary) array. + pub fn array_boundaries( + &self, + array: &Array, + ) -> Result { + let reader = self.array_reader(array)?; + let m = schemas::Boundary::check(&reader)?; + let inclusive = reader.iter_column("inclusive")?; + Ok(match m { + schemas::Boundary::F32 => Boundaries::F32(GenericBoundaries::new( + reader.iter_column("value")?, + inclusive, + )), + schemas::Boundary::F64 => Boundaries::F64(GenericBoundaries::new( + reader.iter_column("value")?, + inclusive, + )), + schemas::Boundary::I64 => Boundaries::I64(GenericBoundaries::new( + reader.iter_column("value")?, + inclusive, + )), + schemas::Boundary::Date => Boundaries::Date(GenericBoundaries::new( + reader.iter_column("value")?, + inclusive, + )), + schemas::Boundary::DateTime => Boundaries::DateTime(GenericBoundaries::new( + reader.iter_column("value")?, + inclusive, + )), + }) + } + + /// Read an [`array_type::RegularSubblock`](crate::array_type::RegularSubblock) array. + pub fn array_regular_subblocks( + &self, + array: &Array, + ) -> Result { + let reader = self.array_reader(array)?; + schemas::RegularSubblock::check(&reader)?; + let parents = reader.iter_multi_column(["parent_u", "parent_v", "parent_w"])?; + let corners = reader.iter_multi_column([ + "corner_min_u", + "corner_min_v", + "corner_min_w", + "corner_max_u", + "corner_max_v", + "corner_max_w", + ])?; + Ok(RegularSubblocks::new(parents, corners, array.constraint())) + } + + /// Read an [`array_type::FreeformSubblock`](crate::array_type::FreeformSubblock) array. + pub fn array_freeform_subblocks( + &self, + array: &Array, + ) -> Result { + let reader = self.array_reader(array)?; + let m = schemas::FreeformSubblock::check(&reader)?; + let parents = reader.iter_multi_column(["parent_u", "parent_v", "parent_w"])?; + Ok(match m { + schemas::FreeformSubblock::U32F32 => { + FreeformSubblocks::F32(GenericFreeformSubblocks::new( + parents, + reader.iter_multi_column([ + "corner_min_u", + "corner_min_v", + "corner_min_w", + "corner_max_u", + "corner_max_v", + "corner_max_w", + ])?, + array.constraint(), + )) + } + schemas::FreeformSubblock::U32F64 => { + FreeformSubblocks::F64(GenericFreeformSubblocks::new( + parents, + reader.iter_multi_column([ + "corner_min_u", + "corner_min_v", + "corner_min_w", + "corner_max_u", + "corner_max_v", + "corner_max_w", + ])?, + array.constraint(), + )) + } + }) + } + + /// Read an [`array_type::Number`](crate::array_type::Number) array. + pub fn array_numbers(&self, array: &Array) -> Result { + let reader = self.array_reader(array)?; + Ok(match schemas::Number::check(&reader)? { + schemas::Number::F32 => { + Numbers::F32(GenericNumbers(reader.iter_nullable_column("number")?)) + } + schemas::Number::F64 => { + Numbers::F64(GenericNumbers(reader.iter_nullable_column("number")?)) + } + schemas::Number::I64 => { + Numbers::I64(GenericNumbers(reader.iter_nullable_column("number")?)) + } + schemas::Number::Date => { + Numbers::Date(GenericNumbers(reader.iter_nullable_column("number")?)) + } + schemas::Number::DateTime => { + Numbers::DateTime(GenericNumbers(reader.iter_nullable_column("number")?)) + } + }) + } + + /// Read an [`array_type::Index`](crate::array_type::Index) array. + pub fn array_indices(&self, array: &Array) -> Result { + let reader = self.array_reader(array)?; + Ok(match schemas::Index::check(&reader)? { + schemas::Index::U32 => { + Indices::new(reader.iter_nullable_column("index")?, array.constraint()) + } + }) + } + + /// Read an [`array_type::Vector`](crate::array_type::Vector) array. + pub fn array_vectors(&self, array: &Array) -> Result { + let reader = self.array_reader(array)?; + Ok(match schemas::Vector::check(&reader)? { + schemas::Vector::F32x2 => Vectors::F32x2(GenericOptionalArrays( + reader.iter_nullable_group_column("vector", ["x", "y"])?, + )), + schemas::Vector::F64x2 => Vectors::F64x2(GenericOptionalArrays( + reader.iter_nullable_group_column("vector", ["x", "y"])?, + )), + schemas::Vector::F32x3 => Vectors::F32x3(GenericOptionalArrays( + reader.iter_nullable_group_column("vector", ["x", "y", "z"])?, + )), + schemas::Vector::F64x3 => Vectors::F64x3(GenericOptionalArrays( + reader.iter_nullable_group_column("vector", ["x", "y", "z"])?, + )), + }) + } + + /// Read a [`array_type::Text](crate::array_type::Text) array. + pub fn array_text(&self, array: &Array) -> Result { + let reader = self.array_reader(array)?; + schemas::Text::check(&reader)?; + reader.iter_nullable_column("text").map(Text) + } + + /// Read an [`array_type::Boolean`](crate::array_type::Boolean) array. + pub fn array_booleans(&self, array: &Array) -> Result { + let reader = self.array_reader(array)?; + schemas::Boolean::check(&reader)?; + reader.iter_nullable_column("bool").map(Booleans) + } + + /// Read an [`array_type::Color`](crate::array_type::Color) array. + pub fn array_colors(&self, array: &Array) -> Result { + let reader = self.array_reader(array)?; + Ok(match schemas::Color::check(&reader)? { + schemas::Color::Rgba8 => { + Colors(reader.iter_nullable_group_column("color", ["r", "g", "b", "a"])?) + } + }) + } +} diff --git a/src/file/parquet/schemas.rs b/src/file/parquet/schemas.rs new file mode 100644 index 0000000..b7eedc4 --- /dev/null +++ b/src/file/parquet/schemas.rs @@ -0,0 +1,321 @@ +use std::{fs::File, sync::OnceLock}; + +use crate::{ + file::SubFile, + pqarray::{schema_match, PqArrayMatcher, PqArrayReader}, +}; + +macro_rules! declare_schema { + ($name:ident { $( $variant:ident { $($token:tt)* } )* }) => { + #[derive(Debug, Clone, Copy, PartialEq)] + pub(super) enum $name { + $($variant,)* + } + + impl $name { + fn matcher() -> &'static PqArrayMatcher<$name> { + static MATCHER: OnceLock> = OnceLock::new(); + MATCHER.get_or_init(|| { + schema_match! { + $($name::$variant => schema { $($token)* })* + } + }) + } + + pub fn check( + reader: &PqArrayReader>, + ) -> Result<$name, crate::error::Error> { + reader.matches(Self::matcher()) + } + } + }; +} + +declare_schema! { + Scalar { + F32 { + required float scalar; + } + F64 { + required double scalar; + } + } +} + +declare_schema! { + Vertex { + F32 { + required float x; + required float y; + required float z; + } + F64 { + required double x; + required double y; + required double z; + } + } +} + +declare_schema! { + Segment { + U32 { + required int32 a (integer(32, false)); + required int32 b (integer(32, false)); + } + } +} + +declare_schema! { + Triangle { + U32 { + required int32 a (integer(32, false)); + required int32 b (integer(32, false)); + required int32 c (integer(32, false)); + } + } +} + +declare_schema! { + Name { + String { + required byte_array name (string); + } + } +} + +declare_schema! { + Gradient { + Rgba8 { + required int32 r (integer(8, false)); + required int32 g (integer(8, false)); + required int32 b (integer(8, false)); + required int32 a (integer(8, false)); + } + } +} + +declare_schema! { + Texcoord { + F32 { + required float u; + required float v; + } + F64 { + required double u; + required double v; + } + } +} + +declare_schema! { + Boundary { + F32 { + required float value; + required boolean inclusive; + } + F64 { + required double value; + required boolean inclusive; + } + I64 { + required int64 value; + required boolean inclusive; + } + Date { + required int32 value (date); + required boolean inclusive; + } + DateTime { + required int64 value (timestamp(micros, true)); + required boolean inclusive; + } + } +} + +declare_schema! { + RegularSubblock { + U32U32 { + required int32 parent_u (integer(32, false)); + required int32 parent_v (integer(32, false)); + required int32 parent_w (integer(32, false)); + required int32 corner_min_u (integer(32, false)); + required int32 corner_min_v (integer(32, false)); + required int32 corner_min_w (integer(32, false)); + required int32 corner_max_u (integer(32, false)); + required int32 corner_max_v (integer(32, false)); + required int32 corner_max_w (integer(32, false)); + } + } +} + +declare_schema! { + FreeformSubblock { + U32F32 { + required int32 parent_u (integer(32, false)); + required int32 parent_v (integer(32, false)); + required int32 parent_w (integer(32, false)); + required float corner_min_u; + required float corner_min_v; + required float corner_min_w; + required float corner_max_u; + required float corner_max_v; + required float corner_max_w; + } + U32F64 { + required int32 parent_u (integer(32, false)); + required int32 parent_v (integer(32, false)); + required int32 parent_w (integer(32, false)); + required double corner_min_u; + required double corner_min_v; + required double corner_min_w; + required double corner_max_u; + required double corner_max_v; + required double corner_max_w; + } + } +} + +declare_schema! { + Number { + F32 { + optional float number; + } + F64 { + optional double number; + } + I64 { + optional int64 number; + } + Date { + optional int32 number (date); + } + DateTime { + optional int64 number (timestamp(micros, true)); + } + } +} + +declare_schema! { + Index { + U32 { + optional int32 index (integer(32, false)); + } + } +} + +declare_schema! { + Vector { + F32x2 { + optional group vector { + required float x; + required float y; + } + } + F64x2 { + optional group vector { + required double x; + required double y; + } + } + F32x3 { + optional group vector { + required float x; + required float y; + required float z; + } + } + F64x3 { + optional group vector { + required double x; + required double y; + required double z; + } + } + } +} + +declare_schema! { + Text { + Text { + optional byte_array text (string); + } + } +} + +declare_schema! { + Boolean { + Boolean { + optional boolean bool; + } + } +} + +declare_schema! { + Color { + Rgba8 { + optional group color { + required int32 r (integer(8, false)); + required int32 g (integer(8, false)); + required int32 b (integer(8, false)); + required int32 a (integer(8, false)); + } + } + } +} + +#[cfg(test)] +pub(crate) fn dump_parquet_schemas() { + use std::{ + fs::{create_dir_all, OpenOptions}, + io::Write, + path::Path, + }; + + use parquet::schema::{printer::print_schema, types::Type}; + + fn schema_string(ty: &Type) -> String { + let mut buf = Vec::new(); + print_schema(&mut buf, ty); + String::from_utf8_lossy(&buf).trim_end().to_owned() + } + + let items = [ + ("Scalar.txt", Scalar::matcher().schemas()), + ("Vertex.txt", Vertex::matcher().schemas()), + ("Segment.txt", Segment::matcher().schemas()), + ("Triangle.txt", Triangle::matcher().schemas()), + ("Name.txt", Name::matcher().schemas()), + ("Gradient.txt", Gradient::matcher().schemas()), + ("Texcoord.txt", Texcoord::matcher().schemas()), + ("Boundary.txt", Boundary::matcher().schemas()), + ("RegularSubblock.txt", RegularSubblock::matcher().schemas()), + ( + "FreeformSubblock.txt", + FreeformSubblock::matcher().schemas(), + ), + ("Number.txt", Number::matcher().schemas()), + ("Index.txt", Index::matcher().schemas()), + ("Vector.txt", Vector::matcher().schemas()), + ("Text.txt", Text::matcher().schemas()), + ("Boolean.txt", Boolean::matcher().schemas()), + ("Color.txt", Color::matcher().schemas()), + ]; + let base_dir = Path::new("docs/parquet"); + create_dir_all(base_dir).unwrap(); + for (name, schemas) in items { + let path = base_dir.join(name); + let mut f = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(path) + .unwrap(); + let s = schemas + .into_iter() + .map(schema_string) + .collect::>() + .join("\n\n"); + f.write_all(s.as_bytes()).unwrap(); + } +} diff --git a/src/file/parquet/writer.rs b/src/file/parquet/writer.rs new file mode 100644 index 0000000..ef08279 --- /dev/null +++ b/src/file/parquet/writer.rs @@ -0,0 +1,319 @@ +use crate::{ + array_type, + data::{write_checks::*, *}, + error::Error, + file::zip_container::FileType, + pqarray::{PqArrayWriter, PqWriteOptions}, + Array, ArrayType, +}; + +use super::super::{Compression, Writer}; + +impl From for PqWriteOptions { + fn from(value: Compression) -> Self { + Self { + compression_level: value.level(), + ..Default::default() + } + } +} + +impl Writer { + fn array_writer<'a>(&self) -> PqArrayWriter<'a> { + PqArrayWriter::new(self.compression().into()) + } + + fn array_write(&mut self, writer: PqArrayWriter) -> Result, Error> { + let f = self.builder.open(FileType::Parquet)?; + let name = f.name().to_owned(); + let length = writer.write(f)?; + Ok(Array::new(name, length)) + } + + /// Write an [`array_type::Scalar`](crate::array_type::Scalar) array. + /// + /// Values can be `f32` or `f64`. + pub fn array_scalars(&mut self, data: I) -> Result, Error> + where + I: IntoIterator, + T: FloatType, + { + let mut min = MinimumScalar::new(); + let mut writer = self.array_writer(); + writer.add("scalar", data.into_iter().map(|v| min.visit(v)))?; + Ok(self.array_write(writer)?.add_write_checks(min.get())) + } + + /// Write an [`array_type::Vertex`](crate::array_type::Vertex) array. + pub fn array_vertices(&mut self, data: I) -> Result, Error> + where + I: IntoIterator, + T: FloatType, + { + let mut writer = self.array_writer(); + writer.add_multiple(&["x", "y", "z"], data)?; + self.array_write(writer) + } + + /// Write an [`array_type::Segment`](crate::array_type::Segment) array. + /// + /// Values can be `[u8; 2]`, `[u16; 2]`, or `[u32; 2]` and all indices must be less than the + /// number of vertices. + pub fn array_segments(&mut self, data: I) -> Result, Error> + where + I: IntoIterator, + { + let mut max = MaximumIndex::new(); + let mut writer = self.array_writer(); + writer.add_multiple(&["a", "b"], data.into_iter().map(|v| max.visit_array(v)))?; + Ok(self.array_write(writer)?.add_write_checks(max.get())) + } + + /// Write an [`array_type::Triangle`](crate::array_type::Triangle) array. + /// + /// Values can be `[u8; 3]`, `[u16; 3]`, or `[u32; 3]` and all indices must be less than the + /// number of vertices. + pub fn array_triangles(&mut self, data: I) -> Result, Error> + where + I: IntoIterator, + { + let mut max = MaximumIndex::new(); + let mut writer = self.array_writer(); + writer.add_multiple( + &["a", "b", "c"], + data.into_iter().map(|v| max.visit_array(v)), + )?; + Ok(self.array_write(writer)?.add_write_checks(max.get())) + } + + /// Write an [`array_type::Name`](crate::array_type::Name) array. + pub fn array_names(&mut self, data: I) -> Result, Error> + where + I: IntoIterator, + { + let mut writer = self.array_writer(); + writer.add("name", data)?; + self.array_write(writer) + } + + /// Write an [`array_type::Gradient`](crate::array_type::Gradient) array. + /// + /// Values are `[u8; 4]` with channels in RGBA color. + pub fn array_gradient(&mut self, data: I) -> Result, Error> + where + I: IntoIterator, + { + let mut writer = self.array_writer(); + writer.add_multiple(&["r", "g", "b", "a"], data.into_iter())?; + self.array_write(writer) + } + + /// Write an [`array_type::Texcoord`](crate::array_type::Texcoord) array. + /// + /// Values can be either `[f32; 2]` or `[f64; 2]` containing normalized texture coordinates. + pub fn array_texcoords(&mut self, data: I) -> Result, Error> + where + I: IntoIterator, + T: FloatType, + { + let mut writer = self.array_writer(); + writer.add_multiple(&["u", "v"], data)?; + self.array_write(writer) + } + + /// Write an [`array_type::Boundary`](crate::array_type::Boundary) array. + /// + /// The boundary value type `T` can be `f64`, `i64`, `chrono::NaiveDate`, or `chrono::DateTime`. + pub fn array_boundaries(&mut self, data: I) -> Result, Error> + where + I: IntoIterator>, + T: NumberType, + { + let mut increasing = IncreasingBoundary::new(); + let mut writer = self.array_writer(); + writer.add_multiple( + &["value", "inclusive"], + data.into_iter() + .map(|b| (increasing.visit(b.value()), b.is_inclusive())), + )?; + Ok(self.array_write(writer)?.add_write_checks(increasing.get())) + } + + /// Write an [`array_type::RegularSubblock`](crate::array_type::RegularSubblock) array. + /// + /// The `parent_indices` and `corners` iterators must be the same length. Each row is + /// `[parent_i, parent_j, parent_k, min_corner_i, min_corner_j, min_corner_k, + /// max_corner_i, max_corner_j, max_corner_k]`. The parent and corner indices can + /// be different types. + /// + /// Parent indices can be `[u8; 3]`, `[u16; 3]`, or `[u32; 3]`. Sub-block corners can + /// separately be `[u8; 6]`, `[u16; 6]`, or `[u32; 6]` which each row storing + /// $(u_{min}, v_{min}, w_{min}, u_{max}, v_{max}, w_{max})$ as indices into the regular + /// grid within the parent block. + pub fn array_regular_subblocks( + &mut self, + data: I, + ) -> Result, Error> + where + I: IntoIterator, + { + let mut parents = ParentIndices::new(); + let mut corners = RegularCorners::new(); + let mut writer = self.array_writer(); + writer.add_multiple( + &[ + "parent_u", + "parent_v", + "parent_w", + "corner_min_u", + "corner_min_v", + "corner_min_w", + "corner_max_u", + "corner_max_v", + "corner_max_w", + ], + data.into_iter().map(|(p, c)| { + parents.visit(p); + corners.visit(c); + (p[0], p[1], p[2], c[0], c[1], c[2], c[3], c[4], c[5]) + }), + )?; + Ok(self + .array_write(writer)? + .add_write_checks(parents.get()) + .add_write_checks(corners.get())) + } + + /// Write an [`array_type::FreeformSubblock`](crate::array_type::FreeformSubblock) array. + /// + /// The `parent_indices` and `corners` iterators must be the same length. Each row is + /// `[parent_i, parent_j, parent_k, min_corner_x, min_corner_y, min_corner_z, + /// max_corner_x, max_corner_y, max_corner_z]`. + /// + /// Parent indices can be `[u8; 3]`, `[u16; 3]`, or `[u32; 3]`. Sub-block corners can be + /// `[f32; 6]` or `[f64; 6]` which each row storing + /// $(u_{min}, v_{min}, w_{min}, u_{max}, v_{max}, w_{max})$ in the range [0, 1] relative + /// to the parent block. + pub fn array_freeform_subblocks( + &mut self, + data: I, + ) -> Result, Error> + where + I: IntoIterator, + C: FloatType, + { + let mut corner = FreeformCorners::new(); + let mut parent = ParentIndices::new(); + let mut writer = self.array_writer(); + writer.add_multiple( + &[ + "parent_u", + "parent_v", + "parent_w", + "corner_min_u", + "corner_min_v", + "corner_min_w", + "corner_max_u", + "corner_max_v", + "corner_max_w", + ], + data.into_iter().map(|(p, c)| { + parent.visit(p); + corner.visit(c); + (p[0], p[1], p[2], c[0], c[1], c[2], c[3], c[4], c[5]) + }), + )?; + Ok(self + .array_write(writer)? + .add_write_checks(parent.get()) + .add_write_checks(corner.get())) + } + + /// Write an [`array_type::Number`](crate::array_type::Number) array. + /// + /// Values are `Option` where `T` can be `f32`, `f64`, `i32`, `i64`, `chrono::NaiveDate`, + /// or `chrono::DateTime`. Use `None` to represent null values rather than NaN or + /// any flag values like −9999. + pub fn array_numbers(&mut self, data: I) -> Result, Error> + where + I: IntoIterator>, + T: NumberType, + { + let mut writer = self.array_writer(); + writer.add_nullable("number", data)?; + self.array_write(writer) + } + + /// Write an [`array_type::Index`](crate::array_type::Index) array. + /// + /// Values are `Option` where `T` can be `Option`, `Option`, or `Option`. + /// Smaller types won't compress much better but will let other applications allocate less + /// memory when reading the array. Use `None` to represent null values. + pub fn array_indices(&mut self, data: I) -> Result, Error> + where + I: IntoIterator>, + { + let mut max = MaximumIndex::new(); + let mut writer = self.array_writer(); + writer.add_nullable("index", data.into_iter().map(|v| max.visit_opt(v)))?; + Ok(self.array_write(writer)?.add_write_checks(max.get())) + } + + /// Write a [`array_type::Vector`](crate::array_type::Vector) array. + /// + /// Values are `Option` where `T` can be `[f32; 2]`, `[f64; 2]`, `[f32; 3]`, or `[f64; 3]`. + pub fn array_vectors(&mut self, data: I) -> Result, Error> + where + I: IntoIterator>, + V: VectorSource, + T: FloatType, + { + let mut writer = self.array_writer(); + if V::IS_3D { + writer.add_nullable_group( + "vector", + &["x", "y", "z"], + data.into_iter().map(|o| o.map(V::into_3d)), + )?; + } else { + writer.add_nullable_group( + "vector", + &["x", "y"], + data.into_iter().map(|o| o.map(V::into_2d)), + )?; + } + self.array_write(writer) + } + + /// Write a [`array_type::Text`](crate::array_type::Text) array. + pub fn array_text(&mut self, data: I) -> Result, Error> + where + I: IntoIterator>, + { + let mut writer = self.array_writer(); + writer.add_nullable("text", data)?; + self.array_write(writer) + } + + /// Write a [`array_type::Boolean`](crate::array_type::Boolean) array. + pub fn array_booleans(&mut self, data: I) -> Result, Error> + where + I: IntoIterator>, + { + let mut writer = self.array_writer(); + writer.add_nullable("bool", data)?; + self.array_write(writer) + } + + /// Write a [`array_type::Color`](crate::array_type::Color) array. + /// + /// Values are `Option<[u8; 4]>` with channels in RGBA order. + pub fn array_colors(&mut self, data: I) -> Result, Error> + where + I: IntoIterator>, + { + let mut writer = self.array_writer(); + writer.add_nullable_group("color", &["r", "g", "b", "a"], data.into_iter())?; + self.array_write(writer) + } +} diff --git a/src/file/reader.rs b/src/file/reader.rs new file mode 100644 index 0000000..043fb9b --- /dev/null +++ b/src/file/reader.rs @@ -0,0 +1,221 @@ +use std::{ + fs::File, + io::{BufReader, Read}, + path::Path, +}; + +use flate2::read::GzDecoder; + +use crate::{ + array, + error::{Error, Limit}, + validate::{Problems, Validate, Validator}, + Project, FORMAT_VERSION_MAJOR, FORMAT_VERSION_MINOR, FORMAT_VERSION_PRERELEASE, +}; + +use super::{ + zip_container::{Archive, INDEX_NAME}, + SubFile, +}; + +pub const DEFAULT_VALIDATION_LIMIT: u32 = 100; +pub const DEFAULT_JSON_LIMIT: u64 = 1024 * 1024; +#[cfg(target_pointer_width = "32")] +pub const DEFAULT_MEMORY_LIMIT: u64 = 1024 * 1024 * 1024; +#[cfg(not(target_pointer_width = "32"))] +pub const DEFAULT_MEMORY_LIMIT: u64 = 16 * 1024 * 1024 * 1024; + +/// Memory limits for reading OMF files. +#[derive(Debug, Clone, Copy)] +pub struct Limits { + /// Maximum uncompressed size for the JSON index. + /// + /// Default is 1 MB. + pub json_bytes: Option, + /// Maximum uncompressed image size. + /// + /// Default is 1 GB on 32-bit systems or 16 GB on 64-bit systems. + pub image_bytes: Option, + /// Maximum image width or height, default unlimited. + pub image_dim: Option, + /// Maximum number of validation errors. + /// + /// Errors beyond this limit will be discarded. Default is 100. + pub validation: Option, +} + +impl Limits { + /// Creates an object with no limits set. + /// + /// Running without limits is not recommended. + pub fn no_limits() -> Self { + Self { + json_bytes: None, + image_bytes: None, + image_dim: None, + validation: None, + } + } +} + +impl Default for Limits { + /// The default limits. + fn default() -> Self { + Self { + json_bytes: Some(DEFAULT_JSON_LIMIT), + image_bytes: Some(DEFAULT_MEMORY_LIMIT), + image_dim: None, + validation: Some(DEFAULT_VALIDATION_LIMIT), + } + } +} + +/// OMF reader object. +/// +/// Typical usage pattern is: +/// +/// 1. Create the reader object. +/// 1. Optional: retrieve the file version with `reader.version()`. +/// 1. Optional: adjust the limits with `reader.set_limits(...)`. +/// 1. Read the project from the file with `reader.project()`. +/// 1. Iterate through the project's contents to find the elements and attributes you want to load. +/// 1. For each of those items load the array or image data. +/// +/// > **Warning:** +/// > When loading arrays and images from OMF files, beware of "zip bombs" +/// > where data is maliciously crafted to expand to an excessive size when decompressed, +/// > leading to a potential denial of service attack. +/// > Use the limits provided check arrays sizes before allocating memory. + +pub struct Reader { + archive: Archive, + version: [u32; 2], + limits: Limits, +} + +impl Reader { + /// Creates the reader from a `SeekRead` implementation. + /// + /// Makes only the minimum number of reads to check the file header and footer. + /// Fails with an error if an IO error occurs or the file isn't in OMF 2 format. + pub fn new(file: File) -> Result { + let archive = Archive::new(file)?; + let (version, pre_release) = archive.version(); + if let Some(pre) = pre_release { + if Some(pre) != FORMAT_VERSION_PRERELEASE { + return Err(Error::PreReleaseVersion(version[0], version[1], pre.into())); + } + } + if version > [FORMAT_VERSION_MAJOR, FORMAT_VERSION_MINOR] { + return Err(Error::NewerVersion(version[0], version[1])); + } + Ok(Self { + archive, + version, + limits: Default::default(), + }) + } + + /// Creates a reader by opening the given path. + pub fn open(path: impl AsRef) -> Result { + Self::new(File::open(path)?) + } + + /// Returns the current limits. + pub fn limits(&self) -> Limits { + self.limits + } + + /// Sets the memory limits. + /// + /// These limits prevent the reader from consuming excessive system resources, which might + /// allow denial of service attacks with maliciously crafted files. Running without limits + /// is not recommended. + pub fn set_limits(&mut self, limits: Limits) { + self.limits = limits; + } + + /// Return the version number of the file, which can only be `[2, 0]` right now. + pub fn version(&self) -> [u32; 2] { + self.version + } + + /// Reads, validates, and returns the root `Project` object from the file. + /// + /// Fails with an error if an IO error occurs, the `json_bytes` limit is exceeded, or validation + /// fails. Validation warnings are returned alongside the project if successful or included + /// with the errors if not. + pub fn project(&self) -> Result<(Project, Problems), Error> { + let mut project: Project = serde_json::from_reader(BufReader::new(LimitedRead::new( + GzDecoder::new(self.archive.open(INDEX_NAME)?), + self.limits().json_bytes.unwrap_or(u64::MAX), + ))) + .map_err(Error::DeserializationFailed)?; + let mut val = Validator::new() + .with_filenames(self.archive.filenames()) + .with_limit(self.limits().validation); + project.validate_inner(&mut val); + let warnings = val.finish().into_result()?; + Ok((project, warnings)) + } + + /// Returns the size in bytes of the compressed array. + pub fn array_compressed_size( + &self, + array: &array::Array, + ) -> Result { + Ok(self.archive.span(array.filename())?.size) + } + + /// Returns a sub-file for reading raw bytes from the file. + /// + /// Fails with an error if the range is invalid. The contents are not checked or validated by + /// this method. The caller must ensure they are valid and safe to use. This function doesn't + /// check against any limit. + pub fn array_bytes_reader( + &self, + array: &array::Array, + ) -> Result, Error> { + array.constraint(); // Check that validation has been done. + self.archive.open(array.filename()) + } + + /// Return the compressed bytes of an array. + /// + /// The will allocate memory to store the result. Call `array_compressed_size` to find out how + /// much will be allocated. + pub fn array_bytes( + &self, + array: &array::Array, + ) -> Result, Error> { + let mut buf = Vec::new(); + self.array_bytes_reader(array)?.read_to_end(&mut buf)?; + Ok(buf) + } +} + +struct LimitedRead { + inner: R, + limit: u64, +} + +impl LimitedRead { + fn new(inner: R, limit: u64) -> Self { + Self { inner, limit } + } +} + +impl Read for LimitedRead { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.inner.read(buf)?; + self.limit = self.limit.saturating_sub(n as u64); + if self.limit == 0 { + Err(std::io::Error::new( + std::io::ErrorKind::Other, + Error::LimitExceeded(Limit::JsonBytes), + )) + } else { + Ok(n) + } + } +} diff --git a/src/file/sub_file.rs b/src/file/sub_file.rs new file mode 100644 index 0000000..3076fc3 --- /dev/null +++ b/src/file/sub_file.rs @@ -0,0 +1,125 @@ +use std::io::{Read, Seek, SeekFrom}; + +/// A seek-able sub-file with a start and end point within a larger file. +pub struct SubFile { + inner: T, + offset: u64, + position: u64, + limit: u64, +} + +impl SubFile { + /// Creates a sub-file from seek-able object. + /// + /// This new file will its start and zero position at the current position of `inner` and + /// extend up to `limit` bytes. + pub fn new(mut inner: T, limit: u64) -> std::io::Result { + Ok(Self { + position: 0, + offset: inner.stream_position()?, + inner, + limit, + }) + } + + /// Returns the total length of the sub-file, ignoring the current position. + pub fn len(&self) -> u64 { + self.limit + } + + /// Returns true if the file is empty. + pub fn is_empty(&self) -> bool { + self.limit == 0 + } +} + +impl Read for SubFile { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if self.position == self.limit { + return Ok(0); + } + let max = (buf.len() as u64).min(self.limit - self.position) as usize; + let n = self.inner.read(&mut buf[..max])?; + assert!( + self.position + (n as u64) <= self.limit, + "number of read bytes exceeds limit" + ); + self.position += n as u64; + Ok(n) + } +} + +impl Seek for SubFile { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + let new_position = match pos { + SeekFrom::Start(pos) => pos as i64, + SeekFrom::End(delta) => self.limit as i64 + delta, + SeekFrom::Current(delta) => self.position as i64 + delta, + }; + if new_position < 0 { + return Err(std::io::ErrorKind::InvalidInput.into()); + } + self.position = new_position as u64; + self.inner + .seek(SeekFrom::Start(self.offset + self.position))?; + Ok(self.position) + } + + fn stream_position(&mut self) -> std::io::Result { + Ok(self.position) + } +} + +#[cfg(feature = "parquet")] +impl parquet::file::reader::Length for SubFile { + fn len(&self) -> u64 { + self.limit + } +} + +#[cfg(feature = "parquet")] +impl parquet::file::reader::ChunkReader for SubFile { + type T = ::T; + + fn get_read(&self, start: u64) -> parquet::errors::Result { + self.inner.get_read(self.offset.saturating_add(start)) + } + + fn get_bytes(&self, start: u64, length: usize) -> parquet::errors::Result { + self.inner + .get_bytes(self.offset.saturating_add(start), length) + } +} + +#[cfg(test)] +mod tests { + use std::io::Cursor; + + use super::*; + + #[test] + fn subfile() { + let data = b"0123456789"; + let mut base = Cursor::new(data); + base.seek(SeekFrom::Start(2)).unwrap(); + let mut t = SubFile::new(base, 6).unwrap(); + let mut buf = [0; 5]; + t.read_exact(&mut buf).unwrap(); + assert_eq!(&buf, b"23456"); + let mut buf = [0; 2]; + t.seek(SeekFrom::Current(-2)).unwrap(); + t.read_exact(&mut buf).unwrap(); + assert_eq!(&buf, b"56"); + t.seek(SeekFrom::Current(-3)).unwrap(); + t.read_exact(&mut buf).unwrap(); + assert_eq!(&buf, b"45"); + t.seek(SeekFrom::Start(0)).unwrap(); + t.read_exact(&mut buf).unwrap(); + assert_eq!(&buf, b"23"); + let mut buf = [0; 10]; + let e = t.read_exact(&mut buf).unwrap_err(); + assert_eq!(e.kind(), std::io::ErrorKind::UnexpectedEof); + let e = t.seek(SeekFrom::End(-10)).unwrap_err(); + assert_eq!(e.kind(), std::io::ErrorKind::InvalidInput); + } +} diff --git a/src/file/writer.rs b/src/file/writer.rs new file mode 100644 index 0000000..56cf89b --- /dev/null +++ b/src/file/writer.rs @@ -0,0 +1,215 @@ +use std::{ + fmt::Debug, + fs::{File, OpenOptions}, + io::{Read, Write}, + path::Path, +}; + +use flate2::write::GzEncoder; + +use crate::{ + array::DataType, + array_type, + error::Error, + file::zip_container::FileType, + validate::{Problems, Validate, Validator}, + Array, ArrayType, Project, FORMAT_VERSION_MAJOR, FORMAT_VERSION_MINOR, + FORMAT_VERSION_PRERELEASE, +}; + +use super::zip_container::Builder; + +/// Compression level to use. Applies to Parquet and JSON data in the OMF file. +#[derive(Debug, Clone, Copy)] +pub struct Compression(u32); + +impl Compression { + const MINIMUM: u32 = 0; + const MAXIMUM: u32 = 9; + + /// Create a compression level, clamped to the range `0..=9`. + pub fn new(level: u32) -> Self { + Self(level.clamp(Self::MINIMUM, Self::MAXIMUM)) + } + + /// No compression. + pub const fn none() -> Self { + Self(0) + } + + /// Compress as fast as possible at the cost of file size. + pub const fn fast() -> Self { + Self(1) + } + + /// Take as long as necessary to compress as small as possible. + pub const fn best() -> Self { + Self(9) + } + + /// Returns the compression level. + pub const fn level(&self) -> u32 { + self.0 + } +} + +impl Default for Compression { + /// The default compression level, a balance between speed and file size. + fn default() -> Self { + Self(6) + } +} + +impl From for flate2::Compression { + fn from(value: Compression) -> Self { + Self::new(value.level()) + } +} + +/// OMF writer object. +/// +/// To use the writer: +/// +/// 1. Create the writer object. +/// 1. Create an empty [`Project`] and fill in the details. +/// 1. For each element you want to store: +/// 1. Write the arrays and image with the writer. +/// 1. Fill in the required struct with the array pointers and other details then add it to the project. +/// 1. Repeat for the attributes, adding them to the newly created element. +/// 1. Call `writer.finish(project)` to validate everything inside the the project and write it. +pub struct Writer { + pub(crate) builder: Builder, + compression: Compression, +} + +impl Writer { + /// Creates a writer that writes into a file-like object. + pub fn new(file: File) -> Result { + Ok(Self { + builder: Builder::new(file)?, + compression: Default::default(), + }) + } + + /// Creates a writer by opening a file. + /// + /// The file will be created if it doesn't exist, and truncated and replaced if it does. + pub fn open(path: impl AsRef) -> Result { + Self::new( + OpenOptions::new() + .write(true) + .truncate(true) + .create(true) + .open(path)?, + ) + } + + /// Return the current compression. + pub fn compression(&self) -> Compression { + self.compression + } + + /// Set the compression to use. + /// + /// This affects Parquet data and the JSON index, but not images. + /// The default is `Compression::default()`. + pub fn set_compression(&mut self, compression: Compression) { + self.compression = compression; + } + + /// Write an array from already-encoded bytes. + /// + /// Returns the new [`Array`](crate::Array) on success or an error if file IO fails. + pub fn array_bytes( + &mut self, + length: u64, + bytes: &[u8], + ) -> Result, Error> { + let file_type = check_header::(bytes)?; + let mut f = self.builder.open(file_type)?; + let name = f.name().to_owned(); + f.write_all(bytes)?; + Ok(Array::new(name, length)) + } + + /// Consumes everything from `read` and writes it as a new array. + /// + /// The bytes must already be encoded in Parquet, PNG, or JPEG depending on the array type. + /// Returns the new [`Array`](crate::Array) on success or an error if file IO fails on either + /// side. + pub fn array_bytes_from( + &mut self, + length: u64, + mut read: impl Read, + ) -> Result, Error> { + let mut header = [0_u8; 8]; + read.read_exact(&mut header)?; + let file_type = check_header::(&header)?; + let mut f = self.builder.open(file_type)?; + let name = f.name().to_owned(); + f.write_all(&header)?; + let mut buffer = vec![0_u8; 4096]; + loop { + let n = read.read(&mut buffer)?; + if n == 0 { + break; + } + f.write_all(&buffer)?; + } + Ok(Array::new(name, length)) + } + + /// Write an existing PNG or JPEG image from a slice without re-encoding it. + pub fn image_bytes(&mut self, bytes: &[u8]) -> Result, Error> { + self.array_bytes(0, bytes) + } + + /// Write an existing PNG or JPEG image from a file without re-encoding it. + pub fn image_bytes_from(&mut self, read: impl Read) -> Result, Error> { + self.array_bytes_from(0, read) + } + + /// Validate and write the project and close the file. + /// + /// Returns validation warnings on success or an [`Error`] on failure, which can be a + /// validation failure or a file IO error. + pub fn finish(mut self, mut project: Project) -> Result<(File, Problems), Error> { + let mut val = Validator::new().with_filenames(self.builder.filenames()); + project.validate_inner(&mut val); + let warnings = val.finish().into_result()?; + let gz = GzEncoder::new(self.builder.open(FileType::Index)?, self.compression.into()); + serde_json::to_writer(gz, &project).map_err(Error::SerializationFailed)?; + // In the future we could base the format version on the data, writing backward + // compatible files if new features weren't used. + let file = self.builder.finish( + FORMAT_VERSION_MAJOR, + FORMAT_VERSION_MINOR, + FORMAT_VERSION_PRERELEASE, + )?; + Ok((file, warnings)) + } +} + +fn check_header(bytes: &[u8]) -> Result { + const PNG_MAGIC: &[u8] = &[0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]; + const JPEG_MAGIC: &[u8] = &[0xFF, 0xD8, 0xFF]; + const PARQUET_MAGIC: &[u8] = b"PAR1"; + match A::DATA_TYPE { + DataType::Image => { + if bytes.starts_with(PNG_MAGIC) { + Ok(FileType::Png) + } else if bytes.starts_with(JPEG_MAGIC) { + Ok(FileType::Jpeg) + } else { + Err(Error::NotImageData) + } + } + _ => { + if !bytes.starts_with(PARQUET_MAGIC) || !bytes.ends_with(PARQUET_MAGIC) { + Err(Error::NotParquetData) + } else { + Ok(FileType::Parquet) + } + } + } +} diff --git a/src/file/zip_container.rs b/src/file/zip_container.rs new file mode 100644 index 0000000..6c93abc --- /dev/null +++ b/src/file/zip_container.rs @@ -0,0 +1,222 @@ +use std::{ + collections::HashMap, + fs::File, + io::{Seek, SeekFrom}, +}; + +use zip::{ + read::{ZipArchive, ZipFile}, + write::{FullFileOptions, ZipWriter}, +}; + +use crate::{error::Error, FORMAT_NAME}; + +use super::SubFile; + +pub(crate) const INDEX_NAME: &str = "index.json.gz"; +pub(crate) const PARQUET_EXT: &str = ".parquet"; +pub(crate) const PNG_EXT: &str = ".png"; +pub(crate) const JPEG_EXT: &str = ".jpg"; + +pub(crate) enum FileType { + Index, + Parquet, + Png, + Jpeg, +} + +pub(crate) struct Builder { + zip_writer: ZipWriter, + next_id: u64, + filenames: Vec, +} + +impl Builder { + pub fn new(file: File) -> Result { + Ok(Self { + zip_writer: ZipWriter::new(file), + next_id: 1, + filenames: Vec::new(), + }) + } + + fn id(&mut self) -> u64 { + let i = self.next_id; + self.next_id += 1; + i + } + + pub fn open(&mut self, file_type: FileType) -> Result, Error> { + let name = match file_type { + FileType::Index => INDEX_NAME.to_owned(), + FileType::Parquet => format!("{}{PARQUET_EXT}", self.id()), + FileType::Png => format!("{}{PNG_EXT}", self.id()), + FileType::Jpeg => format!("{}{JPEG_EXT}", self.id()), + }; + self.zip_writer.start_file( + name.clone(), + FullFileOptions::default() + .large_file(true) + .compression_method(zip::CompressionMethod::Stored), + )?; + self.filenames.push(name.clone()); + Ok(SubFileWrite { + name, + inner: &mut self.zip_writer, + }) + } + + pub fn filenames(&self) -> impl Iterator { + self.filenames.iter().map(|s| &**s) + } + + pub fn finish( + mut self, + major: u32, + minor: u32, + pre_release: Option<&str>, + ) -> Result { + use std::fmt::Write; + let mut comment = format!("{FORMAT_NAME} {major}.{minor}"); + if let Some(pre) = pre_release { + _ = write!(&mut comment, "-{pre}"); + } + self.zip_writer.set_comment(comment); + Ok(self.zip_writer.finish()?) + } +} + +pub(crate) struct SubFileWrite<'a> { + name: String, + inner: &'a mut ZipWriter, +} + +impl<'a> SubFileWrite<'a> { + pub fn name(&self) -> &str { + &self.name + } +} + +impl<'a> std::io::Write for SubFileWrite<'a> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.inner.write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + +#[derive(Debug, Default, Clone, Copy)] +pub(crate) struct FileSpan { + pub offset: u64, + pub size: u64, +} + +impl<'a> From> for FileSpan { + fn from(f: ZipFile<'a>) -> Self { + Self { + offset: f.data_start(), + size: f.compressed_size(), + } + } +} + +pub(crate) struct Archive { + file: File, + members: HashMap, + version: [u32; 2], + pre_release: Option, +} + +impl Archive { + pub fn new(file: File) -> Result { + // TODO: get the version from the zip comment. + let mut zip_archive = ZipArchive::new(file)?; + let mut members = HashMap::new(); + let mut index_found = false; + for i in 0..zip_archive.len() { + let f = zip_archive.by_index_raw(i)?; + if f.compression() != zip::CompressionMethod::Stored { + return Err(Error::ZipError("members may not be compressed".into())); + } + index_found = index_found || f.name() == INDEX_NAME; + members.insert(f.name().into(), f.into()); + } + if !index_found { + return Err(Error::ZipMemberMissing(INDEX_NAME.to_owned())); + } + let Some((version, pre_release)) = get_version(zip_archive.comment()) else { + return Err(Error::NotOmf( + String::from_utf8_lossy(zip_archive.comment()).into_owned(), + )); + }; + Ok(Self { + file: zip_archive.into_inner(), + members, + version, + pre_release, + }) + } + + pub fn version(&self) -> ([u32; 2], Option<&str>) { + (self.version, self.pre_release.as_deref()) + } + + pub fn filenames(&self) -> impl Iterator { + self.members.keys().map(|s| &**s) + } + + pub fn span(&self, name: &str) -> Result { + self.members + .get(name) + .ok_or_else(|| Error::ZipMemberMissing(name.to_owned())) + .copied() + } + + pub fn open(&self, name: &str) -> Result, Error> { + let span = self + .members + .get(name) + .ok_or_else(|| Error::ZipMemberMissing(name.to_owned()))?; + let mut f = self.file.try_clone()?; + f.seek(SeekFrom::Start(span.offset))?; + Ok(SubFile::new(f, span.size)?) + } +} + +fn get_version(comment_bytes: &[u8]) -> Option<([u32; 2], Option)> { + let comment = std::str::from_utf8(comment_bytes).ok()?; + let mut dash_parts = comment + .strip_prefix(FORMAT_NAME)? + .strip_prefix(' ')? + .split('-'); + let main = dash_parts.next()?; + let pre_release = dash_parts.next().map(ToOwned::to_owned); + let mut version_parts = main.split('.'); + let major = version_parts.next()?.parse().ok()?; + let minor = version_parts.next()?.parse().ok()?; + if version_parts.next().is_some() { + return None; + } + Some(([major, minor], pre_release)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn versions() { + assert_eq!( + get_version("Open Mining Format 2.0".as_bytes()), + Some(([2, 0], None)) + ); + assert_eq!( + get_version("Open Mining Format 2.0-alpha.1".as_bytes()), + Some(([2, 0], Some("alpha.1".to_string()))) + ); + assert_eq!(get_version("Something else 1.0".as_bytes()), None); + assert_eq!(get_version(b"Something not UTF-8 \xff"), None); + } +} diff --git a/src/geometry.rs b/src/geometry.rs new file mode 100644 index 0000000..27c0572 --- /dev/null +++ b/src/geometry.rs @@ -0,0 +1,370 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::{ + array::Constraint, + array_type, + validate::{Validate, Validator}, + Array, BlockModel, Element, Grid2, Location, Orient2, Vector3, +}; + +pub(crate) fn zero_origin(v: &Vector3) -> bool { + *v == [0.0, 0.0, 0.0] +} + +/// Selects the type of geometry in an [`Element`](crate::Element) from several options. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type")] +pub enum Geometry { + PointSet(PointSet), + LineSet(LineSet), + Surface(Surface), + GridSurface(GridSurface), + BlockModel(BlockModel), + Composite(Composite), +} + +impl Geometry { + /// Returns the valid locations for attributes on this geometry. + pub fn valid_locations(&self) -> &'static [Location] { + match self { + Self::PointSet(_) => &[Location::Vertices], + Self::LineSet(_) => &[Location::Vertices, Location::Primitives], + Self::Surface(_) => &[Location::Vertices, Location::Primitives], + Self::GridSurface(_) => &[Location::Vertices, Location::Primitives], + Self::Composite(_) => &[Location::Elements], + Self::BlockModel(b) if b.has_subblocks() => { + &[Location::Subblocks, Location::Primitives] + } + Self::BlockModel(_) => &[Location::Primitives], + } + } + + /// Returns the length of the given location, if valid. + pub fn location_len(&self, location: Location) -> Option { + if location == Location::Projected { + Some(0) + } else { + match self { + Self::PointSet(p) => p.location_len(location), + Self::LineSet(l) => l.location_len(location), + Self::Surface(s) => s.location_len(location), + Self::GridSurface(t) => t.location_len(location), + Self::BlockModel(b) => b.location_len(location), + Self::Composite(c) => c.location_len(location), + } + } + } + + pub(crate) fn type_name(&self) -> &'static str { + match self { + Self::PointSet(_) => "PointSet", + Self::LineSet(_) => "LineSet", + Self::Surface(_) => "Surface", + Self::GridSurface(_) => "GridSurface", + Self::Composite(_) => "PoinCompositetSet", + Self::BlockModel(b) if b.has_subblocks() => "BlockModel(sub-blocked)", + Self::BlockModel(_) => "BlockModel", + } + } +} + +impl From for Geometry { + fn from(value: PointSet) -> Self { + Self::PointSet(value) + } +} + +impl From for Geometry { + fn from(value: LineSet) -> Self { + Self::LineSet(value) + } +} + +impl From for Geometry { + fn from(value: Surface) -> Self { + Self::Surface(value) + } +} + +impl From for Geometry { + fn from(value: GridSurface) -> Self { + Self::GridSurface(value) + } +} + +impl From for Geometry { + fn from(value: BlockModel) -> Self { + Self::BlockModel(value) + } +} + +impl From for Geometry { + fn from(value: Composite) -> Self { + Self::Composite(value) + } +} + +/// Point set geometry. +/// +/// ### Attribute Locations +/// +/// - [`Vertices`](crate::Location::Vertices) puts attribute values on the points. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct PointSet { + /// Origin of the points relative to the project coordinate reference system. + #[serde(default, skip_serializing_if = "zero_origin")] + pub origin: Vector3, + /// Array with `Vertex` type storing the vertex locations. + /// + /// Add `origin` and the [project](crate::Project) origin to get the locations relative + /// to the project coordinate reference system. + pub vertices: Array, +} + +impl PointSet { + pub fn new(vertices: Array) -> Self { + Self::with_origin(vertices, Default::default()) + } + + pub fn with_origin(vertices: Array, origin: Vector3) -> Self { + Self { origin, vertices } + } + + pub fn location_len(&self, location: Location) -> Option { + match location { + Location::Vertices => Some(self.vertices.item_count()), + _ => None, + } + } +} + +/// A set of line segments. +/// +/// ### Attribute Locations +/// +/// - [`Vertices`](crate::Location::Vertices) puts attribute values on the vertices. +/// +/// - [`Primitives`](crate::Location::Primitives) puts attribute values on the line segments. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct LineSet { + /// Origin of the lines relative to the project coordinate reference system. + #[serde(default, skip_serializing_if = "zero_origin")] + pub origin: Vector3, + /// Array with `Vertex` type storing the vertex locations. + /// + /// Add `origin` and the [project](crate::Project) origin to get the locations relative + /// to the project coordinate reference system. + pub vertices: Array, + /// Array with `Segment` type storing each segment as a pair of indices into `vertices`. + pub segments: Array, +} + +impl LineSet { + pub fn new(vertices: Array, segments: Array) -> Self { + Self::with_origin(vertices, segments, Default::default()) + } + + pub fn with_origin( + vertices: Array, + segments: Array, + origin: Vector3, + ) -> Self { + Self { + origin, + vertices, + segments, + } + } + + pub fn location_len(&self, location: Location) -> Option { + match location { + Location::Vertices => Some(self.vertices.item_count()), + Location::Primitives => Some(self.segments.item_count()), + _ => None, + } + } +} + +/// A surface made up of triangles. +/// +/// ### Attribute Locations +/// +/// - [`Vertices`](crate::Location::Vertices) puts attribute values on the vertices. +/// +/// - [`Primitives`](crate::Location::Primitives) puts attribute values on the triangles. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Surface { + /// Origin of the surface relative to the project coordinate reference system. + #[serde(default, skip_serializing_if = "zero_origin")] + pub origin: Vector3, + /// Array with `Vertex` type storing the vertex locations. + /// + /// Add `origin` and the [project](crate::Project) origin to get the locations relative + /// to the project coordinate reference system. + pub vertices: Array, + /// Array with `Triangle` type storing each triangle as a triple of indices into `vertices`. + /// Triangle winding should be counter-clockwise around an outward-pointing normal. + pub triangles: Array, +} + +impl Surface { + pub fn new( + vertices: Array, + triangles: Array, + ) -> Self { + Self::with_origin(vertices, triangles, Default::default()) + } + + pub fn with_origin( + vertices: Array, + triangles: Array, + origin: Vector3, + ) -> Self { + Self { + origin, + vertices, + triangles, + } + } + + pub fn location_len(&self, location: Location) -> Option { + match location { + Location::Vertices => Some(self.vertices.item_count()), + Location::Primitives => Some(self.triangles.item_count()), + _ => None, + } + } +} + +/// A surface defined by a 2D grid a height on each grid vertex. +/// +/// ### Attribute Locations +/// +/// - [`Vertices`](crate::Location::Vertices) puts attribute values on the grid vertices. +/// +/// - [`Primitives`](crate::Location::Primitives) puts attribute values on the grid cells. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct GridSurface { + /// Position and orientation of the surface. + pub orient: Orient2, + /// 2D grid definition, which can be regular or tensor. + pub grid: Grid2, + /// Array with `Scalar` type storing the offset of each grid vertex from the place. + /// Heights may be positive or negative. Will be absent from flat 2D grids. + pub heights: Option>, +} + +impl GridSurface { + pub fn new(orient: Orient2, grid: Grid2, heights: Option>) -> Self { + Self { + orient, + grid, + heights, + } + } + + pub fn location_len(&self, location: Location) -> Option { + match location { + Location::Vertices => Some(self.grid.flat_corner_count()), + Location::Primitives => Some(self.grid.flat_count()), + _ => None, + } + } +} + +/// A container for sub-elements. +/// +/// ### Attribute Locations +/// +/// - [`Elements`](crate::Location::Elements) puts attribute values on elements. +#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Composite { + #[serde(default)] + pub elements: Vec, +} + +impl Composite { + pub fn new(elements: Vec) -> Self { + Self { elements } + } + + pub fn location_len(&self, location: Location) -> Option { + match location { + Location::Elements => Some(self.elements.len().try_into().expect("usize fits in u64")), + _ => None, + } + } +} + +impl Validate for Geometry { + fn validate_inner(&mut self, val: &mut Validator) { + let v = val.enter("Geometry"); + match self { + Self::PointSet(x) => v.obj(x), + Self::LineSet(x) => v.obj(x), + Self::Surface(x) => v.obj(x), + Self::GridSurface(x) => v.obj(x), + Self::BlockModel(x) => v.obj(x), + Self::Composite(x) => v.obj(x), + }; + } +} + +impl Validate for PointSet { + fn validate_inner(&mut self, val: &mut Validator) { + val.enter("PointSet") + .finite_seq(self.origin, "origin") + .array(&mut self.vertices, Constraint::Vertex, "vertices"); + } +} + +impl Validate for LineSet { + fn validate_inner(&mut self, val: &mut Validator) { + val.enter("LineSet") + .finite_seq(self.origin, "origin") + .array(&mut self.vertices, Constraint::Vertex, "vertices") + .array( + &mut self.segments, + Constraint::Segment(self.vertices.item_count()), + "segments", + ); + } +} + +impl Validate for Surface { + fn validate_inner(&mut self, val: &mut Validator) { + val.enter("Surface") + .finite_seq(self.origin, "origin") + .array(&mut self.vertices, Constraint::Vertex, "vertices") + .array( + &mut self.triangles, + Constraint::Triangle(self.vertices.item_count()), + "triangles", + ); + } +} + +impl Validate for GridSurface { + fn validate_inner(&mut self, val: &mut Validator) { + val.enter("GridSurface") + .obj(&mut self.orient) + .obj(&mut self.grid) + .array_opt(self.heights.as_mut(), Constraint::Scalar, "heights") + .array_size_opt( + self.heights.as_ref().map(|h| h.item_count()), + self.grid.flat_corner_count(), + "heights", + ); + } +} + +impl Validate for Composite { + fn validate_inner(&mut self, val: &mut Validator) { + val.enter("Composite").objs(&mut self.elements).unique( + self.elements.iter().map(|e| &e.name), + "elements[..]::name", + false, + ); + } +} diff --git a/src/grid.rs b/src/grid.rs new file mode 100644 index 0000000..b91d431 --- /dev/null +++ b/src/grid.rs @@ -0,0 +1,308 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::{ + array::Constraint, + array_type, + validate::{Validate, Validator}, + Array, Vector3, +}; + +/// Defines a 2D grid spacing and size. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type")] +pub enum Grid2 { + /// Regularly spaced cells. + /// + /// ![Diagram of a 2D regular grid](../images/grid2_regular.svg "A 2D regular grid") + Regular { + /// The cell size in the U and V axes. Both must be greater than zero. + size: [f64; 2], + /// The number of cells in the U and V axes. Both must be greater than zero. + count: [u32; 2], + }, + /// Tensor cells, where each row and column can have a different size. + /// + /// ![Diagram of a 3D tensor grid](../images/grid2_tensor.svg "A 2D tensor grid") + Tensor { + /// Array with `Scalar` type storing the size of each cell along the U axis. + /// These sizes must be greater than zero. + u: Array, + /// Array with `Scalar` type storing the size of each cell along the V axis. + /// These sizes must be greater than zero. + v: Array, + }, +} + +impl Grid2 { + /// Create a 2D regular grid from the cell size and count. + pub fn from_size_and_count(size: [f64; 2], count: [u32; 2]) -> Self { + Self::Regular { size, count } + } + + /// Create a 2D tensor grid from the size arrays. + pub fn from_arrays(u: Array, v: Array) -> Self { + Self::Tensor { u, v } + } + + /// Returns the number of cells in each axis. + pub fn count(&self) -> [u32; 2] { + match self { + Self::Regular { count, .. } => *count, + Self::Tensor { u, v } => [u.item_count() as u32, v.item_count() as u32], + } + } + + /// Returns the total number of cells. + pub fn flat_count(&self) -> u64 { + self.count().into_iter().map(u64::from).product() + } + + /// Returns the total number of cell corners. + pub fn flat_corner_count(&self) -> u64 { + self.count().into_iter().map(|n| u64::from(n) + 1).product() + } +} + +impl Default for Grid2 { + /// Creates a regular grid with size `[1.0, 1.0]` and count `[1, 1]`. + fn default() -> Self { + Self::Regular { + size: [1.0, 1.0], + count: [1, 1], + } + } +} + +/// Defines a 3D grid spacing and size. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type")] +#[allow(clippy::large_enum_variant)] +pub enum Grid3 { + /// Regularly spaced cells. + /// + /// ![Diagram of a 3D regular grid](../images/grid3_regular.svg "A 3D regular grid") + Regular { + /// The block size in the U and V axes. All must be greater than zero. + size: Vector3, + /// The number of cells in the U, V, and W axes. All must be greater than zero. + count: [u32; 3], + }, + /// Tensor cells, where each row, column, and layer can have a different size. All sizes + /// must be greater than zero. + /// + /// ![Diagram of a 3D tensor grid](../images/grid3_tensor.svg "A 3D tensor grid") + Tensor { + /// Array with `Scalar` type storing the size of each cell along the U axis. + /// These sizes must be greater than zero. + u: Array, + /// Array with `Scalar` type storing the size of each cell along the V axis. + /// These sizes must be greater than zero. + v: Array, + /// Array with `Scalar` type storing the size of each cell along the W axis. + /// These sizes must be greater than zero. + w: Array, + }, +} + +impl Grid3 { + /// Create a 3D regular grid from the block size and count. + pub fn from_size_and_count(size: Vector3, count: [u32; 3]) -> Self { + Self::Regular { size, count } + } + + /// Create a 3D tensor grid from the size arrays. + pub fn from_arrays( + u: Array, + v: Array, + w: Array, + ) -> Self { + Self::Tensor { u, v, w } + } + + /// Returns the number of blocks in each axis. + pub fn count(&self) -> [u32; 3] { + match self { + Self::Regular { count, .. } => *count, + // validation checks that this cast is valid + Self::Tensor { u, v, w } => [ + u.item_count() as u32, + v.item_count() as u32, + w.item_count() as u32, + ], + } + } + + /// Returns the total number of blocks. + pub fn flat_count(&self) -> u64 { + self.count().iter().map(|n| u64::from(*n)).product() + } + + /// Returns the total number of block corners. + pub fn flat_corner_count(&self) -> u64 { + self.count().iter().map(|n| u64::from(*n) + 1).product() + } +} + +impl Default for Grid3 { + fn default() -> Self { + Self::Regular { + size: [1.0, 1.0, 1.0], + count: [1, 1, 1], + } + } +} + +const fn i() -> Vector3 { + [1.0, 0.0, 0.0] +} + +const fn j() -> Vector3 { + [0.0, 1.0, 0.0] +} + +const fn k() -> Vector3 { + [0.0, 0.0, 1.0] +} + +/// Defines the position and orientation of a 2D plane in 3D space. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, JsonSchema)] +#[repr(C)] +pub struct Orient2 { + /// Origin point relative to the project origin and coordinate reference. + pub origin: Vector3, + /// The direction of the U axis of the plane. Must be a unit vector perpendicular to `v`. + /// Default [1, 0, 0]. + #[serde(default = "i")] + pub u: Vector3, + /// The direction of the V axis of the plane. Must be a unit vector perpendicular to `u`. + /// Default [0, 1, 0]. + #[serde(default = "j")] + pub v: Vector3, +} + +impl Orient2 { + /// Creates a new 2D orientation. + pub fn new(origin: Vector3, u: Vector3, v: Vector3) -> Self { + Self { origin, u, v } + } + + /// Creates a new axis-aligned 2D orientation. + pub fn from_origin(origin: Vector3) -> Self { + Self::new(origin, i(), j()) + } +} + +impl Default for Orient2 { + /// Creates a new axis-aligned 2D orientation at the origin. + fn default() -> Self { + Self { + origin: [0.0; 3], + u: i(), + v: j(), + } + } +} + +/// Defines the position and orientation of a 3D sub-space. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, JsonSchema)] +#[repr(C)] +pub struct Orient3 { + /// Origin point relative to the project origin and coordinate reference. + pub origin: Vector3, + /// The direction of the U axis of the grid. Must be a unit vector perpendicular to + /// `v` and 'w'. Default [1, 0, 0]. + #[serde(default = "i")] + pub u: Vector3, + /// The direction of the V axis of the grid. Must be a unit vector perpendicular to + /// `u` and 'w'. Default [0, 1, 0]. + #[serde(default = "j")] + pub v: Vector3, + /// The direction of the W axis of the grid. Must be a unit vector perpendicular to + /// `u` and 'v'. Default [0, 0, 1]. + #[serde(default = "k")] + pub w: Vector3, +} + +impl Orient3 { + /// Creates a new 3D orientation. + pub fn new(origin: Vector3, u: Vector3, v: Vector3, w: Vector3) -> Self { + Self { origin, u, v, w } + } + + /// Creates a new axis-aligned 3D orientation. + pub fn from_origin(origin: Vector3) -> Self { + Self::new(origin, i(), j(), k()) + } +} + +impl Default for Orient3 { + fn default() -> Self { + Self { + origin: [0.0; 3], + u: i(), + v: j(), + w: k(), + } + } +} + +impl Validate for Grid2 { + fn validate_inner(&mut self, val: &mut Validator) { + match self { + Grid2::Regular { size, count } => { + val.enter("Grid2::Regular") + .finite_seq(*size, "size") + .above_zero_seq(*size, "size") + .above_zero_seq(*count, "count"); + } + Grid2::Tensor { u, v } => { + val.enter("Grid2::Tensor") + .grid_count(&[u.item_count(), v.item_count()]) + .array(u, Constraint::Size, "u") + .array(v, Constraint::Size, "v"); + } + } + } +} + +impl Validate for Grid3 { + fn validate_inner(&mut self, val: &mut Validator) { + match self { + Grid3::Regular { size, count } => { + val.enter("Grid3::Regular") + .finite_seq(*size, "size") + .above_zero_seq(*size, "size") + .above_zero_seq(*count, "count"); + } + Grid3::Tensor { u, v, w } => { + val.enter("Grid3::Tensor") + .grid_count(&[u.item_count(), v.item_count(), w.item_count()]) + .array(u, Constraint::Size, "u") + .array(v, Constraint::Size, "v") + .array(w, Constraint::Size, "w"); + } + } + } +} + +impl Validate for Orient2 { + fn validate_inner(&mut self, val: &mut Validator) { + val.enter("Orient2") + .finite_seq(self.origin, "origin") + .unit_vector(self.u, "u") + .unit_vector(self.v, "v") + .vectors_ortho2(self.u, self.v); + } +} + +impl Validate for Orient3 { + fn validate_inner(&mut self, val: &mut Validator) { + val.enter("Orient3") + .finite_seq(self.origin, "origin") + .unit_vector(self.u, "u") + .unit_vector(self.v, "v") + .unit_vector(self.w, "w") + .vectors_ortho3(self.u, self.v, self.w); + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..aa417f0 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,105 @@ +//! Reader and writer for Open Mining Format version 2, +//! a standard for mining data interchange backed by the +//! [Global Mining Guidelines Group](https://gmggroup.org). +//! +//! > **Warning:** +//! > This is an alpha release of OMF 2. The storage format and libraries might be changed in +//! > backward-incompatible ways and are not subject to any SLA or deprecation policy. +//! > Further, this code is unfinished and may not be secure. +//! > Don't use it to open files you don't trust, and don't use it in production yet. +//! +//! # What is OMF +//! +//! OMF is an open-source serialization format and library to support data interchange +//! across the entire mining community. +//! Its goal is to standardize file formats and promote collaboration. +//! +//! This repository provides a file format specification and a Rust library for reading and writing files, +//! plus wrappers to use that library from C and Python. +//! +//! # Getting Started +//! +//! The [Reader](crate::file::Reader) and [Writer](crate::file::Writer) +//! objects are the starting points for reading and writing files. +//! [Error](crate::error::Error) is the combined error type for everything. +//! [Project] is the root object of the data contained within the file, +//! storing a list of [elements](crate::Element), +//! each containing some [geometry](crate::Geometry) and a list of [attributes](crate::Attribute). +//! +//! Supported element geometries are: +//! +//! - [Points](crate::PointSet). +//! - [Line segments](crate::LineSet). +//! - [Triangulated surfaces](crate::Surface). +//! - [Grid surfaces](crate::GridSurface). +//! - Regular or tensor [grid spacing](crate::Grid2). +//! - Any [orientation](crate::Orient2). +//! - [Block models](crate::BlockModel), with optional [sub-blocks](crate::Subblocks). +//! - Regular or tensor [grid spacing](crate::Grid3). +//! - Any [orientation](crate::Orient3). +//! - Regular sub-blocks that lie on a grid within their parent, with octree or arbitrary layout. +//! - Free-form sub-blocks that don't lie on any grid. +//! - [Composite] elements made out of any of the above. +//! +//! Supported attribute data types are: +//! +//! - [Floating-point or signed integer](crate::AttributeData::Number) values, +//! including date and date-time values. +//! - [Category](crate::AttributeData::Category) values, +//! storing an index used to look up name, color, or other sub-attributes. +//! - [Boolean](crate::AttributeData::Boolean) or filter values. +//! - 2D and 3D [vector](crate::AttributeData::Vector) values. +//! - [Text](crate::AttributeData::Text) values. +//! - [Color](crate::AttributeData::Color) values. +//! - [Projected texture](crate::AttributeData::ProjectedTexture) images. +//! - [UV mapped texture](crate::AttributeData::MappedTexture) images. +//! +//! Attributes values can be valid or null. +//! They can be attached to different [parts](crate::Location) of each element type, +//! such as the vertices vs. faces of a surface, +//! or the parent blocks vs. sub-blocks of a block model. + +#![deny(unsafe_code)] + +mod array; +mod attribute; +mod block_model; +mod colormap; +#[cfg(feature = "parquet")] +pub mod data; +pub mod date_time; +mod element; +pub mod error; +pub mod file; +mod geometry; +mod grid; +#[cfg(feature = "omf1")] +pub mod omf1; +#[cfg(feature = "parquet")] +mod pqarray; +mod project; +mod schema; +#[cfg(test)] +mod schema_doc; +pub mod validate; +mod version; + +pub use array::{array_type, Array, ArrayType, DataType}; +pub use attribute::{Attribute, AttributeData, Location}; +pub use block_model::{BlockModel, SubblockMode, Subblocks}; +pub use colormap::{NumberColormap, NumberRange}; +pub use element::Element; +pub use geometry::{Composite, Geometry, GridSurface, LineSet, PointSet, Surface}; +pub use grid::{Grid2, Grid3, Orient2, Orient3}; +pub use project::Project; +pub use schema::json_schema; +pub use version::{ + crate_full_name, format_full_name, format_version, CRATE_NAME, CRATE_VERSION, FORMAT_EXTENSION, + FORMAT_NAME, FORMAT_VERSION_MAJOR, FORMAT_VERSION_MINOR, FORMAT_VERSION_PRERELEASE, +}; + +/// A 3D vector with `f64` components. +pub type Vector3 = [f64; 3]; + +/// RGBA color with components from 0 to 255. +pub type Color = [u8; 4]; diff --git a/src/omf1/array.rs b/src/omf1/array.rs new file mode 100644 index 0000000..6dd3c44 --- /dev/null +++ b/src/omf1/array.rs @@ -0,0 +1,301 @@ +use crate::{error::Error, file::Writer}; + +use super::{ + objects::{ + Array, DataType, Int2Array, Int3Array, Key, ScalarArray, Vector2Array, Vector3Array, + }, + reader::Omf1Reader, + Omf1Error, +}; + +pub fn vertices_array( + r: &Omf1Reader, + w: &mut Writer, + array: &Key, +) -> Result, Error> { + let mut iter = HoldError::new( + ByteChunks(r.array_decompressed_bytes(&r.model(array)?.array)?) + .map(|r| r.map(vector3_from_le_bytes)), + ); + let array = w.array_vertices(iter.by_ref())?; + iter.finish()?; + Ok(array) +} + +pub fn scalars_array( + r: &Omf1Reader, + w: &mut Writer, + array_key: &Key, +) -> Result, Error> { + let array = &r.model(array_key)?.array; + // The OMF1 code doesn't require scalar arrays to be float, so allow int too. + let is_float = array.dtype == DataType::Float; + let mut iter = items(r, array, move |b| { + if is_float { + Ok(f64::from_le_bytes(b)) + } else { + Ok(i64::from_le_bytes(b) as f64) + } + })?; + let array = w.array_scalars(iter.by_ref())?; + iter.finish()?; + Ok(array) +} + +pub enum ScalarValues { + Float(Vec), + Int(Vec), +} + +pub fn load_scalars(r: &Omf1Reader, array: &ScalarArray) -> Result { + if array.array.dtype == DataType::Float { + let mut iter = items(r, &array.array, |b| Ok(f64::from_le_bytes(b)))?; + let out = iter.by_ref().collect(); + iter.finish()?; + Ok(ScalarValues::Float(out)) + } else { + let mut iter = items(r, &array.array, |b| Ok(i64::from_le_bytes(b)))?; + let out = iter.by_ref().collect(); + iter.finish()?; + Ok(ScalarValues::Int(out)) + } +} + +pub fn numbers_array( + r: &Omf1Reader, + w: &mut Writer, + array_key: &Key, +) -> Result, Error> { + let array = &r.model(array_key)?.array; + if array.dtype == DataType::Float { + let mut iter = items(r, array, |b| Ok(none_if_nan(f64::from_le_bytes(b))))?; + let array = w.array_numbers(iter.by_ref())?; + iter.finish()?; + Ok(array) + } else { + let mut iter = items(r, array, |b| Ok(Some(i64::from_le_bytes(b))))?; + let array = w.array_numbers(iter.by_ref())?; + iter.finish()?; + Ok(array) + } +} + +pub fn segments_array( + r: &Omf1Reader, + w: &mut Writer, + array: &Key, +) -> Result, Error> { + let mut iter = items(r, &r.model(array)?.array, segment_from_le_bytes)?; + let array = w.array_segments(iter.by_ref())?; + iter.finish()?; + Ok(array) +} + +pub fn triangles_array( + r: &Omf1Reader, + w: &mut Writer, + array: &Key, +) -> Result, Error> { + let mut iter = items(r, &r.model(array)?.array, triangle_from_le_bytes)?; + let array = w.array_triangles(iter.by_ref())?; + iter.finish()?; + Ok(array) +} + +pub fn color_array( + r: &Omf1Reader, + w: &mut Writer, + array: &Int3Array, +) -> Result, Error> { + let mut iter = items(r, &array.array, color_from_le_bytes)?; + let array = w.array_colors(iter.by_ref())?; + iter.finish()?; + Ok(array) +} + +pub fn vectors2_array( + r: &Omf1Reader, + w: &mut Writer, + array: &Key, +) -> Result, Error> { + let mut iter = items(r, &r.model(array)?.array, |b| { + Ok(none_if_any_nan(vector2_from_le_bytes(b))) + })?; + let array = w.array_vectors(iter.by_ref())?; + iter.finish()?; + Ok(array) +} + +pub fn vectors3_array( + r: &Omf1Reader, + w: &mut Writer, + array: &Key, +) -> Result, Error> { + let mut iter = items(r, &r.model(array)?.array, |b| { + Ok(none_if_any_nan(vector3_from_le_bytes(b))) + })?; + let array = w.array_vectors(iter.by_ref())?; + iter.finish()?; + Ok(array) +} + +pub fn index_array( + r: &Omf1Reader, + w: &mut Writer, + array_key: &Key, +) -> Result<(u32, crate::Array), Error> { + let array = &r.model(array_key)?.array; + let is_float = array.dtype == DataType::Float; + let mut maximum = 0_u32; + let mut iter = items(r, array, |b| { + let n = if is_float { + let x = f64::from_le_bytes(b); + if x.floor() != x { + return Err(Omf1Error::NonIntegerArray.into()); + } + x as i64 + } else { + i64::from_le_bytes(b) + }; + if n == -1 { + Ok(None) + } else if n < 0 || n > (u32::MAX as i64) { + Err(Omf1Error::IndexOutOfRange { index: n }.into()) + } else { + maximum = maximum.max(n as u32); + Ok(Some(n as u32)) + } + })?; + let array = w.array_indices(iter.by_ref())?; + iter.finish()?; + Ok((maximum, array)) +} + +fn items( + r: &Omf1Reader, + array: &Array, + mut func: impl FnMut([u8; N]) -> Result, +) -> Result>, Error>, Error> +where + [u8; N]: Default, +{ + Ok(HoldError::new( + ByteChunks(r.array_decompressed_bytes(array)?) + .map(move |r| r.map_err(Error::from).and_then(&mut func)), + )) +} + +fn from_bytes( + bytes: [u8; B], + func: impl Fn([u8; N]) -> T, +) -> [T; M] { + debug_assert_eq!(M, B / N); + std::array::from_fn(|i| func((&bytes[(i * N)..((i + 1) * N)]).try_into().unwrap())) +} + +fn segment_from_le_bytes(bytes: [u8; 16]) -> Result<[u32; 2], Error> { + let [a, b] = from_bytes(bytes, |b| { + let n = i64::from_le_bytes(b); + n.try_into() + .map_err(|_| Omf1Error::IndexOutOfRange { index: n }) + }); + Ok([a?, b?]) +} + +fn triangle_from_le_bytes(bytes: [u8; 24]) -> Result<[u32; 3], Error> { + let [a, b, c] = from_bytes(bytes, |b| { + let n = i64::from_le_bytes(b); + n.try_into() + .map_err(|_| Omf1Error::IndexOutOfRange { index: n }) + }); + Ok([a?, b?, c?]) +} + +fn color_from_le_bytes(bytes: [u8; 24]) -> Result, Error> { + let [r, g, b] = from_bytes(bytes, |b| { + i64::from_le_bytes(b).clamp(0, u8::MAX as i64) as u8 + }); + Ok(Some([r, g, b, u8::MAX])) +} + +fn vector2_from_le_bytes(bytes: [u8; 16]) -> [f64; 2] { + from_bytes(bytes, f64::from_le_bytes) +} + +fn vector3_from_le_bytes(bytes: [u8; 24]) -> [f64; 3] { + from_bytes(bytes, f64::from_le_bytes) +} + +fn none_if_nan(input: f64) -> Option { + if input.is_nan() { + None + } else { + Some(input) + } +} + +fn none_if_any_nan(input: [f64; N]) -> Option<[f64; N]> { + if input.into_iter().any(f64::is_nan) { + None + } else { + Some(input) + } +} + +struct ByteChunks(I); + +impl Iterator for ByteChunks +where + [u8; N]: Default + Copy, + I: Iterator>, +{ + type Item = Result<[u8; N], std::io::Error>; + + fn next(&mut self) -> Option { + let mut item = [0_u8; N]; + for byte in item.iter_mut() { + match self.0.next() { + Some(Ok(b)) => *byte = b, + Some(Err(e)) => return Some(Err(e)), + None => return None, + } + } + Some(Ok(item)) + } +} + +pub struct HoldError { + inner: I, + result: Result<(), E>, +} + +impl HoldError { + fn new(inner: I) -> Self { + Self { + inner, + result: Ok(()), + } + } + + pub fn finish(self) -> Result<(), E> { + self.result + } +} + +impl Iterator for HoldError +where + I: Iterator>, +{ + type Item = T; + + fn next(&mut self) -> Option { + match self.inner.next() { + None => None, + Some(Err(e)) => { + self.result = Err(e); + None + } + Some(Ok(byte)) => Some(byte), + } + } +} diff --git a/src/omf1/attributes.rs b/src/omf1/attributes.rs new file mode 100644 index 0000000..56b2f9c --- /dev/null +++ b/src/omf1/attributes.rs @@ -0,0 +1,211 @@ +use crate::{error::Error, file::Writer}; + +use super::{ + array::{color_array, index_array, numbers_array, vectors2_array, vectors3_array}, + category_handler::CategoryHandler, + model::ColorArrayModel, + objects::*, + reader::Omf1Reader, +}; + +impl ScalarData { + pub fn convert(&self, r: &Omf1Reader, w: &mut Writer) -> Result { + attribute( + &self.content, + self.location, + crate::AttributeData::Number { + values: numbers_array(r, w, &self.array)?, + colormap: self + .colormap + .as_ref() + .map(|key| r.model(key)?.convert(r, w)) + .transpose()?, + }, + ) + } +} + +impl DateTimeData { + pub fn convert(&self, r: &Omf1Reader, w: &mut Writer) -> Result { + attribute( + &self.content, + self.location, + crate::AttributeData::Number { + values: w.array_numbers(r.model(&self.array)?.array.iter().copied())?, + colormap: self + .colormap + .as_ref() + .map(|key| r.model(key)?.convert(r, w)) + .transpose()?, + }, + ) + } +} + +impl Vector2Data { + pub fn convert(&self, r: &Omf1Reader, w: &mut Writer) -> Result { + attribute( + &self.content, + self.location, + crate::AttributeData::Vector { + values: vectors2_array(r, w, &self.array)?, + }, + ) + } +} + +impl Vector3Data { + pub fn convert(&self, r: &Omf1Reader, w: &mut Writer) -> Result { + attribute( + &self.content, + self.location, + crate::AttributeData::Vector { + values: vectors3_array(r, w, &self.array)?, + }, + ) + } +} + +impl ColorData { + pub fn convert(&self, r: &Omf1Reader, w: &mut Writer) -> Result { + match r.model(&self.array)? { + ColorArrayModel::Int3Array(array) => attribute( + &self.content, + self.location, + crate::AttributeData::Color { + values: color_array(r, w, array)?, + }, + ), + ColorArrayModel::ColorArray(ColorArray { array, .. }) => attribute( + &self.content, + self.location, + crate::AttributeData::Color { + values: w + .array_colors(array.iter().map(|&[r, g, b]| Some([r, g, b, u8::MAX])))?, + }, + ), + } + } +} + +impl StringData { + pub fn convert(&self, r: &Omf1Reader, w: &mut Writer) -> Result { + let strings = &r.model(&self.array)?.array; + attribute( + &self.content, + self.location, + crate::AttributeData::Text { + values: w.array_text(strings.iter().cloned().map(|s| { + if s.is_empty() { + None + } else { + Some(s) + } + }))?, + }, + ) + } +} + +impl MappedData { + pub fn convert(&self, r: &Omf1Reader, w: &mut Writer) -> Result { + let (max_index, values) = index_array(r, w, &self.array)?; + let mut handler = CategoryHandler::new(max_index); + for key in &self.legends { + let legend = r.model(key)?; + handler.add( + r, + &legend.content.name, + &legend.content.description, + &legend.values, + )?; + } + attribute(&self.content, self.location, handler.write(w, values)?) + } +} + +impl ImageTexture { + pub fn convert(&self, r: &Omf1Reader, w: &mut Writer) -> Result { + let (u, width) = projection_axis(self.axis_u); + let (v, height) = projection_axis(self.axis_v); + Ok(crate::Attribute { + name: self.content.name.clone(), + description: self.content.description.clone(), + units: Default::default(), + metadata: self.content.uid.metadata(), + location: crate::Location::Projected, + data: crate::AttributeData::ProjectedTexture { + image: w.image_bytes_from(r.image(&self.image)?)?, + orient: crate::Orient2 { + origin: self.origin, + u, + v, + }, + width, + height, + }, + }) + } +} + +impl ScalarColormap { + pub fn convert(&self, r: &Omf1Reader, w: &mut Writer) -> Result { + let [min, max] = self.limits; + Ok(crate::NumberColormap::Continuous { + range: (min, max).into(), + gradient: gradient(r, w, &self.gradient)?, + }) + } +} + +impl DateTimeColormap { + pub fn convert(&self, r: &Omf1Reader, w: &mut Writer) -> Result { + let [min, max] = self.limits; + Ok(crate::NumberColormap::Continuous { + range: (min, max).into(), + gradient: gradient(r, w, &self.gradient)?, + }) + } +} + +fn gradient( + r: &Omf1Reader, + w: &mut Writer, + colors: &Key, +) -> Result, Error> { + w.array_gradient( + r.model(colors)? + .array + .iter() + .map(|&[r, g, b]| [r, g, b, u8::MAX]), + ) +} + +pub(super) fn attribute( + content: &ContentModel, + location: DataLocation, + data: crate::AttributeData, +) -> Result { + Ok(crate::Attribute { + name: content.name.clone(), + description: content.description.clone(), + units: Default::default(), + metadata: content.uid.metadata(), + location: match location { + DataLocation::Vertices => crate::Location::Vertices, + DataLocation::Segments | DataLocation::Faces | DataLocation::Cells => { + crate::Location::Primitives + } + }, + data, + }) +} + +fn projection_axis(axis: [f64; 3]) -> ([f64; 3], f64) { + let length: f64 = axis.iter().map(|x| x * x).sum(); + if length == 0.0 { + ([0.0, 0.0, 0.0], 0.0) + } else { + (axis.map(|x| x / length), length) + } +} diff --git a/src/omf1/category_handler.rs b/src/omf1/category_handler.rs new file mode 100644 index 0000000..fb42717 --- /dev/null +++ b/src/omf1/category_handler.rs @@ -0,0 +1,236 @@ +use core::panic; +use std::collections::HashSet; + +use chrono::{DateTime, Utc}; + +use crate::{error::Error, file::Writer}; + +use super::{ + array::{load_scalars, ScalarValues}, + model::{LegendArrayModel, LegendArrays}, + objects::{ColorArray, DateTimeArray, Key, ScalarArray, StringArray}, + reader::Omf1Reader, +}; + +struct Legend { + name: String, + description: String, + data: Data, +} + +enum Data { + Color(Vec<[u8; 3]>), + DateTime(Vec>>), + String(Vec), + Float(Vec), + Int(Vec), +} + +impl Data { + fn len(&self) -> usize { + match self { + Data::Color(x) => x.len(), + Data::DateTime(x) => x.len(), + Data::String(x) => x.len(), + Data::Float(x) => x.len(), + Data::Int(x) => x.len(), + } + } + + fn names_score(&self) -> Option<(usize, usize)> { + if let Self::String(strings) = self { + let unique_count = strings + .iter() + .filter(|s| !s.is_empty()) + .collect::>() + .len(); + let total_len = strings.iter().map(|s| s.len()).sum(); + Some((unique_count, total_len)) + } else { + None + } + } + + fn gradient_score(&self) -> Option { + if let Self::Color(colors) = self { + Some(colors.iter().collect::>().len()) + } else { + None + } + } +} + +impl Legend { + fn write(self, w: &mut Writer, len: usize) -> Result { + let data = match self.data { + Data::Color(colors) => crate::AttributeData::Color { + values: w.array_colors(iter_to_len( + len, + colors.into_iter().map(|[r, g, b]| Some([r, g, b, u8::MAX])), + None, + ))?, + }, + Data::DateTime(date_times) => crate::AttributeData::Number { + values: w.array_numbers(iter_to_len(len, date_times.into_iter(), None))?, + colormap: None, + }, + Data::String(strings) => crate::AttributeData::Text { + values: w.array_text(iter_to_len( + len, + strings + .into_iter() + .map(|s| if s.is_empty() { None } else { Some(s) }), + None, + ))?, + }, + Data::Float(numbers) => crate::AttributeData::Number { + values: w.array_numbers(iter_to_len( + len, + numbers + .into_iter() + .map(|x| if x.is_nan() { None } else { Some(x) }), + None, + ))?, + colormap: None, + }, + Data::Int(numbers) => crate::AttributeData::Number { + values: w.array_numbers(iter_to_len(len, numbers.into_iter().map(Some), None))?, + colormap: None, + }, + }; + Ok(crate::Attribute { + name: self.name, + description: self.description, + units: Default::default(), + metadata: Default::default(), + location: crate::Location::Categories, + data, + }) + } +} + +pub struct CategoryHandler { + max_index: u32, + len: usize, + legends: Vec, + names: Vec, + gradient: Option>, +} + +impl CategoryHandler { + pub fn new(max_index: u32) -> Self { + Self { + max_index, + len: 0, + legends: Vec::new(), + names: Vec::new(), + gradient: None, + } + } + + pub fn add( + &mut self, + r: &Omf1Reader, + name: &str, + description: &str, + key: &Key, + ) -> Result<(), Error> { + let data = match r.model(key)? { + LegendArrayModel::ColorArray(ColorArray { array, .. }) => Data::Color(array.clone()), + LegendArrayModel::DateTimeArray(DateTimeArray { array, .. }) => { + Data::DateTime(array.clone()) + } + LegendArrayModel::StringArray(StringArray { array, .. }) => Data::String(array.clone()), + LegendArrayModel::ScalarArray(a @ ScalarArray { .. }) => match load_scalars(r, a)? { + ScalarValues::Float(x) => Data::Float(x), + ScalarValues::Int(x) => Data::Int(x), + }, + }; + self.legends.push(Legend { + name: name.to_owned(), + description: description.to_owned(), + data, + }); + Ok(()) + } + + fn process(&mut self) { + self.len = self + .legends + .iter() + .map(|l| l.data.len()) + .max() + .unwrap_or(0) + .max(self.max_index as usize); + if let Some((_, index)) = self + .legends + .iter() + .enumerate() + .filter_map(|(i, l)| l.data.names_score().map(|score| (score, i))) + .max() + { + let Legend { + data: Data::String(names), + .. + } = self.legends.remove(index) + else { + panic!("expected string legend"); + }; + self.names = names; + } else { + self.names = (0..self.len).map(|i| format!("Category{i}")).collect(); + } + if let Some((_, index)) = self + .legends + .iter() + .enumerate() + .filter_map(|(i, l)| l.data.gradient_score().map(|score| (score, i))) + .max() + { + let Legend { + data: Data::Color(colors), + .. + } = self.legends.remove(index) + else { + panic!("expected color legend"); + }; + self.gradient = Some(colors); + } + } + + pub fn write( + mut self, + w: &mut Writer, + values: crate::Array, + ) -> Result { + self.process(); + Ok(crate::AttributeData::Category { + values, + names: w.array_names(iter_to_len(self.len, self.names.into_iter(), String::new()))?, + gradient: self + .gradient + .map(|g| { + w.array_gradient(iter_to_len( + self.len, + g.into_iter().map(|[r, g, b]| [r, g, b, u8::MAX]), + [128, 128, 128, 255], + )) + }) + .transpose()?, + attributes: self + .legends + .into_iter() + .map(|l| l.write(w, self.len)) + .collect::>()?, + }) + } +} + +pub fn iter_to_len( + len: usize, + input: impl Iterator, + pad: T, +) -> impl Iterator { + let padding = Some(pad).into_iter().cycle(); + input.chain(padding).take(len) +} diff --git a/src/omf1/converter.rs b/src/omf1/converter.rs new file mode 100644 index 0000000..a8aa961 --- /dev/null +++ b/src/omf1/converter.rs @@ -0,0 +1,100 @@ +use std::{ + fs::{File, OpenOptions}, + io::Read, + path::Path, +}; + +use crate::{ + error::Error, + file::{Compression, Limits}, + validate::Problems, +}; + +use super::reader::Omf1Reader; + +/// Returns true if the file looks more like OMF1 than OMF2. +/// +/// Does not guarantee that the file will load. Returns an error in file read fails. +pub fn detect(read: &mut impl Read) -> Result { + const PREFIX: [u8; 8] = [0x84, 0x83, 0x82, 0x81, b'O', b'M', b'F', b'-']; + let mut prefix = [0; PREFIX.len()]; + read.read_exact(&mut prefix)?; + Ok(prefix == PREFIX) +} + +/// Returns true if the path looks more like OMF1 than OMF2. +/// +/// Does not guarantee that the file will load. Returns an error in file open or read fails. +pub fn detect_open(path: &Path) -> Result { + detect(&mut File::open(path)?) +} + +/// Converts a OMF1 files to OMF2. +/// +/// This object allows you to set up the desired parameters then convert one or more files. +#[derive(Debug, Default)] +pub struct Converter { + limits: Limits, + compression: Compression, +} + +impl Converter { + /// Creates a new default converter. + pub fn new() -> Self { + Self::default() + } + + /// Returns the current limits. + pub fn limits(&self) -> Limits { + self.limits + } + + /// Set the limits to use during conversion. + pub fn set_limits(&mut self, limits: Limits) { + self.limits = limits; + } + + /// Returns the current compression level. + pub fn compression(&self) -> Compression { + self.compression + } + + /// Set the compression level to use when writing. + pub fn set_compression(&mut self, compression: Compression) { + self.compression = compression; + } + + /// Runs a conversion from one open file to another file. + /// + /// `input` must support read and seek, while `output` must support write. + /// On success the validation warnings are returned. + /// + /// May be called more than once to convert multiple files with the same parameters. + pub fn convert(&self, input: File, output: File) -> Result { + let reader = Omf1Reader::new(input, self.limits.json_bytes)?; + let mut writer = crate::file::Writer::new(output)?; + writer.set_compression(self.compression); + let project = reader.project()?.convert(&reader, &mut writer)?; + writer.finish(project).map(|(_, p)| p) + } + + /// Runs a conversion from one filename to another. + /// + /// The output file will be created if it does not exist, and truncated if it does. + /// On success the validation warnings are returned. + /// + /// May be called more than once to convert multiple files with the same parameters. + pub fn convert_open( + &self, + input_path: impl AsRef, + output_path: impl AsRef, + ) -> Result { + let input = File::open(input_path)?; + let output = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(output_path)?; + self.convert(input, output) + } +} diff --git a/src/omf1/elements.rs b/src/omf1/elements.rs new file mode 100644 index 0000000..dd22c26 --- /dev/null +++ b/src/omf1/elements.rs @@ -0,0 +1,242 @@ +use chrono::SecondsFormat; + +use crate::{crate_full_name, date_time::utc_now, error::Error, file::Writer}; + +use super::{ + array::{scalars_array, segments_array, triangles_array, vertices_array}, + model::*, + objects::*, + reader::Omf1Reader, +}; + +impl UidModel { + pub fn metadata(&self) -> serde_json::Map { + let mut map = serde_json::Map::new(); + insert_metadata(&mut map, "date_created", &self.date_created); + insert_metadata(&mut map, "date_modified", &self.date_modified); + map + } +} + +impl Project { + pub fn convert(&self, r: &Omf1Reader, w: &mut Writer) -> Result { + let mut conversion_details = serde_json::Map::new(); + conversion_details.insert("from".to_owned(), r.version().into()); + conversion_details.insert("by".to_owned(), crate_full_name().into()); + conversion_details.insert( + "on".to_owned(), + crate::date_time::utc_now() + .to_rfc3339_opts(SecondsFormat::Secs, true) + .into(), + ); + let mut metadata = self.content.uid.metadata(); + metadata.insert("OMF1 conversion".to_owned(), conversion_details.into()); + Ok(crate::Project { + name: self.content.name.clone(), + description: self.content.description.clone(), + coordinate_reference_system: Default::default(), + units: self.units.clone(), + origin: self.origin, + author: self.author.clone(), + application: Default::default(), + date: self.content.uid.date_created.parse().unwrap_or(utc_now()), + metadata, + elements: self + .elements + .iter() + .map(|k| { + let element = r.model(k)?; + element.convert(r, w) + }) + .collect::>()?, + }) + } +} + +impl<'a> ElementModel<'a> { + pub fn convert(&self, r: &Omf1Reader, w: &mut Writer) -> Result { + match *self { + Self::PointSetElement(x) => x.convert(r, w), + Self::LineSetElement(x) => x.convert(r, w), + Self::SurfaceElement(x) => x.convert(r, w), + Self::VolumeElement(x) => x.convert(r, w), + } + } +} + +impl PointSetElement { + pub fn convert(&self, r: &Omf1Reader, w: &mut Writer) -> Result { + let geometry = r.model(&self.geometry)?; + let mut e = element( + &self.content, + self.color, + attributes_and_textures(r, w, &self.data, &self.textures)?, + crate::PointSet { + origin: geometry.origin, + vertices: vertices_array(r, w, &geometry.vertices)?, + }, + )?; + e.metadata.insert( + "subtype".to_owned(), + match self.subtype { + PointSetSubtype::Point => "point", + PointSetSubtype::Collar => "collar", + PointSetSubtype::BlastHole => "blasthole", + } + .into(), + ); + Ok(e) + } +} + +impl LineSetElement { + pub fn convert(&self, r: &Omf1Reader, w: &mut Writer) -> Result { + let geometry = r.model(&self.geometry)?; + let mut e = element( + &self.content, + self.color, + attributes(r, w, &self.data)?, + crate::LineSet { + origin: geometry.origin, + vertices: vertices_array(r, w, &geometry.vertices)?, + segments: segments_array(r, w, &geometry.segments)?, + }, + )?; + e.metadata.insert( + "subtype".to_owned(), + match self.subtype { + LineSetSubtype::Line => "line", + LineSetSubtype::BoreHole => "borehole", + } + .into(), + ); + Ok(e) + } +} + +impl SurfaceElement { + pub fn convert(&self, r: &Omf1Reader, w: &mut Writer) -> Result { + match r.model(&self.geometry)? { + SurfaceGeometryModel::SurfaceGeometry(geometry) => element( + &self.content, + self.color, + attributes_and_textures(r, w, &self.data, &self.textures)?, + crate::Surface { + origin: geometry.origin, + vertices: vertices_array(r, w, &geometry.vertices)?, + triangles: triangles_array(r, w, &geometry.triangles)?, + }, + ), + SurfaceGeometryModel::SurfaceGridGeometry(geometry) => element( + &self.content, + self.color, + attributes(r, w, &self.data)?, + crate::GridSurface { + orient: crate::Orient2 { + origin: geometry.origin, + u: geometry.axis_u, + v: geometry.axis_v, + }, + grid: crate::Grid2::Tensor { + u: w.array_scalars(geometry.tensor_u.iter().copied())?, + v: w.array_scalars(geometry.tensor_v.iter().copied())?, + }, + heights: geometry + .offset_w + .as_ref() + .map(|a| scalars_array(r, w, a)) + .transpose()?, + }, + ), + } + } +} + +impl VolumeElement { + pub fn convert(&self, r: &Omf1Reader, w: &mut Writer) -> Result { + let geometry = r.model(&self.geometry)?; + element( + &self.content, + self.color, + attributes(r, w, &self.data)?, + crate::BlockModel { + orient: crate::Orient3 { + origin: geometry.origin, + u: geometry.axis_u, + v: geometry.axis_v, + w: geometry.axis_w, + }, + grid: crate::Grid3::Tensor { + u: w.array_scalars(geometry.tensor_u.iter().copied())?, + v: w.array_scalars(geometry.tensor_v.iter().copied())?, + w: w.array_scalars(geometry.tensor_w.iter().copied())?, + }, + subblocks: None, + }, + ) + } +} + +impl<'a> DataModel<'a> { + pub fn convert(&self, r: &Omf1Reader, w: &mut Writer) -> Result { + match *self { + DataModel::ScalarData(x) => x.convert(r, w), + DataModel::DateTimeData(x) => x.convert(r, w), + DataModel::Vector2Data(x) => x.convert(r, w), + DataModel::Vector3Data(x) => x.convert(r, w), + DataModel::ColorData(x) => x.convert(r, w), + DataModel::StringData(x) => x.convert(r, w), + DataModel::MappedData(x) => x.convert(r, w), + } + } +} + +fn element( + content: &ContentModel, + color: Option<[u8; 3]>, + attributes: Vec, + geometry: impl Into, +) -> Result { + Ok(crate::Element { + name: content.name.clone(), + description: content.description.clone(), + color: color.map(|[r, g, b]| [r, g, b, u8::MAX]), + metadata: content.uid.metadata(), + attributes, + geometry: geometry.into(), + }) +} + +fn attributes( + r: &Omf1Reader, + w: &mut Writer, + data: &[Key], +) -> Result, Error> { + data.iter() + .map(|key| r.model(key)?.convert(r, w)) + .collect::, _>>() +} + +fn attributes_and_textures( + r: &Omf1Reader, + w: &mut Writer, + data: &[Key], + textures: &[Key], +) -> Result, Error> { + let mut data_attributes = data + .iter() + .map(|key| r.model(key)?.convert(r, w)) + .collect::, _>>()?; + let mut texture_attributes = textures + .iter() + .map(|key| r.model(key)?.convert(r, w)) + .collect::, _>>()?; + data_attributes.append(&mut texture_attributes); + Ok(data_attributes) +} + +fn insert_metadata(map: &mut serde_json::Map, key: &str, value: &str) { + if !value.is_empty() { + map.insert(key.to_owned(), value.into()); + } +} diff --git a/src/omf1/error.rs b/src/omf1/error.rs new file mode 100644 index 0000000..b2730d7 --- /dev/null +++ b/src/omf1/error.rs @@ -0,0 +1,48 @@ +use super::ModelType; + +/// Errors specific to the OMF v1 conversion process. +/// +/// Converted fails may also fail validation during conversion, as the checks in OMF v1 weren't +/// as strict. +#[derive(Debug, thiserror::Error)] +pub enum Omf1Error { + /// Tried to load a file that is not in OMF1 format. + #[error("this is not an OMF1 file")] + NotOmf1, + /// The OMF version is not supported. + #[error("version '{version}' is not supported")] + UnsupportedVersion { version: String }, + /// A record in the JSON data has the wrong type. + #[error("wrong value type, found {found} when expecting {}", join(expected))] + WrongType { + found: ModelType, + expected: &'static [ModelType], + }, + /// A record is missing from the JSON data. + #[error("item '{key}' is missing from the file")] + MissingItem { key: String }, + /// Non-integer values from in what should be an integer array. + #[error("an integer array was expected, but floating-point was found")] + NonIntegerArray, + /// An integer index is invalid. + #[error("index {index} is outside the range 0 to 4294967295, or -1 for null categories")] + IndexOutOfRange { index: i64 }, + /// Forwards `serde_json` errors when deserializing OMF1. + #[error("JSON deserialization error: {0}")] + DeserializationFailed(#[from] serde_json::Error), +} + +fn join(types: &[ModelType]) -> String { + match types { + [] => "none".to_owned(), + [t] => t.to_string(), + [t, s] => format!("{t} or {s}"), + [s @ .., t] => format!( + "{}, or {t}", + s.iter() + .map(ToString::to_string) + .collect::>() + .join(", ") + ), + } +} diff --git a/src/omf1/mod.rs b/src/omf1/mod.rs new file mode 100644 index 0000000..eb0a6a9 --- /dev/null +++ b/src/omf1/mod.rs @@ -0,0 +1,50 @@ +//! Convert existing OMF v1 files to OMF v2. +//! +//! ## Conversion details +//! +//! There are a few parts of OMF1 that don't map directly to OMF2. +//! +//! ### Elements +//! +//! - The `date_created` and `date_modified` fields are moved into the metadata. +//! - The `subtype` field on point-sets and line-sets is moved into the metadata. +//! On other elements, where it only had one valid value, it is discarded. +//! - Line-sets and surfaces with invalid vertex indices will cause conversion to fail. +//! - Line-sets and surfaces with more than 4,294,967,295 vertices will cause conversion to fail. +//! +//! ### Data to Attributes +//! +//! - Scalar data becomes a number attribute, preserving the float/int type of the array. +//! - In number data, NaN becomes null. +//! - In 2D or 3D vector data, if any component is NaN the vector becomes null. +//! - In string data, empty strings become nulls. +//! OMF2 supports both null and empty string so we can only guess which was intended. +//! - In date-time data, empty strings become null. +//! - Date-times outside the range of approximately ±262,000 years CE will cause conversion to fail. +//! +//! ### Mapped Data to Category Attribute +//! +//! The exact layout of mapped data from OMF v1 can't be stored in OMF v2. +//! It is transformed to a category attribute by following these rules: +//! +//! - Indices equal to minus one become null. +//! - Indices outside the range 0 to 4,294,967,295 will cause conversion to fail. +//! - The most unique, least empty, and shortest string legend becomes the category names, +//! padded with empty strings if necessary. +//! - The most unique and least empty color legend becomes the category colors, padded with +//! gray if necessary. +//! - Other legends become extra attributes, padded with nulls if necessary. + +mod array; +mod attributes; +mod category_handler; +mod converter; +mod elements; +mod error; +mod model; +mod objects; +mod reader; + +pub use converter::{detect, detect_open, Converter}; +pub use error::Omf1Error; +pub use model::ModelType; diff --git a/src/omf1/model.rs b/src/omf1/model.rs new file mode 100644 index 0000000..5a77757 --- /dev/null +++ b/src/omf1/model.rs @@ -0,0 +1,173 @@ +use std::fmt::Display; + +use serde::Deserialize; + +use super::{objects::*, Omf1Error}; + +/// Converts a `&Model` into either a reference to the individual item, or into +/// a subset enum. +/// +/// This is used by `Omf1Root::get` to check variants on load. +pub trait FromModel { + type Output<'a>; + + fn from_model(model: &Model) -> Result, Omf1Error>; +} + +/// Creates enums and `FromModel` implementations for the UidModel objects in OMF v1. +macro_rules! model { + ($( $variant:ident )*) => { + /// Contains an OMF v1 top-level object. + #[derive(Debug, Deserialize)] + #[serde(tag = "__class__")] + pub enum Model { + $( $variant($variant), )* + } + + /// The types of object allowed at the top level of OMF v1. + #[derive(Debug)] + pub enum ModelType { + $( $variant, )* + } + + impl Model { + /// Return the model type. + fn model_type(&self) -> ModelType { + match self { + $( Self::$variant(_) => ModelType::$variant, )* + } + } + } + + $( + impl FromModel for $variant { + type Output<'a> = &'a $variant; + + fn from_model(model: &Model) -> Result, Omf1Error> { + match model { + Model::$variant(x) => Ok(x), + _ => Err(Omf1Error::WrongType { + found: model.model_type(), + expected: &[ModelType::$variant], + }), + } + } + } + )* + }; +} + +/// Creates marker type, a subset of `Model`, and a `FromModel` implementation to tie them +/// together. +/// +/// This lets us have type-tagged keys for a subset of model types in the objects that +/// `Omf1Root::get` can load and check automatically. The loading code can then match +/// exhaustively without worrying about the incorrect types. +macro_rules! model_subset { + ($model_name:ident $enum_name:ident { $( $variant:ident )* }) => { + #[derive(Debug)] + pub struct $model_name {} + + #[derive(Debug, Clone, Copy)] + #[allow(clippy::enum_variant_names)] + pub enum $enum_name<'a> { + $( $variant(&'a $variant), )* + } + + impl FromModel for $model_name { + type Output<'a> = $enum_name<'a>; + + fn from_model(model: &Model) -> Result, Omf1Error> { + match model { + $( Model::$variant(x) => Ok($enum_name::$variant(x)), )* + _ => Err(Omf1Error::WrongType { + found: model.model_type(), + expected: &[$( ModelType::$variant ),*], + }), + } + } + } + }; +} + +model! { + Project + PointSetElement + PointSetGeometry + LineSetElement + LineSetGeometry + SurfaceElement + SurfaceGeometry + SurfaceGridGeometry + VolumeElement + VolumeGridGeometry + ScalarColormap + DateTimeColormap + Legend + ScalarData + DateTimeData + Vector2Data + Vector3Data + ColorData + StringData + MappedData + ImageTexture + ScalarArray + Vector2Array + Vector3Array + Int2Array + Int3Array + StringArray + DateTimeArray + ColorArray +} + +impl Display for ModelType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } +} + +model_subset! { + Elements ElementModel { + PointSetElement + LineSetElement + SurfaceElement + VolumeElement + } +} + +model_subset! { + Data DataModel { + ScalarData + DateTimeData + Vector2Data + Vector3Data + ColorData + StringData + MappedData + } +} + +model_subset! { + SurfaceGeometries SurfaceGeometryModel { + SurfaceGeometry + SurfaceGridGeometry + } +} + +model_subset! { + LegendArrays LegendArrayModel { + ColorArray + DateTimeArray + StringArray + ScalarArray + } +} + +model_subset! { + ColorArrays ColorArrayModel { + Int3Array + ColorArray + } +} diff --git a/src/omf1/objects.rs b/src/omf1/objects.rs new file mode 100644 index 0000000..0a56d18 --- /dev/null +++ b/src/omf1/objects.rs @@ -0,0 +1,468 @@ +#![allow(dead_code)] // Many attributes in here exist for the JSON loading, but aren't used. +use std::marker::PhantomData; + +use chrono::{DateTime, Utc}; +use serde::Deserialize; + +use super::model::{ColorArrays, Data, Elements, LegendArrays, SurfaceGeometries}; + +/// Stores a string, while `T` can control what types of model as accepted. +#[derive(Debug)] +pub struct Key { + pub value: String, + _phantom: PhantomData, +} + +impl Key { + pub fn from_bytes(bytes: [u8; 16]) -> Self { + let mut value = format!("{:032x}", u128::from_be_bytes(bytes)); + for i in [20, 16, 12, 8] { + value.insert(i, '-'); + } + Self { + value, + _phantom: Default::default(), + } + } +} + +struct KeyVisitor { + _phantom: PhantomData, +} + +impl<'de, T> serde::de::Visitor<'de> for KeyVisitor { + type Value = Key; + + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("an id string") + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + Ok(Key { + value: v.to_owned(), + _phantom: Default::default(), + }) + } +} + +impl<'de, T> Deserialize<'de> for Key { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_str(KeyVisitor:: { + _phantom: Default::default(), + }) + } +} + +#[derive(Debug, Deserialize)] +pub struct UidModel { + #[serde(default)] + pub date_created: String, + #[serde(default)] + pub date_modified: String, +} + +#[derive(Debug, Deserialize)] +pub struct ContentModel { + #[serde(flatten)] + pub uid: UidModel, + #[serde(default)] + pub name: String, + #[serde(default)] + pub description: String, +} + +#[derive(Debug, Deserialize)] +pub struct Project { + #[serde(flatten)] + pub content: ContentModel, + #[serde(default)] + pub author: String, + #[serde(default)] + pub revision: String, + #[serde(default)] + pub date: DateTime, + #[serde(default)] + pub units: String, + #[serde(default)] + pub origin: [f64; 3], + #[serde(default)] + pub elements: Vec>, +} + +// Elements + +#[derive(Debug, Deserialize)] +pub struct PointSetElement { + #[serde(flatten)] + pub content: ContentModel, + #[serde(default)] + pub color: Option<[u8; 3]>, + #[serde(default)] + pub data: Vec>, + #[serde(default)] + pub textures: Vec>, + #[serde(default)] + pub subtype: PointSetSubtype, + pub geometry: Key, +} + +#[derive(Debug, Deserialize)] +pub struct PointSetGeometry { + #[serde(flatten)] + pub uid: UidModel, + #[serde(default)] + pub origin: [f64; 3], + pub vertices: Key, +} + +#[derive(Debug, Deserialize)] +pub struct LineSetElement { + #[serde(flatten)] + pub content: ContentModel, + #[serde(default)] + pub color: Option<[u8; 3]>, + #[serde(default)] + pub data: Vec>, + #[serde(default)] + pub subtype: LineSetSubtype, + pub geometry: Key, +} + +#[derive(Debug, Deserialize)] +pub struct LineSetGeometry { + #[serde(flatten)] + pub uid: UidModel, + #[serde(default)] + pub origin: [f64; 3], + pub vertices: Key, + pub segments: Key, +} + +#[derive(Debug, Deserialize)] +pub struct SurfaceElement { + #[serde(flatten)] + pub content: ContentModel, + #[serde(default)] + pub color: Option<[u8; 3]>, + #[serde(default)] + pub data: Vec>, + #[serde(default)] + pub textures: Vec>, + #[serde(default)] + pub subtype: SurfaceSubtype, + pub geometry: Key, +} + +#[derive(Debug, Deserialize)] +pub struct SurfaceGeometry { + #[serde(flatten)] + pub uid: UidModel, + #[serde(default)] + pub origin: [f64; 3], + pub vertices: Key, + pub triangles: Key, +} + +#[derive(Debug, Deserialize)] +pub struct SurfaceGridGeometry { + #[serde(flatten)] + pub uid: UidModel, + #[serde(default)] + pub origin: [f64; 3], + pub tensor_u: Vec, + pub tensor_v: Vec, + #[serde(default = "i")] + pub axis_u: [f64; 3], + #[serde(default = "j")] + pub axis_v: [f64; 3], + #[serde(default)] + pub offset_w: Option>, +} + +#[derive(Debug, Deserialize)] +pub struct VolumeElement { + #[serde(flatten)] + pub content: ContentModel, + #[serde(default)] + pub color: Option<[u8; 3]>, + #[serde(default)] + pub data: Vec>, + #[serde(default)] + pub subtype: VolumeSubtype, + pub geometry: Key, +} + +#[derive(Debug, Deserialize)] +pub struct VolumeGridGeometry { + #[serde(flatten)] + pub uid: UidModel, + #[serde(default)] + pub origin: [f64; 3], + pub tensor_u: Vec, + pub tensor_v: Vec, + pub tensor_w: Vec, + #[serde(default = "i")] + pub axis_u: [f64; 3], + #[serde(default = "j")] + pub axis_v: [f64; 3], + #[serde(default = "k")] + pub axis_w: [f64; 3], + #[serde(default)] + pub offset_w: Option>, +} + +// Colormaps + +#[derive(Debug, Deserialize)] +pub struct ScalarColormap { + #[serde(flatten)] + pub content: ContentModel, + pub gradient: Key, + pub limits: [f64; 2], +} + +#[derive(Debug, Deserialize)] +pub struct DateTimeColormap { + #[serde(flatten)] + pub content: ContentModel, + pub gradient: Key, + pub limits: [DateTime; 2], +} + +#[derive(Debug, Deserialize)] +pub struct Legend { + #[serde(flatten)] + pub content: ContentModel, + pub values: Key, +} + +// Data + +#[derive(Debug, Deserialize)] +pub struct ScalarData { + #[serde(flatten)] + pub content: ContentModel, + pub location: DataLocation, + pub colormap: Option>, + pub array: Key, +} + +#[derive(Debug, Deserialize)] +pub struct DateTimeData { + #[serde(flatten)] + pub content: ContentModel, + pub location: DataLocation, + pub colormap: Option>, + pub array: Key, +} + +#[derive(Debug, Deserialize)] +pub struct Vector2Data { + #[serde(flatten)] + pub content: ContentModel, + pub location: DataLocation, + pub array: Key, +} + +#[derive(Debug, Deserialize)] +pub struct Vector3Data { + #[serde(flatten)] + pub content: ContentModel, + pub location: DataLocation, + pub array: Key, +} + +#[derive(Debug, Deserialize)] +pub struct ColorData { + #[serde(flatten)] + pub content: ContentModel, + pub location: DataLocation, + pub array: Key, +} + +#[derive(Debug, Deserialize)] +pub struct StringData { + #[serde(flatten)] + pub content: ContentModel, + pub location: DataLocation, + pub array: Key, +} + +#[derive(Debug, Deserialize)] +pub struct MappedData { + #[serde(flatten)] + pub content: ContentModel, + pub location: DataLocation, + pub array: Key, + #[serde(default)] + pub legends: Vec>, +} + +// Texture + +#[derive(Debug, Deserialize)] +pub struct ImageTexture { + #[serde(flatten)] + pub content: ContentModel, + #[serde(default)] + pub origin: [f64; 3], + #[serde(default = "i")] + pub axis_u: [f64; 3], + #[serde(default = "j")] + pub axis_v: [f64; 3], + pub image: Image, +} + +// Arrays + +#[derive(Debug, Deserialize)] +pub struct Array { + pub start: u64, + pub length: u64, + pub dtype: DataType, +} + +#[derive(Debug, Deserialize)] +pub struct Image { + pub start: u64, + pub length: u64, + pub dtype: ImageType, +} + +#[derive(Debug, Deserialize, Clone, Copy, PartialEq)] +pub enum DataType { + #[serde(rename = ", +} + +#[derive(Debug, Deserialize)] +pub struct DateTimeArray { + #[serde(flatten)] + pub uid: UidModel, + pub array: Vec>>, +} + +#[derive(Debug, Deserialize)] +pub struct ColorArray { + #[serde(flatten)] + pub uid: UidModel, + pub array: Vec<[u8; 3]>, +} + +// Enums + +#[derive(Debug, Default, Deserialize, Clone, Copy, PartialEq)] +pub enum PointSetSubtype { + #[serde(rename = "point")] + #[default] + Point, + #[serde(rename = "collar")] + Collar, + #[serde(rename = "blasthole")] + BlastHole, +} + +#[derive(Debug, Default, Deserialize, Clone, Copy, PartialEq)] +pub enum LineSetSubtype { + #[serde(rename = "line")] + #[default] + Line, + #[serde(rename = "borehole")] + BoreHole, +} + +#[derive(Debug, Default, Deserialize, Clone, Copy, PartialEq)] +pub enum SurfaceSubtype { + #[serde(rename = "surface")] + #[default] + Surface, +} + +#[derive(Debug, Default, Deserialize, Clone, Copy, PartialEq)] +pub enum VolumeSubtype { + #[serde(rename = "volume")] + #[default] + Volume, +} + +#[derive(Debug, Deserialize, Clone, Copy, PartialEq)] +pub enum DataLocation { + #[serde(rename = "vertices")] + Vertices, + #[serde(rename = "segments")] + Segments, + #[serde(rename = "faces")] + Faces, + #[serde(rename = "cells")] + Cells, +} + +// Default factories + +fn i() -> [f64; 3] { + [1.0, 0.0, 0.0] +} + +fn j() -> [f64; 3] { + [0.0, 1.0, 0.0] +} + +fn k() -> [f64; 3] { + [0.0, 0.0, 1.0] +} diff --git a/src/omf1/reader.rs b/src/omf1/reader.rs new file mode 100644 index 0000000..b8e062e --- /dev/null +++ b/src/omf1/reader.rs @@ -0,0 +1,117 @@ +use std::{ + collections::HashMap, + fs::File, + io::{BufReader, Read, Seek, SeekFrom}, +}; + +use flate2::bufread::ZlibDecoder; + +use crate::{ + error::{Error, Limit}, + file::SubFile, +}; + +use super::{ + model::{FromModel, Model}, + objects::{Array, Image, Key, Project}, + Omf1Error, +}; + +/// The OMF1 file loader. +#[derive(Debug)] +pub struct Omf1Reader { + file: File, + project: Key, + models: HashMap, + version: String, +} + +impl Omf1Reader { + pub fn new(mut file: File, limit: Option) -> Result { + file.rewind()?; + let (project, json_start, version) = read_header(&mut file)?; + let stream_len = file.seek(SeekFrom::End(0))?; + if let Some(lim) = limit { + if stream_len.saturating_sub(json_start) > lim { + return Err(Error::LimitExceeded(Limit::JsonBytes)); + } + } + file.seek(SeekFrom::Start(json_start))?; + let models: HashMap = + serde_json::from_reader(&mut file).map_err(Omf1Error::DeserializationFailed)?; + Ok(Self { + file, + project, + models, + version, + }) + } + + pub fn version(&self) -> &str { + &self.version + } + + pub fn model(&self, key: &Key) -> Result, Error> + where + T: FromModel, + { + match self.models.get(&key.value) { + Some(model) => Ok(T::from_model(model)?), + None => Err(Omf1Error::MissingItem { + key: key.value.to_owned(), + } + .into()), + } + } + + pub fn project(&self) -> Result<&Project, Error> { + self.model(&self.project) + } + + pub fn image(&self, array: &Image) -> Result { + let mut f = self.file.try_clone()?; + f.seek(SeekFrom::Start(array.start))?; + Ok(ZlibDecoder::new(BufReader::new(SubFile::new( + f, + array.length, + )?))) + } + + pub fn array_decompressed_bytes( + &self, + array: &Array, + ) -> Result>, Error> { + let mut f = self.file.try_clone()?; + f.seek(SeekFrom::Start(array.start))?; + Ok(ZlibDecoder::new(BufReader::new(SubFile::new(f, array.length)?)).bytes()) + } +} + +fn read_header(read: &mut impl Read) -> Result<(Key, u64, String), Error> { + const MAGIC: [u8; 4] = [0x84, 0x83, 0x82, 0x81]; + const SUPPORTED_VERSION: &str = "OMF-v0.9.0"; + + let mut header = [0_u8; 60]; + match read.read_exact(&mut header) { + Ok(_) => (), + Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => { + return Err(Omf1Error::NotOmf1.into()) + } + Err(e) => return Err(e.into()), + }; + if header[..4] != MAGIC { + return Err(Omf1Error::NotOmf1.into()); + } + let version = String::from_utf8_lossy(&header[4..36]) + .trim_end_matches('\0') + .to_owned(); + if !version.starts_with("OMF-") { + return Err(Omf1Error::NotOmf1.into()); + } + if version != SUPPORTED_VERSION { + return Err(Omf1Error::UnsupportedVersion { version }.into()); + } + let project = Key::from_bytes((&header[36..52]).try_into().expect("16 bytes")); + let json_start = u64::from_le_bytes((&header[52..60]).try_into().expect("8 bytes")); + Ok((project, json_start, version)) +} diff --git a/src/pqarray/array_type.rs b/src/pqarray/array_type.rs new file mode 100644 index 0000000..27f57ae --- /dev/null +++ b/src/pqarray/array_type.rs @@ -0,0 +1,288 @@ +use chrono::prelude::*; +use parquet::{ + basic::{Encoding, LogicalType, Type as PhysicalType}, + data_type::{AsBytes, ByteArray, ByteArrayType, DataType, Int32Type, Int64Type}, + format::{MicroSeconds, TimeUnit}, +}; + +use crate::date_time::*; + +pub trait PqArrayType: Default + 'static { + type DataType: DataType; + + fn physical_type() -> PhysicalType { + ::get_physical_type() + } + + fn logical_type() -> Option { + None + } + + fn check_logical_type(file_type: Option) -> bool { + file_type == Self::logical_type() + } + + fn encoding() -> Option { + None + } + + fn from_parquet( + value: ::T, + _logical_type: &Option, + ) -> Self; + + fn to_parquet(self) -> ::T; +} + +macro_rules! simple { + ($t:ty, $dt:ident) => { + impl PqArrayType for $t { + type DataType = parquet::data_type::$dt; + + fn to_parquet(self) -> ::T { + self + } + + fn from_parquet( + value: ::T, + _logical_type: &Option, + ) -> Self { + value + } + } + }; +} + +macro_rules! physical_integer { + ($t:ty, $dt:ident, $bits:literal, $signed:literal) => { + impl PqArrayType for $t { + type DataType = parquet::data_type::$dt; + + fn to_parquet(self) -> ::T { + self as ::T + } + + fn from_parquet( + value: ::T, + _logical_type: &Option, + ) -> Self { + value as Self + } + } + }; +} + +macro_rules! logical_integer { + ($t:ty, $dt:ident, $bits:literal, $signed:literal) => { + impl PqArrayType for $t { + type DataType = parquet::data_type::$dt; + + fn logical_type() -> Option { + Some(LogicalType::Integer { + bit_width: $bits, + is_signed: $signed, + }) + } + + fn to_parquet(self) -> ::T { + self as ::T + } + + fn from_parquet( + value: ::T, + _logical_type: &Option, + ) -> Self { + value as Self + } + } + }; +} + +simple!(f64, DoubleType); +simple!(f32, FloatType); +simple!(bool, BoolType); +logical_integer!(u8, Int32Type, 8, false); +logical_integer!(u16, Int32Type, 16, false); +logical_integer!(u32, Int32Type, 32, false); +logical_integer!(u64, Int64Type, 64, false); +logical_integer!(i8, Int32Type, 8, true); +logical_integer!(i16, Int32Type, 16, true); +physical_integer!(i32, Int32Type, 32, true); +physical_integer!(i64, Int64Type, 64, true); + +impl PqArrayType for String { + type DataType = ByteArrayType; + + fn logical_type() -> Option { + Some(LogicalType::String) + } + + fn to_parquet(self) -> ByteArray { + self.into_bytes().into() + } + + fn from_parquet(value: ByteArray, _logical_type: &Option) -> Self { + String::from_utf8_lossy(value.as_bytes()).into_owned() + } +} + +impl PqArrayType for Vec { + type DataType = ByteArrayType; + + fn to_parquet(self) -> ByteArray { + self.into() + } + + fn from_parquet(value: ByteArray, _logical_type: &Option) -> Self { + value.as_bytes().into() + } +} + +impl PqArrayType for NaiveDate { + type DataType = Int32Type; + + fn logical_type() -> Option { + Some(LogicalType::Date) + } + + fn to_parquet(self) -> i32 { + date_to_i64(self).clamp(i32::MIN as i64, i32::MAX as i64) as i32 + } + + fn from_parquet(value: i32, _logical_type: &Option) -> Self { + i64_to_date(value.into()) + } +} + +impl PqArrayType for DateTime { + type DataType = Int64Type; + + fn logical_type() -> Option { + Some(LogicalType::Timestamp { + is_adjusted_to_u_t_c: true, + unit: TimeUnit::MICROS(MicroSeconds::new()), + }) + } + + fn check_logical_type(file_type: Option) -> bool { + matches!(file_type, Some(LogicalType::Timestamp { .. })) + } + + fn to_parquet(self) -> i64 { + self.timestamp_micros() + } + + fn from_parquet( + value: ::T, + logical_type: &Option, + ) -> Self { + match logical_type { + Some(LogicalType::Timestamp { unit, .. }) => match unit { + TimeUnit::MILLIS(_) => i64_milli_to_date_time(value), + TimeUnit::MICROS(_) => i64_to_date_time(value), + TimeUnit::NANOS(_) => i64_nano_to_date_time(value), + }, + _ => unreachable!("the logical type was checked earlier"), + } + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use parquet::format::{MilliSeconds, NanoSeconds}; + + use super::*; + + const DATE_TIME_MILLI: Option = Some(LogicalType::Timestamp { + is_adjusted_to_u_t_c: true, + unit: TimeUnit::MILLIS(MilliSeconds {}), + }); + const DATE_TIME_MICRO: Option = Some(LogicalType::Timestamp { + is_adjusted_to_u_t_c: true, + unit: TimeUnit::MICROS(MicroSeconds {}), + }); + const DATE_TIME_NANO: Option = Some(LogicalType::Timestamp { + is_adjusted_to_u_t_c: true, + unit: TimeUnit::NANOS(NanoSeconds {}), + }); + + #[test] + fn source_data_identical() { + let n: f64 = 13.5_f64.to_parquet(); + assert_eq!(n, 13.5_f64); + } + + #[test] + fn source_data_unsigned() { + let i: i32 = 4_294_967_295_u32.to_parquet(); + let j = u32::from_ne_bytes(i.to_ne_bytes()); + assert_eq!(j, 4_294_967_295_u32); + } + + #[test] + fn source_data_smaller_unsigned() { + let i: i32 = 50000_u16.to_parquet(); + assert_eq!(i, 50000); + } + + #[test] + fn source_data_smaller() { + let i: i32 = 16000_i16.to_parquet(); + assert_eq!(i, 16000); + } + + #[test] + fn date_conversion() { + let zero = NaiveDate::from_ymd_opt(1970, 01, 01).unwrap(); + let one = NaiveDate::from_ymd_opt(1970, 01, 02).unwrap(); + let minus_one = NaiveDate::from_ymd_opt(1969, 12, 31).unwrap(); + assert_eq!(zero.to_parquet(), 0); + assert_eq!(one.to_parquet(), 1); + assert_eq!(minus_one.to_parquet(), -1); + assert_eq!(NaiveDate::from_parquet(0, &None), zero); + assert_eq!(NaiveDate::from_parquet(1, &None), one); + assert_eq!(NaiveDate::from_parquet(-1, &None), minus_one); + } + + #[test] + fn date_overflow() { + let min = NaiveDate::from_parquet(i32::MIN, &None); + assert_eq!(min.to_string(), "-262143-01-01"); + let max = NaiveDate::from_parquet(i32::MAX, &None); + assert_eq!(max.to_string(), "+262142-12-31"); + } + + #[test] + fn date_time_conversion() { + let zero_ms = DateTime::::from_parquet(0, &DATE_TIME_MILLI); + let zero_us = DateTime::::from_parquet(0, &DATE_TIME_MICRO); + let zero_ns = DateTime::::from_parquet(0, &DATE_TIME_NANO); + let e = DateTime::::default(); + assert_eq!(e.to_parquet(), 0); + assert_eq!(zero_ms, e); + assert_eq!(zero_us, e); + assert_eq!(zero_ns, e); + let noon_ms = DateTime::::from_parquet(43_200_000, &DATE_TIME_MILLI); + let noon_us = DateTime::::from_parquet(43_200_000_000, &DATE_TIME_MICRO); + let noon_ns = DateTime::::from_parquet(43_200_000_000_000, &DATE_TIME_NANO); + let n = DateTime::::from_str("1970-01-01T12:00:00Z").unwrap(); + assert_eq!(n.to_parquet(), 43_200_000_000); + assert_eq!(noon_ms, n); + assert_eq!(noon_us, n); + assert_eq!(noon_ns, n); + let bc0 = DateTime::::from_str("-1000-08-20 03:00:00.123456 UTC").unwrap(); + assert_eq!(bc0.to_parquet(), -93704158799876544); + let bc1 = DateTime::::from_parquet(bc0.to_parquet(), &DATE_TIME_MICRO); + assert_eq!(bc0, bc1); + } + + #[test] + fn date_time_overflow() { + let min = DateTime::::from_parquet(i64::MIN, &DATE_TIME_MILLI); + assert_eq!(min.to_string(), "-262143-01-01 00:00:00 UTC"); + let max = DateTime::::from_parquet(i64::MAX, &DATE_TIME_MILLI); + assert_eq!(max.to_string(), "+262142-12-31 23:59:59.999999999 UTC"); + } +} diff --git a/src/pqarray/mod.rs b/src/pqarray/mod.rs new file mode 100644 index 0000000..b1c98d2 --- /dev/null +++ b/src/pqarray/mod.rs @@ -0,0 +1,15 @@ +mod array_type; +pub mod read; +pub mod schema; +mod source; +#[cfg(test)] +mod tests; +mod write; + +pub(crate) use array_type::PqArrayType; +pub(crate) use read::PqArrayReader; +pub(crate) use schema::{ + schema, schema_field, schema_fields, schema_logical_type, schema_match, schema_physical_type, + schema_repetition, PqArrayMatcher, +}; +pub(crate) use write::{PqArrayWriter, PqWriteOptions}; diff --git a/src/pqarray/read.rs b/src/pqarray/read.rs new file mode 100644 index 0000000..6de778c --- /dev/null +++ b/src/pqarray/read.rs @@ -0,0 +1,481 @@ +use std::{fmt::Debug, marker::PhantomData, ops::Range, sync::Arc}; + +use parquet::{ + basic::{LogicalType, Repetition}, + column::reader::{ColumnReader, ColumnReaderImpl}, + data_type::DataType, + errors::ParquetError, + file::{ + metadata::ParquetMetaData, + reader::{ChunkReader, FileReader}, + serialized_reader::SerializedFileReader, + }, + schema::types::{ColumnPath, Type}, +}; + +use crate::error::Error; + +use super::{array_type::PqArrayType, PqArrayMatcher}; + +const CHUNK_SIZE: usize = 8192; + +#[derive(Debug, Clone)] +struct Info { + column_index: usize, + size_hint: (usize, Option), + logical_type: Option, +} + +fn invalid(message: impl Into) -> Error { + ParquetError::General(message.into()).into() +} + +/// Checks the type of a column so it at least shouldn't panic. +/// For better errors use [`PqArrayMatcher`](super::PqArrayMatcher) before reading. +fn check( + metadata: &ParquetMetaData, + path: ColumnPath, + nullable: bool, +) -> Result { + let schema = metadata.file_metadata().schema_descr(); + // Find column. + let column_index = schema + .columns() + .iter() + .enumerate() + .find_map(|(i, c)| if c.path() == &path { Some(i) } else { None }) + .ok_or_else(|| invalid(format!("Parquet column not found: {}", path)))?; + // Check primitive and logical types. + let ty = schema.column(column_index).self_type_ptr(); + if !ty.is_primitive() + || ty.get_physical_type() != P::physical_type() + || !P::check_logical_type(ty.get_basic_info().logical_type()) + { + return Err(invalid("column type mismatch")); + } + // Check repetition. + match (ty.get_basic_info().repetition(), nullable) { + (Repetition::REQUIRED, true | false) => (), + (Repetition::OPTIONAL, true) => (), + _ => return Err(invalid("column repetition mismatch")), + } + Ok(Info { + column_index, + logical_type: ty.get_basic_info().logical_type(), + size_hint: if let Ok(n) = metadata.file_metadata().num_rows().try_into() { + (n, Some(n)) + } else { + (usize::MAX, None) + }, + }) +} + +#[derive(Debug, Default)] +struct Counter(usize); + +impl Counter { + fn new() -> Self { + Self(0) + } + + fn incr(&mut self) -> usize { + let out = self.0; + self.0 += 1; + out + } + + fn reset(&mut self) { + self.0 = 0; + } + + fn reached(&self, limit: usize) -> bool { + self.0 >= limit + } +} + +/// Reads a block of data from a column reader. +fn read_column_chunk( + column: &mut ColumnReaderImpl, + values: &mut Vec, + mut def_levels: Option<&mut Vec>, +) -> Result<(usize, usize), Error> { + values.clear(); + let mut max_records = values.capacity(); + if let Some(d) = &mut def_levels { + d.clear(); + max_records = max_records.max(d.capacity()); + } + let (n_val, n_def, _n_rep) = column.read_records(max_records, def_levels, None, values)?; + Ok((n_val, n_def)) +} + +/// Iterates over all the values in one column within one row group. +/// +/// Two implementations support required and nullable values. +pub trait GroupValues { + type Item; + const NULLABLE: bool; + + fn new(logical_type: Option) -> Self; + fn set_column_reader(&mut self, column: Result, Error>); + fn next(&mut self) -> Option>; +} + +pub struct RequiredGroupValues { + column: Result, Option>, + len: usize, + index: Counter, + values: Vec<::T>, + logical_type: Option, +} + +impl GroupValues

for RequiredGroupValues

{ + type Item = P; + const NULLABLE: bool = false; + + fn new(logical_type: Option) -> Self { + Self { + column: Err(None), + len: 0, + index: Counter::new(), + values: vec![Default::default(); CHUNK_SIZE], + logical_type, + } + } + + fn set_column_reader(&mut self, column: Result, Error>) { + self.column = column.map_err(Some); + self.len = 0; + self.index.reset(); + } + + fn next(&mut self) -> Option> { + let column = match &mut self.column { + Ok(x) => x, + Err(None) => return None, + Err(e @ Some(_)) => return Some(Err(e.take().unwrap())), + }; + if self.index.reached(self.len) { + match read_column_chunk(column, &mut self.values, None) { + Ok((n_read, _)) => { + self.len = n_read; + self.index.reset(); + if n_read == 0 { + return None; + } + } + Err(e) => { + return Some(Err(e)); + } + } + } + let value = self.values[self.index.incr()].clone(); + Some(Ok(P::from_parquet(value, &self.logical_type))) + } +} + +/// Iterates over all the values in one column within one row group. +pub struct NullableGroupValues { + column: Result, Option>, + len: usize, + index: Counter, + value_index: Counter, + values: Vec<::T>, + def_levels: Vec, + logical_type: Option, +} + +impl GroupValues

for NullableGroupValues

{ + type Item = Option

; + const NULLABLE: bool = true; + + fn new(logical_type: Option) -> Self { + Self { + column: Err(None), + len: 0, + index: Counter::new(), + value_index: Counter::new(), + values: Vec::with_capacity(CHUNK_SIZE), + def_levels: Vec::with_capacity(CHUNK_SIZE), + logical_type, + } + } + + fn set_column_reader(&mut self, column: Result, Error>) { + self.column = column.map_err(Some); + self.len = 0; + self.index.reset(); + self.value_index.reset(); + } + + fn next(&mut self) -> Option, Error>> { + let column = match &mut self.column { + Ok(x) => x, + Err(None) => return None, + Err(e @ Some(_)) => return Some(Err(e.take().unwrap())), + }; + if self.index.reached(self.len) { + match read_column_chunk(column, &mut self.values, Some(&mut self.def_levels)) { + Ok((n_read, _)) => { + self.len = n_read; + self.index.reset(); + self.value_index.reset(); + if n_read == 0 { + return None; + } + } + Err(e) => { + return Some(Err(e)); + } + } + } + let value = if self.def_levels[self.index.incr()] == 0 { + None + } else { + Some(P::from_parquet( + self.values[self.value_index.incr()].clone(), + &self.logical_type, + )) + }; + Some(Ok(value)) + } +} + +/// Iterates over row groups in a file, generating column readers. +struct Groups { + file_reader: Arc>, + column_index: usize, + indices: Range, +} + +impl Iterator for Groups { + type Item = Result; + + fn next(&mut self) -> Option { + let index = self.indices.next()?; + Some( + self.file_reader + .get_row_group(index) + .and_then(|rg| rg.get_column_reader(self.column_index)) + .map_err(Into::into), + ) + } +} + +/// Iterates over a Parquet file object, generating all the values in a single column. +pub struct ColumnIter> { + info: Info, + first: bool, + groups: Groups, + group_values: G, + _phantom: PhantomData

, +} + +impl> ColumnIter { + fn new(file_reader: Arc>, path: ColumnPath) -> Result { + let info = check::

(file_reader.metadata(), path, G::NULLABLE)?; + Ok(Self { + first: true, + groups: Groups { + indices: 0..file_reader.num_row_groups(), + file_reader, + column_index: info.column_index, + }, + group_values: G::new(info.logical_type.clone()), + info, + _phantom: PhantomData, + }) + } + + fn advance_group(&mut self) -> Option<()> { + self.group_values.set_column_reader( + self.groups + .next()? + .map(|c| P::DataType::get_column_reader(c).expect("matching column reader type")), + ); + Some(()) + } +} + +impl> Iterator for ColumnIter { + type Item = Result; + + fn next(&mut self) -> Option { + if self.first { + self.first = false; + self.advance_group()?; + } + loop { + if let Some(value) = self.group_values.next() { + return Some(value); + } + self.advance_group()?; + } + } + + fn size_hint(&self) -> (usize, Option) { + self.info.size_hint + } +} + +impl> Debug for ColumnIter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ColumnIter") + .field("info", &self.info) + .field("first", &self.first) + .finish() + } +} + +/// Combines multiple column iterators into one. +#[derive(Debug)] +pub struct ColumnZip { + iters: [I; N], + _phantom: PhantomData, +} + +impl Iterator for ColumnZip +where + T: Default, + I: Iterator>, +{ + type Item = Result<[T; N], Error>; + + fn next(&mut self) -> Option { + let items: [_; N] = std::array::from_fn(|i| self.iters[i].next()); + let mut output = std::array::from_fn(|_| T::default()); + for (item, out) in items.into_iter().zip(&mut output) { + match item { + None => return None, + Some(Err(error)) => return Some(Err(error)), + Some(Ok(value)) => *out = value, + } + } + Some(Ok(output)) + } +} + +/// Combines multiple nullable column iterators into one. +#[derive(Debug)] +pub struct NullableColumnZip { + iters: [I; N], + _phantom: PhantomData, +} + +impl Iterator for NullableColumnZip +where + T: Default, + I: Iterator, Error>>, +{ + type Item = Result, Error>; + + fn next(&mut self) -> Option { + let items: [_; N] = std::array::from_fn(|i| self.iters[i].next()); + let mut output = std::array::from_fn(|_| T::default()); + for (item, out) in items.into_iter().zip(&mut output) { + match item { + None => return None, + Some(Err(error)) => return Some(Err(error)), + Some(Ok(None)) => return Some(Ok(None)), + Some(Ok(Some(value))) => *out = value, + } + } + Some(Ok(Some(output))) + } +} + +/// Provides an interface for reading OMF arrays from Parquet files. +pub struct PqArrayReader { + file_reader: Arc>, +} + +impl PqArrayReader { + pub fn new(read: R) -> Result { + Ok(Self { + file_reader: SerializedFileReader::new(read)?.into(), + }) + } + + pub fn len(&self) -> u64 { + self.file_reader + .metadata() + .file_metadata() + .num_rows() + .try_into() + .unwrap_or_default() + } + + pub fn matches(&self, matcher: &PqArrayMatcher) -> Result { + matcher.check(self.schema()) + } + + pub fn schema(&self) -> Arc { + self.file_reader + .metadata() + .file_metadata() + .schema_descr() + .root_schema_ptr() + } + + pub fn iter_column(&self, name: &str) -> Result, Error> { + ColumnIter::new( + self.file_reader.clone(), + ColumnPath::new(vec![name.to_owned()]), + ) + } + + pub fn iter_nullable_column( + &self, + name: &str, + ) -> Result, Error> { + ColumnIter::new( + self.file_reader.clone(), + ColumnPath::new(vec![name.to_owned()]), + ) + } + + pub fn iter_multi_column( + &self, + names: [&str; N], + ) -> Result, Error> { + let iters: [_; N] = names + .iter() + .map(|n| self.iter_column::

(n)) + .collect::, _>>()? + .try_into() + .expect("correct length"); + Ok(ColumnZip { + iters, + _phantom: PhantomData, + }) + } + + pub fn iter_nullable_group_column( + &self, + group_name: &str, + field_names: [&str; N], + ) -> Result, Error> { + let iters: [_; N] = field_names + .into_iter() + .map(|name| { + ColumnIter::new( + self.file_reader.clone(), + ColumnPath::new(vec![group_name.to_owned(), name.to_owned()]), + ) + }) + .collect::, _>>()? + .try_into() + .expect("correct length"); + Ok(NullableColumnZip { + iters, + _phantom: PhantomData, + }) + } +} + +pub type SimpleIter = ColumnIter>; +pub type NullableIter = ColumnIter>; +pub type MultiIter = + ColumnZip>, N>; +pub type NullableGroupIter = + NullableColumnZip>, N>; diff --git a/src/pqarray/schema.rs b/src/pqarray/schema.rs new file mode 100644 index 0000000..b3d982f --- /dev/null +++ b/src/pqarray/schema.rs @@ -0,0 +1,243 @@ +use std::sync::Arc; + +use parquet::schema::types::Type; + +use crate::error::Error; + +#[derive(Debug, Default, Clone)] +pub struct PqArrayMatcher { + values: Vec, + expected: Arc>, +} + +impl PqArrayMatcher { + /// Creates a matcher from a list of field types. + /// + /// Types with the same name will be grouped together. + pub fn new(items: [(T, Type); N]) -> Self { + let (values, expected) = items.into_iter().unzip(); + Self { + values, + expected: Arc::new(expected), + } + } + + /// Returns the schemas of this matcher. + #[cfg(test)] + pub fn schemas(&self) -> &[Type] { + &self.expected + } + + /// Checks a file schema against this matcher, returning the matched index on success. + pub fn check(&self, schema: Arc) -> Result { + let index = self + .expected + .iter() + .enumerate() + .find_map(|(i, s)| if schema.as_ref() == s { Some(i) } else { None }) + .ok_or_else(|| Error::ParquetSchemaMismatch(schema, self.expected.clone()))?; + Ok(self.values[index]) + } +} + +/// Creates a schema matcher. If a field can have multiple types then repeat the name in +/// multiple fields. +macro_rules! schema_match { + // -> PqArrayMatcher + ($($value:expr => schema $body:tt)*) => { + crate::pqarray::PqArrayMatcher::new( + [ + $( ($value, crate::pqarray::schema! $body), )* + ], + ) + }; +} + +/// Creates a schema. +macro_rules! schema { // -> Type + ($($token:tt)*) => { + parquet::schema::types::Type::group_type_builder("schema") + .with_fields( + crate::pqarray::schema_fields!($($token)*) + .into_iter() + .map(std::sync::Arc::new) + .collect() + ) + .build() + .expect("valid type") + }; +} + +macro_rules! schema_fields { // -> Vec + ($( $rep:ident $phys:ident $name:ident $( ($($log:tt)*) )? ; )+) => { + vec![ + $( crate::pqarray::schema_field!( + $rep $phys $name + $(( $($log)* ))? + ) ),* + ] + }; + ($( $rep:ident group $name:ident $( ($($log:tt)*) )? { $($gr:tt)* } )+) => { + vec![ + $( crate::pqarray::schema_field!( + $rep group $name + $(( $($log)* ))? + { $($gr)* } + ) ),* + ] + }; +} + +macro_rules! schema_field { // -> Type (single field) + ($rep:ident $phys:ident $name:ident $( ($log:ident $($log_detail:tt)?) )?) => { + parquet::schema::types::Type::primitive_type_builder( + stringify!($name), + crate::pqarray::schema_physical_type!($phys), + ) + .with_repetition(crate::pqarray::schema_repetition!($rep)) + .with_logical_type(crate::pqarray::schema_logical_type!($( $log $($log_detail)? )?)) + .build() + .expect("valid type") + }; + ($rep:ident group $name:ident { $($token:tt)* }) => { + parquet::schema::types::Type::group_type_builder(stringify!($name)) + .with_fields( + crate::pqarray::schema_fields!($($token)*) + .into_iter() + .map(std::sync::Arc::new) + .collect() + ) + .with_repetition(crate::pqarray::schema_repetition!($rep)) + .build() + .expect("valid type") + }; +} + +macro_rules! schema_repetition { + (required) => { + parquet::basic::Repetition::REQUIRED + }; + (optional) => { + parquet::basic::Repetition::OPTIONAL + }; +} + +macro_rules! schema_physical_type { + (int32) => { + parquet::basic::Type::INT32 + }; + (int64) => { + parquet::basic::Type::INT64 + }; + (float) => { + parquet::basic::Type::FLOAT + }; + (double) => { + parquet::basic::Type::DOUBLE + }; + (boolean) => { + parquet::basic::Type::BOOLEAN + }; + (byte_array) => { + parquet::basic::Type::BYTE_ARRAY + }; +} + +macro_rules! schema_logical_type { + () => { + None + }; + (string) => { + Some(parquet::basic::LogicalType::String) + }; + (integer($bits:literal, $signed:literal)) => { + Some(parquet::basic::LogicalType::Integer { + bit_width: $bits, + is_signed: $signed, + }) + }; + (date) => { + Some(parquet::basic::LogicalType::Date) + }; + (timestamp(millis, true)) => { + Some(parquet::basic::LogicalType::Timestamp { + is_adjusted_to_u_t_c: true, + unit: TimeUnit::MILLIS(Default::default()), + }) + }; + (timestamp(micros, true)) => { + Some(parquet::basic::LogicalType::Timestamp { + is_adjusted_to_u_t_c: true, + unit: parquet::basic::TimeUnit::MICROS(Default::default()), + }) + }; + (timestamp(nanos, true)) => { + Some(Timestamp { + is_adjusted_to_u_t_c: true, + unit: TimeUnit::NANOS(Default::default()), + }) + }; +} + +pub(crate) use { + schema, schema_field, schema_fields, schema_logical_type, schema_match, schema_physical_type, + schema_repetition, +}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parquet_schema_match() { + let matcher = schema_match! { + 0 => schema { + required float x; + optional int32 y (date); + } + 1 => schema { + required double x; + optional int32 y (date); + } + }; + let schema = schema! { + required double x; + optional int32 y (date); + }; + let index = matcher.check(schema.into()).unwrap(); + assert_eq!(index, 1); + } + + #[test] + fn parquet_schema_no_match() { + let matcher = schema_match! { + 0 => schema { + required float x; + } + 1 => schema { + required double x; + } + }; + let schema = schema! { + required int32 x; + }; + let err = matcher.check(schema.into()).unwrap_err(); + assert_eq!( + err.to_string(), + "\ + Parquet schema mismatch, found: +message schema { + REQUIRED INT32 x; +} + +Expected one of: +message schema { + REQUIRED FLOAT x; +} +message schema { + REQUIRED DOUBLE x; +} +" + ); + } +} diff --git a/src/pqarray/source.rs b/src/pqarray/source.rs new file mode 100644 index 0000000..eeef2d8 --- /dev/null +++ b/src/pqarray/source.rs @@ -0,0 +1,283 @@ +use std::{collections::TryReserveError, io::Write}; + +use parquet::{ + basic::Repetition, + data_type::DataType, + errors::ParquetError, + file::writer::{SerializedColumnWriter, SerializedRowGroupWriter}, + schema::types::Type, +}; + +use super::array_type::PqArrayType; + +pub trait RowGrouper { + fn next_column(&mut self) -> Result>, ParquetError>; +} + +impl<'a, W: Write + Send> RowGrouper for SerializedRowGroupWriter<'a, W> { + fn next_column(&mut self) -> Result>, ParquetError> { + SerializedRowGroupWriter::next_column(self) + } +} + +pub trait Source { + /// Parquet schema types that this source writes. + fn types(&self) -> Vec; + /// Read up to `size` items into internal buffers, returning the number of items read. + fn buffer(&mut self, size: usize) -> usize; + /// Write data from the last `buffer()` call. + fn write(&mut self, row_group: &mut dyn RowGrouper) -> Result<(), ParquetError>; +} + +fn single_type(name: &str, nullable: bool) -> Type { + Type::primitive_type_builder(name, P::physical_type()) + .with_repetition(if nullable { + Repetition::OPTIONAL + } else { + Repetition::REQUIRED + }) + .with_logical_type(P::logical_type()) + .build() + .expect("valid type") +} + +fn row_group_vec(n: usize) -> Result, TryReserveError> { + let mut v = Vec::new(); + v.try_reserve_exact(n)?; + Ok(v) +} + +fn row_group_vec_init(value: T, n: usize) -> Result, TryReserveError> { + let mut v = row_group_vec(n)?; + v.resize(n, value); + Ok(v) +} + +pub trait PqArrayRow: 'static { + type Buffer: Sized; + const WIDTH: usize; + + fn types(names: &[&str]) -> Vec; + fn make_buffer(row_group_size: usize) -> Result; + fn clear_buffer(buffer: &mut Self::Buffer); + fn add_to_buffer(self, buffer: &mut Self::Buffer); + fn write_buffer( + buffer: &Self::Buffer, + row_group: &mut dyn RowGrouper, + def_levels: &[i16], + ) -> Result<(), ParquetError>; +} + +pub struct RowSource> { + row_types: Vec, + iter: I, + buffer: R::Buffer, + def_levels: Vec, + count: usize, +} + +impl> RowSource { + pub fn new(names: &[&str], iter: I, row_group_size: usize) -> Result { + Ok(Self { + row_types: R::types(names), + iter, + buffer: R::make_buffer(row_group_size)?, + def_levels: row_group_vec_init(1, row_group_size)?, + count: 0, + }) + } +} + +impl> Source for RowSource { + fn types(&self) -> Vec { + self.row_types.clone() + } + + fn buffer(&mut self, size: usize) -> usize { + R::clear_buffer(&mut self.buffer); + self.count = 0; + for row in self.iter.by_ref().take(size) { + row.add_to_buffer(&mut self.buffer); + self.count += 1; + } + self.count + } + + fn write(&mut self, row_group: &mut dyn RowGrouper) -> Result<(), ParquetError> { + R::write_buffer(&self.buffer, row_group, &self.def_levels[..self.count]) + } +} + +pub struct NullableRowSource>> { + ty: Type, + iter: I, + buffer: R::Buffer, + def_levels: Vec, +} + +impl>> NullableRowSource { + pub fn new( + group_name: &str, + names: &[&str], + iter: I, + row_group_size: usize, + ) -> Result { + Ok(Self { + ty: Type::group_type_builder(group_name) + .with_repetition(Repetition::OPTIONAL) + .with_fields(R::types(names).into_iter().map(Into::into).collect()) + .build() + .expect("valid type"), + iter, + buffer: R::make_buffer(row_group_size)?, + def_levels: row_group_vec_init(1, row_group_size)?, + }) + } + + pub fn new_single(name: &str, iter: I, row_group_size: usize) -> Result { + assert_eq!(R::WIDTH, 1); + let ty = R::types(&["tmp"]).into_iter().next().unwrap(); + Ok(Self { + ty: Type::primitive_type_builder(name, ty.get_physical_type()) + .with_repetition(Repetition::OPTIONAL) + .with_logical_type(ty.get_basic_info().logical_type()) + .build() + .expect("valid type"), + iter, + buffer: R::make_buffer(row_group_size)?, + def_levels: row_group_vec_init(1, row_group_size)?, + }) + } +} + +impl>> Source for NullableRowSource { + fn types(&self) -> Vec { + vec![self.ty.clone()] + } + + fn buffer(&mut self, size: usize) -> usize { + self.def_levels.clear(); + R::clear_buffer(&mut self.buffer); + for opt_row in self.iter.by_ref().take(size) { + if let Some(row) = opt_row { + row.add_to_buffer(&mut self.buffer); + self.def_levels.push(1); + } else { + self.def_levels.push(0); + } + } + self.def_levels.len() + } + + fn write(&mut self, row_group: &mut dyn RowGrouper) -> Result<(), ParquetError> { + R::write_buffer( + &self.buffer, + row_group, + &self.def_levels[..self.def_levels.len()], + ) + } +} + +impl PqArrayRow for [P; N] { + type Buffer = [Vec<::T>; N]; + const WIDTH: usize = N; + + fn types(names: &[&str]) -> Vec { + assert_eq!(names.len(), N); + names.iter().map(|n| single_type::

(n, false)).collect() + } + + fn make_buffer(row_group_size: usize) -> Result { + let mut buffer = std::array::from_fn(|_| Vec::new()); + for b in &mut buffer { + *b = row_group_vec(row_group_size)?; + } + Ok(buffer) + } + + fn clear_buffer(buffer: &mut Self::Buffer) { + for b in buffer { + b.clear(); + } + } + + fn add_to_buffer(self, buffer: &mut Self::Buffer) { + for (b, a) in buffer.iter_mut().zip(self.into_iter()) { + b.push(a.to_parquet()); + } + } + + fn write_buffer( + buffer: &Self::Buffer, + row_group: &mut dyn RowGrouper, + def_levels: &[i16], + ) -> Result<(), ParquetError> { + for b in buffer { + let mut column = row_group.next_column()?.expect("columns to match schema"); + column + .typed::() + .write_batch(b, Some(def_levels), None)?; + column.close()?; + } + Ok(()) + } +} + +macro_rules! row { + ($width:literal, { $($i:tt $P:ident),* }) => { + impl<$( $P: PqArrayType ),*> PqArrayRow for ($( $P, )*) { + type Buffer = ( + $( Vec<<$P::DataType as DataType>::T>, )* + ); + const WIDTH: usize = $width; + + fn types(names: &[&str]) -> Vec { + assert_eq!(names.len(), $width); + let mut names_iter = names.into_iter(); + vec![$( + single_type::<$P>(names_iter.next().unwrap(), false), + )*] + } + + fn make_buffer(row_group_size: usize) -> Result { + Ok(($( + row_group_vec::<<$P::DataType as DataType>::T>(row_group_size)?, + )*)) + } + + fn clear_buffer(buffer: &mut Self::Buffer) { + $( buffer.$i.clear(); )* + } + + fn add_to_buffer(self, buffer: &mut Self::Buffer) { + $( buffer.$i.push(self.$i.to_parquet()); )* + } + + fn write_buffer( + buffer: &Self::Buffer, + row_group: &mut dyn RowGrouper, + def_levels: &[i16], + ) -> Result<(), ParquetError> { + $( + let mut column = row_group.next_column()?.expect("columns to match schema"); + column + .typed::<$P::DataType>() + .write_batch(&buffer.$i, Some(def_levels), None)?; + column.close()?; + )* + Ok(()) + } + } + }; +} + +row!(1, { 0 P0 }); +row!(2, { 0 P0, 1 P1 }); +row!(3, { 0 P0, 1 P1, 2 P2 }); +row!(4, { 0 P0, 1 P1, 2 P2, 3 P3 }); +row!(5, { 0 P0, 1 P1, 2 P2, 3 P3, 4 P4 }); +row!(6, { 0 P0, 1 P1, 2 P2, 3 P3, 4 P4, 5 P5 }); +row!(7, { 0 P0, 1 P1, 2 P2, 3 P3, 4 P4, 5 P5, 6 P6 }); +row!(8, { 0 P0, 1 P1, 2 P2, 3 P3, 4 P4, 5 P5, 6 P6, 7 P7 }); +row!(9, { 0 P0, 1 P1, 2 P2, 3 P3, 4 P4, 5 P5, 6 P6, 7 P7, 8 P8 }); +row!(10, { 0 P0, 1 P1, 2 P2, 3 P3, 4 P4, 5 P5, 6 P6, 7 P7, 8 P8, 9 P9 }); diff --git a/src/pqarray/tests.rs b/src/pqarray/tests.rs new file mode 100644 index 0000000..5d8b10d --- /dev/null +++ b/src/pqarray/tests.rs @@ -0,0 +1,374 @@ +use std::str::FromStr; + +use bytes::Bytes; +use chrono::prelude::*; + +use super::{schema::schema, *}; + +fn write_read(writer: PqArrayWriter) -> PqArrayReader { + let mut buffer = Vec::new(); + writer.write(&mut buffer).unwrap(); + PqArrayReader::new(Bytes::from(buffer)).unwrap() +} + +fn read_column(reader: &PqArrayReader, name: &str) -> Vec

{ + reader + .iter_column::

(name) + .unwrap() + .collect::, _>>() + .unwrap() +} + +fn read_nullable_column( + reader: &PqArrayReader, + name: &str, +) -> Vec> { + reader + .iter_nullable_column::

(name) + .unwrap() + .collect::, _>>() + .unwrap() +} + +fn read_multi_column( + reader: &PqArrayReader, + names: [&str; N], +) -> Vec<[P; N]> { + reader + .iter_multi_column::(names) + .unwrap() + .collect::, _>>() + .unwrap() +} + +fn read_nullable_group_column( + reader: &PqArrayReader, + group_name: &str, + field_names: [&str; N], +) -> Vec> { + reader + .iter_nullable_group_column::(group_name, field_names) + .unwrap() + .collect::, _>>() + .unwrap() +} + +#[test] +fn parquet_array_bool() { + let values = [true, true, false, true]; + let mut writer = PqArrayWriter::new(Default::default()); + writer.add("filter", values).unwrap(); + let reader = write_read(writer); + assert_eq!( + reader.schema().as_ref(), + &schema! { + required boolean filter; + } + ); + assert_eq!(read_column::(&reader, "filter"), values); +} + +#[test] +fn parquet_array_f64() { + let values = [0.0_f64, 1.0, 2.0, 3.0, 4.0]; + let mut writer = PqArrayWriter::new(Default::default()); + writer.add("value", values).unwrap(); + let reader = write_read(writer); + assert_eq!( + reader.schema().as_ref(), + &schema! { + required double value; + } + ); + assert_eq!(read_column::(&reader, "value"), values); +} + +#[test] +fn parquet_array_u32() { + let values = [0_u32, 1, 2, 3, 4_294_967_295]; + let mut writer = PqArrayWriter::new(Default::default()); + writer.add("value", values).unwrap(); + let reader = write_read(writer); + assert_eq!( + reader.schema().as_ref(), + &schema! { + required int32 value(integer(32,false)); + } + ); + assert_eq!(read_column::(&reader, "value"), values); +} + +#[test] +fn parquet_array_u8() { + let values = [0_u8, 1, 2, 3, 255]; + let mut writer = PqArrayWriter::new(Default::default()); + writer.add("a", values).unwrap(); + let reader = write_read(writer); + assert_eq!( + reader.schema().as_ref(), + &schema! { + required int32 a(integer(8,false)); + } + ); + assert_eq!(read_column::(&reader, "a"), values); +} + +#[test] +fn parquet_array_i64() { + let values = [-1_000_000_000_000_i64, 1, 2, 3, 1_000_000_000_000]; + let mut writer = PqArrayWriter::new(PqWriteOptions { + row_group_size: 3, + compression_level: 10, + ..Default::default() + }); + writer.add("x", values).unwrap(); + let reader = write_read(writer); + assert_eq!( + reader.schema().as_ref(), + &schema! { + required int64 x; + } + ); + assert_eq!(read_column::(&reader, "x"), values); +} + +#[test] +fn parquet_array_nullable_f64() { + let values = [Some(0.0_f64), Some(1.0), Some(2.0), None, None, Some(4.0)]; + let mut writer = PqArrayWriter::new(Default::default()); + writer.add_nullable("value", values).unwrap(); + let reader = write_read(writer); + assert_eq!( + reader.schema().as_ref(), + &schema! { + optional double value; + } + ); + assert_eq!(read_nullable_column::(&reader, "value"), values); +} + +#[test] +fn parquet_array_multi() { + let values = [ + [0.0, 0.1, 0.2], + [1.0, 1.1, 1.2], + [2.0, 2.1, 2.2], + [3.0, 3.1, 3.2], + ]; + let mut writer = PqArrayWriter::new(Default::default()); + writer.add_multiple(&["x", "y", "z"], values).unwrap(); + let reader = write_read(writer); + assert_eq!( + reader.schema().as_ref(), + &schema! { + required double x; + required double y; + required double z; + } + ); + assert_eq!( + read_multi_column::(&reader, ["x", "y", "z"]), + values + ); +} + +#[test] +fn parquet_array_nullable_group() { + let values = [ + Some([0.0_f32, 0.1]), + None, + Some([10.0, 0.2]), + Some([20.0, 0.3]), + Some([30.0, 0.4]), + ]; + let mut writer = PqArrayWriter::new(Default::default()); + writer + .add_nullable_group("vectors", &["x", "y"], values) + .unwrap(); + let reader = write_read(writer); + assert_eq!( + reader.schema().as_ref(), + &schema! { + optional group vectors { + required float x; + required float y; + } + } + ); + assert_eq!( + read_nullable_group_column::(&reader, "vectors", ["x", "y"]), + values + ); +} + +#[test] +fn parquet_array_separate_columns() { + let values0 = [0_i64, 1, 2, 3, 4]; + let values1 = [Some(0.0_f64), Some(0.1), Some(0.2), Some(0.3), None]; + let mut writer = PqArrayWriter::new(Default::default()); + writer.add("a", values0).unwrap(); + writer.add_nullable("b", values1).unwrap(); + let reader = write_read(writer); + assert_eq!( + reader.schema().as_ref(), + &schema! { + required int64 a; + optional double b; + } + ); + assert_eq!(read_column::(&reader, "a"), values0); + assert_eq!(read_nullable_column::(&reader, "b"), values1); +} + +#[test] +fn parquet_array_separate_columns_uneven() { + let values0 = [0_i64, 1, 2, 3, 4]; + let values1 = [0.0_f64, 0.1, 0.2, 0.3]; + let mut writer = PqArrayWriter::new(Default::default()); + writer.add("a", values0).unwrap(); + writer.add("b", values1).unwrap(); + let mut buf = Vec::new(); + let e = writer.write(&mut buf).unwrap_err(); + assert_eq!( + e.to_string(), + "Parquet error: uneven iterator lengths after 4 items" + ); +} + +#[test] +fn parquet_array_string() { + let values = [ + "foo".to_owned(), + "bar".to_owned(), + "".to_owned(), + "♫".to_owned(), + ]; + let mut writer = PqArrayWriter::new(Default::default()); + writer.add("s", values.clone()).unwrap(); + let reader = write_read(writer); + assert_eq!( + reader.schema().as_ref(), + &schema! { + required byte_array s(string); + } + ); + assert_eq!(read_column::(&reader, "s"), values); +} + +#[test] +fn parquet_array_binary() { + let values = [ + vec![0, 1, 2, 3], + b"foo".to_vec(), + b"".to_vec(), + vec![254, 255], + ]; + let mut writer = PqArrayWriter::new(Default::default()); + writer.add("b", values.clone()).unwrap(); + let reader = write_read(writer); + assert_eq!( + reader.schema().as_ref(), + &schema! { + required byte_array b; + } + ); + assert_eq!(read_column::>(&reader, "b"), values); +} + +#[test] +fn parquet_nullable_array_string() { + let values = [ + Some("foo".to_owned()), + Some("bar".to_owned()), + None, + Some("".to_owned()), + ]; + let mut writer = PqArrayWriter::new(Default::default()); + writer.add_nullable("a", values.clone()).unwrap(); + let reader = write_read(writer); + assert_eq!( + reader.schema().as_ref(), + &schema! { + optional byte_array a(string); + } + ); + assert_eq!(read_nullable_column::(&reader, "a"), values); +} + +#[ignore = "for performance testing"] +#[test] +fn parquet_array_large() { + let values: Vec<_> = (0..10_000_000).map(|i| f64::from(i)).collect(); + + let mut writer = PqArrayWriter::new(Default::default()); + writer.add("value", values).unwrap(); + + let mut start = std::time::Instant::now(); + let mut buf = Vec::new(); + writer.write(&mut buf).unwrap(); + println!("file size is {} B", buf.len()); + println!("wrote in {:?}", start.elapsed()); + + start = std::time::Instant::now(); + let reader = PqArrayReader::new(Bytes::from(buf)).unwrap(); + let _read_column = read_column::(&reader, "value"); + println!("read in {:?}", start.elapsed()); + + assert!(false); +} + +#[test] +fn parquet_array_date() { + let values = ["1970-01-01", "-1000-08-20", "2023-09-14", "+10000-12-31"]; + let mut writer = PqArrayWriter::new(Default::default()); + writer + .add("d", values.iter().map(|s| NaiveDate::from_str(s).unwrap())) + .unwrap(); + let reader = write_read(writer); + assert_eq!( + reader.schema().as_ref(), + &schema! { + required int32 d (date); + } + ); + assert_eq!( + reader + .iter_column::("d") + .unwrap() + .map(|d| d.unwrap().to_string()) + .collect::>(), + values + ); +} + +#[test] +fn parquet_array_date_time() { + let values = [ + "1970-01-01 00:00:00 UTC", + "-1000-08-20 03:00:00.123456 UTC", + "2023-09-14 12:00:00 UTC", + "+10000-12-31 23:59:59 UTC", + ]; + let mut writer = PqArrayWriter::new(Default::default()); + writer + .add( + "t", + values.iter().map(|s| DateTime::::from_str(s).unwrap()), + ) + .unwrap(); + let reader = write_read(writer); + assert_eq!( + reader.schema().as_ref(), + &schema! { + required int64 t (timestamp(micros, true)); + } + ); + assert_eq!( + reader + .iter_column::>("t") + .unwrap() + .map(|d| d.unwrap().to_string()) + .collect::>(), + values + ); +} diff --git a/src/pqarray/write.rs b/src/pqarray/write.rs new file mode 100644 index 0000000..db8c448 --- /dev/null +++ b/src/pqarray/write.rs @@ -0,0 +1,186 @@ +use parquet::{ + basic::{Compression, Encoding, GzipLevel}, + errors::ParquetError, + file::{ + properties::{ + EnabledStatistics, WriterProperties, WriterVersion, DEFAULT_STATISTICS_ENABLED, + }, + writer::SerializedFileWriter, + }, + schema::types::Type, +}; + +use crate::error::Error; + +use super::{array_type::PqArrayType, source::*}; + +#[derive(Debug, Clone)] +pub struct PqWriteOptions { + pub row_group_size: usize, + pub compression_level: u32, + pub statistics: bool, +} + +impl Default for PqWriteOptions { + fn default() -> Self { + Self { + // Small row groups are massively slower. + row_group_size: 1024 * 1024, + // Default gzip compression level. + compression_level: 6, + // Statistics just waste time because we don't read them. + statistics: false, + } + } +} + +impl PqWriteOptions { + fn properties(&self) -> WriterProperties { + WriterProperties::builder() + .set_writer_version(WriterVersion::PARQUET_2_0) + .set_compression(Compression::GZIP( + GzipLevel::try_new(self.compression_level.clamp(0, 10)) + .expect("valid compression level"), + )) + .set_statistics_enabled(if self.statistics { + DEFAULT_STATISTICS_ENABLED + } else { + EnabledStatistics::None + }) + .set_max_row_group_size(self.row_group_size) + // We don't need any fancy encodings, and not using them is faster. + .set_dictionary_enabled(false) + .set_encoding(Encoding::PLAIN) + .build() + } +} + +#[derive(Default)] +pub struct PqArrayWriter<'a> { + options: PqWriteOptions, + sources: Vec>, + total_written: u64, +} + +impl<'a> PqArrayWriter<'a> { + pub fn new(options: PqWriteOptions) -> Self { + Self { + options, + sources: Vec::new(), + total_written: 0, + } + } + + fn push_source(&mut self, source: impl Source + 'a) { + self.sources.push(Box::new(source)) + } + + pub fn add + 'a>( + &mut self, + name: &str, + data: I, + ) -> Result<(), Error> { + self.push_source(RowSource::new( + &[name], + data.into_iter().map(|item| (item,)), + self.options.row_group_size, + )?); + Ok(()) + } + + pub fn add_nullable> + 'a>( + &mut self, + name: &str, + data: I, + ) -> Result<(), Error> { + self.push_source(NullableRowSource::new_single( + name, + data.into_iter() + .map(|opt_item| opt_item.map(|item| (item,))), + self.options.row_group_size, + )?); + Ok(()) + } + + pub fn add_multiple(&mut self, names: &[&str], data: I) -> Result<(), Error> + where + R: PqArrayRow, + I: IntoIterator + 'a, + { + self.push_source(RowSource::new( + names, + data.into_iter(), + self.options.row_group_size, + )?); + Ok(()) + } + + pub fn add_nullable_group> + 'a>( + &mut self, + group_name: &str, + field_names: &[&str], + data: I, + ) -> Result<(), Error> { + self.push_source(NullableRowSource::new( + group_name, + field_names, + data.into_iter(), + self.options.row_group_size, + )?); + Ok(()) + } + + fn schema(&self) -> Result { + let fields = self + .sources + .iter() + .flat_map(|s| s.types()) + .map(Into::into) + .collect(); + Type::group_type_builder("schema") + .with_fields(fields) + .build() + .map_err(Into::into) + } + + fn check_counts(&mut self, counts: &[u64]) -> Result { + let (min, max) = counts + .iter() + .fold((u64::MAX, 0), |(a, b), &c| (a.min(c), b.max(c))); + self.total_written += min; + if min != max { + return Err(ParquetError::General(format!( + "uneven iterator lengths after {} items", + self.total_written + )) + .into()); + } + Ok(min == 0) + } + + pub fn write(mut self, w: impl std::io::Write + Send) -> Result { + // Create Parquet writer. + let mut writer = + SerializedFileWriter::new(w, self.schema()?.into(), self.options.properties().into())?; + // Write data. + let mut counts = vec![0_u64; self.sources.len()]; + loop { + // Buffer data, collecting counts. + for (source, n) in self.sources.iter_mut().zip(&mut counts) { + *n = source.buffer(self.options.row_group_size) as u64; + } + // Check that sources buffered the same amount. + if self.check_counts(&counts)? { + break; + } + // Write that data. + let mut row_group = writer.next_row_group()?; + for source in &mut self.sources { + source.write(&mut row_group)?; + } + row_group.close()?; + } + writer.close()?; + Ok(self.total_written) + } +} diff --git a/src/project.rs b/src/project.rs new file mode 100644 index 0000000..3addc58 --- /dev/null +++ b/src/project.rs @@ -0,0 +1,132 @@ +use chrono::{DateTime, Utc}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::{ + date_time::utc_now, + geometry::zero_origin, + validate::{Validate, Validator}, + Element, Vector3, +}; + +/// Root object of an OMF file. +/// +/// This is the root element of an OMF file, holding global metadata and a list of +/// [Elements](crate::Element) that describe the objects or shapes within the file. +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, JsonSchema)] +pub struct Project { + /// Project name. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub name: String, + /// Optional project description. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub description: String, + /// Optional [EPSG](https://epsg.io/) or [PROJ](https://proj.org/) local transformation + /// string, default empty. + /// + /// Exactly what is supported depends on the application reading the file. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub coordinate_reference_system: String, + /// Optional unit for distances and locations within the file. + /// + /// Typically "meters", "metres", "feet", or empty because the coordinate reference system + /// defines it. If both are empty then applications may assume meters. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub units: String, + /// Optional project origin, default [0, 0, 0]. + /// + /// Most geometries also have their own origin field. To get the real location add this + /// origin and the geometry origin to all locations within each element. + #[serde(default, skip_serializing_if = "zero_origin")] + pub origin: Vector3, + /// Optional name or email address of the person that created the file, default empty. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub author: String, + /// Optional name and version of the application that created the file, default empty. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub application: String, + /// Optional file or data creation date, default empty. + pub date: DateTime, + /// Arbitrary metadata. + /// + /// This is the place to put anything that doesn't fit in the other fields. + /// Application-specific data should use a prefix that identifies the application, like + /// `"lf-something"` for Leapfrog. + #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] + pub metadata: serde_json::Map, + /// List of elements. + #[serde(default)] + pub elements: Vec, +} + +impl Project { + /// Create a new project with just the name set. + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + ..Default::default() + } + } +} + +impl Default for Project { + fn default() -> Self { + Self { + name: Default::default(), + description: Default::default(), + coordinate_reference_system: Default::default(), + units: Default::default(), + origin: Default::default(), + author: Default::default(), + application: Default::default(), + date: utc_now(), + metadata: Default::default(), + elements: Default::default(), + } + } +} + +impl Validate for Project { + fn validate_inner(&mut self, val: &mut Validator) { + val.enter("Project") + .name(&self.name) + .finite_seq(self.origin, "origin") + .objs(&mut self.elements) + .unique( + self.elements.iter().map(|e| &e.name), + "elements[..]::name", + false, + ); + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use serde_json::Value; + + use super::*; + + #[test] + fn serde_empty_project() { + let mut p = Project::new("Test"); + p.name = "Foo".to_owned(); + p.units = "meters".to_owned(); + p.origin = [1e6, 0.0, 0.0]; + p.date = chrono::DateTime::from_str("2022-10-31T09:00:00.594Z").unwrap(); + p.metadata.insert("other".to_owned(), Value::Bool(true)); + let s = serde_json::to_string(&p).unwrap(); + let q = serde_json::from_str(&s).unwrap(); + assert_eq!(p, q); + assert_eq!( + s, + concat!( + r#"{"name":"Foo","units":"meters","origin":[1000000.0,0.0,0.0],"#, + r#""date":"2022-10-31T09:00:00.594Z","metadata":{"other":true},"#, + r#""elements":[]}"# + ) + ); + } +} diff --git a/src/schema.rs b/src/schema.rs new file mode 100644 index 0000000..a298e56 --- /dev/null +++ b/src/schema.rs @@ -0,0 +1,151 @@ +use std::fmt::Write; + +use schemars::{ + gen::SchemaSettings, + schema::{ + InstanceType, Metadata, RootSchema, Schema, SchemaObject, SingleOrVec, SubschemaValidation, + }, + visit::visit_schema_object, + JsonSchema, +}; +use serde_json::Value; + +use crate::{format_full_name, Project}; + +use schemars::visit::Visitor; + +fn simple_enum_variant(outer_schema: &Schema) -> Option<(String, String)> { + if let Schema::Object(schema) = outer_schema { + let Some([Value::String(variant)]) = schema.enum_values.as_deref() else { + return None; + }; + let Some(Metadata { + description: Some(descr), + .. + }) = schema.metadata.as_deref() + else { + return None; + }; + Some((variant.clone(), descr.clone())) + } else { + None + } +} + +#[derive(Debug, Clone, Default)] +struct TweakSchema { + remove_descr: bool, +} + +impl Visitor for TweakSchema { + fn visit_schema_object(&mut self, schema: &mut SchemaObject) { + // Add a maximum for uint8 values. + if schema.format.as_deref() == Some("uint8") { + schema.number().maximum = Some(255.0); + } + // Move descriptions of simple enum values into the parent. + if let Some(SubschemaValidation { + one_of: Some(variants), + .. + }) = schema.subschemas.as_deref() + { + if let Some(v) = variants + .iter() + .map(simple_enum_variant) + .collect::>>() + { + schema.subschemas = None; + schema.enum_values = Some(v.iter().map(|(name, _)| (&name[..]).into()).collect()); + schema.instance_type = Some(SingleOrVec::Single(Box::new(InstanceType::String))); + let mut descr = schema.metadata().description.clone().unwrap_or_default(); + descr += "\n\n### Values\n\n"; + for (n, d) in v { + let body = d.replace("\n\n", "\n\n "); + write!(&mut descr, "`{n}`\n: {body}\n\n").unwrap(); + } + schema.metadata().description = Some(descr); + } + } + // Optionally remove descriptions. These get transformed into the documentation, + // they're a bit too complex to be readable in the schema itself. + if self.remove_descr { + let mut empty = false; + if let Some(m) = schema.metadata.as_deref_mut() { + m.description = None; + empty = m == &Metadata::default(); + } + if empty { + schema.metadata = None; + } + } + // Change references to the generics Array_for_* to just Array. + if let Some(r) = schema.reference.as_mut() { + if r.starts_with("#/definitions/Array_for_") { + "#/definitions/Array".clone_into(r); + } + } + // Then delegate to default implementation to visit any subschemas. + visit_schema_object(self, schema); + } +} + +pub(crate) fn schema_for(remove_descr: bool) -> RootSchema { + SchemaSettings::draft2019_09() + .with_visitor(TweakSchema { remove_descr }) + .into_generator() + .into_root_schema_for::() +} + +pub(crate) fn project_schema(remove_descr: bool) -> RootSchema { + let mut root = schema_for::(remove_descr); + root.schema.metadata().title = Some(format_full_name()); + root.schema.metadata().id = + Some("https://github.com/gmggroup/omf-rust/blob/main/omf.schema.json".to_owned()); + let array_def = root.definitions.get("Array_for_Boolean").unwrap().clone(); + root.definitions + .retain(|name, _| !name.starts_with("Array_for")); + root.definitions.insert("Array".to_owned(), array_def); + root +} + +pub fn json_schema() -> RootSchema { + project_schema(true) +} + +#[cfg(test)] +pub(crate) mod tests { + use schemars::schema::RootSchema; + + use crate::schema::json_schema; + + const SCHEMA: &str = "omf.schema.json"; + + #[ignore = "used to get schema"] + #[test] + fn update_schema() { + std::fs::write( + SCHEMA, + serde_json::to_string_pretty(&json_schema()) + .unwrap() + .as_bytes(), + ) + .unwrap(); + crate::schema_doc::update_schema_docs(); + } + + #[ignore = "used to get schema docs"] + #[test] + fn update_schema_docs() { + crate::schema_doc::update_schema_docs(); + #[cfg(feature = "parquet")] + crate::file::parquet::schemas::dump_parquet_schemas(); + } + + #[test] + fn schema() { + let schema = json_schema(); + let expected: RootSchema = + serde_json::from_reader(std::fs::File::open(SCHEMA).unwrap()).unwrap(); + assert!(schema == expected, "schema has changed"); + } +} diff --git a/src/schema_doc.rs b/src/schema_doc.rs new file mode 100644 index 0000000..bb03e3f --- /dev/null +++ b/src/schema_doc.rs @@ -0,0 +1,386 @@ +// Note: this outputs Python Markdown for mkdocs rather than the CommonMark that rustdoc uses. +use core::panic; +use std::{ + collections::BTreeMap, + fs::{create_dir_all, OpenOptions}, + io::Write, + path::Path, + sync::OnceLock, +}; + +use schemars::{ + schema::{ + ArrayValidation, InstanceType, Metadata, ObjectValidation, Schema, SchemaObject, + SingleOrVec, SubschemaValidation, + }, + visit::{visit_schema_object, Visitor}, +}; +use serde_json::Value; + +use crate::schema::{project_schema, schema_for}; + +pub(crate) fn update_schema_docs() { + let schema = project_schema(false); + let base_dir = Path::new("docs/schema"); + create_dir_all(Path::new(base_dir)).unwrap(); + object(base_dir, "Project", &schema.schema).unwrap(); + for (name, def) in &schema.definitions { + let Schema::Object(schema) = def else { + panic!("unknown definition: \"{name}\" = {def:#?}"); + }; + if name == "Geometry" { + geometry(base_dir, schema).unwrap(); + } else if name == "NumberRange" { + number_colormap_range(base_dir, name, schema).unwrap(); + } else { + object(base_dir, name, schema).unwrap(); + } + } +} + +fn geometry(base_dir: &Path, schema: &SchemaObject) -> std::io::Result<()> { + let mut f = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(base_dir.join("Geometry.md"))?; + write!(f, "\n\n")?; + // Title. + write!(f, "# Geometry\n\n")?; + // Description paragraph. + let descr = description(&schema); + write!(f, "{descr}\n\n")?; + // List of options. + write!(f, "## Options\n\n")?; + let Some(SubschemaValidation { + one_of: Some(variants), + .. + }) = schema.subschemas.as_deref() + else { + panic!("unknown geometry = {schema:#?}"); + }; + for item in variants { + let Schema::Object(child_schema) = item else { + panic!("unknown geometry option = {item:#?}"); + }; + let name = variant_name(child_schema); + write!(f, "- [{name}]({name}.md)\n")?; + object(base_dir, name, child_schema)?; + } + write!(f, "\n")?; + // Code. + schema_object_code(&mut f, schema) +} + +fn object(base_dir: &Path, name: &str, schema: &SchemaObject) -> std::io::Result<()> { + let mut f = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(base_dir.join(format!("{name}.md")))?; + write!(f, "\n\n")?; + // Title. + write!(f, "# {name}\n\n")?; + // Description paragraph. + let descr = description(&schema); + write!(f, "{descr}\n\n")?; + if let Some(ObjectValidation { + properties, + additional_properties, + .. + }) = schema.object.as_deref() + { + // Struct. + write!(f, "## Fields\n\n")?; + struct_fields(&mut f, properties, additional_properties.is_some())?; + } else if let Some(SubschemaValidation { + one_of: Some(variants), + .. + }) = schema.subschemas.as_deref() + { + // Enum with rich variants. + enum_variants(&mut f, variants)?; + } else if let Some(values) = &schema.enum_values { + // Simple enum: no data and no docs on individual items. Don't add options if the + // description already has them because `json_schema` can add them too. + if !descr.contains("## Values") { + simple_enum_values(&mut f, values)?; + } + } else { + // Something we don't understand. + panic!("unknown schema: {schema:#?}"); + } + // Code. + schema_object_code(&mut f, schema) +} + +fn number_colormap_range( + base_dir: &Path, + name: &str, + schema: &SchemaObject, +) -> std::io::Result<()> { + let mut f = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(base_dir.join(format!("{name}.md")))?; + write!(f, "\n\n")?; + // Title. + write!(f, "# {name}\n\n")?; + // Description paragraph. + let descr = description(&schema); + write!(f, "{descr}\n\n")?; + ty_defn_list_item( + &mut f, + "min", + "double, integer, date, or date-time", + "Minimum value of the range. The type should match the associated number array type.", + )?; + ty_defn_list_item( + &mut f, + "max", + "double, integer, date, or date-time", + "Maximum value of the range. Must have the same type as `min`.", + )?; + // Code. + schema_object_code(&mut f, schema) +} + +fn description(schema: &SchemaObject) -> String { + use regex::{Captures, Regex}; + + static RE: OnceLock = OnceLock::new(); + let re = + RE.get_or_init(|| Regex::new(r#"\]\(crate::(?\w+)(::(?\w+))?\)"#).unwrap()); + + let Some(Metadata { + description: Some(descr), + .. + }) = schema.metadata.as_deref() + else { + return String::new(); + }; + re.replace_all(descr, |caps: &Captures| { + let page = caps.name("page").unwrap().as_str(); + if let Some(anchor) = caps.name("anchor") { + format!("]({page}.md#{anchor})", anchor = anchor.as_str()) + } else { + format!("]({page}.md)") + } + }) + .into_owned() +} + +fn struct_fields( + f: &mut impl Write, + properties: &BTreeMap, + additional: bool, +) -> std::io::Result<()> { + for (name, child) in properties { + if name == "type" { + continue; + } + if let Schema::Object(schema) = child { + ty_defn_list_item(f, name, &field_type(schema), &description(schema))?; + } else { + defn_list_item(f, name, "")?; + } + } + if additional { + write!(f, "- Plus any custom values.\n\n")?; + } + Ok(()) +} + +fn simple_enum_values(f: &mut impl Write, values: &Vec) -> std::io::Result<()> { + write!(f, "## Values\n\n")?; + for value in values { + if let Value::String(s) = value { + let fixed = s.replace("\n\n", "\n\n "); + write!(f, "- {fixed}\n")?; + } else { + panic!("enum variant not a string: {value:#?}") + } + } + write!(f, "\n") +} + +fn enum_variants(f: &mut impl Write, variants: &Vec) -> std::io::Result<()> { + for variant in variants { + if let Schema::Object(schema) = variant { + let name = variant_name(schema); + write!(f, "## {name}\n\n")?; + write!(f, "{}\n\n", description(&schema))?; + if let Some(ObjectValidation { + properties, + additional_properties, + .. + }) = schema.object.as_deref() + { + write!(f, "### Fields\n\n")?; + defn_list_item(f, "type", &format!("`\"{name}\"`"))?; + struct_fields(f, properties, additional_properties.is_some())?; + } + } + } + Ok(()) +} + +fn variant_name(schema: &SchemaObject) -> &str { + if let Some(ObjectValidation { properties, .. }) = schema.object.as_deref() { + for (name, value) in properties { + if let Schema::Object(SchemaObject { + enum_values: Some(v), + .. + }) = value + { + if name == "type" && v.len() == 1 { + return v[0].as_str().expect("string for enum type"); + } + } + } + } + panic!("expected enum variant: {schema:#?}"); +} + +fn field_type(schema: &SchemaObject) -> String { + if let Some(ty) = &schema.reference { + let name = ty.strip_prefix("#/definitions/").unwrap_or(ty); + format!("[`{name}`]({name}.md)") + } else if let Some(name) = known_type(schema) { + name + } else if let Some(name) = optional_type(schema) { + name + } else if let Some(name) = array_type(schema) { + name + } else if let Some(SingleOrVec::Single(t)) = &schema.instance_type { + instance_type(**t) + .unwrap_or_else(|| panic!("unknown array: {schema:#?}")) + .to_owned() + } else { + panic!("unknown type: {schema:#?}"); + } +} + +fn instance_type(t: InstanceType) -> Option<&'static str> { + match t { + InstanceType::Object => Some("object"), + InstanceType::Null => Some("null"), + InstanceType::Boolean => Some("bool"), + InstanceType::Number => Some("number"), + InstanceType::String => Some("string"), + InstanceType::Integer => Some("integer"), + InstanceType::Array => None, + } +} + +fn known_type(schema: &SchemaObject) -> Option { + fn no_meta(schema: &SchemaObject) -> SchemaObject { + let mut copy = schema.clone(); + copy.metadata = None; + copy + } + + static TYPES: OnceLock<[(&str, SchemaObject); 3]> = OnceLock::new(); + let t = TYPES.get_or_init(|| { + [ + ( + "RGB color (uint8, 3 items)", + no_meta(&schema_for::<[u8; 3]>(true).schema), + ), + ( + "RGB color (uint8, 3 items) or null", + no_meta(&schema_for::>(true).schema), + ), + ("3D vector", no_meta(&schema_for::<[f64; 3]>(true).schema)), + ] + }); + + let s = no_meta(schema); + t.iter().find_map(|(name, ty)| { + if ty == &s { + Some((*name).to_owned()) + } else { + None + } + }) +} + +fn optional_type(schema: &SchemaObject) -> Option { + let null = SchemaObject { + instance_type: Some(SingleOrVec::Single(Box::new(InstanceType::Null))), + ..Default::default() + }; + if let Some(SubschemaValidation { + any_of: Some(items), + .. + }) = schema.subschemas.as_deref() + { + if let [Schema::Object(first), Schema::Object(second)] = &items[..] { + if second == &null { + return Some(format!("{} or null", field_type(first))); + } + } + } + if let Some(SingleOrVec::Vec(types)) = schema.instance_type.as_ref() { + if let [first, InstanceType::Null] = &types[..] { + let t = instance_type(*first)?; + return Some(format!("{t} or null")); + } + } + None +} + +fn array_type(schema: &SchemaObject) -> Option { + if let Some(ArrayValidation { + items: Some(SingleOrVec::Single(item)), + additional_items: None, + max_items, + min_items, + unique_items: None, + contains: None, + }) = schema.array.as_deref() + { + let Schema::Object(ty) = item.as_ref() else { + panic!("not an object"); + }; + let ty = field_type(ty); + match (min_items, max_items) { + (None, None) => Some(format!("array of {ty}")), + (None, Some(m)) => Some(format!("array of {ty}, up to {m} items")), + (Some(n), None) => Some(format!("array of {ty}, at least {n} items")), + (Some(n), Some(m)) if n == m => Some(format!("array of {ty}, {n} items")), + (Some(n), Some(m)) => Some(format!("array of {ty}, {n} to {m} items")), + } + } else { + None + } +} + +fn defn_list_item(f: &mut impl Write, title: &str, body: &str) -> std::io::Result<()> { + let fixed_body = body.trim().replace("\n\n", "\n\n "); + write!(f, "`{title}`\n: {fixed_body}\n\n") +} + +fn ty_defn_list_item(f: &mut impl Write, title: &str, ty: &str, body: &str) -> std::io::Result<()> { + let fixed_body = body.trim().replace("\n\n", "\n\n "); + write!(f, "`{title}`: {ty}\n: {fixed_body}\n\n") +} + +struct RemoveDescriptions {} + +impl Visitor for RemoveDescriptions { + fn visit_schema_object(&mut self, schema: &mut SchemaObject) { + schema.metadata().description = None; + schemars::visit::visit_schema_object(self, schema) + } +} + +fn schema_object_code(f: &mut impl Write, schema: &SchemaObject) -> std::io::Result<()> { + write!(f, "## Schema\n\n")?; + let mut no_descr = schema.clone(); + no_descr.metadata().description = None; + visit_schema_object(&mut RemoveDescriptions {}, &mut no_descr); + let code = serde_json::to_string_pretty(&no_descr).unwrap(); + write!(f, "```json\n{code}\n```\n") +} diff --git a/src/validate/mod.rs b/src/validate/mod.rs new file mode 100644 index 0000000..64d8be8 --- /dev/null +++ b/src/validate/mod.rs @@ -0,0 +1,7 @@ +//! Validation tools. + +mod problem; +mod validator; + +pub use problem::{Problem, Problems, Reason}; +pub use validator::{Validate, Validator}; diff --git a/src/validate/problem.rs b/src/validate/problem.rs new file mode 100644 index 0000000..9761ccc --- /dev/null +++ b/src/validate/problem.rs @@ -0,0 +1,199 @@ +use std::{fmt::Debug, fmt::Display}; + +use crate::{colormap::NumberRange, error::InvalidData, Location}; + +/// Validation failure reason. +#[derive(Debug, Clone, PartialEq, thiserror::Error)] +pub enum Reason { + /// A floating-point number is NaN, Inf, or -Inf. + #[error("must be finite")] + NotFinite, + /// A size is zero or less. + #[error("must be greater than zero")] + NotGreaterThanZero, + /// Vector must have length one. + #[error("must be a unit vector but {0:?} length is {1}")] + NotUnitVector([f64; 3], f64), + /// Vectors must be at right angles. + #[error("vectors are not orthogonal: {0:?} {1:?}")] + NotOrthogonal([f64; 3], [f64; 3]), + /// A sub-blocked model says it uses octree mode but the sub-block counts are not + /// powers of two. + #[error("sub-block counts {0:?} must be powers of two for octree mode")] + OctreeNotPowerOfTwo([u32; 3]), + /// A grid or block model has size greater than 2³² in any direction. + #[error("grid count {0:?} exceeds maximum of 4,294,967,295")] + GridTooLarge(Vec), + /// Attribute using a location that doesn't exist on the containing geometry. + #[error("is {0:?} which is not valid on {1} geometry")] + AttrLocationWrongForGeom(Location, &'static str), + /// Attribute using a location that is impossible for the attribute data. + #[error("is {0:?} which is not valid on {1} attributes")] + AttrLocationWrongForAttr(Location, &'static str), + /// Attribute length doesn't match the geometry and location. + #[error("length {0} does not match geometry ({1})")] + AttrLengthMismatch(u64, u64), + /// Minimum is greater than maximum. + #[error("minimum is greater than maximum in {0}")] + MinMaxOutOfOrder(NumberRange), + /// The data inside an array is invalid. + #[error("array contains invalid data: {0}")] + InvalidData(InvalidData), + /// A data file or index is missing from the zip. + #[error("refers to non-existent archive member '{0}'")] + ZipMemberMissing(String), + /// A field that must be unique is duplicated. + #[error("must be unique but {0} is repeated")] + NotUnique(String), + /// A field that should be unique is duplicated. + #[error("contains duplicate of {0}")] + SoftNotUnique(String), + /// Ran into the validation message limit. + #[error("{0} more errors")] + MoreErrors(u32), + /// Ran into the validation message limit. + #[error("{0} more warnings")] + MoreWarnings(u32), +} + +impl Reason { + /// True if the reason is an error, false if it is a warning. + pub fn is_error(&self) -> bool { + !matches!(self, Self::SoftNotUnique(_) | Reason::MoreWarnings(_)) + } +} + +/// A single validation problem. +#[derive(Debug, Clone, PartialEq)] +pub struct Problem { + /// Reason for the problem. + pub reason: Reason, + /// Type name of the failed object. + pub ty: &'static str, + /// Optional field name where the failure is. + pub field: Option<&'static str>, + /// Optional name of the containing object. + pub name: Option, +} + +impl Problem { + /// True if the reason is an error, false if it is a warning. + pub fn is_error(&self) -> bool { + self.reason.is_error() + } +} + +impl Display for Problem { + /// Formats a validation problem. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let severity = if self.reason.is_error() { + "Error" + } else { + "Warning" + }; + write!(f, "{severity}: '{}", self.ty)?; + if let Some(field) = self.field { + write!(f, "::{field}'")?; + } else { + write!(f, "'")?; + } + write!(f, " {}", self.reason)?; + if let Some(name) = &self.name { + write!(f, ", inside '{name}'")?; + } + Ok(()) + } +} + +/// A container of validation problems. +#[derive(Debug, Default, Clone, PartialEq)] +pub struct Problems(Vec); + +impl Problems { + /// True if there are no problems. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// The number of problems. + pub fn len(&self) -> usize { + self.0.len() + } + + /// Converts to a vec without copying. + pub fn into_vec(self) -> Vec { + self.0 + } + + /// Iterates over the problems. + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } + + /// Ok if there are only warnings, Err if there are errors. + pub(crate) fn into_result(self) -> Result { + if self.0.iter().any(|p| p.is_error()) { + Err(self) + } else { + Ok(self) + } + } + + pub(crate) fn push( + &mut self, + reason: Reason, + ty: &'static str, + field: Option<&'static str>, + name: Option, + ) { + self.0.push(Problem { + reason, + ty, + field, + name, + }) + } +} + +impl Display for Problems { + /// Formats a list of validation problems. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let n_errors = self.0.iter().filter(|p| p.reason.is_error()).count(); + let n_warnings = self.0.len() - n_errors; + match (n_errors, n_warnings) { + (0, 0) => write!(f, "OMF validation passed")?, + (0, _) => write!(f, "OMF validation passed with warnings:")?, + _ => write!(f, "OMF validation failed:")?, + } + for problem in self { + write!(f, "\n {problem}")?; + } + Ok(()) + } +} + +impl std::error::Error for Problems {} + +impl IntoIterator for Problems { + type Item = Problem; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +impl<'a> IntoIterator for &'a Problems { + type Item = &'a Problem; + type IntoIter = std::slice::Iter<'a, Problem>; + + fn into_iter(self) -> Self::IntoIter { + self.0.iter() + } +} + +impl From for Vec { + fn from(value: Problems) -> Self { + value.into_vec() + } +} diff --git a/src/validate/validator.rs b/src/validate/validator.rs new file mode 100644 index 0000000..50a0d25 --- /dev/null +++ b/src/validate/validator.rs @@ -0,0 +1,583 @@ +use std::{ + cell::RefCell, + collections::{HashMap, HashSet}, + fmt::Debug, + hash::Hash, + rc::Rc, +}; + +use crate::{ + array, colormap::NumberRange, Attribute, AttributeData, Geometry, Location, SubblockMode, + Vector3, +}; + +use super::{Problems, Reason}; + +pub trait Validate { + #[doc(hidden)] + fn validate_inner(&mut self, _val: &mut Validator); + + /// Call to validate the object, returning errors and warnings. + /// + /// The `Ok` value is a `Problems` object contain that is either empty or contains + /// only warnings. The `Err` value is a `Problems` object containing at least one + /// error. + fn validate(&mut self) -> Result { + let mut val = Validator::new(); + self.validate_inner(&mut val); + val.finish().into_result() + } +} + +impl Validate for Option { + #[doc(hidden)] + fn validate_inner(&mut self, val: &mut Validator) { + if let Some(obj) = self { + obj.validate_inner(val); + } + } +} + +fn normalise([x, y, z]: Vector3) -> Vector3 { + let mag = (x * x + y * y + z * z).sqrt(); + if mag == 0.0 { + [0.0, 0.0, 0.0] + } else { + [x / mag, y / mag, z / mag] + } +} + +fn ortho(a: Vector3, b: Vector3) -> bool { + const THRESHOLD: f64 = 1e-6; + let [x0, y0, z0] = normalise(a); + let [x1, y1, z1] = normalise(b); + (x0 * x1 + y0 * y1 + z0 * z1).abs() < THRESHOLD +} + +#[derive(Debug)] +pub struct Validator<'n> { + filenames: Rc>>, + problems: Rc>, + ty: &'static str, + name: Option<&'n str>, + limit: Option, + extra_errors: u32, + extra_warnings: u32, +} + +impl<'n> Validator<'n> { + pub(crate) fn new() -> Self { + Self { + filenames: Default::default(), + problems: Default::default(), + ty: "", + name: None, + limit: None, + extra_errors: 0, + extra_warnings: 0, + } + } + + pub(crate) fn finish(mut self) -> Problems { + if self.extra_warnings > 0 { + self.push(Reason::MoreWarnings(self.extra_warnings), None); + } + if self.extra_errors > 0 { + self.push(Reason::MoreErrors(self.extra_errors), None); + } + self.problems.take() + } + + fn push_full( + &mut self, + reason: Reason, + ty: &'static str, + field: Option<&'static str>, + name: Option<&str>, + ) { + let mut problems = self.problems.borrow_mut(); + match self.limit { + Some(limit) if problems.len() >= limit => { + if reason.is_error() { + self.extra_errors += 1; + } else { + self.extra_warnings += 1; + } + } + _ => { + problems.push(reason, ty, field, name.map(ToOwned::to_owned)); + } + } + } + + fn push(&mut self, reason: Reason, field: Option<&'static str>) { + self.push_full(reason, self.ty, field, self.name); + } + + pub(crate) fn with_filenames(mut self, filenames: I) -> Self + where + I: IntoIterator, + T: Into, + { + self.filenames = Some(filenames.into_iter().map(Into::into).collect()).into(); + self + } + + pub(crate) fn with_limit(mut self, limit: Option) -> Self { + self.limit = limit.map(|n| n.try_into().expect("u32 fits in usize")); + self + } + + pub(crate) fn enter(&mut self, ty: &'static str) -> Self { + Validator { + filenames: self.filenames.clone(), + problems: self.problems.clone(), + ty, + name: self.name, + limit: None, + extra_errors: 0, + extra_warnings: 0, + } + } + + pub(crate) fn name(mut self, name: &'n str) -> Self { + self.name = Some(name); + self + } + + pub(crate) fn obj(mut self, obj: &mut impl Validate) -> Self { + obj.validate_inner(&mut self); + self + } + + pub(crate) fn array( + mut self, + array: &mut array::Array, + constraint: array::Constraint, + field: &'static str, + ) -> Self { + _ = array + .constrain(constraint) + .map_err(|reason| self.push(reason, Some(field))); + for reason in array.run_write_checks() { + self.push(reason, Some(field)); + } + if let Some(filenames) = self.filenames.as_ref() { + if !filenames.contains(array.filename()) { + self.push( + Reason::ZipMemberMissing(array.filename().to_owned()), + Some(field), + ); + } + } + self + } + + pub(crate) fn array_opt( + self, + array_opt: Option<&mut array::Array>, + constraint: array::Constraint, + field: &'static str, + ) -> Self { + if let Some(array) = array_opt { + self.array(array, constraint, field) + } else { + self + } + } + + pub(crate) fn objs<'c>( + mut self, + objs: impl IntoIterator, + ) -> Self { + for obj in objs { + self = self.obj(obj); + } + self + } + + pub(crate) fn grid_count(mut self, count: &[u64]) -> Self { + const MAX: u64 = u32::MAX as u64; + if count.iter().any(|n| *n > MAX) { + self.push(Reason::GridTooLarge(count.to_vec()), None); + } + self + } + + pub(crate) fn subblock_mode_and_count( + mut self, + mode: Option, + count: [u32; 3], + ) -> Self { + if mode == Some(SubblockMode::Octree) && !count.iter().all(|n| n.is_power_of_two()) { + self.push(Reason::OctreeNotPowerOfTwo(count), None); + } + self + } + + pub(crate) fn finite(mut self, value: f64, field: &'static str) -> Self { + if !value.is_finite() { + self.push(Reason::NotFinite, Some(field)); + } + self + } + + pub(crate) fn finite_seq( + mut self, + values: impl IntoIterator, + field: &'static str, + ) -> Self { + for value in values { + if !value.is_finite() { + self.push(Reason::NotFinite, Some(field)); + break; + } + } + self + } + + pub(crate) fn above_zero(mut self, value: T, field: &'static str) -> Self + where + T: Default + PartialOrd, + { + if value <= T::default() { + self.push(Reason::NotGreaterThanZero, Some(field)); + } + self + } + + pub(crate) fn above_zero_seq(mut self, values: I, field: &'static str) -> Self + where + T: Default + PartialOrd, + I: IntoIterator, + { + for value in values { + if value <= T::default() { + self.push(Reason::NotGreaterThanZero, Some(field)); + break; + } + } + self + } + + pub(crate) fn min_max(mut self, range: NumberRange) -> Self { + let ok = match range { + NumberRange::Float { min, max } => { + self = self.finite(min, "min").finite(max, "max"); + !min.is_finite() || !max.is_finite() || min <= max + } + NumberRange::Integer { min, max } => min <= max, + NumberRange::Date { min, max } => min <= max, + NumberRange::DateTime { min, max } => min <= max, + }; + if !ok { + self.push(Reason::MinMaxOutOfOrder(range), Some("range")); + } + self + } + + pub(crate) fn unique( + mut self, + values: impl IntoIterator, + field: &'static str, + is_error: bool, + ) -> Self { + let mut seen = HashMap::new(); + for value in values { + let count = seen.entry(value).or_insert(0_usize); + if *count == 1 { + if is_error { + self.push(Reason::NotUnique(format!("{value:?}")), Some(field)); + } else { + self.push(Reason::SoftNotUnique(format!("{value:?}")), Some(field)); + } + } + *count += 1; + } + self + } + + pub(crate) fn unit_vector(mut self, [x, y, z]: Vector3, field: &'static str) -> Self { + const THRESHOLD: f64 = 1e-6; + let mag2 = x * x + y * y + z * z; + if (1.0 - mag2).abs() >= THRESHOLD { + let len = (mag2.sqrt() * 1e7).floor() / 1e7; + self.push(Reason::NotUnitVector([x, y, z], len), Some(field)); + } + self + } + + pub(crate) fn vectors_ortho2(mut self, u: Vector3, v: Vector3) -> Self { + if !ortho(u, v) { + self.push(Reason::NotOrthogonal(u, v), None); + } + self + } + + pub(crate) fn vectors_ortho3(mut self, u: Vector3, v: Vector3, w: Vector3) -> Self { + for (a, b) in [(u, v), (u, w), (v, w)] { + if !ortho(u, v) { + self.push(Reason::NotOrthogonal(a, b), None); + break; + } + } + self + } + + pub(crate) fn array_size(mut self, size: u64, required: u64, field: &'static str) -> Self { + if size != required { + self.push(Reason::AttrLengthMismatch(size, required), Some(field)); + } + self + } + + pub(crate) fn array_size_opt( + self, + size_opt: Option, + required: u64, + field: &'static str, + ) -> Self { + if let Some(size) = size_opt { + self.array_size(size, required, field) + } else { + self + } + } + + pub(crate) fn attrs_on_geometry(mut self, attrs: &Vec, geometry: &Geometry) -> Self { + for attr in attrs { + if matches!(attr.data, AttributeData::ProjectedTexture { .. }) + != (attr.location == Location::Projected) + { + self.push_full( + Reason::AttrLocationWrongForAttr(attr.location, attr.data.type_name()), + "Attribute", + Some("location"), + Some(&attr.name), + ); + } else if let Some(geom_len) = geometry.location_len(attr.location) { + if geom_len != attr.len() { + self.push_full( + Reason::AttrLengthMismatch(attr.len(), geom_len), + "Attribute", + None, + Some(&attr.name), + ); + } + } else { + self.push_full( + Reason::AttrLocationWrongForGeom(attr.location, geometry.type_name()), + "Attribute", + Some("location"), + Some(&attr.name), + ); + } + } + self + } + + pub(crate) fn attrs_on_attribute(mut self, attrs: &Vec, n_categories: u64) -> Self { + for attr in attrs { + if attr.location != Location::Categories { + self.push_full( + Reason::AttrLocationWrongForGeom(attr.location, "AttributeData::Categories"), + "Attribute", + Some("location"), + Some(&attr.name), + ); + } else if attr.len() != n_categories { + self.push_full( + Reason::AttrLengthMismatch(attr.len(), n_categories), + "Attribute", + None, + Some(&attr.name), + ); + } + } + self + } +} + +#[cfg(test)] +mod tests { + use crate::{array_type, Array, PointSet}; + + use super::*; + + /// Test that if you have only warnings the result is `Ok`. + #[test] + fn problems_into_result() { + let mut problems = Problems::default(); + problems.push( + Reason::SoftNotUnique("x".to_owned()), + "Test", + Some("field"), + None, + ); + assert_eq!(problems.into_result().unwrap().len(), 1); + } + + #[test] + fn validator_basics() { + let mut v = Validator::new().enter("Test"); + v.push(Reason::NotFinite, None); + v.push(Reason::NotFinite, Some("field")); + v = v.name("name"); + v.push(Reason::NotFinite, None); + v.push(Reason::NotFinite, Some("field")); + let errors: Vec<_> = v + .finish() + .into_iter() + .map(|prob| prob.to_string()) + .collect(); + assert_eq!( + errors, + vec![ + "Error: 'Test' must be finite", + "Error: 'Test::field' must be finite", + "Error: 'Test' must be finite, inside 'name'", + "Error: 'Test::field' must be finite, inside 'name'", + ] + ) + } + + #[test] + fn validator_checks() { + let attrs = vec![ + Attribute::new( + "a", + Location::Vertices, + AttributeData::Number { + values: Array::new("1.parquet".to_owned(), 100).into(), + colormap: None, + }, + ), + Attribute::new( + "b", + Location::Primitives, // location error + AttributeData::Number { + values: Array::new("2.parquet".to_owned(), 100).into(), + colormap: None, + }, + ), + Attribute::new( + "c", + Location::Vertices, + AttributeData::Number { + values: Array::new("3.parquet".to_owned(), 101).into(), // length error + colormap: None, + }, + ), + Attribute::new( + "c", + Location::Vertices, // error + AttributeData::ProjectedTexture { + orient: Default::default(), + width: 10.0, + height: 10.0, + image: Array::new("4.jpeg".to_owned(), 100), + }, + ), + Attribute::new( + "d", + Location::Projected, + AttributeData::ProjectedTexture { + orient: Default::default(), + width: 10.0, + height: 10.0, + image: Array::new("6.png".to_owned(), 100), // missing file error + }, + ), + ]; + let results: Vec<_> = Validator::new() + .with_filenames(["1.parquet"]) + .enter("Test") + .finite(0.0, "zero") + .finite(f64::INFINITY, "inf") // error + .finite(f64::NAN, "nan") // error + .finite_seq([0.0, f64::NEG_INFINITY, f64::NAN], "seq") // error + .above_zero_seq([1.0, 2.0, 3.0], "normal") + .above_zero_seq([1.0, 0.0, -1.0], "seq") // error + .above_zero_seq([1.0, f64::NAN], "seq_nan") + .min_max(NumberRange::Float { + // error + min: f64::NAN, + max: 100.0, + }) + .min_max(NumberRange::Float { + min: 100.0, + max: 100.0, + }) + .min_max(NumberRange::Float { + min: 101.5, + max: 100.0, + }) // error + .unit_vector([1.0, 0.0, 0.0], "i") + .unit_vector([0.5 * 2.0_f64.sqrt(), 0.5 * 2.0_f64.sqrt(), 0.0], "angled") + .unit_vector([0.5, 0.0, 0.0], "short") // error + .vectors_ortho2([1.0, 0.0, 0.0], [0.0, 1.0, 0.0]) + .vectors_ortho2([0.8, 0.0, 0.0], [0.0, 0.0, 1.0]) + .vectors_ortho2([0.0, 1.0, 0.0], [0.0, 0.0, -1.0]) + .vectors_ortho2([1.0, 0.0, 0.0], [0.8, 0.2, 0.0]) // error + .vectors_ortho3([1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]) + .vectors_ortho3([1.0, 0.001, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]) // error + .attrs_on_geometry( + &attrs, + &PointSet::new(Array::new("5.parquet".to_owned(), 100)).into(), + ) // 3 errors + .array( + &mut Array::::new("1.parquet".to_owned(), 10), + array::Constraint::String, + "fine", + ) + .array( + // error + &mut Array::::new("2.parquet".to_owned(), 10), + array::Constraint::String, + "missing", + ) + .subblock_mode_and_count(None, [16, 8, 15]) + .subblock_mode_and_count(Some(SubblockMode::Full), [16, 8, 5]) + .subblock_mode_and_count(Some(SubblockMode::Octree), [16, 8, 5]) // error + .subblock_mode_and_count(Some(SubblockMode::Octree), [16, 8, 4]) + .unique([0; 0], "empty", true) + .unique([1], "single", true) + .unique([1, 2, 3, 4], "normal", false) + .unique([1, 2, 3, 4, 2], "dupped", true) // warning + .unique(["a", "b", "c", "d", "c", "a", "a"], "multiple", false) // 2 warnings + .finish() + .into_iter() + .map(|p| p.to_string()) + .collect(); + let mut expected = vec![ + "Error: 'Test::inf' must be finite", + "Error: 'Test::nan' must be finite", + "Error: 'Test::seq' must be finite", + "Error: 'Test::seq' must be greater than zero", + "Error: 'Test::min' must be finite", + "Error: 'Test::range' minimum is greater than maximum in [101.5, 100]", + "Error: 'Test::short' must be a unit vector but [0.5, 0.0, 0.0] length is 0.5", + "Error: 'Test' vectors are not orthogonal: [1.0, 0.0, 0.0] [0.8, 0.2, 0.0]", + "Error: 'Test' vectors are not orthogonal: [1.0, 0.001, 0.0] [0.0, 1.0, 0.0]", + "Error: 'Attribute::location' is Primitives which is not valid on PointSet geometry, inside 'b'", + "Error: 'Attribute' length 101 does not match geometry (100), inside 'c'", + "Error: 'Attribute::location' is Vertices which is not valid on ProjectedTexture attributes, inside 'c'", + "Error: 'Test::missing' refers to non-existent archive member '2.parquet'", + "Error: 'Test' sub-block counts [16, 8, 5] must be powers of two for octree mode", + "Error: 'Test::dupped' must be unique but 2 is repeated", + "Warning: 'Test::multiple' contains duplicate of \"c\"", + "Warning: 'Test::multiple' contains duplicate of \"a\"", + ]; + let mut unexpected = Vec::new(); + for s in results { + if let Some(index) = expected.iter().position(|e| *e == &s) { + expected.remove(index); + } else { + unexpected.push(s.to_owned()); + } + } + if !unexpected.is_empty() || !expected.is_empty() { + panic!("unexpected problems: {unexpected:#?}\nexpected but not found: {expected:#?}"); + } + } +} diff --git a/src/version.rs b/src/version.rs new file mode 100644 index 0000000..f739245 --- /dev/null +++ b/src/version.rs @@ -0,0 +1,43 @@ +/// Library name. +pub const CRATE_NAME: &str = env!("CARGO_PKG_NAME"); + +/// Library version. +pub const CRATE_VERSION: &str = env!("CARGO_PKG_VERSION"); + +/// File format name. +pub const FORMAT_NAME: &str = "Open Mining Format"; + +/// File format extension. +pub const FORMAT_EXTENSION: &str = "omf"; + +/// File format major version number. +pub const FORMAT_VERSION_MAJOR: u32 = 2; + +/// File format minor version number. +pub const FORMAT_VERSION_MINOR: u32 = 0; + +/// File format pre-release version suffix. +/// +/// This will always be `None` in release versions of the crate. Pre-release formats +/// may contain experimental changes so can't be opened in by release versions. +pub const FORMAT_VERSION_PRERELEASE: Option<&str> = Some("beta.1"); + +/// Returns a string containing the file format version that this crate produces. +pub fn format_version() -> String { + let mut v = format!("{FORMAT_VERSION_MAJOR}.{FORMAT_VERSION_MINOR}"); + if let Some(pre) = FORMAT_VERSION_PRERELEASE { + v = format!("{v}-{pre}"); + } + v +} + +/// Returns a string containing the full name and version of the file format that this +/// crate produces. +pub fn format_full_name() -> String { + format!("{} {}", FORMAT_NAME, format_version()) +} + +/// Returns the crate name and version. +pub fn crate_full_name() -> String { + format!("{CRATE_NAME} {CRATE_VERSION}") +} diff --git a/tests/conversion_tests.rs b/tests/conversion_tests.rs new file mode 100644 index 0000000..4a878f8 --- /dev/null +++ b/tests/conversion_tests.rs @@ -0,0 +1,84 @@ +#![cfg(feature = "omf1")] + +use std::{fs::read_dir, path::Path, time::Instant}; + +use omf::{error::Error, file::Reader, omf1::Converter}; + +#[test] +fn convert_omf1() { + let output_path = Path::new(env!("CARGO_TARGET_TMPDIR")).join("test_proj.2.omf"); + let converter = Converter::new(); + let warnings = converter + .convert_open("tests/omf1/test_proj.omf", &output_path) + .unwrap(); + let warning_strings: Vec<_> = warnings.into_iter().map(|p| p.to_string()).collect(); + assert_eq!( + warning_strings, + vec!["Warning: 'Project::elements[..]::name' contains duplicate of \"\", inside ''"] + ); + let project = Reader::open(&output_path).unwrap().project().unwrap().0; + let metadata = serde_json::to_string_pretty(&project.metadata).unwrap(); + dbg!(&metadata); + assert!(metadata.starts_with( + r#"{ + "OMF1 conversion": { + "by": "omf 0.1.0-beta.1", + "from": "OMF-v0.9.0", + "on": "# + )); + assert!(metadata.ends_with( + r#"Z" + }, + "date_created": "2017-10-04T21:46:17Z", + "date_modified": "2017-10-04T21:46:17Z" +}"# + )); +} + +#[ignore = "requires local files"] +#[test] +fn convert_external_files() { + let dirs = &[ + Path::new("C:/Work/data/OMF"), + Path::new("C:/Work/data/OMF/Geosoft Voxel OMF"), + ]; + + let mut converter = Converter::new(); + let mut limits = omf::file::Limits::default(); + limits.json_bytes = None; + converter.set_limits(limits); + let mut success = true; + for dir in dirs { + for file in read_dir(dir).unwrap().map(Result::unwrap) { + let name = file.file_name().into_string().unwrap(); + if name.ends_with(".omf") { + let len = file.metadata().unwrap().len(); + let size = if len > 1024 * 1024 { + format!("{:.1} MB", (len as f64) / 1024.0 / 1024.0) + } else { + format!("{:.1} KB", (len as f64) / 1024.0) + }; + println!("{name} ({size})"); + let output = Path::new(env!("CARGO_TARGET_TMPDIR")).join(file.file_name()); + let start = Instant::now(); + match converter.convert_open(file.path(), output) { + Ok(_) => { + println!(" succeeded in {:.3} s", start.elapsed().as_secs_f64()) + } + Err(Error::ValidationFailed(problems)) => { + success = false; + println!(" FAILED"); + for problem in problems { + println!(" {problem}"); + } + } + Err(e) => { + success = false; + println!(" FAILED {e}"); + } + } + } + } + } + assert!(success); +} diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs new file mode 100644 index 0000000..8324c2e --- /dev/null +++ b/tests/integration_tests.rs @@ -0,0 +1,1172 @@ +#![allow(dead_code, unused_imports)] + +use std::{ + fs::{File, OpenOptions}, + io::{Cursor, Read, Seek, Write}, + path::Path, + str::FromStr, +}; + +use chrono::DateTime; +#[cfg(feature = "image")] +use image; + +#[cfg(feature = "parquet")] +use omf::data::*; +use omf::{ + file::{Reader, Writer}, + validate::Validate, + *, +}; + +const PYRAMID_VERTICES: [[f32; 3]; 5] = [ + [-1.0, -1.0, 0.0], + [1.0, -1.0, 0.0], + [1.0, 1.0, 0.0], + [-1.0, 1.0, 0.0], + [0.0, 0.0, 1.0], +]; +const PYRAMID_TRIANGLES: [[u32; 3]; 6] = [ + [0, 1, 4], + [1, 2, 4], + [2, 3, 4], + [3, 0, 4], + [0, 2, 1], + [0, 3, 2], +]; +const PYRAMID_SEGMENTS: [[u32; 2]; 8] = [ + [0, 1], + [1, 2], + [2, 3], + [3, 0], + [0, 4], + [1, 4], + [2, 4], + [3, 4], +]; +const CUBE_VERTICES: [[f64; 3]; 8] = [ + [0.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [1.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 0.0, 1.0], + [1.0, 0.0, 1.0], + [1.0, 1.0, 1.0], + [0.0, 1.0, 1.0], +]; +const CUBE_TRIANGLES: [[u32; 3]; 12] = [ + [0, 2, 1], + [0, 3, 2], + [0, 1, 5], + [0, 5, 4], + [1, 2, 6], + [1, 6, 5], + [2, 3, 7], + [2, 7, 6], + [3, 0, 4], + [3, 4, 7], + [4, 5, 6], + [4, 6, 7], +]; +const CUBE_SEGMENTS: [[u32; 2]; 12] = [ + [0, 1], + [1, 2], + [2, 3], + [3, 0], + [0, 4], + [1, 5], + [2, 6], + [3, 7], + [4, 5], + [5, 6], + [6, 7], + [7, 4], +]; +const OCTREE_SUBBLOCKS: [([u32; 3], [u32; 6]); 11] = [ + ([0, 0, 0], [0, 0, 0, 4, 4, 4]), + ([1, 0, 0], [0, 0, 0, 4, 4, 4]), + ([0, 1, 0], [0, 0, 0, 4, 4, 4]), + ([1, 1, 0], [0, 0, 0, 4, 4, 4]), + ([0, 0, 1], [0, 0, 0, 4, 4, 4]), + ([1, 0, 1], [0, 0, 0, 4, 4, 4]), + ([0, 1, 1], [0, 0, 0, 4, 4, 4]), + ([1, 1, 1], [0, 0, 0, 2, 2, 2]), + ([1, 1, 1], [0, 2, 0, 1, 3, 1]), + ([1, 1, 1], [2, 0, 0, 3, 1, 1]), + ([1, 1, 1], [0, 0, 2, 1, 1, 3]), +]; +const FREEFORM_SUBBLOCKS: [([u32; 3], [f32; 6]); 10] = [ + ([0, 0, 0], [0.0, 0.0, 0.0, 1.0, 1.0, 1.0]), + ([1, 0, 0], [0.0, 0.0, 0.0, 1.0, 1.0, 1.0]), + ([0, 1, 0], [0.0, 0.0, 0.0, 1.0, 1.0, 1.0]), + ([1, 1, 0], [0.0, 0.0, 0.0, 1.0, 1.0, 1.0]), + ([0, 0, 1], [0.0, 0.0, 0.0, 1.0, 1.0, 1.0]), + ([1, 0, 1], [0.0, 0.0, 0.0, 1.0, 1.0, 1.0]), + ([0, 1, 1], [0.0, 0.0, 0.0, 1.0, 1.0, 1.0]), + ([1, 1, 1], [0.0, 0.0, 0.0, 1.0, 1.0, 0.3333]), + ([1, 1, 1], [0.0, 0.0, 0.3333, 0.75, 0.75, 0.6666]), + ([1, 1, 1], [0.0, 0.0, 0.6666, 0.5, 0.5, 1.0]), +]; +const IMAGE_BYTES: &[u8] = include_bytes!("test.png"); + +fn temp_file(name: &str, contents: &[u8]) -> File { + let path = Path::new(env!("CARGO_TARGET_TMPDIR")).join(name); + let mut f = OpenOptions::new() + .truncate(true) + .read(true) + .write(true) + .create(true) + .open(path) + .unwrap(); + f.write_all(contents).unwrap(); + f +} + +fn file_contents(mut file: File) -> Vec { + file.rewind().unwrap(); + let mut buf = Vec::new(); + file.read_to_end(&mut buf).unwrap(); + buf +} + +/// Creates an OMF file containing approximately one of everything. +/// +/// project +// metadata +// surface (pyramid) +// metadata +// per-vertex numbers +// per-primitive colors +// per-vertex date-time +// point-set (pyramid vertices) +// per-vertex categories +// metadata +// integer sub-attribute +// per-vertex vector2 +// per-vertex vector3 +// line-set (pyramid edges) +// per-primitive text +// grid surface (2x2) +// block-model (2x2x2) +// per-block filter +// tensor block-model (2x2x2) +// block-model regular sub-blocks (2x2x2 with 4x4x4 octree sub-blocks in one parent) +// block-model free-form sub-blocks (2x2x2 with one parent split) +// composite +// surface (cube faces) +// line-set (cube outline) +// surface (rectangle) +// projected texture +// mapped texture +#[cfg(feature = "parquet")] +fn one_of_everything() -> (Vec, Project) { + let mut writer = Writer::new(temp_file("one_of_everything.omf", b"")).unwrap(); + // Project. + let mut project = Project::new("One of everything"); + project.description = + "An OMF 2.0 project containing roughly one of every different type.".to_owned(); + project.author = "Tim Evans".to_owned(); + project.date = chrono::DateTime::from_str("1970-01-01T00:00:00Z").unwrap(); + project.metadata.insert("null".to_owned(), ().into()); + project.metadata.insert("bool".to_owned(), true.into()); + project.metadata.insert("string".to_owned(), "value".into()); + project.metadata.insert("number".to_owned(), 42.into()); + project + .metadata + .insert("array".to_owned(), vec![1, 2, 3].into()); + let mut obj: serde_json::Map = Default::default(); + obj.insert("a".to_owned(), 1.into()); + obj.insert("b".to_owned(), 2.into()); + project.metadata.insert("object".to_owned(), obj.into()); + // Surface element. + let mut surface = Element::new( + "Pyramid surface", + Surface::new( + writer + .array_vertices(PYRAMID_VERTICES.iter().copied()) + .unwrap(), + writer + .array_triangles(PYRAMID_TRIANGLES.iter().copied()) + .unwrap(), + ), + ); + surface.description = "A surface forming a pyramid".to_owned(); + surface.color = Some([255, 128, 0, 255]); + surface.attributes.push(Attribute::from_numbers( + "Numbers", + Location::Vertices, + writer + .array_numbers([Some(1.0), Some(2.0), Some(3.0), Some(4.0), None]) + .unwrap(), + )); + surface.attributes.push(Attribute::from_colors( + "Colors", + Location::Primitives, + writer + .array_colors( + [ + [255_u8, 0, 0, 255], + [255, 255, 0, 255], + [0, 255, 0, 255], + [0, 0, 255, 255], + [255, 255, 255, 255], + [255, 255, 255, 255], + ] + .into_iter() + .map(Some), + ) + .unwrap(), + )); + surface + .attributes + .push(Attribute::from_numbers_discrete_colormap( + "Date-times", + Location::Vertices, + writer + .array_numbers( + [ + DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + DateTime::from_str("2000-01-01T01:00:00Z").unwrap(), + DateTime::from_str("2000-01-01T02:00:00Z").unwrap(), + DateTime::from_str("2000-01-01T03:00:00Z").unwrap(), + DateTime::from_str("2000-01-01T04:00:00Z").unwrap(), + ] + .map(Some), + ) + .unwrap(), + writer + .array_boundaries([ + Boundary::Less(DateTime::from_str("2000-01-01T01:00:00Z").unwrap()), + Boundary::Less(DateTime::from_str("2000-01-01T02:00:00Z").unwrap()), + Boundary::LessEqual(DateTime::from_str("2000-01-01T03:00:00Z").unwrap()), + ]) + .unwrap(), + writer + .array_gradient([ + [0, 0, 255, 255], + [0, 255, 0, 255], + [255, 0, 0, 255], + [255, 255, 255, 255], + ]) + .unwrap(), + )); + project.elements.push(surface); + // PointSet element. + let mut points = Element::new( + "Pyramid points", + PointSet::new(writer.array_vertices(PYRAMID_VERTICES.clone()).unwrap()), + ); + let mut categories = Attribute::from_categories( + "Categories", + Location::Vertices, + writer.array_indices([0_u32, 0, 0, 0, 1].map(Some)).unwrap(), + writer + .array_names(["Base", "Top"].map(|s| s.to_owned())) + .unwrap(), + Some( + writer + .array_gradient([[255_u8, 128, 0, 255], [0, 128, 255, 255]]) + .unwrap(), + ), + [Attribute::from_numbers( + "Layer", + Location::Categories, + writer.array_numbers([Some(1_i64), Some(2)]).unwrap(), + )], + ); + categories.description = "Divides the points into top and base.".to_owned(); + categories.units = "whatever".to_owned(); + categories.metadata.insert("key".to_owned(), "value".into()); + points.attributes.push(categories); + points.attributes.push(Attribute::from_vectors( + "2D Vectors", + Location::Vertices, + writer + .array_vectors([ + Some([1.0_f32, 0.0]), + Some([1.0, 1.0]), + Some([0.0, 1.0]), + Some([0.0, 0.0]), + None, + ]) + .unwrap(), + )); + points.attributes.push(Attribute::from_vectors( + "3D Vectors", + Location::Vertices, + writer + .array_vectors([None, None, None, None, Some([0.0_f32, 0.0, 1.0])]) + .unwrap(), + )); + project.elements.push(points); + // LineSet element. + let mut line_set = Element::new( + "Pyramid lines", + LineSet::new( + writer + .array_vertices(PYRAMID_VERTICES.iter().copied()) + .unwrap(), + writer + .array_segments(PYRAMID_SEGMENTS.iter().copied()) + .unwrap(), + ), + ); + line_set.attributes.push(Attribute::from_strings( + "Strings", + Location::Primitives, + writer + .array_text([ + None, + None, + None, + None, + Some("sw".to_owned()), + Some("se".to_owned()), + Some("ne".to_owned()), + Some("nw".to_owned()), + ]) + .unwrap(), + )); + project.elements.push(line_set); + // GridSurface element. + project.elements.push(Element::new( + "Pyramid grid surface", + GridSurface::new( + Orient2 { + origin: [-1.5, -1.5, 0.0], + u: [1.0, 0.0, 0.0], + v: [0.0, 1.0, 0.0], + }, + Grid2::Tensor { + u: writer.array_scalars([1.0, 2.0]).unwrap(), + v: writer.array_scalars([1.5, 1.5]).unwrap(), + }, + Some( + writer + .array_scalars([0.0_f32, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 0.0]) + .unwrap(), + ), + ), + )); + // Regular block model element. + let mut block_model = Element::new( + "Regular block model", + BlockModel::new( + Orient3::from_origin([-1.0, -1.0, -1.0]), + Grid3::Regular { + size: [1.0, 1.0, 1.0], + count: [2, 2, 2], + }, + ), + ); + block_model.attributes.push(Attribute::from_booleans( + "Filter", + Location::Primitives, + writer + .array_booleans([false, false, false, false, false, false, false, true].map(Some)) + .unwrap() + .into(), + )); + project.elements.push(block_model); + // Tensor block model element. + project.elements.push(Element::new( + "Tensor block model", + BlockModel::new( + Orient3::from_origin([-1.0, -1.0, -1.0]), + Grid3::Tensor { + u: writer.array_scalars([0.6666, 1.333]).unwrap(), + v: writer.array_scalars([0.6666, 1.333]).unwrap(), + w: writer.array_scalars([1.0, 1.0]).unwrap(), + }, + ), + )); + // Block model element, with regular sub-blocks. + project.elements.push(Element::new( + "Sub-blocked block model, regular", + BlockModel::with_subblocks( + Orient3::from_origin([-1.0, -1.0, -1.0]), + Grid3::Regular { + size: [1.0, 1.0, 1.0], + count: [2, 2, 2], + }, + Subblocks::Regular { + count: [4, 4, 4], + subblocks: writer.array_regular_subblocks(OCTREE_SUBBLOCKS).unwrap(), + mode: Some(SubblockMode::Octree), + }, + ), + )); + // Block model element, with free-form sub-blocks. + project.elements.push(Element::new( + "Sub-blocked block model, free-form", + BlockModel::with_subblocks( + Orient3::from_origin([-1.0, -1.0, -1.0]), + Grid3::Regular { + size: [1.0, 1.0, 1.0], + count: [2, 2, 2], + }, + Subblocks::Freeform { + subblocks: writer.array_freeform_subblocks(FREEFORM_SUBBLOCKS).unwrap(), + }, + ), + )); + // Composite elememnt. + project.elements.push(Element::new( + "Composite", + Composite::new(vec![ + Element::new( + "Cube faces", + Surface::new( + writer.array_vertices(CUBE_VERTICES).unwrap(), + writer.array_triangles(CUBE_TRIANGLES).unwrap(), + ), + ), + Element::new( + "Cube edges", + LineSet::new( + writer.array_vertices(CUBE_VERTICES).unwrap(), + writer.array_segments(CUBE_SEGMENTS).unwrap(), + ), + ), + ]), + )); + // Projected texture. + let mut rectangle = Element::new( + "Textured", + Surface::new( + writer + .array_vertices([ + [0.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [1.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + ]) + .unwrap(), + writer.array_triangles([[0_u32, 1, 2], [0, 2, 3]]).unwrap(), + ), + ); + rectangle.attributes.push(Attribute::from_texture_project( + "Projected", + writer.image_bytes(IMAGE_BYTES).unwrap(), + Orient2 { + origin: [0.0, 0.0, 0.0], + u: [1.0, 0.0, 0.0], + v: [0.0, 1.0, 0.0], + }, + 1.0, + 1.0, + )); + rectangle.attributes.push(Attribute::from_texture_map( + "Mapped", + writer.image_bytes(IMAGE_BYTES).unwrap(), + Location::Vertices, + writer + .array_texcoords([[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0]]) + .unwrap(), + )); + project.elements.push(rectangle); + // Done. + let project_copy = project.clone(); + let (f, warnings) = writer.finish(project).unwrap(); + assert!(warnings.is_empty()); + (file_contents(f), project_copy) +} + +#[cfg(feature = "parquet")] +fn one_of_everything_but_wrong() -> Project { + let mut writer = file::Writer::new(temp_file("one_of_everything_but_wrong.omf", b"")).unwrap(); + // Project. + let mut project = Project::new("One of everything but wrong"); + // Error: infinite. + project.origin = [f64::INFINITY, 0.0, f64::NEG_INFINITY]; + // Surface element. + let mut triangles = PYRAMID_TRIANGLES.clone(); + triangles[2][0] = 5; // Error: vertex index out of range. + let mut surface = Element::new( + "Pyramid surface", + Surface::with_origin( + writer.array_vertices(PYRAMID_VERTICES).unwrap(), + writer.array_triangles(triangles).unwrap(), + [f64::NAN, 0.0, 0.0], // Error: NaN in origin. + ), + ); + surface.attributes.push(Attribute::from_numbers( + "Numbers", + Location::Vertices, + // Error: too many values. + writer + .array_numbers([ + Some(1.0_f64), + Some(2.0), + Some(3.0), + Some(4.0), + None, + Some(5.0), + ]) + .unwrap(), + )); + surface.attributes.push(Attribute::from_colors( + "Numbers", // Warning: duplicate name. + Location::Primitives, + writer + .array_colors([ + Some([255_u8, 0, 0, 255]), + Some([255, 255, 0, 255]), + Some([0, 255, 0, 255]), + Some([0, 0, 255, 255]), + Some([255, 255, 255, 255]), + // Error: too few values. + ]) + .unwrap(), + )); + project.elements.push(surface); + // PointSet element. + let mut points = Element::new( + "Pyramid points", + PointSet::new(writer.array_vertices(PYRAMID_VERTICES).unwrap()), + ); + points.attributes.push(Attribute::from_categories( + "Categories", + Location::Vertices, + // Error: category out of range. + writer + .array_indices([0_u32, 0, 0, 10, 1].map(Some)) + .unwrap() + .into(), + writer + .array_names(["Base".to_owned(), "Top".to_owned()]) + .unwrap(), + Some( + writer + .array_gradient([[255_u8, 128, 0, 255], [0, 128, 255, 255]]) + .unwrap(), + ), + [Attribute::from_numbers( + "Layer", + Location::Categories, + writer + .array_numbers([Some(1_i64), Some(2), Some(3)]) + .unwrap(), // Error: array too long + )], + )); + points.attributes.push(Attribute::from_vectors( + "2D Vectors", + Location::Primitives, // Error: invalid location. + writer + .array_vectors([ + Some([1.0_f32, 0.0]), + Some([1.0, 1.0]), + Some([0.0, 1.0]), + Some([0.0, 0.0]), + None, + ]) + .unwrap(), + )); + points.attributes.push(Attribute::from_vectors( + "3D Vectors", + Location::Vertices, + // Error: too many values. + writer + .array_vectors([Some([0.0_f32, 0.0, 1.0]), None, None, None, None, None]) + .unwrap(), + )); + project.elements.push(points); + // LineSet element. + let mut segments = PYRAMID_SEGMENTS.clone(); + segments[0][0] = 10; // Error: vertex index out of range. + let mut line_set = Element::new( + "Pyramid lines", + LineSet::new( + writer.array_vertices(PYRAMID_VERTICES).unwrap(), + writer.array_segments(segments).unwrap(), + ), + ); + line_set.attributes.push(Attribute::from_strings( + "Strings", + Location::Primitives, + writer + .array_text([ + None, + None, + None, + None, + Some("sw".to_owned()), + Some("se".to_owned()), + Some("ne".to_owned()), + ]) + .unwrap(), + )); + project.elements.push(line_set); + // GridSurface element. + project.elements.push(Element::new( + "Pyramid grid surface", + GridSurface::new( + Orient2 { + origin: [-1.5, -1.5, 0.0], + // Error: not unit vector. + u: [0.8, 0.0, 0.0], + // Error: not orthogonal. + v: [0.2, 1.0, 0.0], + }, + Grid2::Tensor { + u: writer.array_scalars([1.0, 2.0]).unwrap(), + v: writer.array_scalars([1.5, 1.5]).unwrap(), + }, + // Error: wrong length. + Some( + writer + .array_scalars([0.0_f32, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0]) + .unwrap(), + ), + ), + )); + // Regular block model element. + let mut block_model = Element::new( + "Regular block model", + BlockModel::new( + Orient3::from_origin([-1.0, -1.0, -1.0]), + Grid3::Regular { + size: [1.0, 0.0, 1.0], // Error: zero size. + count: [2, 2, 0], // Error: zero count. + }, + ), + ); + block_model.attributes.push(Attribute::from_booleans( + "Filter", + Location::Primitives, + writer + .array_booleans( + // Error: too few values. + [false, false, false, false, false, false, false, true].map(Some), + ) + .unwrap(), + )); + project.elements.push(block_model); + // Tensor block model element. + project.elements.push(Element::new( + "Tensor block model", + BlockModel::new( + Orient3::from_origin([-1.0, -1.0, f64::NAN]), // Error: Nan origin + Grid3::Tensor { + u: writer.array_scalars([0.6666, 1.333]).unwrap(), + v: writer.array_scalars([0.6666, 1.333]).unwrap(), + w: writer.array_scalars([1.0, 0.0]).unwrap(), // Error: zero size + }, + ), + )); + // Block model element, with regular sub-blocks. + let mut subblocks = OCTREE_SUBBLOCKS.clone(); + subblocks[4].0[2] = 2; // Error: parent index out of range + subblocks[2].1[5] = 10; // Error: corner out of range + project.elements.push(Element::new( + "Sub-blocked block model, regular", + BlockModel::with_subblocks( + Orient3::from_origin([-1.0, -1.0, -1.0]), + Grid3::Regular { + size: [1.0, 1.0, 1.0], + count: [2, 2, 2], + }, + Subblocks::Regular { + count: [4, 4, 4], + subblocks: writer.array_regular_subblocks(subblocks).unwrap(), + mode: Some(SubblockMode::Octree), + }, + ), + )); + // Block model element, with free-form sub-blocks. + project.elements.push(Element::new( + "Sub-blocked block model, free-form", + BlockModel::with_subblocks( + Orient3::from_origin([-1.0, -1.0, -1.0]), + Grid3::Regular { + size: [1.0, 1.0, 1.0], + count: [2, 2, 2], + }, + Subblocks::Freeform { + subblocks: writer.array_freeform_subblocks(FREEFORM_SUBBLOCKS).unwrap(), + }, + ), + )); + // Composite elememnt. + project.elements.push(Element::new( + "Composite", + Composite::new(vec![ + Element::new( + "Cube faces", + Surface::new( + writer.array_vertices(CUBE_VERTICES).unwrap(), + writer.array_triangles(CUBE_TRIANGLES).unwrap(), + ), + ), + Element::new( + "Cube faces", // Warning: duplicate name. + LineSet::new( + writer.array_vertices(CUBE_VERTICES).unwrap(), + writer.array_segments(CUBE_SEGMENTS).unwrap(), + ), + ), + ]), + )); + // Projected texture. + let mut rectangle = Element::new( + "Textured", + Surface::new( + writer + .array_vertices([ + [0.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [1.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + ]) + .unwrap(), + writer.array_triangles([[0_u32, 1, 2], [0, 2, 3]]).unwrap(), + ), + ); + rectangle.attributes.push(Attribute::from_texture_project( + "Projected", + writer.image_bytes(IMAGE_BYTES).unwrap(), + Orient2 { + origin: [0.0, f64::INFINITY, 0.0], // Error: infinite. + u: [1.0, 0.0, 0.0], + v: [0.0, 1.0, 0.0], + }, + 1.0, + 1.0, + )); + rectangle.attributes.push(Attribute::from_texture_map( + "Mapped", + writer.image_bytes(IMAGE_BYTES).unwrap(), + Location::Vertices, + // Error: too many values. + writer + .array_texcoords([[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0], [2.0, 2.0]]) + .unwrap(), + )); + project.elements.push(rectangle); + // Done. + project +} + +fn check_loc_lens(element: &Element, required: &[(Location, u64)]) { + for loc in [ + Location::Vertices, + Location::Primitives, + Location::Primitives, + Location::Subblocks, + Location::Elements, + ] { + let req = required + .iter() + .chain(Some(&(Location::Projected, 0))) + .find_map(|(l, n)| if *l == loc { Some(*n) } else { None }); + let val = element.location_len(loc); + if val != req { + let name = format!("{:?}", &element.geometry) + .split('(') + .next() + .unwrap() + .to_owned(); + panic!("{name}::location_len({loc:?}) was {val:?}, should have been {req:?}",) + } + } +} + +#[cfg(feature = "parquet")] +#[test] +fn location_lengths() { + let (_, project) = one_of_everything(); + macro_rules! check { + ($index:literal, $( $loc:ident = $n:literal ),*) => { + check_loc_lens( + &&project.elements[$index], + &[ $( (Location::$loc, $n) ),* ], + ) + }; + } + check!(0, Vertices = 5, Primitives = 6); // surface + check!(1, Vertices = 5); // point-set + check!(2, Vertices = 5, Primitives = 8); // line-set + check!(3, Vertices = 9, Primitives = 4); // grid-surface + check!(4, Primitives = 8, Vertices = 27); // block-model + check!(5, Primitives = 8, Vertices = 27); // tensor block-model + check!(6, Primitives = 8, Vertices = 27, Subblocks = 11); // sub-blocks + check!(7, Primitives = 8, Vertices = 27, Subblocks = 10); // free-form sub-blocks + check!(8, Elements = 2); // composite +} + +#[cfg(feature = "parquet")] +#[test] +fn load_one_of_everything() { + let (_, project) = one_of_everything(); + let bytes = std::fs::read("tests/one_of_everything.omf.json").unwrap(); + let loaded_project: Project = serde_json::from_slice(&bytes).unwrap(); + assert!( + loaded_project == project, + "one_of_everything project differs from tests/one_of_everything.omf" + ); +} + +#[cfg(feature = "parquet")] +#[ignore = "used to update benchmark"] +#[test] +fn update_one_of_everything() { + let (bytes, project) = one_of_everything(); + std::fs::write("tests/one_of_everything.omf", bytes).unwrap(); + std::fs::write( + "tests/one_of_everything.omf.json", + serde_json::to_string_pretty(&project).unwrap().as_bytes(), + ) + .unwrap(); +} + +#[cfg(feature = "parquet")] +#[test] +fn write_and_read_one_of_everything() { + let (bytes, base_project) = one_of_everything(); + let f = temp_file("write_and_read_one_of_everything.omf", &bytes); + let reader = Reader::new(f).unwrap(); + let (project, warnings) = reader.project().unwrap(); + let warning_strings: Vec = warnings.into_iter().map(|p| p.to_string()).collect(); + assert_eq!(warning_strings, Vec::::new()); + assert!(project == base_project, "read project differs"); + macro_rules! geometry { + ($name:ident: $geom_type:ident = $source:ident[$index:literal]) => { + let Element { + geometry: Geometry::$geom_type($name), + .. + } = &$source.elements[$index] + else { + panic!("wrong geometry"); + }; + }; + } + macro_rules! check_array { + ($load:ident($array:expr): $enum:ident :: $var:ident, $expected:expr) => { + let $enum::$var(iter) = reader.$load($array).unwrap() else { + panic!("wrong type"); + }; + let data = iter.collect::, _>>().unwrap(); + assert_eq!(data, $expected); + }; + ($load:ident($array:expr), $expected:expr) => { + let iter = reader.$load($array).unwrap(); + let data = iter.collect::, _>>().unwrap(); + assert_eq!(data, $expected); + }; + } + macro_rules! check_attr { + ($load:ident($e:literal, $a:literal, $attr:ident): $enum:ident :: $var:ident, $expected:expr) => { + let AttributeData::$attr { values, .. } = &project.elements[$e].attributes[$a].data + else { + panic!("wrong attribute type"); + }; + check_array!($load(values): $enum :: $var, $expected); + }; + ($load:ident($e:literal, $a:literal, $attr:ident), $expected:expr) => { + let AttributeData::$attr { values, .. } = &project.elements[$e].attributes[$a].data + else { + panic!("wrong attribute type"); + }; + check_array!($load(values), $expected); + }; + } + // Surface element. + geometry!(surface: Surface = project[0]); + check_array!(array_vertices(&surface.vertices): Vertices::F32, PYRAMID_VERTICES); + check_array!(array_triangles(&surface.triangles), PYRAMID_TRIANGLES); + check_attr!(array_numbers(0, 0, Number): Numbers::F64, [ + Some(1.0), Some(2.0), Some(3.0), Some(4.0), None, + ]); + check_attr!( + array_colors(0, 1, Color), + [ + Some([255, 0, 0, 255]), + Some([255, 255, 0, 255]), + Some([0, 255, 0, 255]), + Some([0, 0, 255, 255]), + Some([255, 255, 255, 255]), + Some([255, 255, 255, 255]), + ] + ); + let AttributeData::Number { + values, + colormap: + Some(NumberColormap::Discrete { + boundaries, + gradient, + }), + } = &project.elements[0].attributes[2].data + else { + panic!("wrong type"); + }; + check_array!(array_numbers(values): Numbers::DateTime, [ + DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + DateTime::from_str("2000-01-01T01:00:00Z").unwrap(), + DateTime::from_str("2000-01-01T02:00:00Z").unwrap(), + DateTime::from_str("2000-01-01T03:00:00Z").unwrap(), + DateTime::from_str("2000-01-01T04:00:00Z").unwrap(), + ].map(Some)); + check_array!( + array_boundaries(boundaries): Boundaries::DateTime, + [ + Boundary::Less(DateTime::from_str("2000-01-01T01:00:00Z").unwrap()), + Boundary::Less(DateTime::from_str("2000-01-01T02:00:00Z").unwrap()), + Boundary::LessEqual(DateTime::from_str("2000-01-01T03:00:00Z").unwrap()), + ] + ); + check_array!( + array_gradient(gradient), + [ + [0, 0, 255, 255], + [0, 255, 0, 255], + [255, 0, 0, 255], + [255, 255, 255, 255], + ] + ); + // PointSet element. + geometry!(point_set: PointSet = project[1]); + check_array!(array_vertices(&point_set.vertices): Vertices::F32, PYRAMID_VERTICES); + check_attr!(array_indices(1, 0, Category), [0, 0, 0, 0, 1].map(Some)); + check_attr!(array_vectors(1, 1, Vector): Vectors::F32x2, [ + Some([1.0, 0.0]), Some([1.0, 1.0]), Some([0.0, 1.0]), Some([0.0, 0.0]), None, + ]); + check_attr!(array_vectors(1, 2, Vector): Vectors::F32x3, [ + None, None, None, None, Some([0.0, 0.0, 1.0]), + ]); + let AttributeData::Category { + names, + gradient: Some(c), + attributes, + .. + } = &project.elements[1].attributes[0].data + else { + panic!("wrong attr type"); + }; + check_array!(array_names(names), ["Base", "Top"]); + check_array!( + array_gradient(c), + [[255_u8, 128, 0, 255], [0, 128, 255, 255]] + ); + let AttributeData::Number { + values, + colormap: None, + } = &attributes[0].data + else { + panic!("wrong attribute data type"); + }; + check_array!(array_numbers(values): Numbers::I64, [Some(1), Some(2)]); + // LineSet element. + geometry!(line_set: LineSet = project[2]); + check_array!(array_vertices(&line_set.vertices): Vertices::F32, PYRAMID_VERTICES); + check_array!(array_segments(&line_set.segments), PYRAMID_SEGMENTS); + let AttributeData::Text { values } = &project.elements[2].attributes[0].data else { + panic!("wrong attr type"); + }; + check_array!( + array_text(values), + [ + None, + None, + None, + None, + Some("sw".to_owned()), + Some("se".to_owned()), + Some("ne".to_owned()), + Some("nw".to_owned()), + ] + ); + // GridSurface element. + geometry!(grid_surface: GridSurface = project[3]); + check_array!( + array_scalars(grid_surface.heights.as_ref().unwrap()): Scalars::F32, + [0.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 0.0] + ); + let Grid2::Tensor { u, v } = &grid_surface.grid else { + panic!("not tensor"); + }; + check_array!(array_scalars(u): Scalars::F64, [1.0, 2.0]); + check_array!(array_scalars(v): Scalars::F64, [1.5, 1.5]); + // Regular block model element. + geometry!(_regular_bm: BlockModel = project[4]); + check_attr!( + array_booleans(4, 0, Boolean), + [false, false, false, false, false, false, false, true].map(Some) + ); + geometry!(tensor_bm: BlockModel = project[5]); + let Grid3::Tensor { u, v, w } = &tensor_bm.grid else { + panic!("not tensor"); + }; + check_array!(array_scalars(u): Scalars::F64, [0.6666, 1.333]); + check_array!(array_scalars(v): Scalars::F64, [0.6666, 1.333]); + check_array!(array_scalars(w): Scalars::F64, [1.0, 1.0]); + // Block model element, with regular sub-blocks. + geometry!(regular_subblocks: BlockModel = project[6]); + let Some(Subblocks::Regular { subblocks, .. }) = ®ular_subblocks.subblocks else { + panic!("not regular sub-blocked"); + }; + check_array!( + array_regular_subblocks(subblocks), + [ + ([0, 0, 0], [0, 0, 0, 4, 4, 4]), + ([1, 0, 0], [0, 0, 0, 4, 4, 4]), + ([0, 1, 0], [0, 0, 0, 4, 4, 4]), + ([1, 1, 0], [0, 0, 0, 4, 4, 4]), + ([0, 0, 1], [0, 0, 0, 4, 4, 4]), + ([1, 0, 1], [0, 0, 0, 4, 4, 4]), + ([0, 1, 1], [0, 0, 0, 4, 4, 4]), + ([1, 1, 1], [0, 0, 0, 2, 2, 2]), + ([1, 1, 1], [0, 2, 0, 1, 3, 1]), + ([1, 1, 1], [2, 0, 0, 3, 1, 1]), + ([1, 1, 1], [0, 0, 2, 1, 1, 3]), + ] + ); + // Block model element, with free-form sub-blocks. + geometry!(freeform_subblocks: BlockModel = project[7]); + let Some(Subblocks::Freeform { subblocks, .. }) = &freeform_subblocks.subblocks else { + panic!("not free-form sub-blocked"); + }; + check_array!( + array_freeform_subblocks(subblocks): FreeformSubblocks::F32, + [ + ([0, 0, 0], [0.0, 0.0, 0.0, 1.0, 1.0, 1.0]), + ([1, 0, 0], [0.0, 0.0, 0.0, 1.0, 1.0, 1.0]), + ([0, 1, 0], [0.0, 0.0, 0.0, 1.0, 1.0, 1.0]), + ([1, 1, 0], [0.0, 0.0, 0.0, 1.0, 1.0, 1.0]), + ([0, 0, 1], [0.0, 0.0, 0.0, 1.0, 1.0, 1.0]), + ([1, 0, 1], [0.0, 0.0, 0.0, 1.0, 1.0, 1.0]), + ([0, 1, 1], [0.0, 0.0, 0.0, 1.0, 1.0, 1.0]), + ([1, 1, 1], [0.0, 0.0, 0.0, 1.0, 1.0, 0.3333]), + ([1, 1, 1], [0.0, 0.0, 0.3333, 0.75, 0.75, 0.6666]), + ([1, 1, 1], [0.0, 0.0, 0.6666, 0.5, 0.5, 1.0]), + ] + ); + // Composite element. + geometry!(comp: Composite = project[8]); + geometry!(comp_surface: Surface = comp[0]); + check_array!(array_vertices(&comp_surface.vertices): Vertices::F64, CUBE_VERTICES); + check_array!(array_triangles(&comp_surface.triangles), CUBE_TRIANGLES); + geometry!(comp_line_set: LineSet = comp[1]); + check_array!(array_vertices(&comp_line_set.vertices): Vertices::F64, CUBE_VERTICES); + check_array!(array_segments(&comp_line_set.segments), CUBE_SEGMENTS); + // Textures. + geometry!(_rectangle: Surface = project[9]); + let AttributeData::ProjectedTexture { image, .. } = &project.elements[9].attributes[0].data + else { + panic!("wrong attr type"); + }; + let bytes = reader.array_bytes(image).unwrap(); + assert_eq!(bytes, IMAGE_BYTES); + let AttributeData::MappedTexture { + image, texcoords, .. + } = &project.elements[9].attributes[1].data + else { + panic!("wrong attr type"); + }; + let bytes = reader.array_bytes(image).unwrap(); + assert_eq!(bytes, IMAGE_BYTES); + check_array!(array_texcoords(texcoords): Texcoords::F64, + [[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0]] + ); +} + +#[cfg(feature = "parquet")] +#[test] +fn validate_one_of_everything_but_wrong() { + let mut project = one_of_everything_but_wrong(); + let problems = project.validate().unwrap_err(); + let mut expected = vec![ + "Error: 'Project::origin' must be finite, inside 'One of everything but wrong'", + "Error: 'Surface::origin' must be finite, inside 'Pyramid surface'", + "Warning: 'Element::attributes[..]::name' contains duplicate of \"Numbers\", inside 'Pyramid surface'", + "Error: 'Attribute' length 6 does not match geometry (5), inside 'Numbers'", + "Error: 'Attribute' length 5 does not match geometry (6), inside 'Numbers'", + "Error: 'Attribute::location' is Primitives which is not valid on PointSet geometry, inside '2D Vectors'", + "Error: 'Attribute' length 6 does not match geometry (5), inside '3D Vectors'", + "Error: 'Attribute' length 7 does not match geometry (8), inside 'Strings'", + "Error: 'Orient2::u' must be a unit vector but [0.8, 0.0, 0.0] length is 0.8, inside 'Pyramid grid surface'", + "Error: 'Orient2::v' must be a unit vector but [0.2, 1.0, 0.0] length is 1.0198039, inside 'Pyramid grid surface'", + "Error: 'Orient2' vectors are not orthogonal: [0.8, 0.0, 0.0] [0.2, 1.0, 0.0], inside 'Pyramid grid surface'", + "Error: 'GridSurface::heights' length 8 does not match geometry (9), inside 'Pyramid grid surface'", + "Error: 'Grid3::Regular::size' must be greater than zero, inside 'Regular block model'", + "Error: 'Grid3::Regular::count' must be greater than zero, inside 'Regular block model'", + "Error: 'Attribute' length 8 does not match geometry (0), inside 'Filter'", + "Error: 'Orient3::origin' must be finite, inside 'Tensor block model'", + "Warning: 'Composite::elements[..]::name' contains duplicate of \"Cube faces\", inside 'Composite'", + "Error: 'Orient2::origin' must be finite, inside 'Projected'", + "Error: 'Attribute' length 5 does not match geometry (4), inside 'Mapped'", + "Error: 'Attribute' length 3 does not match geometry (2), inside 'Layer'", + "Error: 'Surface::triangles' array contains invalid data: index value 5 exceeds the maximum index 4, inside 'Pyramid surface'", + "Error: 'AttributeData::Category::values' array contains invalid data: index value 10 exceeds the maximum index 1, inside 'Categories'", + "Error: 'LineSet::segments' array contains invalid data: index value 10 exceeds the maximum index 4, inside 'Pyramid lines'", + "Error: 'Grid3::Tensor::w' array contains invalid data: size value 0 is zero or less, inside 'Tensor block model'", + "Error: 'Subblocks::Regular::subblocks' array contains invalid data: block index [0, 0, 2] exceeds the maximum index [1, 1, 1], inside 'Sub-blocked block model, regular'", + "Error: 'Subblocks::Regular::subblocks' array contains invalid data: sub-block [0, 0, 0] to [4, 4, 10] exceeds the maximum [4, 4, 4], inside 'Sub-blocked block model, regular'", + "Error: 'Subblocks::Regular::subblocks' array contains invalid data: sub-block [0, 0, 0] to [4, 4, 10] is invalid for Octree mode, inside 'Sub-blocked block model, regular'", + ]; + let mut unexpected = Vec::new(); + for p in problems.into_vec() { + let s = p.to_string(); + if let Some(index) = expected.iter().position(|e| *e == &s) { + expected.remove(index); + } else { + unexpected.push(p.to_string()); + } + } + if !unexpected.is_empty() || !expected.is_empty() { + panic!("unexpected problems: {unexpected:#?}\nexpected but not found: {expected:#?}"); + } +} + +#[cfg(all(feature = "image", feature = "parquet"))] +#[test] +fn write_and_read_images() { + let points = [[1_f32, 2.0, 3.0], [4.0, 5.0, 6.0]]; + let test_image = image::open("tests/test.png").unwrap(); + let image::DynamicImage::ImageRgb8(test_image_rgb8) = &test_image else { + panic!("expected Rgb8"); + }; + let mut writer = file::Writer::new(temp_file("write_and_read_images.omf", b"")).unwrap(); + let mut project = Project::new("test"); + let arr = writer.array_vertices(points).unwrap(); + let jpeg = writer.image_jpeg(&test_image_rgb8, 85).unwrap(); + let png = writer.image_png(&test_image).unwrap(); + let mut points = Element::new("points", PointSet::new(arr)); + points.attributes.push(Attribute::from_texture_project( + "jpeg texture", + jpeg, + Default::default(), + 10.0, + 10.0, + )); + points.attributes.push(Attribute::from_texture_project( + "png texture", + png, + Default::default(), + 10.0, + 10.0, + )); + project.elements.push(points); + let original = project.clone(); + let (mut file, warnings) = writer.finish(project).unwrap(); + assert!(warnings.is_empty()); + + file.rewind().unwrap(); + let reader = Reader::new(file).unwrap(); + let (loaded, warnings) = reader.project().unwrap(); + assert!(warnings.is_empty()); + assert_eq!(loaded, original); + let AttributeData::ProjectedTexture { image: i, .. } = &loaded.elements[0].attributes[0].data + else { + panic!("wrong project structure"); + }; + let AttributeData::ProjectedTexture { image: j, .. } = &loaded.elements[0].attributes[1].data + else { + panic!("wrong project structure"); + }; + let loaded_jpeg = reader.image(&i).unwrap(); + let loaded_png = reader.image(&j).unwrap(); + // jpeg will be different due to lossy compression + assert!(loaded_jpeg != test_image); + // png will be exactly the same + assert!(loaded_png == test_image); +} diff --git a/tests/omf1/test_proj.omf b/tests/omf1/test_proj.omf new file mode 100644 index 0000000..14aa2c2 Binary files /dev/null and b/tests/omf1/test_proj.omf differ diff --git a/tests/one_of_everything.omf b/tests/one_of_everything.omf new file mode 100644 index 0000000..7abf133 Binary files /dev/null and b/tests/one_of_everything.omf differ diff --git a/tests/one_of_everything.omf.json b/tests/one_of_everything.omf.json new file mode 100644 index 0000000..197a494 --- /dev/null +++ b/tests/one_of_everything.omf.json @@ -0,0 +1,522 @@ +{ + "name": "One of everything", + "description": "An OMF 2.0 project containing roughly one of every different type.", + "author": "Tim Evans", + "date": "1970-01-01T00:00:00Z", + "metadata": { + "array": [ + 1, + 2, + 3 + ], + "bool": true, + "null": null, + "number": 42, + "object": { + "a": 1, + "b": 2 + }, + "string": "value" + }, + "elements": [ + { + "name": "Pyramid surface", + "description": "A surface forming a pyramid", + "color": [ + 255, + 128, + 0, + 255 + ], + "attributes": [ + { + "name": "Numbers", + "location": "Vertices", + "data": { + "type": "Number", + "values": { + "filename": "3.parquet", + "item_count": 5 + } + } + }, + { + "name": "Colors", + "location": "Primitives", + "data": { + "type": "Color", + "values": { + "filename": "4.parquet", + "item_count": 6 + } + } + }, + { + "name": "Date-times", + "location": "Vertices", + "data": { + "type": "Number", + "values": { + "filename": "5.parquet", + "item_count": 5 + }, + "colormap": { + "type": "Discrete", + "boundaries": { + "filename": "6.parquet", + "item_count": 3 + }, + "gradient": { + "filename": "7.parquet", + "item_count": 4 + } + } + } + } + ], + "geometry": { + "type": "Surface", + "vertices": { + "filename": "1.parquet", + "item_count": 5 + }, + "triangles": { + "filename": "2.parquet", + "item_count": 6 + } + } + }, + { + "name": "Pyramid points", + "attributes": [ + { + "name": "Categories", + "description": "Divides the points into top and base.", + "units": "whatever", + "metadata": { + "key": "value" + }, + "location": "Vertices", + "data": { + "type": "Category", + "values": { + "filename": "9.parquet", + "item_count": 5 + }, + "names": { + "filename": "10.parquet", + "item_count": 2 + }, + "gradient": { + "filename": "11.parquet", + "item_count": 2 + }, + "attributes": [ + { + "name": "Layer", + "location": "Categories", + "data": { + "type": "Number", + "values": { + "filename": "12.parquet", + "item_count": 2 + } + } + } + ] + } + }, + { + "name": "2D Vectors", + "location": "Vertices", + "data": { + "type": "Vector", + "values": { + "filename": "13.parquet", + "item_count": 5 + } + } + }, + { + "name": "3D Vectors", + "location": "Vertices", + "data": { + "type": "Vector", + "values": { + "filename": "14.parquet", + "item_count": 5 + } + } + } + ], + "geometry": { + "type": "PointSet", + "vertices": { + "filename": "8.parquet", + "item_count": 5 + } + } + }, + { + "name": "Pyramid lines", + "attributes": [ + { + "name": "Strings", + "location": "Primitives", + "data": { + "type": "Text", + "values": { + "filename": "17.parquet", + "item_count": 8 + } + } + } + ], + "geometry": { + "type": "LineSet", + "vertices": { + "filename": "15.parquet", + "item_count": 5 + }, + "segments": { + "filename": "16.parquet", + "item_count": 8 + } + } + }, + { + "name": "Pyramid grid surface", + "geometry": { + "type": "GridSurface", + "orient": { + "origin": [ + -1.5, + -1.5, + 0.0 + ], + "u": [ + 1.0, + 0.0, + 0.0 + ], + "v": [ + 0.0, + 1.0, + 0.0 + ] + }, + "grid": { + "type": "Tensor", + "u": { + "filename": "18.parquet", + "item_count": 2 + }, + "v": { + "filename": "19.parquet", + "item_count": 2 + } + }, + "heights": { + "filename": "20.parquet", + "item_count": 9 + } + } + }, + { + "name": "Regular block model", + "attributes": [ + { + "name": "Filter", + "location": "Primitives", + "data": { + "type": "Boolean", + "values": { + "filename": "21.parquet", + "item_count": 8 + } + } + } + ], + "geometry": { + "type": "BlockModel", + "orient": { + "origin": [ + -1.0, + -1.0, + -1.0 + ], + "u": [ + 1.0, + 0.0, + 0.0 + ], + "v": [ + 0.0, + 1.0, + 0.0 + ], + "w": [ + 0.0, + 0.0, + 1.0 + ] + }, + "grid": { + "type": "Regular", + "size": [ + 1.0, + 1.0, + 1.0 + ], + "count": [ + 2, + 2, + 2 + ] + } + } + }, + { + "name": "Tensor block model", + "geometry": { + "type": "BlockModel", + "orient": { + "origin": [ + -1.0, + -1.0, + -1.0 + ], + "u": [ + 1.0, + 0.0, + 0.0 + ], + "v": [ + 0.0, + 1.0, + 0.0 + ], + "w": [ + 0.0, + 0.0, + 1.0 + ] + }, + "grid": { + "type": "Tensor", + "u": { + "filename": "22.parquet", + "item_count": 2 + }, + "v": { + "filename": "23.parquet", + "item_count": 2 + }, + "w": { + "filename": "24.parquet", + "item_count": 2 + } + } + } + }, + { + "name": "Sub-blocked block model, regular", + "geometry": { + "type": "BlockModel", + "orient": { + "origin": [ + -1.0, + -1.0, + -1.0 + ], + "u": [ + 1.0, + 0.0, + 0.0 + ], + "v": [ + 0.0, + 1.0, + 0.0 + ], + "w": [ + 0.0, + 0.0, + 1.0 + ] + }, + "grid": { + "type": "Regular", + "size": [ + 1.0, + 1.0, + 1.0 + ], + "count": [ + 2, + 2, + 2 + ] + }, + "subblocks": { + "type": "Regular", + "count": [ + 4, + 4, + 4 + ], + "subblocks": { + "filename": "25.parquet", + "item_count": 11 + }, + "mode": "Octree" + } + } + }, + { + "name": "Sub-blocked block model, free-form", + "geometry": { + "type": "BlockModel", + "orient": { + "origin": [ + -1.0, + -1.0, + -1.0 + ], + "u": [ + 1.0, + 0.0, + 0.0 + ], + "v": [ + 0.0, + 1.0, + 0.0 + ], + "w": [ + 0.0, + 0.0, + 1.0 + ] + }, + "grid": { + "type": "Regular", + "size": [ + 1.0, + 1.0, + 1.0 + ], + "count": [ + 2, + 2, + 2 + ] + }, + "subblocks": { + "type": "Freeform", + "subblocks": { + "filename": "26.parquet", + "item_count": 10 + } + } + } + }, + { + "name": "Composite", + "geometry": { + "type": "Composite", + "elements": [ + { + "name": "Cube faces", + "geometry": { + "type": "Surface", + "vertices": { + "filename": "27.parquet", + "item_count": 8 + }, + "triangles": { + "filename": "28.parquet", + "item_count": 12 + } + } + }, + { + "name": "Cube edges", + "geometry": { + "type": "LineSet", + "vertices": { + "filename": "29.parquet", + "item_count": 8 + }, + "segments": { + "filename": "30.parquet", + "item_count": 12 + } + } + } + ] + } + }, + { + "name": "Textured", + "attributes": [ + { + "name": "Projected", + "location": "Projected", + "data": { + "type": "ProjectedTexture", + "image": { + "filename": "33.png", + "item_count": 0 + }, + "orient": { + "origin": [ + 0.0, + 0.0, + 0.0 + ], + "u": [ + 1.0, + 0.0, + 0.0 + ], + "v": [ + 0.0, + 1.0, + 0.0 + ] + }, + "width": 1.0, + "height": 1.0 + } + }, + { + "name": "Mapped", + "location": "Vertices", + "data": { + "type": "MappedTexture", + "image": { + "filename": "34.png", + "item_count": 0 + }, + "texcoords": { + "filename": "35.parquet", + "item_count": 4 + } + } + } + ], + "geometry": { + "type": "Surface", + "vertices": { + "filename": "31.parquet", + "item_count": 4 + }, + "triangles": { + "filename": "32.parquet", + "item_count": 2 + } + } + } + ] +} \ No newline at end of file diff --git a/tests/test.png b/tests/test.png new file mode 100644 index 0000000..7e734e9 Binary files /dev/null and b/tests/test.png differ