JPBergsma/partial data

.github/workflows/ci.yml

@@ -26,10 +26,10 @@ jobs:
       with:
         submodules: true
 
-    - name: Set up Python 3.10
+    - name: Set up Python 3.9
       uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.9'
         cache: 'pip'
         cache-dependency-path: |
           requirements*.txt
@@ -56,10 +56,10 @@ jobs:
       with:
         submodules: true
 
-    - name: Set up Python 3.10
+    - name: Set up Python 3.9
       uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.9'
         cache: 'pip'
         cache-dependency-path: |
           requirements*.txt
@@ -227,27 +227,27 @@ jobs:
       run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/adapters/
 
     - name: Run tests for validator only to assess coverage (mongomock)
-      if: matrix.python-version == 3.10
+      if: matrix.python-version == 3.9
       run: pytest -rs --cov=./optimade/ --cov-report=xml:validator_cov.xml --cov-append tests/server/test_server_validation.py
       env:
         OPTIMADE_DATABASE_BACKEND: 'mongomock'
 
     - name: Run tests for validator only to assess coverage (Elasticsearch)
-      if: matrix.python-version == 3.10
+      if: matrix.python-version == 3.9
       run: pytest -rs --cov=./optimade/ --cov-report=xml:validator_cov.xml --cov-append tests/server/test_server_validation.py
       env:
         OPTIMADE_DATABASE_BACKEND: 'elastic'
         OPTIMADE_INSERT_TEST_DATA: false # Must be specified as previous steps will have already inserted the test data
 
     - name: Run tests for validator only to assess coverage (MongoDB)
-      if: matrix.python-version == 3.10
+      if: matrix.python-version == 3.9
       run: pytest -rs --cov=./optimade/ --cov-report=xml:validator_cov.xml --cov-append tests/server/test_server_validation.py
       env:
         OPTIMADE_DATABASE_BACKEND: 'mongodb'
         OPTIMADE_INSERT_TEST_DATA: false # Must be specified as previous steps will have already inserted the test data
 
     - name: Run the OPTIMADE Client CLI
-      if: matrix.python-version == 3.10
+      if: matrix.python-version == 3.9
       run: |
         coverage run --append --source optimade optimade/client/cli.py \
            --filter 'nsites = 1' \
@@ -275,15 +275,15 @@ jobs:
         coverage xml
 
     - name: Upload coverage to Codecov
-      if: matrix.python-version == '3.10' && github.repository == 'Materials-Consortia/optimade-python-tools'
+      if: matrix.python-version == '3.9' && github.repository == 'Materials-Consortia/optimade-python-tools'
       uses: codecov/codecov-action@v3
       with:
         name: project
         file: ./coverage.xml
         flags: project
 
     - name: Upload validator coverage to Codecov
-      if: matrix.python-version == '3.10' && github.repository == 'Materials-Consortia/optimade-python-tools'
+      if: matrix.python-version == '3.9' && github.repository == 'Materials-Consortia/optimade-python-tools'
       uses: codecov/codecov-action@v3
       with:
         name: validator
@@ -300,7 +300,7 @@ jobs:
 
     - uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.9'
         cache: 'pip'
         cache-dependency-path: |
           requirements*.txt
@@ -330,10 +330,10 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v4
 
-    - name: Set up Python 3.10
+    - name: Set up Python 3.9
       uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.9'
         cache: 'pip'
         cache-dependency-path: |
           requirements*.txt

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 default_language_version:
-  python: python3.10
+  python: python3.9
 
 repos:
   - repo: https://github.com/ambv/black
@@ -9,7 +9,7 @@ repos:
       name: Blacken
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
     - id: check-symlinks
     - id: check-yaml
@@ -23,11 +23,17 @@ repos:
     - id: trailing-whitespace
       args: [--markdown-linebreak-ext=md]
 
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.15.0
+    hooks:
+    - id: pyupgrade
+      args: ["--py39-plus"]
+
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.0.291'
+    rev: 'v0.0.292'
     hooks:
     - id: ruff
-      args: [--fix]
+      args: [--fix, --exit-non-zero-on-fix]
 
   - repo: local
     hooks:
@@ -46,7 +52,7 @@ repos:
       description: Update the API Reference documentation whenever a Python file is touched in the code base.
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.5.1
+    rev: v1.6.0
     hooks:
       - id: mypy
         name: "MyPy"

docs/api_reference/adapters/jsonl.md

@@ -0,0 +1,3 @@
+# jsonl
+
+::: optimade.adapters.jsonl

docs/api_reference/models/partial_data.md

@@ -0,0 +1,5 @@
+# partial_data
+
+::: optimade.models.partial_data
+    options:
+      show_if_no_docstring: true

docs/api_reference/server/mappers/partial_data.md

@@ -0,0 +1,3 @@
+# partial_data
+
+::: optimade.server.mappers.partial_data

docs/api_reference/server/routers/partial_data.md

@@ -0,0 +1,3 @@
+# partial_data
+
+::: optimade.server.routers.partial_data

openapi/index_openapi.json

@@ -467,6 +467,29 @@
         "title": "BaseRelationshipResource",
         "description": "Minimum requirements to represent a relationship resource"
       },
+      "EntryMetadata": {
+        "properties": {
+          "property_metadata": {
+            "type": "object",
+            "title": "Property Metadata",
+            "description": "An object containing per-entry and per-property metadata. The keys are the names of the fields in attributes for which metadata is available. The values belonging to these keys are dictionaries containing the relevant metadata fields. See also [Metadata properties](https://github.com/Materials-Consortia/OPTIMADE/blob/develop/optimade.rst#metadata-properties)"
+          },
+          "partial_data_links": {
+            "additionalProperties": {
+              "items": {
+                "$ref": "#/components/schemas/PartialDataLink"
+              },
+              "type": "array"
+            },
+            "type": "object",
+            "title": "Partial Data Links",
+            "description": "A dictionary, where the keys are the names of the properties in the attributes field for which the value is too large to be shared by default.\n        For each property one or more links are provided from which the value of the attribute can be retrieved."
+          }
+        },
+        "type": "object",
+        "title": "EntryMetadata",
+        "description": "Contains the metadata for the attributes of an entry"
+      },
       "EntryRelationships": {
         "properties": {
           "references": {
@@ -520,11 +543,11 @@
           "meta": {
             "allOf": [
               {
-                "$ref": "#/components/schemas/Meta"
+                "$ref": "#/components/schemas/EntryMetadata"
               }
             ],
             "title": "Meta",
-            "description": "a meta object containing non-standard meta-information about a resource that can not be represented as an attribute or relationship."
+            "description": "A [JSON API meta object](https://jsonapi.org/format/1.1/#document-meta) that is used to communicate metadata."
           },
           "attributes": {
             "allOf": [
@@ -1129,11 +1152,11 @@
           "meta": {
             "allOf": [
               {
-                "$ref": "#/components/schemas/Meta"
+                "$ref": "#/components/schemas/EntryMetadata"
               }
             ],
             "title": "Meta",
-            "description": "a meta object containing non-standard meta-information about a resource that can not be represented as an attribute or relationship."
+            "description": "A [JSON API meta object](https://jsonapi.org/format/1.1/#document-meta) that is used to communicate metadata."
           },
           "attributes": {
             "allOf": [
@@ -1393,6 +1416,29 @@
         "title": "OptimadeError",
         "description": "detail MUST be present"
       },
+      "PartialDataLink": {
+        "properties": {
+          "link": {
+            "type": "string",
+            "maxLength": 65536,
+            "minLength": 1,
+            "format": "uri",
+            "title": "Link",
+            "description": "String. A JSON API link that points to a location from which the omitted data can be fetched. There is no requirement on the syntax or format for the link URL."
+          },
+          "format": {
+            "type": "string",
+            "title": "Format",
+            "description": "String. The name of the format provided via this link. For one of the objects this format field SHOULD have the value \"jsonlines\", which refers to the format in OPTIMADE JSON lines partial data format."
+          }
+        },
+        "type": "object",
+        "required": [
+          "link",
+          "format"
+        ],
+        "title": "PartialDataLink"
+      },
       "Provider": {
         "properties": {
           "name": {