Merge branch 'master' into JPBergsma_add_trajectory

# Conflicts:
#	openapi/openapi.json
#	optimade/server/entry_collections/entry_collections.py
#	optimade/server/routers/utils.py
#	providers
#	requirements.txt

.github/workflows/ci.yml

@@ -194,6 +194,12 @@ jobs:
       env:
         OPTIMADE_DATABASE_BACKEND: 'mongomock'
 
+    - name: Run server tests with no API validation (using `mongomock`)
+      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server tests/filtertransformers
+      env:
+        OPTIMADE_DATABASE_BACKEND: 'mongomock'
+        OPTIMADE_VALIDATE_API_RESPONSE: false
+
     - name: Run server tests (using a real MongoDB)
       run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server tests/filtertransformers
       env:

.pre-commit-config.yaml

@@ -23,10 +23,11 @@ repos:
     - id: trailing-whitespace
       args: [--markdown-linebreak-ext=md]
 
-  - repo: https://github.com/pycqa/flake8
-    rev: '6.0.0'
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: 'v0.0.254'
     hooks:
-    - id: flake8
+    - id: ruff
+      args: [--fix]
 
   - repo: local
     hooks:
@@ -45,14 +46,8 @@ repos:
       description: Update the API Reference documentation whenever a Python file is touched in the code base.
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.991
+    rev: v1.1.1
     hooks:
       - id: mypy
         name: "MyPy"
         additional_dependencies: ["types-all", "pydantic~=1.10"]
-
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-        name: "isort"

CHANGELOG.md

@@ -2,9 +2,91 @@
 
 ## [Unreleased](https://github.com/Materials-Consortia/optimade-python-tools/tree/HEAD)
 
-[Full Changelog](https://github.com/Materials-Consortia/optimade-python-tools/compare/v0.20.3...HEAD)
+[Full Changelog](https://github.com/Materials-Consortia/optimade-python-tools/compare/v0.22.1...HEAD)
 
-This release primarily adds compatibility for the newest FastAPI releases (`>=0.87`) by updating our test client to work with both `httpx` and `requests`.
+**Closed issues:**
+
+- Add docs examples for client callbacks [\#1527](https://github.com/Materials-Consortia/optimade-python-tools/issues/1527)
+- Cannot use the `OptimadeClient` in async mode with an existing event loop [\#1195](https://github.com/Materials-Consortia/optimade-python-tools/issues/1195)
+
+## [v0.22.1](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.22.1) (2023-02-28)
+
+[Full Changelog](https://github.com/Materials-Consortia/optimade-python-tools/compare/v0.22.0...v0.22.1)
+
+This patch release changes the synchronous functionality of `OptimadeClient` to use "vanilla" requests rather than httpx. This enables easier use inside of Jupyter notebooks.
+
+**Merged pull requests:**
+
+- Replace httpx usage in synchronous `OptimadeClient` with requests  [\#1536](https://github.com/Materials-Consortia/optimade-python-tools/pull/1536) ([ml-evs](https://github.com/ml-evs))
+
+## [v0.22.0](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.22.0) (2023-02-27)
+
+[Full Changelog](https://github.com/Materials-Consortia/optimade-python-tools/compare/v0.21.0...v0.22.0)
+
+This release adds a new feature to the reference server for avoiding validating data on the way out of the API (useful in cases where a small amount of leniency is desirable, and will have a minor performance bump). It also fixes a bug in the schema where `page_below` and `page_above` were codified as integers rather than strings.
+
+**Implemented enhancements:**
+
+- Add server config option to disable validation of outgoing data [\#1530](https://github.com/Materials-Consortia/optimade-python-tools/pull/1530) ([ml-evs](https://github.com/ml-evs))
+
+**Fixed bugs:**
+
+- Make `page_above/below` strings rather than ints [\#1529](https://github.com/Materials-Consortia/optimade-python-tools/pull/1529) ([ml-evs](https://github.com/ml-evs))
+
+## [v0.21.0](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.21.0) (2023-02-20)
+
+[Full Changelog](https://github.com/Materials-Consortia/optimade-python-tools/compare/v0.20.3...v0.21.0)
+
+This minor release contains new client functionality and improved support for ASE.
+
+## New features:
+
+- Ability to specify async callback functions that are called after every client response.
+  This can be used for e.g., iteratively saving to file or a database. For example:
+
+  ```python
+  from optimade.client import OptimadeClient
+
+  DATABASE = pymongo.MongoClient().database.collection
+
+  def save_callback(url, results) -> None:
+      for structure in results["data"]:
+          DATABASE.insert_one(structure)
+
+  client = OptimadeClient(callbacks=[save_callback])
+  client.get()
+  ```
+
+- Ability to create OPTIMADE structure objects from ASE atoms:
+  ```python
+  from optimade.adapters import Structure
+  from ase import Atoms
+
+  co = Atoms('CO', positions=[(0, 0, 0), (0, 0, 1.1)])
+
+  structure = Structure.from_ase_atoms(co)
+  ```
+
+- Added ability to mute the client progress bars with `--silent`/`silent=True` and increased default response timeouts to better reflect those required for practical queries.
+
+**Implemented enhancements:**
+
+- Add customisable callback functions to client [\#1515](https://github.com/Materials-Consortia/optimade-python-tools/issues/1515)
+- Add ability to specify callbacks to run after each client request [\#1519](https://github.com/Materials-Consortia/optimade-python-tools/pull/1519) ([ml-evs](https://github.com/ml-evs))
+- Increase client timeouts and tweak `response_fields` behaviour [\#1514](https://github.com/Materials-Consortia/optimade-python-tools/pull/1514) ([ml-evs](https://github.com/ml-evs))
+- Add ASE ingester and generalize other ingestion utilities [\#1509](https://github.com/Materials-Consortia/optimade-python-tools/pull/1509) ([ml-evs](https://github.com/ml-evs))
+
+**Closed issues:**
+
+- Increase default client timeouts [\#1513](https://github.com/Materials-Consortia/optimade-python-tools/issues/1513)
+- Provide basic support for older pymatgen versions in adapters [\#1490](https://github.com/Materials-Consortia/optimade-python-tools/issues/1490)
+
+**Merged pull requests:**
+
+- Replace several linters and fixers \(flake8, isort etc.\) with ruff [\#1526](https://github.com/Materials-Consortia/optimade-python-tools/pull/1526) ([ml-evs](https://github.com/ml-evs))
+- Add `--silent` option to suppress client output until results [\#1518](https://github.com/Materials-Consortia/optimade-python-tools/pull/1518) ([ml-evs](https://github.com/ml-evs))
+- Update tests and client to properly test async mode [\#1517](https://github.com/Materials-Consortia/optimade-python-tools/pull/1517) ([ml-evs](https://github.com/ml-evs))
+- Refresh docs style and associated tweaks [\#1516](https://github.com/Materials-Consortia/optimade-python-tools/pull/1516) ([ml-evs](https://github.com/ml-evs))
 
 ## [v0.20.3](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.20.3) (2023-01-09)
 

Dockerfile

@@ -8,13 +8,13 @@ ENV PYTHONUNBUFFERED 1
 WORKDIR /app
 
 # Copy repo contents
-COPY setup.py setup.cfg requirements.txt LICENSE MANIFEST.in README.md .docker/run.sh ./
+COPY setup.py pyproject.toml requirements.txt requirements-server.txt LICENSE MANIFEST.in README.md .docker/run.sh ./
 COPY optimade ./optimade
 COPY providers/src/links/v1/providers.json ./optimade/server/data/
 RUN apt-get purge -y --auto-remove \
     && rm -rf /var/lib/apt/lists/* \
     && pip install --no-cache-dir --trusted-host pypi.org --trusted-host files.pythonhosted.org -U pip setuptools wheel flit \
-    && pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org -r requirements.txt \
+    && pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org -r requirements.txt -r requirements-server.txt \
     && pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org .[server]
 
 # Setup server configuration

README.md

@@ -4,14 +4,11 @@
 <img width="100px" align="center" src="https://matsci.org/uploads/default/original/2X/b/bd2f59b3bf14fb046b74538750699d7da4c19ac1.svg">
 </div>
 
-<h1 align="center">
-OPTIMADE Python tools
-</h1>
-
+# <div align="center">OPTIMADE Python tools</div>
 
 <div align="center">
 
-<a href="https://doi.org/10.21105/joss.03458"><img alt="JOSS DOI" src="https://joss.theoj.org/papers/10.21105/joss.03458/status.svg"></a>
+<a href="https://doi.org/10.21105/joss.03458"><img alt="JOSS DOI" src="https://img.shields.io/badge/JOSS-10.21105%2Fjoss.03458-blueviolet"></a>
 </div>
 
 <div align="center">
@@ -30,6 +27,7 @@ OPTIMADE Python tools
   </td>
   <td align="center">
     <a href="https://github.com/Materials-Consortia/optimade-python-tools/actions?query=branch%3Amaster+"><img alt="Build Status" src="https://img.shields.io/github/actions/workflow/status/Materials-Consortia/optimade-python-tools/ci.yml?logo=github"></a><br>
+    <a href="https://optimade.org/optimade-python-tools"><img alt="Docs" src="https://img.shields.io/github/actions/workflow/status/Materials-Consortia/optimade-python-tools/ci_cd_updated_master.yml?label=docs&logo=github"></a><br>
     <a href="https://codecov.io/gh/Materials-Consortia/optimade-python-tools"><img alt="Codecov" src="https://img.shields.io/codecov/c/github/Materials-Consortia/optimade-python-tools?logo=codecov&logoColor=white&token=UJAtmqkZZO"></a><br>
   </td>
   <td align="center">
@@ -49,6 +47,7 @@ This is to enable interoperability among databases that serve crystal structures
 This repository contains a library of tools for implementing and consuming [OPTIMADE APIs](https://www.optimade.org) using Python:
 
 1. [pydantic](https://github.com/pydantic/pydantic) data models for all [OPTIMADE entry types](https://www.optimade.org/optimade-python-tools/latest/all_models/) and endpoint responses, and a [Lark](https://github.com/lark-parser/lark) [EBNF grammar](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) implementation for the OPTIMADE filter language.
+1. Adapters to map OPTIMADE data to and from many commonly used atomistic Python frameworks (e.g., [pymatgen](https://pymatgen.org/), [ASE](https://wiki.fysik.dtu.dk/ase/)) and crystallographic file types (e.g., [CIF](https://www.iucr.org/resources/cif)), using the `optimade.adapters` module.
 1. A configurable reference server implementation that can make use of either MongoDB or Elasticsearch database backends out-of-the-box, and is readily extensible to other backends. Try it out on the [demo site](https://optimade.fly.dev)! The OpenAPI schemas of the server are used to construct the [OPTIMADE schemas](https://schemas.optimade.org/) site.
 1. An [OPTIMADE client](https://www.optimade.org/optimade-python-tools/latest/getting_started/client/) (`optimade-get`) that can query multiple [OPTIMADE providers](https://optimade.org/providers-dashboard) concurrently with a given filter, at the command-line or from Python code.
 1. A fuzzy API validator tool, which may be called from the shell (`optimade-validator`) or used as a GitHub Action from [optimade-validator-action](https://github.com/Materials-Consortia/optimade-validator-action); this validator is used to construct the [providers dashboard](https://optimade.org/providers-dashboard).
@@ -65,8 +64,17 @@ The release history and changelog can be found in [the changelog](CHANGELOG.md).
 
 Detailed installation instructions for different use cases (e.g., using the library or running a server) can be found in [the installation documentation](INSTALL.md).
 
-The latest stable version of this package can be obtained from [PyPI](https://pypi.org/project/optimade) `pip install optimade`.
-The latest development version of this package can be obtained from the master branch of this repository `git clone https://github.com/Materials-Consortia/optimade-python-tools`.
+The latest stable version of this package can be obtained from [PyPI](https://pypi.org/project/optimade):
+
+```shell
+pip install optimade
+```
+
+The latest development version of this package can be obtained from the master branch of this repository:
+
+```shell
+git clone https://github.com/Materials-Consortia/optimade-python-tools
+```
 
 ## Supported OPTIMADE versions
 

docs/css/reference.css

@@ -11,3 +11,7 @@ td code {
 code {
     max-height: 20em;
 }
+
+.md-grid {
+    max-width: 70rem;
+}

docs/getting_started/client.md

@@ -13,9 +13,9 @@ This list outlines the current and planned features for the client:
 - [x] Automatically paginate through the results for each query.
 - [x] Validate filters against the OPTIMADE grammar before they are sent to each database.
 - [x] Count the number of results for a query without downloading them all.
+- [x] Cache the results for queries to disk, and use them in future sessions without making new requests (achieved via callbacks).
 - [ ] Valiate the results against the optimade-python-tools models and export into other supported formats (ASE, pymatgen, CIF, AiiDA).
 - [ ] Enable asynchronous use in cases where there is already a running event loop (e.g., inside a Jupyter notebook).
-- [ ] Cache the results for queries to disk, and use them in future sessions without making new requests.
 - [ ] Support for querying databases indirectly via an [OPTIMADE gateway server](https://github.com/Materials-Consortia/optimade-gateway).
 
 ## Installation
@@ -382,3 +382,33 @@ which, at the timing of writing, yields the results:
   }
 }
 ```
+
+### Callbacks
+
+In Python, the client can also be initialized with a list of callbacks.
+These will be executed after every request and will have access to the JSON response and the request URL.
+
+For example, callbacks be used, to save to a file or write to a database, without having to pull all the results into memory.
+Care should be taken if combining an asynchronous client with callbacks, as the callbacks may depend on the execution order of various asynchronous requests and the callbacks themselves may be blocking.
+
+
+=== "Python"
+    ```python
+    # Write a callback that saves into a MongoDB (fake or otherwise)
+    import mongomock as pymongo
+    from optimade.client import OptimadeClient
+
+    db = pymongo.MongoClient().database.structures
+
+    def write_to_db(url, results):
+        for entry in results["data"]:
+            entry.update(entry.pop("attributes"))
+            entry["immutable_id"] = url
+            db.insert_one(entry)
+
+    client = OptimadeClient(callbacks=[write_to_db], silent=True)
+
+    client.get()
+
+    print(db.find_one())
+    ```

docs/static/default_config.json

@@ -12,7 +12,7 @@
     "base_url": null,
     "implementation": {
         "name": "OPTIMADE Python Tools",
-        "version": "0.20.3",
+        "version": "0.22.1",
         "source_url": "https://github.com/Materials-Consortia/optimade-python-tools",
         "maintainer": {"email": "[email protected]"}
     },

mkdocs.yml

@@ -5,13 +5,34 @@ copyright: Built by the Materials-Consortia
 
 theme:
   name: material
+  font:
+    text: 'Karla'
+    code: 'Share Tech Mono'
+    features:
+      - content.code.copy
   palette:
-    scheme: default
-    primary: black
-    accent: red
+
+    # Palette toggle for light mode
+    - media: "(prefers-color-scheme: light)"
+      scheme: default
+      toggle:
+        icon: material/brightness-7
+        name: Switch to dark mode
+      primary: black
+      accent: red
+
+    # Palette toggle for dark mode
+    - media: "(prefers-color-scheme: dark)"
+      scheme: slate
+      toggle:
+        icon: material/brightness-4
+        name: Switch to light mode
+      primary: black
+      accent: red
+
   icon:
     repo: fontawesome/brands/github
-  logo: images/optimade_logo_180x180.svg
+  logo: https://matsci.org/uploads/default/original/2X/b/bd2f59b3bf14fb046b74538750699d7da4c19ac1.svg
   favicon: images/favicon.png
   language: en