diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..30e663db --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,24 @@ +# This CODEOWNERS file denotes the project leads +# and encodes their responsibilities for code review. + +# Instructions: At a minimum, replace the '@GITHUB_USER_NAME_GOES_HERE' +# here with at least one project lead. + +# Lines starting with '#' are comments. +# Each line is a file pattern followed by one or more owners. +# The format is described: https://github.blog/2017-07-06-introducing-code-owners/ + +# These owners will be the default owners for everything in the repo. +* @myronmarston +* @BrianSigafoos-SQ + +# ----------------------------------------------- +# BELOW THIS LINE ARE TEMPLATES, UNUSED +# ----------------------------------------------- +# Order is important. The last matching pattern has the most precedence. +# So if a pull request only touches javascript files, only these owners +# will be requested to review. +# *.js @octocat @github/js + +# You can also use email addresses if you prefer. +# docs/* docs@example.com diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 00000000..6a2f00e2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,30 @@ +--- +name: 🐛 Bug Report +about: Thank you for taking the time, please report a reproducible bug +title: "[Bug] " +labels: bug +assignees: myronmarston, BrianSigafoos-SQ +--- + +**Describe the bug** +*A clear and concise description of what the bug is.* + +**To Reproduce:** +*Steps to reproduce the behavior:* +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior:** +*A clear and concise description of what you expected to happen.* + +**Supporting Material** +*If applicable, add screenshots, output log and/or other documentation to help explain your problem.* + +**Environment (please complete the following information):** + - OS: [ex: iOS] + - Version + +**Additional context** +Add any other context that you feel is relevant about the problem here. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..ce71f103 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,4 @@ +contact_links: + - name: ❓ Questions and Help 🤔 + url: https://discord.gg/tbd (/add your discord channel if applicable) + about: This issue tracker is not for support questions. Please refer to the community for more help. diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..dcdd3c01 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,68 @@ +# This file is generated by `script/update_ci_yaml` based on input from `config/tested_datastore_versions.yaml`. +# To edit it, make changes to the template at the bottom of `script/update_ci_yaml` and run it. +name: ElasticGraph CI + +on: + push: + branches: + - main + pull_request: + +env: + # It's recommended to run ElasticGraph with this option to get better performance. We want to run + # our CI builds with it to ensure that the option always works. + RUBYOPT: "--enable-frozen-string-literal" + # We use the VCR gem as a local "test accelerator" which caches datastore requests/responses for us. + # But in our CI build we don't want to use it at all, so we disable it here. + NO_VCR: "1" + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + build_part: + - run_each_gem_spec + ruby: + - "3.2" + - "3.3" + datastore: + - "elasticsearch:8.15.1" + - "opensearch:2.16.0" + - "opensearch:2.7.0" + include: + # We have 4 build parts. The "primary" one is `run_each_gem_spec`, and we need that to be run on + # every supported Ruby version and against every supported datastore. It's not necessary to run + # these others against every combination of `ruby` and `datastore` so we just run each with one + # configuration here. + - build_part: "run_misc_checks" + ruby: "3.3" + datastore: "elasticsearch:8.15.1" + - build_part: "run_most_specs_with_vcr" + ruby: "3.3" + datastore: "elasticsearch:8.15.1" + - build_part: "run_specs_file_by_file" + ruby: "3.3" + datastore: "elasticsearch:8.15.1" + + steps: + - uses: actions/checkout@v4 + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + + - uses: actions/setup-node@v3 + with: + node-version: "23.x" + + - uses: KengoTODA/actions-setup-docker-compose@main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Note: the `10` argument on the end is a number of seconds to sleep after booting the datastore. + # We've found that there is a minor race condition where the shards aren't fully ready for the tests + # to hit them if we don't wait a bit after booting. + - run: script/ci_parts/${{ matrix.build_part }} ${{ matrix.datastore }} 10 diff --git a/.github/workflows/publish-site.yaml b/.github/workflows/publish-site.yaml new file mode 100644 index 00000000..cb252db7 --- /dev/null +++ b/.github/workflows/publish-site.yaml @@ -0,0 +1,47 @@ +name: Publish Site + +on: + push: + branches: + - main + +jobs: + publish-docs: + runs-on: ubuntu-latest + + permissions: + contents: write + + concurrency: + # Ensures only one job per workflow and branch runs at a time + group: ${{ github.workflow }}-${{ github.ref }} + + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.3" + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + + - name: Set up Node + - uses: actions/setup-node@v3 + with: + node-version: "23.x" + + - name: Build YARD docs and Jekyll site + run: bundle exec rake site:build + + - name: Deploy to GitHub Pages + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: peaceiris/actions-gh-pages@v4 + with: + # The GitHub Actions runner automatically creates this `GITHUB_TOKEN` secret + github_token: ${{ secrets.GITHUB_TOKEN }} + # The output directory for Jekyll + publish_dir: config/site/_site + # The branch to push to for GitHub Pages + publish_branch: gh-pages + enable_jekyll: true diff --git a/.github/workflows/push_gem.yaml b/.github/workflows/push_gem.yaml new file mode 100644 index 00000000..0924b72f --- /dev/null +++ b/.github/workflows/push_gem.yaml @@ -0,0 +1,96 @@ +# This workflow was generated by https://github.com/rubygems/configure_trusted_publisher +name: Push Gem + +on: + workflow_dispatch: + inputs: + version: + description: "Version to release (e.g. 1.2.0)" + required: true + type: string + dry-run: + description: "Dry Run" + required: false + type: boolean + default: false + +permissions: + contents: read + +jobs: + push: + # Limit who can run run this action to core maintainers. + if: github.repository == 'block/elasticgraph' && contains('["myronmarston", "BrianSigafoos-SQ"]', github.actor) + runs-on: ubuntu-latest + + environment: + name: rubygems.org + url: https://rubygems.org/search?query=elasticgraph + + permissions: + contents: write + id-token: write + pull-requests: write + + steps: + # Set up + - name: Harden Runner + uses: step-security/harden-runner@a4aa98b93cab29d9b1101a6143fb8bce00e2eac4 # v2.7.1 + with: + egress-policy: audit + + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - name: Set up Ruby + uses: ruby/setup-ruby@cacc9f1c0b3f4eb8a16a6bb0ed10897b43b9de49 # v1.176.0 + with: + working-directory: config/release + bundler-cache: true + ruby-version: ruby + + - name: Setup Git Config + # https://github.com/orgs/community/discussions/26560#discussioncomment-3531273 + run: | + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config user.name "github-actions[bot]" + + - name: Target Release Rakefile + run: | + rm Rakefile + cp config/release/Rakefile . + git update-index --skip-worktree Rakefile + echo "BUNDLE_GEMFILE=config/release/Gemfile" >> "$GITHUB_ENV" + + - name: Bump the ElasticGraph version + run: bundle exec rake bump_version[${{ inputs.version }}] + + # Note: we put this after bumping the version because really bumping the version is safe to do in dry-run mode. + - name: Enable Dry Run Mode + if: ${{ inputs.dry-run }} + # As per docs[^1], this is the syntax for setting ENV vars for later steps. + # [^1]: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-an-environment-variable + run: echo "GEM_RELEASE_PRETEND=true" >> "$GITHUB_ENV" + + # Release + - name: Release to rubygems.org + uses: rubygems/release-gem@612653d273a73bdae1df8453e090060bb4db5f31 # v1 + with: + await-release: ${{ ! inputs.dry-run }} + + # Note: this must come after we release the gem because it resets git back to the same SHA we started on + # (before bumping the version), but the RubyGems release depends on the version having been bumped. + - name: Create pull request for the version bump + uses: peter-evans/create-pull-request@v6 + with: + branch: release-v${{ inputs.version }} + title: "Release v${{ inputs.version }}" + body: | + - [ ] Confirm the [push-gem action](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) succeeded before merging + - [ ] Confirm this version bump should be merged into [${{ github.ref_name }}](https://github.com/${{ github.repository }}/tree/${{ github.ref_name }}) or change the base branch + - [ ] Review and edit the [GitHub Draft Release](https://github.com/${{ github.repository }}/releases) (can be done after this PR is merged) + + - name: Create GitHub Release + uses: ncipollo/release-action@v1 + with: + draft: true + generateReleaseNotes: true + tag: v${{ inputs.version }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..8ea1a9d3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,73 @@ +# We support running commands from the repository root or from a gem subdirectory. +# To support that, many/most of our ignores are duplicated at both levels. + +# Ignore bundler and steep artifacts. +/bin/ +/bundle/ +/.bundle +Gemfile.lock +rbs_collection.lock.yaml +/*/bin/ +/*/bundle/ +/*/.bundle +/*/Gemfile.lock +/*/rbs_collection.lock.yaml + +# Allows customization of the bundle for things like pry, debugger, etc. +Gemfile-custom + +# Ignore log and temp files. +/log/* +/tmp/* +/*/log/* +/*/tmp/* + +# Ignore artifacts from building the gems. +/pkg/ +/*/pkg/ + +# Ignore Byebug command history file. +.byebug_history +/*/.byebug.history + +# Ignore local rspec config +.rspec-local +/*/.rspec-local + +# Ignore RBS collection directory--it can be regenerated from `rbs_collection.lockl.yaml`. +.gem_rbs_collection + +# Mac OSX file. +.DS_Store +/*/.DS_Store + +# This is git-ignored because we dynamically gnerate it from test.yaml.template on each test run +# based on which datastore backend is being used. +/config/settings/test.yaml + +# We don't want to commit these artifacts to source control +/elasticgraph-apollo/apollo_tests_implementation/config/schema/artifacts/ + +# Allow specifying a Ruby version locally +.ruby-version + +# Ignore Bundler artifacts for our `config/release/Gemfile` bundle. +/config/release/.bundle +/config/release/vendor/bundle + +# Ignore YARD documentation artifacts +/**/.yardoc/ +config/site/doc/ + +# Ignore Jekyll generated site artifacts +config/site/_site +config/site/src/docs +config/site/src/_data/*_queries.yaml +config/site/.jekyll-metadata +config/site/package-lock.json +config/site/examples/*/schema_artifacts +config/site/examples/*/queries/*/*.variables.yaml + +# Ignore the generated CSS +config/site/src/assets/css/main.css +config/site/src/assets/css/highlight.css diff --git a/.rspec b/.rspec new file mode 100644 index 00000000..5998bb16 --- /dev/null +++ b/.rspec @@ -0,0 +1 @@ +--require ./spec_support/spec_helper diff --git a/.standard.yml b/.standard.yml new file mode 100644 index 00000000..0187cac3 --- /dev/null +++ b/.standard.yml @@ -0,0 +1,20 @@ +ruby_version: 3.2 +format: progress +ignore: + - adr/**/* + - bin/**/* + - bundle/**/* + - .bundle/**/* + - log/**/* + - tmp/**/* + - "*/vendor/**/*" + - "**/*.rb": + # This cop forces syntax that steep does not yet support. (e.g. `def foo(**); bar(**); end`) + # Once steep supports that, we can stop ignoring this one. + - Style/ArgumentsForwarding + # This cop forces syntax that steep does not yet support. (e.g. bare `super` which implicitly forwards args) + # Once steep supports that, we can stop ignoring this one. + - Style/SuperArguments + +extend_config: + - config/linting/custom.yaml diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..88e832ae --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,135 @@ + +# Block Code of Conduct + +Block's mission is Economic Empowerment. This means opening the global economy to everyone. We extend the same principles of inclusion to our developer ecosystem. We are excited to build with you. So we will ensure our community is truly open, transparent and inclusive. Because of the global nature of our project, diversity and inclusivity is paramount to our success. We not only welcome diverse perspectives, we **need** them! + +The code of conduct below reflects the expectations for ourselves and for our community. + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, physical appearance, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful and welcoming of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +The Block Open Source Governance Committee (GC) is responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +The GC has the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all project spaces, and it also applies when an individual is representing the project or its community in public spaces. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event, or any space where the project is listed as part of your profile. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the Block Open Source Governance Committee (GC) at +`TODO Create Google Group` (Issue #1) +All complaints will be reviewed and investigated promptly and fairly. + +The GC is obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +The GC will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from the GC, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media and forums. + +Although this list cannot be exhaustive, we explicitly honor diversity in age, culture, ethnicity, gender identity or expression, language, national origin, political beliefs, profession, race, religion, sexual orientation, socioeconomic status, and technical ability. We will not tolerate discrimination based on any of the protected characteristics above, including participants with disabilities. + +Violating these terms may lead to a temporary or permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..6f0bbc78 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,97 @@ +# Contribution Guide + +There are many ways to be an open source contributor, and we're here to help you on your way! You may: + +* Propose ideas in our discord +* Raise an issue or feature request in our [issue tracker](https://github.com/block/elasticgraph/issues) +* Help another contributor with one of their questions, or a code review +* Suggest improvements to our Getting Started documentation by supplying a Pull Request +* Evangelize our work together in conferences, podcasts, and social media spaces. + +This guide is for you. + +## Development Prerequisites + +| Requirement | Tested Version | Installation Instructions | +|----------------|----------------|---------------------------------------------------------------------------| +| Ruby | 3.2.x, 3.3.x | [ruby-lang.org](https://www.ruby-lang.org/en/documentation/installation/) | +| Docker Engine | 27.x | [docker.com](https://docs.docker.com/engine/install/) | +| Docker Compose | 2.29.x | [docker.com](https://docs.docker.com/compose/install/) | + +### Ruby + +This project is written in Ruby, a dynamic, open source programming language with a focus on simplicity and productivity. + +You may verify your `ruby` installation via the terminal: + +``` +$ ruby -v +ruby 3.3.4 (2024-07-09 revision be1089c8ec) [arm64-darwin23] +``` + +If you do not have Ruby, we recommend installing it using one of the following: + +* [RVM](https://rvm.io/) +* [asdf](https://asdf-vm.com/) +* [rbenv](https://rbenv.org/) +* [ruby-install](https://github.com/postmodern/ruby-install) + +Once you have Ruby installed, install the development dependencies by running `bundle install`. + +### Docker and Docker Compose + +This project uses Docker Engine and Docker Compose to run Elasticsearch and OpenSearch locally. We recommend installing +[Docker Desktop](https://docs.docker.com/desktop/) to get both Docker dependencies. + +--- + +## Communications + +### Issues + +Anyone from the community is welcome (and encouraged!) to raise issues via +[GitHub Issues](https://github.com/block/elasticgraph/issues). + +### Discussions + +Design discussions and proposals take place in our discord. + +We advocate an asynchronous, written debate model - so write up your thoughts and invite the community to join in! + +### Continuous Integration + +Build and Test cycles are run on every commit to every branch on [GitHub Actions](https://github.com/block/elasticgraph/actions). + +## Contribution + +We review contributions to the codebase via GitHub's Pull Request mechanism. We have +the following guidelines to ease your experience and help our leads respond quickly +to your valuable work: + +* Start by proposing a change either in Issues (most appropriate for small + change requests or bug fixes) or in Discussions (most appropriate for design + and architecture considerations, proposing a new feature, or where you'd + like insight and feedback) +* Cultivate consensus around your ideas; the project leads will help you + pre-flight how beneficial the proposal might be to the project. Developing early + buy-in will help others understand what you're looking to do, and give you a + greater chance of your contributions making it into the codebase! No one wants to + see work done in an area that's unlikely to be incorporated into the codebase. +* Fork the repo into your own namespace/remote +* Work in a dedicated feature branch. Atlassian wrote a great + [description of this workflow](https://www.atlassian.com/git/tutorials/comparing-workflows/feature-branch-workflow) +* When you're ready to offer your work to the project, first: +* Squash your commits into a single one (or an appropriate small number of commits), and + rebase atop the upstream `main` branch. This will limit the potential for merge + conflicts during review, and helps keep the audit trail clean. A good writeup for + how this is done is + [here](https://medium.com/@slamflipstrom/a-beginners-guide-to-squashing-commits-with-git-rebase-8185cf6e62ec), and if you're + having trouble - feel free to ask a member or the community for help or leave the commits as-is, and flag that you'd like + rebasing assistance in your PR! We're here to support you. +* Open a PR in the project to bring in the code from your feature branch. +* The maintainers noted in the [CODEOWNERS file](https://github.com/block/elasticgraph/blob/main/.github/CODEOWNERS) + will review your PR and optionally open a discussion about its contents before moving forward. +* Remain responsive to follow-up questions, be open to making requested changes, and... + You're a contributor! +* And remember to respect everyone in our global development community. Guidelines + are established in our [Code of Conduct](https://github.com/block/elasticgraph/blob/main/CODE_OF_CONDUCT.md). diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 00000000..6762a7db --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,65 @@ +# Block Open Source Project Governance + + + +* [Contributors](#contributors) +* [Maintainers](#maintainers) +* [Governance Committee](#governance-committee) + + + +## Contributors + +Anyone may be a contributor to Block open source projects. Contribution may take the form of: + +* Asking and answering questions on the Discord or GitHub Issues +* Filing an issue +* Offering a feature or bug fix via a Pull Request +* Suggesting documentation improvements +* ...and more! + +Anyone with a GitHub account may use the project issue trackers and communications channels. We welcome newcomers, so don't hesitate to say hi! + +## Maintainers + +Maintainers have write access to GitHub repositories and act as project administrators. They approve and merge pull requests, cut releases, and guide collaboration with the community. They have: + +* Commit access to their project's repositories +* Write access to continuous integration (CI) jobs + +Both maintainers and non-maintainers may propose changes to +source code. The mechanism to propose such a change is a GitHub pull request. Maintainers review and merge (_land_) pull requests. + +If a maintainer opposes a proposed change, then the change cannot land. The exception is if the Governance Committee (GC) votes to approve the change despite the opposition. Usually, involving the GC is unnecessary. + +See: + +* [List of maintainers - `MAINTAINERS.md`](./MAINTAINERS.md) +* [Contribution Guide - `CONTRIBUTING.md`](./CONTRIBUTING.md) + +### Maintainer activities + +* Helping users and novice contributors +* Contributing code and documentation changes that improve the project +* Reviewing and commenting on issues and pull requests +* Participation in working groups +* Merging pull requests + +## Governance Committee + +The Block Open Source Governance Committee (GC) has final authority over this project, including: + +* Technical direction +* Project governance and process (including this policy) +* Contribution policy +* GitHub repository hosting +* Conduct guidelines +* Maintaining the list of maintainers + +The current GC members are: + +* Manik Surtani, Head of Open Source Program Office, Block +* Andrew Lee Rubinger, Head of Open Source, TBD +* Nidhi Nahar, Head of Patents and Open Source, Block + +Members are not to be contacted individually. The GC may be reached through `TODO Update this` and is an available resource in mediation or for sensitive cases beyond the scope of project maintainers. It operates as a "Self-appointing council or board" as defined by Red Hat: [Open Source Governance Models](https://www.redhat.com/en/blog/understanding-open-source-governance-models). diff --git a/Gemfile b/Gemfile new file mode 100644 index 00000000..9c342df1 --- /dev/null +++ b/Gemfile @@ -0,0 +1,104 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +source "https://rubygems.org" + +# Since this file gets symlinked both at the repo root and into each Gem directory, we have +# to dynamically detect the repo root, by looking for the `.git` directory. +repo_root = ::Pathname.new(::Dir.pwd).ascend.find { |dir| ::Dir.exist?("#{dir}/.git") }.to_s + +# `tmp` and `log` are git-ignored but many of our build tasks and scripts expect them to exist. +# We create them here since `Gemfile` evaluation happens before anything else. +::FileUtils.mkdir_p("#{repo_root}/log") +::FileUtils.mkdir_p("#{repo_root}/tmp") + +# Identify the gems that live in the ElasticGraph repository. +require "#{repo_root}/script/list_eg_gems" +gems_in_this_repo = ::ElasticGraphGems.list.to_set + +# Here we override the `gem` method to automatically add the ElasticGraph version +# to all ElasticGraph gems. If we don't do this, we can get confusing bundler warnings +# like: +# +# A gemspec development dependency (elasticgraph-schema_definition, = 0.17.1.0) is being overridden by a Gemfile dependency (elasticgraph-schema_definition, >= 0). +# This behaviour may change in the future. Please remove either of them, or make sure they both have the same requirement +# +# This is necessary because our `gemspec` call below registers a `gem` for the gem defined by the gemspec, but it does not include +# a version requirement, and bundler gets confused when other gems have dependencies on the same gem with a version requirement. +# This ensures that we always have the same version requirements for all ElasticGraph gems. +define_singleton_method :gem do |name, *args| + if gems_in_this_repo.include?(name) + args.unshift ::ElasticGraph::VERSION unless args.first.include?(::ElasticGraph::VERSION) + end + + super(name, *args) +end + +# This file is symlinked from the repo root into each gem directory. To detect which case we're in, +# we can compare the the current directory to the repo root. +if repo_root == __dir__ + # When we are at the root, we want to load the gemspecs for each ElasticGraph gem in the repository. + gems_in_this_repo.sort.each do |gem_name| + gemspec path: gem_name + end +else + # Otherwise, we just load the local `.gemspec` file in the current directory. + gemspec + + # After loading the gemspec, we want to explicitly tell bundler where to find each of the ElasticGraph + # gems that live in this repository. Otherwise, it will try to look in system gems or on a remote + # gemserver for them. + # + # Bundler stores all loaded gemspecs in `@gemspecs` so here we get the gemspec that was just loaded + if (loaded_gemspec = @gemspecs.last) + + # This set will keep track of which gems have been registered so far, so we never register an + # ElasticGraph gem more than once. + registered_gems = ::Set.new + + register_gemspec_gems_with_path = lambda do |deps| + deps.each do |dep| + next unless gems_in_this_repo.include?(dep.name) && !registered_gems.include?(dep.name) + + dep_path = "#{repo_root}/#{dep.name}" + gem dep.name, path: dep_path + + # record the fact that this gem has been registered so that we don't try calling `gem` for it again. + registered_gems << dep.name + + # Finally, load the gemspec and recursively apply this process to its runtime dependencies. + # Notably, we avoid using `.dependencies` because we do not want development dependencies to + # be registered as part of this. + runtime_dependencies = ::Bundler.load_gemspec("#{dep_path}/#{dep.name}.gemspec").runtime_dependencies + register_gemspec_gems_with_path.call(runtime_dependencies) + end + end + + # Ensure that the recursive lambda above doesn't try to re-register the loaded gemspec's gem. + registered_gems << loaded_gemspec.name + + # Here we begin the process of registering the ElasticGraph gems we need to include in the current + # bundle. We use `loaded_gemspec.dependencies` to include development and runtime dependencies. + # For the "outer" gem identified by our loaded gemspec, we need the bundle to include both its + # runtime and development dependencies. In contrast, when we recurse, we only look at runtime + # dependencies. We are ok with transitive runtime dependencies being pulled in but we don't want + # transitive development dependencies. + register_gemspec_gems_with_path.call(loaded_gemspec.dependencies) + end +end + +# Documentation generation gems +group :site do + gem "filewatcher", "~> 2.1" + gem "jekyll", "~> 4.3" + gem "yard", "~> 0.9", ">= 0.9.36" + gem "yard-doctest", "~> 0.1", ">= 0.1.17" +end + +custom_gem_file = ::File.join(repo_root, "Gemfile-custom") +eval_gemfile(custom_gem_file) if ::File.exist?(custom_gem_file) diff --git a/Gemfile-custom.sample b/Gemfile-custom.sample new file mode 100644 index 00000000..47de7892 --- /dev/null +++ b/Gemfile-custom.sample @@ -0,0 +1,3 @@ +# Make a copy of this at `Gemfile-custom` and edit as needed to include whatever local development gems you want. +gem "debug" +gem "solargraph" diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 00000000..11aabcab --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,24 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +Part of the distributed code (elasticgraph-rack/lib/elastic_graph/rack/graphiql/index.html) +comes from the GraphiQL project, licensed under the MIT License. Copyright (c) GraphQL Contributors. diff --git a/README.md b/README.md new file mode 100644 index 00000000..c82e2fcb --- /dev/null +++ b/README.md @@ -0,0 +1,249 @@ +# ElasticGraph + +ElasticGraph is a general purpose, near real-time data query and search platform that is scalable and performant, +serves rich interactive queries, and dramatically simplifies the creation of complex reports. The platform combines +the power of indexing and search of Elasticsearch or OpenSearch with the query flexibility of GraphQL language. +Optimized for AWS cloud, it also offers scale and reliability. + +ElasticGraph is a naturally flexible framework with many different possible applications. However, the main motivation we have for +building it is to power various data APIs, UIs and reports. These modern reports require filtering and aggregations across a body of ever +growing data sets. Modern APIs allow us to: + +- Minimize network trips to retrieve your data +- Get exactly what you want in a single query. No over- or under-serving the data. +- Push filtering complex calculations to the backend. + +## Libraries + +ElasticGraph is designed to be modular, with a small core, and many built-in extensions that extend that core +for specific use cases. This minimizes exposure to vulnerabilities, reduces bloat, and makes ongoing upgrades +easier. The libraries that ship with ElasticGraph can be broken down into several categories. + +### Core Libraries (8 gems) + +These libraries form the core backbone of ElasticGraph that is designed to run in a production deployment. Every ElasticGraph deployment will need to use all of these. + +* [elasticgraph](elasticgraph/README.md): ElasticGraph meta-gem that pulls in all the core ElasticGraph gems. +* [elasticgraph-admin](elasticgraph-admin/README.md): ElasticGraph gem that provides datastore administrative tasks, to keep a datastore up-to-date with an ElasticGraph schema. +* [elasticgraph-datastore_core](elasticgraph-datastore_core/README.md): ElasticGraph gem containing the core datastore support types and logic. +* [elasticgraph-graphql](elasticgraph-graphql/README.md): The ElasticGraph GraphQL query engine. +* [elasticgraph-indexer](elasticgraph-indexer/README.md): ElasticGraph gem that provides APIs to robustly index data into a datastore. +* [elasticgraph-json_schema](elasticgraph-json_schema/README.md): ElasticGraph gem that provides JSON Schema validation. +* [elasticgraph-schema_artifacts](elasticgraph-schema_artifacts/README.md): ElasticGraph gem containing code related to generated schema artifacts. +* [elasticgraph-support](elasticgraph-support/README.md): ElasticGraph gem providing support utilities to the other ElasticGraph gems. + +#### Dependency Diagram + +```mermaid +graph LR; + elasticgraph --> elasticgraph-admin & elasticgraph-graphql & elasticgraph-indexer & elasticgraph-local + elasticgraph-admin --> elasticgraph-datastore_core & elasticgraph-indexer & elasticgraph-schema_artifacts & elasticgraph-support & rake + elasticgraph-datastore_core --> elasticgraph-schema_artifacts & elasticgraph-support + elasticgraph-graphql --> elasticgraph-datastore_core & elasticgraph-schema_artifacts & graphql + elasticgraph-indexer --> elasticgraph-datastore_core & elasticgraph-json_schema & elasticgraph-schema_artifacts & elasticgraph-support & hashdiff + elasticgraph-json_schema --> elasticgraph-support & json_schemer + elasticgraph-schema_artifacts --> elasticgraph-support + elasticgraph-support --> logger + style elasticgraph color: DodgerBlue; + style elasticgraph-admin color: DodgerBlue; + style elasticgraph-datastore_core color: DodgerBlue; + style elasticgraph-graphql color: DodgerBlue; + style elasticgraph-indexer color: DodgerBlue; + style elasticgraph-json_schema color: DodgerBlue; + style elasticgraph-schema_artifacts color: DodgerBlue; + style elasticgraph-support color: DodgerBlue; + style elasticgraph-local color: Green; + style rake color: Red; + style graphql color: Red; + style hashdiff color: Red; + style json_schemer color: Red; + style logger color: Red; +click graphql href "https://rubygems.org/gems/graphql" +click hashdiff href "https://rubygems.org/gems/hashdiff" +click json_schemer href "https://rubygems.org/gems/json_schemer" +click logger href "https://rubygems.org/gems/logger" +click rake href "https://rubygems.org/gems/rake" +``` + +### AWS Lambda Integration Libraries (5 gems) + +These libraries wrap the the core ElasticGraph libraries so that they can be deployed using AWS Lambda. + +* [elasticgraph-admin_lambda](elasticgraph-admin_lambda/README.md): ElasticGraph gem that wraps elasticgraph-admin in an AWS Lambda. +* [elasticgraph-graphql_lambda](elasticgraph-graphql_lambda/README.md): ElasticGraph gem that wraps elasticgraph-graphql in an AWS Lambda. +* [elasticgraph-indexer_autoscaler_lambda](elasticgraph-indexer_autoscaler_lambda/README.md): ElasticGraph gem that monitors OpenSearch CPU utilization to autoscale indexer lambda concurrency. +* [elasticgraph-indexer_lambda](elasticgraph-indexer_lambda/README.md): Provides an AWS Lambda interface for an elasticgraph API +* [elasticgraph-lambda_support](elasticgraph-lambda_support/README.md): ElasticGraph gem that supports running ElasticGraph using AWS Lambda. + +#### Dependency Diagram + +```mermaid +graph LR; + elasticgraph-admin_lambda --> rake & elasticgraph-admin & elasticgraph-lambda_support + elasticgraph-graphql_lambda --> elasticgraph-graphql & elasticgraph-lambda_support + elasticgraph-indexer_autoscaler_lambda --> elasticgraph-datastore_core & elasticgraph-lambda_support & aws-sdk-lambda & aws-sdk-sqs & ox + elasticgraph-indexer_lambda --> elasticgraph-indexer & elasticgraph-lambda_support & aws-sdk-s3 & ox + elasticgraph-lambda_support --> elasticgraph-opensearch & faraday_middleware-aws-sigv4 + style elasticgraph-admin_lambda color: DodgerBlue; + style elasticgraph-graphql_lambda color: DodgerBlue; + style elasticgraph-indexer_autoscaler_lambda color: DodgerBlue; + style elasticgraph-indexer_lambda color: DodgerBlue; + style elasticgraph-lambda_support color: DodgerBlue; + style rake color: Red; + style elasticgraph-admin color: Green; + style elasticgraph-graphql color: Green; + style elasticgraph-datastore_core color: Green; + style aws-sdk-lambda color: Red; + style aws-sdk-sqs color: Red; + style ox color: Red; + style elasticgraph-indexer color: Green; + style aws-sdk-s3 color: Red; + style elasticgraph-opensearch color: Green; + style faraday_middleware-aws-sigv4 color: Red; +click aws-sdk-lambda href "https://rubygems.org/gems/aws-sdk-lambda" +click aws-sdk-s3 href "https://rubygems.org/gems/aws-sdk-s3" +click aws-sdk-sqs href "https://rubygems.org/gems/aws-sdk-sqs" +click faraday_middleware-aws-sigv4 href "https://rubygems.org/gems/faraday_middleware-aws-sigv4" +click ox href "https://rubygems.org/gems/ox" +click rake href "https://rubygems.org/gems/rake" +``` + +### Extensions (4 gems) + +These libraries extend ElasticGraph to provide optional but commonly needed functionality. + +* [elasticgraph-apollo](elasticgraph-apollo/README.md): An ElasticGraph extension that implements the Apollo federation spec. +* [elasticgraph-health_check](elasticgraph-health_check/README.md): An ElasticGraph extension that provides a health check for high availability deployments. +* [elasticgraph-query_interceptor](elasticgraph-query_interceptor/README.md): An ElasticGraph extension for intercepting datastore queries. +* [elasticgraph-query_registry](elasticgraph-query_registry/README.md): An ElasticGraph extension that supports safer schema evolution by limiting GraphQL queries based on a registry and validating registered queries against the schema. + +#### Dependency Diagram + +```mermaid +graph LR; + elasticgraph-apollo --> elasticgraph-graphql & elasticgraph-support & graphql & apollo-federation + elasticgraph-health_check --> elasticgraph-datastore_core & elasticgraph-graphql & elasticgraph-support + elasticgraph-query_interceptor --> elasticgraph-graphql & elasticgraph-schema_artifacts + elasticgraph-query_registry --> elasticgraph-graphql & elasticgraph-support & graphql & rake + style elasticgraph-apollo color: DodgerBlue; + style elasticgraph-health_check color: DodgerBlue; + style elasticgraph-query_interceptor color: DodgerBlue; + style elasticgraph-query_registry color: DodgerBlue; + style elasticgraph-graphql color: Green; + style elasticgraph-support color: Green; + style graphql color: Red; + style apollo-federation color: Red; + style elasticgraph-datastore_core color: Green; + style elasticgraph-schema_artifacts color: Green; + style rake color: Red; +click apollo-federation href "https://rubygems.org/gems/apollo-federation" +click graphql href "https://rubygems.org/gems/graphql" +click rake href "https://rubygems.org/gems/rake" +``` + +### Datastore Adapters (2 gems) + +These libraries adapt ElasticGraph to your choice of datastore (Elasticsearch or OpenSearch). + +* [elasticgraph-elasticsearch](elasticgraph-elasticsearch/README.md): Wraps the Elasticsearch client for use by ElasticGraph. +* [elasticgraph-opensearch](elasticgraph-opensearch/README.md): Wraps the OpenSearch client for use by ElasticGraph. + +#### Dependency Diagram + +```mermaid +graph LR; + elasticgraph-elasticsearch --> elasticgraph-support & elasticsearch & faraday & faraday-retry + elasticgraph-opensearch --> elasticgraph-support & faraday & faraday-retry & opensearch-ruby + style elasticgraph-elasticsearch color: DodgerBlue; + style elasticgraph-opensearch color: DodgerBlue; + style elasticgraph-support color: Green; + style elasticsearch color: Red; + style faraday color: Red; + style faraday-retry color: Red; + style opensearch-ruby color: Red; +click elasticsearch href "https://rubygems.org/gems/elasticsearch" +click faraday href "https://rubygems.org/gems/faraday" +click faraday-retry href "https://rubygems.org/gems/faraday-retry" +click opensearch-ruby href "https://rubygems.org/gems/opensearch-ruby" +``` + +### Local Development Libraries (3 gems) + +These libraries are used for local development of ElasticGraph applications, but are not intended to be deployed to production (except for `elasticgraph-rack`). +`elasticgraph-rack` is used to boot ElasticGraph locally but can also be used to run ElasticGraph in any rack-compatible server (including a Rails application). + +* [elasticgraph-local](elasticgraph-local/README.md): Provides support for developing and running ElasticGraph applications locally. +* [elasticgraph-rack](elasticgraph-rack/README.md): ElasticGraph gem for serving an ElasticGraph GraphQL endpoint using rack. +* [elasticgraph-schema_definition](elasticgraph-schema_definition/README.md): ElasticGraph gem that provides the schema definition API and generates schema artifacts. + +#### Dependency Diagram + +```mermaid +graph LR; + elasticgraph-local --> elasticgraph-admin & elasticgraph-graphql & elasticgraph-indexer & elasticgraph-rack & elasticgraph-schema_definition & rackup & rake + elasticgraph-rack --> elasticgraph-graphql & rack + elasticgraph-schema_definition --> elasticgraph-graphql & elasticgraph-indexer & elasticgraph-json_schema & elasticgraph-schema_artifacts & elasticgraph-support & graphql & rake + style elasticgraph-local color: DodgerBlue; + style elasticgraph-rack color: DodgerBlue; + style elasticgraph-schema_definition color: DodgerBlue; + style elasticgraph-admin color: Green; + style elasticgraph-graphql color: Green; + style elasticgraph-indexer color: Green; + style rackup color: Red; + style rake color: Red; + style rack color: Red; + style elasticgraph-json_schema color: Green; + style elasticgraph-schema_artifacts color: Green; + style elasticgraph-support color: Green; + style graphql color: Red; +click graphql href "https://rubygems.org/gems/graphql" +click rack href "https://rubygems.org/gems/rack" +click rackup href "https://rubygems.org/gems/rackup" +click rake href "https://rubygems.org/gems/rake" +``` + + +## Versioning Policy + +ElasticGraph does _not_ strictly follow the [SemVer](https://semver.org/) spec. We followed that early in the project's life +cycle and realized that it obscures some important compatibility information. + +ElasticGraph's versioning policy is designed to communicate compatibility information related to the following stakeholders: + +* **Application maintainers**: engineers that define an ElasticGraph schema, maintain project configuration, and perform upgrades. +* **Data publishers**: systems that publish data into an ElasticGraph application for ingestion by an ElasticGraph indexer. +* **GraphQL clients**: clients of the GraphQL API of an ElasticGraph application. + +We use the following versioning scheme: + +* Version numbers are in a `0.MAJOR.MINOR.PATCH` format. (The `0.` prefix is there in order to reserve `1.0.0` and all later versions + for after ElasticGraph has been open-sourced). +* Increments to the PATCH version indicate that the new release contains no backwards incompatibilities for any stakeholders. + It may contain bug fixes, new features, internal refactorings, and dependency upgrades, among other things. You can expect that + PATCH level upgrades are always safe--just update the version in your bundle, generate new schema artifacts, and you should be done. +* Increments to the MINOR version indicate that the new release contains some backwards incompatibilities that may impact the + **application maintainers** of some ElasticGraph applications. MINOR releases may include renames to configuration settings, + changes to the schema definition API, and new schema definition requirements, among other things. You can expect that MINOR + level upgrades can usually be done in 30 minutes or less (usually in a single commit!), with release notes and clear errors + from ElasticGraph command line tasks providing guidance on how to upgrade. +* Increments to the MAJOR version indicate that the new release contains some backwards incompatibilities that may impact the + **data publishers** or **GraphQL clients** of some ElasticGraph applications. MAJOR releases may include changes to the GraphQL + schema that require careful migration of **GraphQL clients** or changes to how indexing is done that require a dataset to be + re-indexed from scratch (e.g. by having **data publishers** republish their data into an ElasticGraph indexer running the new + version). You can expect that the release notes will include detailed instructions on how to perform a MAJOR version upgrade. + +Deprecation warnings may be included at any of these levels--for example, a PATCH release may contain a deprecation warning +for a breaking change that may impact **application maintainers** in an upcoming MINOR release, and a MINOR release may +contain deprecation warnings for breaking changes that may impact **data publishers** or **GraphQL clients** in an upcoming +MAJOR release. + +Each version level is cumulative over the prior levels. That is, a MINOR release may include PATCH-level changes in addition +to backwards incompatibilities that may impact **application maintainers**. A MAJOR release may include PATCH-level or +MINOR-level changes in addition to backwards incompatibilities that may impact **data publishers** or **GraphQL clients**. + +Note that this policy was first adopted in the `v0.15.1.0` release. All prior releases aimed (with some occasional mistakes!) +to follow SemVer with a `0.MAJOR.MINOR.PATCH` versioning scheme. + +Note that _all_ gems in this repository share the same version number. Every time we cut a release, we increment the version +for _all_ gems and release _all_ gems, even if a gem has had no changes since the last release. This is simpler to work with +than the alternatives. diff --git a/Rakefile b/Rakefile new file mode 100644 index 00000000..89f23cc1 --- /dev/null +++ b/Rakefile @@ -0,0 +1,182 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" +require "elastic_graph/apollo/schema_definition/api_extension" +require "elastic_graph/local/rake_tasks" +require "elastic_graph/schema_definition/rake_tasks" +require "yaml" + +project_root = File.expand_path(__dir__) + +# Load tasks from config/site/Rakefile +load "#{project_root}/config/site/Rakefile" + +test_port = "#{project_root}/config/settings/test.yaml.template" + .then { |f| ::YAML.safe_load_file(f, aliases: true).fetch("datastore").fetch("clusters").fetch("main").fetch("url") } + .then { |url| Integer(url[/localhost:(\d+)$/, 1]) } + +schema_def_output = ::SimpleDelegator.new($stdout) +def schema_def_output.puts(*objects) + # There's an edge case (involving a definition of a `count` field competing with a `count` field that ElasticGraph + # wants to define) that ElasticGraph warns about, and our schema definition intentionally exercises that place. We + # don't want to see the warning in our output here since it's by design, so here we silence it. + return if /WARNING: Since a `\w+\.count` field exists/.match?(objects.first.to_s) + + super +end + +module TrackLastNewInstance + attr_reader :last_new_instance + + def new(...) + super.tap do |instance| + @last_new_instance = instance + end + end + + ElasticGraph::SchemaDefinition::RakeTasks.extend self +end + +configure_local_rake_tasks = ->(tasks) do + tasks.schema_element_name_form = :snake_case + tasks.enforce_json_schema_version = false + tasks.index_document_sizes = true + tasks.env_port_mapping = {test: test_port} + tasks.output = schema_def_output + + tasks.define_fake_data_batch_for(:widgets) do |batch| + require "rspec/core" # the factories file expects RSpec to be loaded, so load it. + + # spec_support is not a full-fledged gem and is not on the load path, so we have to + # add it's lib dir to the load path before we can require things from it. + $LOAD_PATH.unshift ::File.join(__dir__, "spec_support", "lib") + require "elastic_graph/spec_support/factories" + + batch.concat(manufacturers = Array.new(10) { FactoryBot.build(:manufacturer) }) + batch.concat(electrical_parts = Array.new(10) { FactoryBot.build(:electrical_part, manufacturer: manufacturers.sample) }) + batch.concat(mechanical_parts = Array.new(10) { FactoryBot.build(:mechanical_part, manufacturer: manufacturers.sample) }) + batch.concat(components = Array.new(10) { FactoryBot.build(:component, parts: (electrical_parts + mechanical_parts).sample(rand(5))) }) + + components = components.shuffle + + batch.concat(Array.new(10) { FactoryBot.build(:address, manufacturer: manufacturers.sample) }) + batch.concat(Array.new(10) do + # Since we now use `sourced_from` to copy `Widget` fields onto `Component` documents, we need to make + # sure that we don't have conflicting widget <-> component relationships defined. Each component can + # have at most 1 widget, so we use `components.shift` here to ensure that a component isn't re-assigned. + widget_components = Array.new(rand(3)) { components.shift }.compact + FactoryBot.build(:widget, components: widget_components) + end) + + batch.concat(sponsors = Array.new(10) { FactoryBot.build(:sponsor) }) + batch.concat(Array.new(10) { FactoryBot.build(:team, sponsors: sponsors.sample(rand(3))) }) + end +end + +ElasticGraph::Local::RakeTasks.new( + local_config_yaml: "config/settings/development.yaml", + path_to_schema: "config/schema.rb", + &configure_local_rake_tasks +) + +schema_def_rake_tasks = ElasticGraph::SchemaDefinition::RakeTasks.last_new_instance + +namespace :apollo do + ElasticGraph::Local::RakeTasks.new( + local_config_yaml: "config/settings/development_with_apollo.yaml", + path_to_schema: "config/schema.rb" + ) do |tasks| + configure_local_rake_tasks.call(tasks) + tasks.schema_definition_extension_modules = [ElasticGraph::Apollo::SchemaDefinition::APIExtension] + end +end + +# When we update our `update_index_data.painless` script, the process can be pretty annoying. +# After updating the script, you have to: +# +# 1. Run `rake schema_artifacts:dump` to generate the `datastore_scripts.yaml` with the new script and to get the script's new id. +# 2. Copy that new id and set `INDEX_DATA_UPDATE_SCRIPT_ID` to it in `constants.rb`. +# 3. Run `rake schema_artifacts:dump` a second time so that `runtime_metadata.yaml`--which depends on the constant--generates correctly. +# +# Here we hook into the `schema_artifacts:dump` task and automate this process. Each time we dump the artifacts, +# this will do a "pre-flight" of the generation of the datastore scripts so that we can get the new constant value. +# We then update it in `constants.rb` as needed, and then allow `schema_artifacts:dump` to proceed, so that it runs +# with the new constant value. +# standard:disable Rake/Desc -- we are just hooking into an existing task here, not defining a new one. +task "schema_artifacts:check" => "apollo:schema_artifacts:check" +task "schema_artifacts:dump" => [:update_artifact_derived_constants, "apollo:schema_artifacts:dump"] +task :update_artifact_derived_constants do + script_id_pattern = "update_index_data_[0-9a-f]+" + + update_index_data_script_id = schema_def_rake_tasks + .send(:schema_definition_results) + .datastore_scripts.keys + .grep(/\A#{script_id_pattern}\z/) + .first + + constants_path = ::File.join(project_root, "elasticgraph-support/lib/elastic_graph/constants.rb") + constants_contents = ::File.read(constants_path) + + if (old_line_match = /^ *INDEX_DATA_UPDATE_SCRIPT_ID = "(#{script_id_pattern})"$/.match(constants_contents)) + old_script_id = old_line_match.captures.first + if update_index_data_script_id == old_script_id + puts "`INDEX_DATA_UPDATE_SCRIPT_ID` in `constants.rb` is already set correctly; leaving unchanged." + else + constants_contents = constants_contents.gsub(old_script_id, update_index_data_script_id) + ::File.write(constants_path, constants_contents) + # Set the constant to the new value so that it takes effect for `schema_artifacts:dump` when that resumes after this. + ElasticGraph.const_set(:INDEX_DATA_UPDATE_SCRIPT_ID, update_index_data_script_id) + puts "Updated `INDEX_DATA_UPDATE_SCRIPT_ID` in `constants.rb` from #{old_script_id} to #{update_index_data_script_id}." + end + else + puts "Warning: could not locate the `INDEX_DATA_UPDATE_SCRIPT_ID =` in `constants.rb` to update." + end +end +# standard:enable Rake/Desc + +# Here we hook into the `[datastore]:test:boot` tasks and do one extra bit of preparation for our tests. +# Each time we boot the datastore, we want to delete the directory that our `ClusterConfigurationManager` +# stores state files in. The state files are used by it to avoid having to reconfigure the datastore when +# the configuration hasn't changed and the datastore hasn't been restarted. That makes the setup for our +# test suite much faster. However, it's essential that we clear out these files every time we boot the datastore +# for our tests or else the `ClusterConfigurationManager` could mistakenly leave the datastore unconfigured +# after we've rebooted it. +# standard:disable Rake/Desc -- these tasks aren't meant to be individually callable; we're just hooking into other tasks here. +task :boot_prep_for_tests do + require "fileutils" + require "rspec/core" + + original_env = ENV.to_h + + begin + ENV.delete("COVERAGE") # so SimpleCov doesn't get loaded when we load `spec_helper` below. + $LOAD_PATH.unshift ::File.join(project_root, "spec_support/lib") + require_relative "spec_support/spec_helper" + require "elastic_graph/spec_support/cluster_configuration_manager" + ensure + ENV.replace(original_env) + end + + state_file_dir = ::File.join(project_root, ElasticGraph::ClusterConfigurationManager::STATE_FILE_DIR) + ::FileUtils.rm_rf(state_file_dir) +end + +::YAML.load_file("#{project_root}/config/tested_datastore_versions.yaml").each do |variant, versions| + namespace variant do + namespace :test do + %i[boot daemon].each { |command| task command => :boot_prep_for_tests } + versions.each do |version| + namespace version do + %i[boot daemon].each { |command| task command => :boot_prep_for_tests } + end + end + end + end +end +# standard:enable Rake/Desc diff --git a/Steepfile b/Steepfile new file mode 100644 index 00000000..7c628ca7 --- /dev/null +++ b/Steepfile @@ -0,0 +1,79 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "script/list_eg_gems" + +target :elasticgraph_gems do + exclude_dirs = %w[ + elasticgraph-rack + spec_support + ].to_set + + ::ElasticGraphGems.list.each do |dir| + next unless Dir.exist?("#{dir}/sig") && Dir.exist?("#{dir}/lib") + next if exclude_dirs.include?(dir) + + signature "#{dir}/sig" + check "#{dir}/lib" + end + + # elasticgraph-admin: existing files that don't type check yet. + ignore(*%w[ + elasticgraph-admin/lib/elastic_graph/admin/datastore_client_dry_run_decorator.rb + elasticgraph-admin/lib/elastic_graph/admin/rake_tasks.rb + ]) + + # elasticgraph-graphql: existing files that don't type check yet. + ignore(*%w[ + elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query.rb + elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query/document_paginator.rb + elasticgraph-graphql/lib/elastic_graph/graphql/datastore_response/search_response.rb + elasticgraph-graphql/lib/elastic_graph/graphql/datastore_search_router.rb + elasticgraph-graphql/lib/elastic_graph/graphql/monkey_patches/schema_field.rb + elasticgraph-graphql/lib/elastic_graph/graphql/monkey_patches/schema_object.rb + elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/get_record_field_value.rb + elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/graphql_adapter.rb + elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/list_records.rb + elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/nested_relationships.rb + elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/query_adapter.rb + elasticgraph-graphql/lib/elastic_graph/graphql/schema.rb + elasticgraph-graphql/lib/elastic_graph/graphql/schema/field.rb + elasticgraph-graphql/lib/elastic_graph/graphql/schema/relation_join.rb + elasticgraph-graphql/lib/elastic_graph/graphql/schema/type.rb + ]) + + # elasticgraph-indexer: existing files that don't type check yet. + ignore(*%w[ + elasticgraph-indexer/lib/elastic_graph/indexer/spec_support/event_matcher.rb + ]) + + # elasticgraph-schema_artifacts: existing files that don't type check yet. + ignore(*%w[ + elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/schema_element_names.rb + ]) + + # elasticgraph-schema_definition: existing files that don't type check yet. + ignore(*%w[ + elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb + elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_indices.rb + elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb + elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enums_for_indexed_types.rb + elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field.rb + elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/union_type.rb + ]) + + library "logger", "time", "json", "base64", "date", "digest", "pathname", "fileutils", "uri", "forwardable", "shellwords", "tempfile", "did_you_mean", "delegate" + + configure_code_diagnostics(::Steep::Diagnostic::Ruby.all_error) do |config| + # Setting these to :hint for now, as some branches are unreachable by steep + # due to the way `Array#[]` and `Hash#[]` work. + # For more detail: https://github.com/soutaro/steep/wiki/Release-Note-1.5#better-flow-sensitive-typing-analysis + config[::Steep::Diagnostic::Ruby::UnreachableBranch] = :hint + config[::Steep::Diagnostic::Ruby::UnreachableValueBranch] = :hint + end +end diff --git a/config/linting/custom.yaml b/config/linting/custom.yaml new file mode 100644 index 00000000..ba0e0de7 --- /dev/null +++ b/config/linting/custom.yaml @@ -0,0 +1,266 @@ +AllCops: + TargetRubyVersion: 3.2 + DisplayCopNames: true + Exclude: + - bin/**/* + - bundle/**/* + - tmp/**/* + - vendor/**/* + - "*/bin/**/*" + - "*/bundle/**/*" + - "*/tmp/**/*" + - "*/vendor/**/*" + +# Put any custom Rubocop configuration here. Note, however, that standardrb (which wraps rubocop) +# does not allow its lint settings to be overriden--we can only add new cops here, so please confirm +# that any new cops added here are actually effective. + +require: + - rubocop-factory_bot + - rubocop-rake + - rubocop-rspec + - ./custom_cops/require_standard_comment_header + +ElasticGraph/RequireStandardCommentHeader: + Enabled: true + + +FactoryBot/AssociationStyle: + Enabled: false +FactoryBot/AttributeDefinedStatically: + Enabled: true +FactoryBot/ConsistentParenthesesStyle: + Enabled: false +FactoryBot/CreateList: + Enabled: false +FactoryBot/FactoryAssociationWithStrategy: + Enabled: false +FactoryBot/FactoryClassName: + Enabled: false +FactoryBot/FactoryNameStyle: + Enabled: true +FactoryBot/IdSequence: + Enabled: false +FactoryBot/RedundantFactoryOption: + Enabled: true +FactoryBot/SyntaxMethods: + Enabled: false + + +Rake/ClassDefinitionInTask: + Enabled: true +Rake/Desc: + Enabled: true +Rake/DuplicateNamespace: + Enabled: true +Rake/DuplicateTask: + Enabled: true +Rake/MethodDefinitionInTask: + Enabled: true + + +RSpec/AlignLeftLetBrace: + Enabled: false +RSpec/AlignRightLetBrace: + Enabled: false +RSpec/AnyInstance: + Enabled: false +RSpec/AroundBlock: + Enabled: true +RSpec/Be: + Enabled: true +RSpec/BeEmpty: + Enabled: false +RSpec/BeEq: + Enabled: false +RSpec/BeEql: + Enabled: false +RSpec/BeNil: + Enabled: false +RSpec/BeforeAfterAll: + Enabled: false +RSpec/ChangeByZero: + Enabled: true +RSpec/ClassCheck: + Enabled: false +RSpec/ContainExactly: + Enabled: false +RSpec/ContextMethod: + Enabled: false +RSpec/ContextWording: + Enabled: false +RSpec/DescribeClass: + Enabled: false +RSpec/DescribeMethod: + Enabled: false +RSpec/DescribeSymbol: + Enabled: true +RSpec/DescribedClass: + Enabled: false +RSpec/DescribedClassModuleWrapping: + Enabled: false +RSpec/Dialect: + Enabled: false +RSpec/DuplicatedMetadata: + Enabled: true +RSpec/EmptyExampleGroup: + Enabled: true +RSpec/EmptyHook: + Enabled: true +RSpec/EmptyLineAfterExample: + Enabled: false +RSpec/EmptyLineAfterExampleGroup: + Enabled: false +RSpec/EmptyLineAfterFinalLet: + Enabled: false +RSpec/EmptyLineAfterHook: + Enabled: false +RSpec/EmptyLineAfterSubject: + Enabled: false +RSpec/EmptyMetadata: + Enabled: true +RSpec/Eq: + Enabled: false +RSpec/ExampleLength: + Enabled: false +RSpec/ExampleWithoutDescription: + Enabled: true + EnforcedStyle: always_allow +RSpec/ExampleWording: + Enabled: false +RSpec/ExcessiveDocstringSpacing: + Enabled: true +RSpec/ExpectActual: + Enabled: true +RSpec/ExpectChange: + Enabled: false +RSpec/ExpectInHook: + Enabled: false +RSpec/ExpectInLet: + Enabled: false +RSpec/ExpectOutput: + Enabled: false +RSpec/Focus: + Enabled: false +RSpec/HookArgument: + Enabled: false +RSpec/HooksBeforeExamples: + Enabled: false +RSpec/IdenticalEqualityAssertion: + Enabled: true +RSpec/ImplicitBlockExpectation: + Enabled: false +RSpec/ImplicitExpect: + Enabled: false +RSpec/ImplicitSubject: + Enabled: false +RSpec/IndexedLet: + Enabled: false +RSpec/InstanceSpy: + Enabled: true +RSpec/InstanceVariable: + Enabled: false +RSpec/ItBehavesLike: + Enabled: false +RSpec/IteratedExpectation: + Enabled: false +RSpec/LeadingSubject: + Enabled: false +RSpec/LeakyConstantDeclaration: + Enabled: false +RSpec/LetBeforeExamples: + Enabled: false +RSpec/LetSetup: + Enabled: false +RSpec/MatchArray: + Enabled: false +RSpec/MessageChain: + Enabled: false +RSpec/MessageExpectation: + Enabled: false +RSpec/MessageSpies: + Enabled: false +RSpec/MetadataStyle: + Enabled: false +RSpec/MissingExampleGroupArgument: + Enabled: false +RSpec/MultipleDescribes: + Enabled: false +RSpec/MultipleExpectations: + Enabled: false +RSpec/MultipleMemoizedHelpers: + Enabled: false +RSpec/MultipleSubjects: + Enabled: true +RSpec/NamedSubject: + Enabled: false +RSpec/NestedGroups: + Enabled: false +RSpec/NoExpectationExample: + Enabled: false +RSpec/NotToNot: + Enabled: false +RSpec/OverwritingSetup: + Enabled: true +RSpec/Pending: + Enabled: false +RSpec/PendingWithoutReason: + Enabled: false +RSpec/PredicateMatcher: + Enabled: false +RSpec/ReceiveCounts: + Enabled: false +RSpec/ReceiveMessages: + Enabled: false +RSpec/ReceiveNever: + Enabled: false +RSpec/RedundantAround: + Enabled: true +RSpec/RepeatedDescription: + Enabled: false +RSpec/RepeatedExample: + Enabled: false +RSpec/RepeatedExampleGroupBody: + Enabled: false +RSpec/RepeatedExampleGroupDescription: + Enabled: false +RSpec/RepeatedIncludeExample: + Enabled: true +RSpec/ReturnFromStub: + Enabled: false +RSpec/ScatteredLet: + Enabled: false +RSpec/ScatteredSetup: + Enabled: false +RSpec/SharedContext: + Enabled: false +RSpec/SharedExamples: + Enabled: false +RSpec/SingleArgumentMessageChain: + Enabled: false +RSpec/SkipBlockInsideExample: + Enabled: true +RSpec/SortMetadata: + Enabled: false +RSpec/SpecFilePathFormat: + Enabled: false +RSpec/SpecFilePathSuffix: + Enabled: true +RSpec/StubbedMock: + Enabled: false +RSpec/SubjectDeclaration: + Enabled: false +RSpec/SubjectStub: + Enabled: false +RSpec/UnspecifiedException: + Enabled: false +RSpec/VariableDefinition: + Enabled: false +RSpec/VariableName: + Enabled: false +RSpec/VerifiedDoubleReference: + Enabled: false +RSpec/VoidExpect: + Enabled: true +RSpec/Yield: + Enabled: false diff --git a/config/linting/custom_cops/require_standard_comment_header.rb b/config/linting/custom_cops/require_standard_comment_header.rb new file mode 100644 index 00000000..32c63dc4 --- /dev/null +++ b/config/linting/custom_cops/require_standard_comment_header.rb @@ -0,0 +1,75 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "rubocop" + +module ElasticGraph + class RequireStandardCommentHeader < ::RuboCop::Cop::Base + extend ::RuboCop::Cop::AutoCorrector + + # At the top of every source file we want to include: + # - The standard Block license header. + # - The frozen string literal magic comment. + # + # The latter can have significant performance benefits, but standardrb (our linter) doesn't enforce it or + # allow you to enforce it with the standard Rubucop Cop--so we are including it here. + # + # For further discussion, see: + # https://github.com/standardrb/standard/pull/181 + STANDARD_COMMENT_HEADER = <<~EOS + # Copyright 2024 Block, Inc. + # + # Use of this source code is governed by an MIT-style + # license that can be found in the LICENSE file or at + # https://opensource.org/licenses/MIT. + # + # frozen_string_literal: true + + EOS + + STANDARD_COMMENT_HEADER_LINES = STANDARD_COMMENT_HEADER.lines.map(&:chomp) + + def on_new_investigation + # Don't mess with files that start with a shebang line -- that must go first and can't be changed. + return if processed_source.lines.first.start_with?("#!") + + last_leading_comment_line_number = find_last_leading_comment_line_number + leading_comment_lines = processed_source.lines[0...last_leading_comment_line_number - 1] + possible_header_lines = leading_comment_lines.first(STANDARD_COMMENT_HEADER_LINES.size) + + unless possible_header_lines == STANDARD_COMMENT_HEADER_LINES + if possible_header_lines.join("\n").include?("Block, Inc.") && possible_header_lines.join("\n").include?("frozen_string_literal: true") + range = processed_source.buffer.line_range(STANDARD_COMMENT_HEADER_LINES.size - 1) + add_offense(range, message: "Standard header is out of date.") do |corrector| + first_line_to_replace = processed_source.buffer.line_range(1) + last_line_to_replace = processed_source.buffer.line_range(STANDARD_COMMENT_HEADER_LINES.size - 1) + range = first_line_to_replace.join(last_line_to_replace) + + replacement = + if processed_source.buffer.line_range(STANDARD_COMMENT_HEADER_LINES.size).source == "" + STANDARD_COMMENT_HEADER.strip + else + STANDARD_COMMENT_HEADER.strip + "\n" + end + + corrector.replace(range, replacement) + end + else + range = processed_source.buffer.line_range(1) + add_offense(range, message: "Missing standard comment header at top of file.") do |corrector| + corrector.insert_before(range, STANDARD_COMMENT_HEADER) + end + end + end + end + + def find_last_leading_comment_line_number + processed_source.tokens.find { |token| !token.comment? }&.line || processed_source.lines.size + end + end +end diff --git a/config/linting/custom_cops/require_standard_comment_header_spec.rb b/config/linting/custom_cops/require_standard_comment_header_spec.rb new file mode 100644 index 00000000..8803a8b5 --- /dev/null +++ b/config/linting/custom_cops/require_standard_comment_header_spec.rb @@ -0,0 +1,201 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "require_standard_comment_header" +require "rubocop/rspec/support" + +module ElasticGraph + RSpec.describe RequireStandardCommentHeader do + include ::RuboCop::RSpec::ExpectOffense + let(:cop) { RequireStandardCommentHeader.new(config) } + let(:config) { ::RuboCop::Config.new("ElasticGraph/RequireStandardCommentHeader" => {"Enabled" => true}) } + + it "autocorrects a file that does not have the standard header" do + expect_offense(<<~RUBY) + module MyClass + ^^^^^^^^^^^^^^ Missing standard comment header at top of file. + end + RUBY + + expect_correction(<<~RUBY) + # Copyright 2024 Block, Inc. + # + # Use of this source code is governed by an MIT-style + # license that can be found in the LICENSE file or at + # https://opensource.org/licenses/MIT. + # + # frozen_string_literal: true + + module MyClass + end + RUBY + end + + it "does not register an offense when a file has the standard header (and no other leading comments)" do + expect_no_offenses(<<~RUBY) + # Copyright 2024 Block, Inc. + # + # Use of this source code is governed by an MIT-style + # license that can be found in the LICENSE file or at + # https://opensource.org/licenses/MIT. + # + # frozen_string_literal: true + + module MyClass + end + RUBY + end + + it "autocorrects a file when the class documentation has no blank line between it and the standard header" do + expect_offense(<<~RUBY) + # Copyright 2024 Block, Inc. + # + # Use of this source code is governed by an MIT-style + # license that can be found in the LICENSE file or at + # https://opensource.org/licenses/MIT. + # + # frozen_string_literal: true + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Standard header is out of date. + # Some documentation for my class. + module MyClass + end + RUBY + + expect_correction(<<~RUBY) + # Copyright 2024 Block, Inc. + # + # Use of this source code is governed by an MIT-style + # license that can be found in the LICENSE file or at + # https://opensource.org/licenses/MIT. + # + # frozen_string_literal: true + + # Some documentation for my class. + module MyClass + end + RUBY + end + + it "does not register an offense when a file has the standard header and other leading comments with a blank line in between" do + expect_no_offenses(<<~RUBY) + # Copyright 2024 Block, Inc. + # + # Use of this source code is governed by an MIT-style + # license that can be found in the LICENSE file or at + # https://opensource.org/licenses/MIT. + # + # frozen_string_literal: true + + # Some documentation for my class. + module MyClass + end + RUBY + end + + it "autocorrects a file that is only comments but lacks the standard header" do + expect_offense(<<~RUBY) + # This is a placeholder file. + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Missing standard comment header at top of file. + # It has some comments. + RUBY + + expect_correction(<<~RUBY) + # Copyright 2024 Block, Inc. + # + # Use of this source code is governed by an MIT-style + # license that can be found in the LICENSE file or at + # https://opensource.org/licenses/MIT. + # + # frozen_string_literal: true + + # This is a placeholder file. + # It has some comments. + RUBY + end + + it "does not register an offense when a file is only comments and has the standard header" do + expect_no_offenses(<<~RUBY) + # Copyright 2024 Block, Inc. + # + # Use of this source code is governed by an MIT-style + # license that can be found in the LICENSE file or at + # https://opensource.org/licenses/MIT. + # + # frozen_string_literal: true + + # This is a placeholder file. + # It has some comments. + RUBY + end + + it "autocorrects a comments-only file when the non-standard comments have no blank comment line between them and the standard header" do + expect_offense(<<~RUBY) + # Copyright 2024 Block, Inc. + # + # Use of this source code is governed by an MIT-style + # license that can be found in the LICENSE file or at + # https://opensource.org/licenses/MIT. + # + # frozen_string_literal: true + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Standard header is out of date. + # This is a placeholder file. + # It has some comments. + RUBY + + expect_correction(<<~RUBY) + # Copyright 2024 Block, Inc. + # + # Use of this source code is governed by an MIT-style + # license that can be found in the LICENSE file or at + # https://opensource.org/licenses/MIT. + # + # frozen_string_literal: true + + # This is a placeholder file. + # It has some comments. + RUBY + end + + it "autocorrects an out-of-date standard header" do + expect_offense(<<~RUBY) + # Copyright 2023 Block, Inc. + # + # Use of this source code is governed by an MIT-style + # license that can be found in the LICENSE file or at + # https://opensource.org/licenses/MIT. + # + # frozen_string_literal: true + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Standard header is out of date. + + module MyClass + end + RUBY + + expect_correction(<<~RUBY) + # Copyright 2024 Block, Inc. + # + # Use of this source code is governed by an MIT-style + # license that can be found in the LICENSE file or at + # https://opensource.org/licenses/MIT. + # + # frozen_string_literal: true + + module MyClass + end + RUBY + end + + it "leaves a file with a shebang line unchanged even if it lacks the standard header since adding it above the shebang would break the script" do + expect_no_offenses(<<~RUBY) + #!/usr/bin/env ruby + + puts "hello world" + RUBY + end + end +end diff --git a/config/release/Gemfile b/config/release/Gemfile new file mode 100644 index 00000000..0bc7fcc1 --- /dev/null +++ b/config/release/Gemfile @@ -0,0 +1,12 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "gem-release", "~> 2.2" +gem "rake", "~> 13.2" diff --git a/config/release/Rakefile b/config/release/Rakefile new file mode 100644 index 00000000..a388784c --- /dev/null +++ b/config/release/Rakefile @@ -0,0 +1,52 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# Note: this Rakefile is designed to work either when run from within config/release or after being copied to the project root. +# That's why we get the project root from git instead of using `__dir__`. +project_root = `git rev-parse --show-toplevel`.strip + +require "gem/release" +require "#{project_root}/elasticgraph-support/lib/elastic_graph/version" +require "#{project_root}/script/list_eg_gems" + +desc "Bumps the ElasticGraph version to the specified new version number" +task :bump_version, [:version] do |_, args| + version = args.fetch(:version) + ::Dir.chdir(project_root) do + sh "bundle exec gem bump elasticgraph-support --file #{project_root}/elasticgraph-support/lib/elastic_graph/version.rb -v #{version} -m 'Release v#{version}.'" + end +end + +desc "Releases ElasticGraph to rubygems.org and tags the release in git" +task :release do + ::Dir.chdir(project_root) do + # `gem release` deletes the built package file as part of its cleanup: + # https://github.com/svenfuchs/gem-release/blob/v2.2.2/lib/gem/release/cmds/release.rb#L115-L122 + # + # However, that interferes with the rubygems release GitHub action: it uses `rubygems-await` + # which expects the pkg file to be at `pkg/*.gem`. + # + # To deal with this, this module disables cleanup. + mkdir_p "#{project_root}/pkg" + disable_cleanup = Module.new do + def cleanup + end + end + ::Gem::Release::Cmds::Release.prepend disable_cleanup + + ElasticGraphGems.list.each do |gem| + ::Gem::Release::Cmds::Runner.new(:release, [gem], {}).run + + unless ENV["GEM_RELEASE_PRETEND"] # If we're dry-running, the gem file won't be there to move. + mv "#{project_root}/#{gem}/#{gem}-#{ElasticGraph::VERSION}.gem", "pkg" + end + end + + sh "bundle exec gem tag elasticgraph-support --push" + end +end diff --git a/config/schema.rb b/config/schema.rb new file mode 100644 index 00000000..235083cd --- /dev/null +++ b/config/schema.rb @@ -0,0 +1,16 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +ElasticGraph.define_schema do |schema| + schema.json_schema_version 1 +end + +# Note: anytime you add a file to load here, you'll also have to update the list here: +# elasticgraph-graphql/spec/acceptance/elasticgraph_graphql_acceptance_support.rb +load File.join(__dir__, "schema/teams.rb") +load File.join(__dir__, "schema/widgets.rb") diff --git a/config/schema/artifacts/datastore_config.yaml b/config/schema/artifacts/datastore_config.yaml new file mode 100644 index 00000000..6788f952 --- /dev/null +++ b/config/schema/artifacts/datastore_config.yaml @@ -0,0 +1,2046 @@ +# Generated by `rake schema_artifacts:dump`. +# DO NOT EDIT BY HAND. Any edits will be lost the next time the rake task is run. +--- +index_templates: + teams: + index_patterns: + - teams_rollover__* + template: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + league: + type: keyword + country_code: + type: keyword + formed_on: + type: date + format: strict_date + current_name: + type: keyword + past_names: + type: keyword + won_championships_at: + type: date + format: strict_date_time + details: + properties: + uniform_colors: + type: keyword + count: + type: integer + stadium_location: + type: geo_point + forbes_valuations: + type: long + forbes_valuation_moneys_nested: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + forbes_valuation_moneys_object: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + current_players_nested: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + __counts: + properties: + nicknames: + type: integer + affiliations|sponsorships_nested: + type: integer + affiliations|sponsorships_object: + type: integer + affiliations|sponsorships_object|sponsor_id: + type: integer + affiliations|sponsorships_object|annual_total: + type: integer + affiliations|sponsorships_object|annual_total|currency: + type: integer + affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + seasons_nested: + type: integer + seasons_object: + type: integer + seasons_object|year: + type: integer + seasons_object|games_played: + type: integer + seasons_object|awards: + type: integer + type: nested + current_players_object: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + type: object + seasons_nested: + properties: + the_record: + properties: + win_count: + type: integer + loss_count: + type: integer + last_win_date: + type: date + format: strict_date + first_win_on: + type: date + format: strict_date + year: + type: integer + notes: + type: keyword + count: + type: integer + started_at: + type: date + format: strict_date_time + won_games_at: + type: date + format: strict_date_time + players_nested: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + __counts: + properties: + nicknames: + type: integer + affiliations|sponsorships_nested: + type: integer + affiliations|sponsorships_object: + type: integer + affiliations|sponsorships_object|sponsor_id: + type: integer + affiliations|sponsorships_object|annual_total: + type: integer + affiliations|sponsorships_object|annual_total|currency: + type: integer + affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + seasons_nested: + type: integer + seasons_object: + type: integer + seasons_object|year: + type: integer + seasons_object|games_played: + type: integer + seasons_object|awards: + type: integer + type: nested + players_object: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + type: object + __counts: + properties: + notes: + type: integer + won_games_at: + type: integer + players_nested: + type: integer + players_object: + type: integer + players_object|name: + type: integer + players_object|nicknames: + type: integer + players_object|affiliations: + type: integer + players_object|affiliations|sponsorships_nested: + type: integer + players_object|affiliations|sponsorships_object: + type: integer + players_object|affiliations|sponsorships_object|sponsor_id: + type: integer + players_object|affiliations|sponsorships_object|annual_total: + type: integer + players_object|affiliations|sponsorships_object|annual_total|currency: + type: integer + players_object|affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + players_object|seasons_nested: + type: integer + players_object|seasons_object: + type: integer + players_object|seasons_object|year: + type: integer + players_object|seasons_object|games_played: + type: integer + players_object|seasons_object|awards: + type: integer + type: nested + seasons_object: + properties: + the_record: + properties: + win_count: + type: integer + loss_count: + type: integer + last_win_date: + type: date + format: strict_date + first_win_on: + type: date + format: strict_date + year: + type: integer + notes: + type: keyword + count: + type: integer + started_at: + type: date + format: strict_date_time + won_games_at: + type: date + format: strict_date_time + players_nested: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + __counts: + properties: + nicknames: + type: integer + affiliations|sponsorships_nested: + type: integer + affiliations|sponsorships_object: + type: integer + affiliations|sponsorships_object|sponsor_id: + type: integer + affiliations|sponsorships_object|annual_total: + type: integer + affiliations|sponsorships_object|annual_total|currency: + type: integer + affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + seasons_nested: + type: integer + seasons_object: + type: integer + seasons_object|year: + type: integer + seasons_object|games_played: + type: integer + seasons_object|awards: + type: integer + type: nested + players_object: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + type: object + type: object + the_nested_fields: + properties: + forbes_valuation_moneys: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + current_players: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + __counts: + properties: + nicknames: + type: integer + affiliations|sponsorships_nested: + type: integer + affiliations|sponsorships_object: + type: integer + affiliations|sponsorships_object|sponsor_id: + type: integer + affiliations|sponsorships_object|annual_total: + type: integer + affiliations|sponsorships_object|annual_total|currency: + type: integer + affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + seasons_nested: + type: integer + seasons_object: + type: integer + seasons_object|year: + type: integer + seasons_object|games_played: + type: integer + seasons_object|awards: + type: integer + type: nested + the_seasons: + properties: + the_record: + properties: + win_count: + type: integer + loss_count: + type: integer + last_win_date: + type: date + format: strict_date + first_win_on: + type: date + format: strict_date + year: + type: integer + notes: + type: keyword + count: + type: integer + started_at: + type: date + format: strict_date_time + won_games_at: + type: date + format: strict_date_time + players_nested: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + __counts: + properties: + nicknames: + type: integer + affiliations|sponsorships_nested: + type: integer + affiliations|sponsorships_object: + type: integer + affiliations|sponsorships_object|sponsor_id: + type: integer + affiliations|sponsorships_object|annual_total: + type: integer + affiliations|sponsorships_object|annual_total|currency: + type: integer + affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + seasons_nested: + type: integer + seasons_object: + type: integer + seasons_object|year: + type: integer + seasons_object|games_played: + type: integer + seasons_object|awards: + type: integer + type: nested + players_object: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + type: object + __counts: + properties: + notes: + type: integer + won_games_at: + type: integer + players_nested: + type: integer + players_object: + type: integer + players_object|name: + type: integer + players_object|nicknames: + type: integer + players_object|affiliations: + type: integer + players_object|affiliations|sponsorships_nested: + type: integer + players_object|affiliations|sponsorships_object: + type: integer + players_object|affiliations|sponsorships_object|sponsor_id: + type: integer + players_object|affiliations|sponsorships_object|annual_total: + type: integer + players_object|affiliations|sponsorships_object|annual_total|currency: + type: integer + players_object|affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + players_object|seasons_nested: + type: integer + players_object|seasons_object: + type: integer + players_object|seasons_object|year: + type: integer + players_object|seasons_object|games_played: + type: integer + players_object|seasons_object|awards: + type: integer + type: nested + nested_fields2: + properties: + forbes_valuation_moneys: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + current_players: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + __counts: + properties: + nicknames: + type: integer + affiliations|sponsorships_nested: + type: integer + affiliations|sponsorships_object: + type: integer + affiliations|sponsorships_object|sponsor_id: + type: integer + affiliations|sponsorships_object|annual_total: + type: integer + affiliations|sponsorships_object|annual_total|currency: + type: integer + affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + seasons_nested: + type: integer + seasons_object: + type: integer + seasons_object|year: + type: integer + seasons_object|games_played: + type: integer + seasons_object|awards: + type: integer + type: nested + the_seasons: + properties: + the_record: + properties: + win_count: + type: integer + loss_count: + type: integer + last_win_date: + type: date + format: strict_date + first_win_on: + type: date + format: strict_date + year: + type: integer + notes: + type: keyword + count: + type: integer + started_at: + type: date + format: strict_date_time + won_games_at: + type: date + format: strict_date_time + players_nested: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + __counts: + properties: + nicknames: + type: integer + affiliations|sponsorships_nested: + type: integer + affiliations|sponsorships_object: + type: integer + affiliations|sponsorships_object|sponsor_id: + type: integer + affiliations|sponsorships_object|annual_total: + type: integer + affiliations|sponsorships_object|annual_total|currency: + type: integer + affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + seasons_nested: + type: integer + seasons_object: + type: integer + seasons_object|year: + type: integer + seasons_object|games_played: + type: integer + seasons_object|awards: + type: integer + type: nested + players_object: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + type: object + __counts: + properties: + notes: + type: integer + won_games_at: + type: integer + players_nested: + type: integer + players_object: + type: integer + players_object|name: + type: integer + players_object|nicknames: + type: integer + players_object|affiliations: + type: integer + players_object|affiliations|sponsorships_nested: + type: integer + players_object|affiliations|sponsorships_object: + type: integer + players_object|affiliations|sponsorships_object|sponsor_id: + type: integer + players_object|affiliations|sponsorships_object|annual_total: + type: integer + players_object|affiliations|sponsorships_object|annual_total|currency: + type: integer + players_object|affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + players_object|seasons_nested: + type: integer + players_object|seasons_object: + type: integer + players_object|seasons_object|year: + type: integer + players_object|seasons_object|games_played: + type: integer + players_object|seasons_object|awards: + type: integer + type: nested + __counts: + properties: + past_names: + type: integer + won_championships_at: + type: integer + details|uniform_colors: + type: integer + forbes_valuations: + type: integer + forbes_valuation_moneys_nested: + type: integer + forbes_valuation_moneys_object: + type: integer + forbes_valuation_moneys_object|currency: + type: integer + forbes_valuation_moneys_object|amount_cents: + type: integer + current_players_nested: + type: integer + current_players_object: + type: integer + current_players_object|name: + type: integer + current_players_object|nicknames: + type: integer + current_players_object|affiliations: + type: integer + current_players_object|affiliations|sponsorships_nested: + type: integer + current_players_object|affiliations|sponsorships_object: + type: integer + current_players_object|affiliations|sponsorships_object|sponsor_id: + type: integer + current_players_object|affiliations|sponsorships_object|annual_total: + type: integer + current_players_object|affiliations|sponsorships_object|annual_total|currency: + type: integer + current_players_object|affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + current_players_object|seasons_nested: + type: integer + current_players_object|seasons_object: + type: integer + current_players_object|seasons_object|year: + type: integer + current_players_object|seasons_object|games_played: + type: integer + current_players_object|seasons_object|awards: + type: integer + seasons_nested: + type: integer + seasons_object: + type: integer + seasons_object|the_record: + type: integer + seasons_object|the_record|win_count: + type: integer + seasons_object|the_record|loss_count: + type: integer + seasons_object|the_record|last_win_date: + type: integer + seasons_object|the_record|first_win_on: + type: integer + seasons_object|year: + type: integer + seasons_object|notes: + type: integer + seasons_object|count: + type: integer + seasons_object|started_at: + type: integer + seasons_object|won_games_at: + type: integer + seasons_object|players_nested: + type: integer + seasons_object|players_object: + type: integer + seasons_object|players_object|name: + type: integer + seasons_object|players_object|nicknames: + type: integer + seasons_object|players_object|affiliations: + type: integer + seasons_object|players_object|affiliations|sponsorships_nested: + type: integer + seasons_object|players_object|affiliations|sponsorships_object: + type: integer + seasons_object|players_object|affiliations|sponsorships_object|sponsor_id: + type: integer + seasons_object|players_object|affiliations|sponsorships_object|annual_total: + type: integer + seasons_object|players_object|affiliations|sponsorships_object|annual_total|currency: + type: integer + seasons_object|players_object|affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + seasons_object|players_object|seasons_nested: + type: integer + seasons_object|players_object|seasons_object: + type: integer + seasons_object|players_object|seasons_object|year: + type: integer + seasons_object|players_object|seasons_object|games_played: + type: integer + seasons_object|players_object|seasons_object|awards: + type: integer + the_nested_fields|forbes_valuation_moneys: + type: integer + the_nested_fields|current_players: + type: integer + the_nested_fields|the_seasons: + type: integer + nested_fields2|forbes_valuation_moneys: + type: integer + nested_fields2|current_players: + type: integer + nested_fields2|the_seasons: + type: integer + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _routing: + required: true + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 + widget_currencies: + index_patterns: + - widget_currencies_rollover__* + template: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + name: + type: keyword + introduced_on: + type: date + format: strict_date + primary_continent: + type: keyword + details: + properties: + unit: + type: keyword + symbol: + type: keyword + widget_names2: + type: keyword + widget_tags: + type: keyword + widget_fee_currencies: + type: keyword + widget_options: + properties: + sizes: + type: keyword + colors: + type: keyword + nested_fields: + properties: + max_widget_cost: + type: integer + oldest_widget_created_at: + type: date + format: strict_date_time + __counts: + properties: + widget_names2: + type: integer + widget_tags: + type: integer + widget_fee_currencies: + type: integer + widget_options|sizes: + type: integer + widget_options|colors: + type: integer + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _routing: + required: true + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 + widgets: + index_patterns: + - widgets_rollover__* + template: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + workspace_id2: + type: keyword + amount_cents: + type: integer + cost: + properties: + currency: + type: keyword + amount_cents: + type: integer + cost_currency_unit: + type: keyword + cost_currency_name: + type: keyword + cost_currency_symbol: + type: keyword + cost_currency_primary_continent: + type: keyword + cost_currency_introduced_on: + type: date + format: strict_date + name: + type: keyword + name_text: + type: text + created_at: + type: date + format: strict_date_time + created_at_time_of_day: + type: date + format: HH:mm:ss||HH:mm:ss.S||HH:mm:ss.SS||HH:mm:ss.SSS + created_on: + type: date + format: strict_date + release_timestamps: + type: date + format: strict_date_time + release_dates: + type: date + format: strict_date + component_ids: + type: keyword + options: + properties: + size: + type: keyword + the_sighs: + type: keyword + color: + type: keyword + the_opts: + properties: + size: + type: keyword + the_sighs: + type: keyword + color: + type: keyword + inventor: + properties: + name: + type: keyword + nationality: + type: keyword + stock_ticker: + type: keyword + __typename: + type: keyword + named_inventor: + properties: + name: + type: keyword + nationality: + type: keyword + stock_ticker: + type: keyword + __typename: + type: keyword + weight_in_ng_str: + type: long + weight_in_ng: + type: long + tags: + type: keyword + amounts: + type: integer + index: false + fees: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + metadata: + type: keyword + workspace_name: + type: keyword + __counts: + properties: + release_timestamps: + type: integer + release_dates: + type: integer + component_ids: + type: integer + tags: + type: integer + amounts: + type: integer + fees: + type: integer + fees|currency: + type: integer + fees|amount_cents: + type: integer + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _routing: + required: true + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 3 +indices: + addresses: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + full_address: + type: keyword + timestamps: + properties: + created_at: + type: date + format: strict_date_time + geo_location: + type: geo_point + shapes: + type: geo_shape + manufacturer_id: + type: keyword + __counts: + properties: + shapes: + type: integer + shapes|type: + type: integer + shapes|coordinates: + type: integer + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 + components: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + name: + type: keyword + created_at: + type: date + format: strict_date_time + position: + properties: + x: + type: double + "y": + type: double + tags: + type: keyword + widget_name: + type: keyword + widget_tags: + type: keyword + widget_workspace_id3: + type: keyword + widget_size: + type: keyword + widget_cost: + properties: + currency: + type: keyword + amount_cents: + type: integer + part_ids: + type: keyword + __counts: + properties: + tags: + type: integer + widget_tags: + type: integer + part_ids: + type: integer + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 + electrical_parts: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + name: + type: keyword + created_at: + type: date + format: strict_date_time + voltage: + type: integer + manufacturer_id: + type: keyword + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 + manufacturers: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + name: + type: keyword + created_at: + type: date + format: strict_date_time + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 + mechanical_parts: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + name: + type: keyword + created_at: + type: date + format: strict_date_time + material: + type: keyword + manufacturer_id: + type: keyword + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 + sponsors: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + name: + type: keyword + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 + widget_workspaces: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + name: + type: keyword + widget: + properties: + id: + type: keyword + created_at: + type: date + format: strict_date_time + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 +scripts: + update_WidgetCurrency_from_Widget_0f26b3e9ea093af29e5cef02a25e75ca: + context: update + script: + lang: painless + source: |- + // Idempotently inserts the given value in the `sortedList`, returning `true` if the list was updated. + boolean appendOnlySet_idempotentlyInsertValue(def value, List sortedList) { + // As per https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/Collections.html#binarySearch(java.util.List,java.lang.Object): + // + // > Returns the index of the search key, if it is contained in the list; otherwise, (-(insertion point) - 1). + // > The insertion point is defined as the point at which the key would be inserted into the list: the index + // > of the first element greater than the key, or list.size() if all elements in the list are less than the + // > specified key. Note that this guarantees that the return value will be >= 0 if and only if the key is found. + int binarySearchResult = Collections.binarySearch(sortedList, value); + + if (binarySearchResult < 0) { + sortedList.add(-binarySearchResult - 1, value); + return true; + } else { + return false; + } + } + + // Wrapper around `idempotentlyInsertValue` that handles a list of values. + // Returns `true` if the list field was updated. + boolean appendOnlySet_idempotentlyInsertValues(List values, List sortedList) { + boolean listUpdated = false; + + for (def value : values) { + listUpdated = appendOnlySet_idempotentlyInsertValue(value, sortedList) || listUpdated; + } + + return listUpdated; + } + + boolean immutableValue_idempotentlyUpdateValue(List scriptErrors, List values, def parentObject, String fullPath, String fieldName, boolean nullable, boolean canChangeFromNull) { + boolean fieldAlreadySet = parentObject.containsKey(fieldName); + + // `values` is always passed to us as a `List` (the indexer normalizes to a list, wrapping single + // values in a list as needed) but we only ever expect at most 1 element. + def newValueCandidate = values.isEmpty() ? null : values[0]; + + if (fieldAlreadySet) { + def currentValue = parentObject[fieldName]; + + // Usually we do not allow `immutable_value` fields to ever change values. However, we make + // a special case for `null`, but only when `can_change_from_null: true` has been configured. + // This can be important when deriving a field that has not always existed on the source events. + // On early events, the value may be `null`, and, when this is enabled, we do not want that to + // interfere with our ability to set the value to the correct non-null value based on a different + // event which has a value for the source field. + if (canChangeFromNull) { + if (currentValue == null) { + parentObject[fieldName] = newValueCandidate; + return true; + } + + // When `can_change_from_null: true` is enabled we also need to ignore NEW `null` values that we + // see _after_ a non-null value. This is necessary because an ElasticGraph invariant is that events + // can be processed in any order. So we might process an old event (predating the existence of the + // source field) after we've already set the field to a non-null value. We must always "converge" + // on the same indexed state regardless, of the order events are seen, so here we just ignore it. + if (newValueCandidate == null) { + return false; + } + } + + // Otherwise, if the values differ, it means we are attempting to mutate the immutable value field, which we cannot allow. + if (currentValue != newValueCandidate) { + if (currentValue == null) { + scriptErrors.add("Field `" + fullPath + "` cannot be changed (" + currentValue + " => " + newValueCandidate + "). Set `can_change_from_null: true` on the `immutable_value` definition to allow this."); + } else { + scriptErrors.add("Field `" + fullPath + "` cannot be changed (" + currentValue + " => " + newValueCandidate + ")."); + } + } + + return false; + } + + if (newValueCandidate == null && !nullable) { + scriptErrors.add("Field `" + fullPath + "` cannot be set to `null`, but the source event contains no value for it. Remove `nullable: false` from the `immutable_value` definition to allow this."); + return false; + } + + parentObject[fieldName] = newValueCandidate; + return true; + } + + boolean maxValue_idempotentlyUpdateValue(List values, def parentObject, String fieldName) { + def currentFieldValue = parentObject[fieldName]; + def maxNewValue = values.isEmpty() ? null : Collections.max(values); + + if (currentFieldValue == null || (maxNewValue != null && maxNewValue.compareTo(currentFieldValue) > 0)) { + parentObject[fieldName] = maxNewValue; + return true; + } + + return false; + } + + boolean minValue_idempotentlyUpdateValue(List values, def parentObject, String fieldName) { + def currentFieldValue = parentObject[fieldName]; + def minNewValue = values.isEmpty() ? null : Collections.min(values); + + if (currentFieldValue == null || (minNewValue != null && minNewValue.compareTo(currentFieldValue) < 0)) { + parentObject[fieldName] = minNewValue; + return true; + } + + return false; + } + + Map data = params.data; + // A variable to accumulate script errors so that we can surface _all_ issues and not just the first. + List scriptErrors = new ArrayList(); + if (ctx._source.details == null) { + ctx._source.details = [:]; + } + if (ctx._source.nested_fields == null) { + ctx._source.nested_fields = [:]; + } + if (ctx._source.widget_fee_currencies == null) { + ctx._source.widget_fee_currencies = []; + } + if (ctx._source.widget_names2 == null) { + ctx._source.widget_names2 = []; + } + if (ctx._source.widget_options == null) { + ctx._source.widget_options = [:]; + } + if (ctx._source.widget_options.colors == null) { + ctx._source.widget_options.colors = []; + } + if (ctx._source.widget_options.sizes == null) { + ctx._source.widget_options.sizes = []; + } + if (ctx._source.widget_tags == null) { + ctx._source.widget_tags = []; + } + + boolean details__symbol_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_symbol"], ctx._source.details, "details.symbol", "symbol", true, true); + boolean details__unit_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_unit"], ctx._source.details, "details.unit", "unit", false, false); + boolean introduced_on_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_introduced_on"], ctx._source, "introduced_on", "introduced_on", true, false); + boolean name_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_name"], ctx._source, "name", "name", true, false); + boolean nested_fields__max_widget_cost_was_noop = !maxValue_idempotentlyUpdateValue(data["cost.amount_cents"], ctx._source.nested_fields, "max_widget_cost"); + boolean oldest_widget_created_at_was_noop = !minValue_idempotentlyUpdateValue(data["created_at"], ctx._source, "oldest_widget_created_at"); + boolean primary_continent_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_primary_continent"], ctx._source, "primary_continent", "primary_continent", true, false); + boolean widget_fee_currencies_was_noop = !appendOnlySet_idempotentlyInsertValues(data["fees.currency"], ctx._source.widget_fee_currencies); + boolean widget_names2_was_noop = !appendOnlySet_idempotentlyInsertValues(data["name"], ctx._source.widget_names2); + boolean widget_options__colors_was_noop = !appendOnlySet_idempotentlyInsertValues(data["options.color"], ctx._source.widget_options.colors); + boolean widget_options__sizes_was_noop = !appendOnlySet_idempotentlyInsertValues(data["options.size"], ctx._source.widget_options.sizes); + boolean widget_tags_was_noop = !appendOnlySet_idempotentlyInsertValues(data["tags"], ctx._source.widget_tags); + + if (!scriptErrors.isEmpty()) { + throw new IllegalArgumentException("Derived index update failed due to bad input data: " + scriptErrors.join(" ")); + } + + // For records with no new values to index, only skip the update if the document itself doesn't already exist. + // Otherwise create an (empty) document to reflect the fact that the id has been seen. + if (ctx._source.id != null && details__symbol_was_noop && details__unit_was_noop && introduced_on_was_noop && name_was_noop && nested_fields__max_widget_cost_was_noop && oldest_widget_created_at_was_noop && primary_continent_was_noop && widget_fee_currencies_was_noop && widget_names2_was_noop && widget_options__colors_was_noop && widget_options__sizes_was_noop && widget_tags_was_noop) { + ctx.op = 'none'; + } else { + // Here we set `_source.id` because if we don't, it'll never be set, making these docs subtly + // different from docs indexed the normal way. + // + // Note also that we MUST use `params.id` instead of `ctx._id`. The latter works on an update + // of an existing document, but is unavailable when we are inserting the document for the first time. + ctx._source.id = params.id; + } + field_as_day_of_week_f2b5c7d9e8f75bf2457b52412bfb6537: + context: field + script: + lang: painless + source: |- + // Check if required params are missing + if (params.offset_ms == null) { + throw new IllegalArgumentException("Missing required parameter: offset_ms"); + } + if (params.time_zone == null) { + throw new IllegalArgumentException("Missing required parameter: time_zone"); + } + + // Set variables used in the loop + ZoneId zoneId = ZoneId.of(params.time_zone); + List results = new ArrayList(); + + for (ZonedDateTime timestamp : doc[params.field]) { + // Convert the timestamp to the specified time zone + ZonedDateTime zonedTimestamp = timestamp.withZoneSameInstant(zoneId); + + // Adjust the timestamp based on the offset_ms parameter + ZonedDateTime adjustedTimestamp = zonedTimestamp.plus(params.offset_ms, ChronoUnit.MILLIS); + + // Format and add the result to the list + results.add(adjustedTimestamp.getDayOfWeek().name()); + } + + return results; + field_as_time_of_day_ed82aba44fc66bff5635bec4305c1c66: + context: field + script: + lang: painless + source: |- + // Check if required params are missing + if (params.offset_ms == null) { + throw new IllegalArgumentException("Missing required parameter: offset_ms"); + } + if (params.time_zone == null) { + throw new IllegalArgumentException("Missing required parameter: time_zone"); + } + if (params.interval == null) { + throw new IllegalArgumentException("Missing required parameter: interval"); + } + + // Set variables used in the loop + ZoneId zoneId = ZoneId.of(params.time_zone); + ChronoUnit intervalUnit; + if (params.interval == "hour") { + intervalUnit = ChronoUnit.HOURS; + } else if (params.interval == "minute") { + intervalUnit = ChronoUnit.MINUTES; + } else if (params.interval == "second") { + intervalUnit = ChronoUnit.SECONDS; + } else { + throw new IllegalArgumentException("Invalid interval value: " + params.interval); + } + DateTimeFormatter formatter = DateTimeFormatter.ISO_LOCAL_TIME; + List results = new ArrayList(); + + for (ZonedDateTime timestamp : doc[params.field]) { + // Convert the timestamp to the specified time zone + ZonedDateTime zonedTimestamp = timestamp.withZoneSameInstant(zoneId); + + // Adjust the timestamp based on the offset_ms parameter + ZonedDateTime adjustedTimestamp = zonedTimestamp.plus(params.offset_ms, ChronoUnit.MILLIS); + + // Truncate the timestamp to the specified interval + adjustedTimestamp = adjustedTimestamp.truncatedTo(intervalUnit); + + // Format and add the result to the list + results.add(adjustedTimestamp.format(formatter)); + } + + return results; + filter_by_time_of_day_ea12d0561b24961789ab68ed38435612: + context: filter + script: + lang: painless + source: |- + ZoneId zoneId = ZoneId.of(params.time_zone); + + for (ZonedDateTime timestamp : doc[params.field]) { + long docValue = timestamp + .withZoneSameInstant(zoneId) + .toLocalTime() + .toNanoOfDay(); + + // Perform comparisons based on whichever params are set. + // ElasticGraph takes care of passing us param values as nano-of-day so that we + // can directly and efficiently compare against `docValue`. + if ((params.gte == null || docValue >= params.gte) && + (params.gt == null || docValue > params.gt) && + (params.lte == null || docValue <= params.lte) && + (params.lt == null || docValue < params.lt) && + (params.equal_to_any_of == null || params.equal_to_any_of.contains(docValue))) { + return true; + } + } + + // No timestamp values matched the params, so return `false`. + return false; + update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413: + context: update + script: + lang: painless + source: |- + Map source = ctx._source; + String sourceId = params.sourceId; + String relationship = params.relationship; + + // Numbers in JSON appear to be parsed as doubles, but we want the version stored as a long, so we need to cast it here. + long eventVersion = (long) params.version; + + if (source.__sources == null) { + source.__sources = []; + } + + if (source.__versions == null) { + source.__versions = [:]; + } + + if (source.__versions[relationship] == null) { + source.__versions[relationship] = [:]; + } + + Map relationshipVersionsMap = source.__versions.get(relationship); + List previousSourceIdsForRelationship = relationshipVersionsMap.keySet().stream().filter(id -> id != sourceId).collect(Collectors.toList()); + + if (previousSourceIdsForRelationship.size() > 0) { + String previousIdDescription = previousSourceIdsForRelationship.size() == 1 ? previousSourceIdsForRelationship.get(0) : previousSourceIdsForRelationship.toString(); + throw new IllegalArgumentException( + "Cannot update document " + params.id + " " + + "with data from related " + relationship + " " + sourceId + " " + + "because the related " + relationship + " has apparently changed (was: " + previousSourceIdsForRelationship + "), " + + "but mutations of relationships used with `sourced_from` are not supported because " + + "allowing it could break ElasticGraph's out-of-order processing guarantees." + ); + } + + // While the version in `__versions` is going to be used for the doc version in the future, for now + // we need to continue getting it from `__sourceVersions`. Both our old version and this versions of this + // script keep the value in `__sourceVersions` up-to-date, whereas the old script only writes it to + // `__sourceVersions`. Until we have completely migrated off of the old script for all ElasticGraph + // clusters, we need to keep using it. + // + // Later, after the old script is no longer used by any clusters, we'll stop using `__sourceVersions`. + // TODO: switch to `__versions` when we no longer need to maintain compatibility with the old version of the script. + Number _versionForSourceType = source.get("__sourceVersions")?.get(params.sourceType)?.get(sourceId); + Number _versionForRelationship = relationshipVersionsMap.get(sourceId); + + // Our JSON schema requires event versions to be non-negative, so we can safely use Long.MIN_VALUE as a stand-in when the value is null. + long versionForSourceType = _versionForSourceType == null ? Long.MIN_VALUE : _versionForSourceType.longValue(); + long versionForRelationship = _versionForRelationship == null ? Long.MIN_VALUE : _versionForRelationship.longValue(); + + // Pick the larger of the two versions as our doc version. Note that `Math.max` didn't work for me here for + // reasons I don't understand, but a simple ternary works fine. + // + // In theory, we could just use `versionForSourceType` as the `docVersion` (and not even check `__versions` at all) + // since both the old version and this version maintain the doc version in `__sourceVersions`. However, that would + // prevent this version of the script from being forward-compatible with the planned next version of this script. + // In the next version, we plan to stop writing to `__sourceVersions`, and as we can't deploy that change atomically, + // this version of the script will continue to run after that has begun to be used. So this version of the script + // must consider which version is greater here, and not simply trust either version value. + long docVersion = versionForSourceType > versionForRelationship ? versionForSourceType : versionForRelationship; + + if (docVersion >= eventVersion) { + throw new IllegalArgumentException("ElasticGraph update was a no-op: [" + + params.id + "]: version conflict, current version [" + + docVersion + "] is higher or equal to the one provided [" + + eventVersion + "]"); + } else { + source.putAll(params.data); + Map __counts = params.__counts; + + if (__counts != null) { + if (source.__counts == null) { + source.__counts = [:]; + } + + source.__counts.putAll(__counts); + } + + source.id = params.id; + source.__versions[relationship][sourceId] = eventVersion; + + // Record the relationship in `__sources` if it's not already there. We maintain it as an append-only set using a sorted list. + // This ensures deterministic ordering of its elements regardless of event ingestion order, and lets us check membership in O(log N) time. + // + // As per https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/Collections.html#binarySearch(java.util.List,java.lang.Object): + // + // > Returns the index of the search key, if it is contained in the list; otherwise, (-(insertion point) - 1). + // > The insertion point is defined as the point at which the key would be inserted into the list: the index + // > of the first element greater than the key, or list.size() if all elements in the list are less than the + // > specified key. Note that this guarantees that the return value will be >= 0 if and only if the key is found. + int sourceBinarySearchResult = Collections.binarySearch(source.__sources, relationship); + if (sourceBinarySearchResult < 0) { + source.__sources.add(-sourceBinarySearchResult - 1, relationship); + } + } diff --git a/config/schema/artifacts/json_schemas.yaml b/config/schema/artifacts/json_schemas.yaml new file mode 100644 index 00000000..b5dbc256 --- /dev/null +++ b/config/schema/artifacts/json_schemas.yaml @@ -0,0 +1,985 @@ +# This is the "public" JSON schema file and is intended to be provided to publishers so that +# they can perform code generation and event validation. +# +# Generated by `rake schema_artifacts:dump`. +# DO NOT EDIT BY HAND. Any edits will be lost the next time the rake task is run. +--- +"$schema": http://json-schema.org/draft-07/schema# +json_schema_version: 1 +"$defs": + ElasticGraphEventEnvelope: + type: object + properties: + op: + type: string + enum: + - upsert + type: + type: string + enum: + - Address + - Component + - ElectricalPart + - Manufacturer + - MechanicalPart + - Sponsor + - Team + - Widget + - WidgetWorkspace + id: + type: string + maxLength: 8191 + version: + type: integer + minimum: 0 + maximum: 9223372036854775807 + record: + type: object + latency_timestamps: + type: object + additionalProperties: false + patternProperties: + "^\\w+_at$": + type: string + format: date-time + json_schema_version: + const: 1 + message_id: + type: string + description: The optional ID of the message containing this event from whatever + messaging system is being used between the publisher and the ElasticGraph + indexer. + additionalProperties: false + required: + - op + - type + - id + - version + - json_schema_version + if: + properties: + op: + const: upsert + then: + required: + - record + Address: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + full_address: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + timestamps: + anyOf: + - "$ref": "#/$defs/AddressTimestamps" + - type: 'null' + geo_location: + anyOf: + - "$ref": "#/$defs/GeoLocation" + - type: 'null' + shapes: + type: array + items: + "$ref": "#/$defs/GeoShape" + manufacturer_id: + anyOf: + - allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + - type: 'null' + __typename: + type: string + const: Address + default: Address + required: + - id + - full_address + - timestamps + - geo_location + - shapes + - manufacturer_id + AddressTimestamps: + type: object + properties: + created_at: + anyOf: + - "$ref": "#/$defs/DateTime" + - type: 'null' + __typename: + type: string + const: AddressTimestamps + default: AddressTimestamps + required: + - created_at + Affiliations: + type: object + properties: + sponsorships_nested: + type: array + items: + "$ref": "#/$defs/Sponsorship" + sponsorships_object: + type: array + items: + "$ref": "#/$defs/Sponsorship" + __typename: + type: string + const: Affiliations + default: Affiliations + required: + - sponsorships_nested + - sponsorships_object + Color: + type: string + enum: + - RED + - BLUE + - GREEN + Company: + type: object + properties: + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + stock_ticker: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + __typename: + type: string + const: Company + default: Company + required: + - name + - stock_ticker + Component: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + created_at: + "$ref": "#/$defs/DateTime" + position: + "$ref": "#/$defs/Position" + tags: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + part_ids: + type: array + items: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + __typename: + type: string + const: Component + default: Component + required: + - id + - name + - created_at + - position + - tags + - part_ids + Date: + type: string + format: date + DateTime: + type: string + format: date-time + ElectricalPart: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + created_at: + "$ref": "#/$defs/DateTime" + voltage: + "$ref": "#/$defs/Int" + manufacturer_id: + anyOf: + - allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + - type: 'null' + __typename: + type: string + const: ElectricalPart + default: ElectricalPart + required: + - id + - name + - created_at + - voltage + - manufacturer_id + Float: + type: number + GeoLocation: + type: object + properties: + latitude: + allOf: + - "$ref": "#/$defs/Float" + - minimum: -90 + maximum: 90 + longitude: + allOf: + - "$ref": "#/$defs/Float" + - minimum: -180 + maximum: 180 + __typename: + type: string + const: GeoLocation + default: GeoLocation + required: + - latitude + - longitude + GeoShape: + type: object + properties: + type: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + coordinates: + type: array + items: + "$ref": "#/$defs/Float" + __typename: + type: string + const: GeoShape + default: GeoShape + required: + - type + - coordinates + ID: + type: string + Int: + type: integer + minimum: -2147483648 + maximum: 2147483647 + Inventor: + required: + - __typename + oneOf: + - "$ref": "#/$defs/Person" + - "$ref": "#/$defs/Company" + JsonSafeLong: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + LocalTime: + type: string + pattern: "^(([0-1][0-9])|(2[0-3])):[0-5][0-9]:[0-5][0-9](\\.[0-9]{1,3})?$" + LongString: + type: integer + minimum: -9223372036854775808 + maximum: 9223372036854775807 + Manufacturer: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + created_at: + "$ref": "#/$defs/DateTime" + __typename: + type: string + const: Manufacturer + default: Manufacturer + required: + - id + - name + - created_at + Material: + type: string + enum: + - ALLOY + - CARBON_FIBER + MechanicalPart: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + created_at: + "$ref": "#/$defs/DateTime" + material: + anyOf: + - "$ref": "#/$defs/Material" + - type: 'null' + manufacturer_id: + anyOf: + - allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + - type: 'null' + __typename: + type: string + const: MechanicalPart + default: MechanicalPart + required: + - id + - name + - created_at + - material + - manufacturer_id + Money: + type: object + properties: + currency: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + amount_cents: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + __typename: + type: string + const: Money + default: Money + required: + - currency + - amount_cents + NamedInventor: + required: + - __typename + oneOf: + - "$ref": "#/$defs/Person" + - "$ref": "#/$defs/Company" + Person: + type: object + properties: + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + nationality: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + __typename: + type: string + const: Person + default: Person + required: + - name + - nationality + Player: + type: object + properties: + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + nicknames: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + affiliations: + "$ref": "#/$defs/Affiliations" + seasons_nested: + type: array + items: + "$ref": "#/$defs/PlayerSeason" + seasons_object: + type: array + items: + "$ref": "#/$defs/PlayerSeason" + __typename: + type: string + const: Player + default: Player + required: + - name + - nicknames + - affiliations + - seasons_nested + - seasons_object + PlayerSeason: + type: object + properties: + year: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + games_played: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + awards: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + __typename: + type: string + const: PlayerSeason + default: PlayerSeason + required: + - year + - games_played + - awards + Position: + type: object + properties: + x: + "$ref": "#/$defs/Float" + "y": + "$ref": "#/$defs/Float" + __typename: + type: string + const: Position + default: Position + required: + - x + - "y" + Size: + type: string + enum: + - SMALL + - MEDIUM + - LARGE + Sponsor: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + __typename: + type: string + const: Sponsor + default: Sponsor + required: + - id + - name + Sponsorship: + type: object + properties: + sponsor_id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + annual_total: + "$ref": "#/$defs/Money" + __typename: + type: string + const: Sponsorship + default: Sponsorship + required: + - sponsor_id + - annual_total + String: + type: string + Team: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + league: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + pattern: "[^ \t\n]+" + country_code: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + formed_on: + "$ref": "#/$defs/Date" + current_name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + past_names: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + won_championships_at: + type: array + items: + "$ref": "#/$defs/DateTime" + details: + anyOf: + - "$ref": "#/$defs/TeamDetails" + - type: 'null' + stadium_location: + anyOf: + - "$ref": "#/$defs/GeoLocation" + - type: 'null' + forbes_valuations: + type: array + items: + "$ref": "#/$defs/JsonSafeLong" + forbes_valuation_moneys_nested: + type: array + items: + "$ref": "#/$defs/Money" + forbes_valuation_moneys_object: + type: array + items: + "$ref": "#/$defs/Money" + current_players_nested: + type: array + items: + "$ref": "#/$defs/Player" + current_players_object: + type: array + items: + "$ref": "#/$defs/Player" + seasons_nested: + type: array + items: + "$ref": "#/$defs/TeamSeason" + seasons_object: + type: array + items: + "$ref": "#/$defs/TeamSeason" + nested_fields: + anyOf: + - "$ref": "#/$defs/TeamNestedFields" + - type: 'null' + nested_fields2: + anyOf: + - "$ref": "#/$defs/TeamNestedFields" + - type: 'null' + __typename: + type: string + const: Team + default: Team + required: + - id + - league + - country_code + - formed_on + - current_name + - past_names + - won_championships_at + - details + - stadium_location + - forbes_valuations + - forbes_valuation_moneys_nested + - forbes_valuation_moneys_object + - current_players_nested + - current_players_object + - seasons_nested + - seasons_object + - nested_fields + - nested_fields2 + TeamDetails: + type: object + properties: + uniform_colors: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + count: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + __typename: + type: string + const: TeamDetails + default: TeamDetails + required: + - uniform_colors + - count + TeamNestedFields: + type: object + properties: + forbes_valuation_moneys: + type: array + items: + "$ref": "#/$defs/Money" + current_players: + type: array + items: + "$ref": "#/$defs/Player" + seasons: + type: array + items: + "$ref": "#/$defs/TeamSeason" + __typename: + type: string + const: TeamNestedFields + default: TeamNestedFields + required: + - forbes_valuation_moneys + - current_players + - seasons + TeamRecord: + type: object + properties: + wins: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + losses: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + last_win_on: + anyOf: + - "$ref": "#/$defs/Date" + - type: 'null' + first_win_on: + anyOf: + - "$ref": "#/$defs/Date" + - type: 'null' + __typename: + type: string + const: TeamRecord + default: TeamRecord + required: + - wins + - losses + - last_win_on + - first_win_on + TeamSeason: + type: object + properties: + record: + anyOf: + - "$ref": "#/$defs/TeamRecord" + - type: 'null' + year: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + notes: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + count: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + started_at: + anyOf: + - "$ref": "#/$defs/DateTime" + - type: 'null' + won_games_at: + type: array + items: + "$ref": "#/$defs/DateTime" + players_nested: + type: array + items: + "$ref": "#/$defs/Player" + players_object: + type: array + items: + "$ref": "#/$defs/Player" + __typename: + type: string + const: TeamSeason + default: TeamSeason + required: + - record + - year + - notes + - count + - started_at + - won_games_at + - players_nested + - players_object + Untyped: + type: + - array + - boolean + - integer + - number + - object + - string + Widget: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + workspace_id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + pattern: "[^ \t\n]+" + amount_cents: + "$ref": "#/$defs/Int" + cost: + anyOf: + - "$ref": "#/$defs/Money" + - type: 'null' + cost_currency_unit: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + cost_currency_name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + cost_currency_symbol: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + cost_currency_primary_continent: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + cost_currency_introduced_on: + anyOf: + - "$ref": "#/$defs/Date" + - type: 'null' + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + name_text: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 104857600 + - type: 'null' + created_at: + "$ref": "#/$defs/DateTime" + created_at_time_of_day: + anyOf: + - "$ref": "#/$defs/LocalTime" + - type: 'null' + created_on: + anyOf: + - "$ref": "#/$defs/Date" + - type: 'null' + release_timestamps: + type: array + items: + "$ref": "#/$defs/DateTime" + release_dates: + type: array + items: + "$ref": "#/$defs/Date" + component_ids: + type: array + items: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + options: + anyOf: + - "$ref": "#/$defs/WidgetOptions" + - type: 'null' + the_options: + anyOf: + - "$ref": "#/$defs/WidgetOptions" + - type: 'null' + inventor: + anyOf: + - "$ref": "#/$defs/Inventor" + - type: 'null' + named_inventor: + anyOf: + - "$ref": "#/$defs/NamedInventor" + - type: 'null' + weight_in_ng_str: + "$ref": "#/$defs/LongString" + weight_in_ng: + "$ref": "#/$defs/JsonSafeLong" + tags: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + amounts: + type: array + items: + "$ref": "#/$defs/Int" + fees: + type: array + items: + "$ref": "#/$defs/Money" + metadata: + anyOf: + - allOf: + - "$ref": "#/$defs/Untyped" + - maxLength: 8191 + - type: 'null' + __typename: + type: string + const: Widget + default: Widget + required: + - id + - workspace_id + - amount_cents + - cost + - cost_currency_unit + - cost_currency_name + - cost_currency_symbol + - cost_currency_primary_continent + - cost_currency_introduced_on + - name + - name_text + - created_at + - created_at_time_of_day + - created_on + - release_timestamps + - release_dates + - component_ids + - options + - the_options + - inventor + - named_inventor + - weight_in_ng_str + - weight_in_ng + - tags + - amounts + - fees + - metadata + WidgetOptions: + type: object + properties: + size: + anyOf: + - "$ref": "#/$defs/Size" + - type: 'null' + the_size: + anyOf: + - "$ref": "#/$defs/Size" + - type: 'null' + color: + anyOf: + - "$ref": "#/$defs/Color" + - type: 'null' + __typename: + type: string + const: WidgetOptions + default: WidgetOptions + required: + - size + - the_size + - color + WidgetWorkspace: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + widget: + anyOf: + - "$ref": "#/$defs/WorkspaceWidget" + - type: 'null' + __typename: + type: string + const: WidgetWorkspace + default: WidgetWorkspace + required: + - id + - name + - widget + WorkspaceWidget: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + created_at: + anyOf: + - "$ref": "#/$defs/DateTime" + - type: 'null' + __typename: + type: string + const: WorkspaceWidget + default: WorkspaceWidget + required: + - id + - created_at diff --git a/config/schema/artifacts/json_schemas_by_version/v1.yaml b/config/schema/artifacts/json_schemas_by_version/v1.yaml new file mode 100644 index 00000000..a836e325 --- /dev/null +++ b/config/schema/artifacts/json_schemas_by_version/v1.yaml @@ -0,0 +1,1351 @@ +# This JSON schema file contains internal ElasticGraph metadata and should be considered private. +# The unversioned JSON schema file is public and intended to be provided to publishers. +# +# Generated by `rake schema_artifacts:dump`. +# DO NOT EDIT BY HAND. Any edits will be lost the next time the rake task is run. +--- +"$schema": http://json-schema.org/draft-07/schema# +json_schema_version: 1 +"$defs": + ElasticGraphEventEnvelope: + type: object + properties: + op: + type: string + enum: + - upsert + type: + type: string + enum: + - Address + - Component + - ElectricalPart + - Manufacturer + - MechanicalPart + - Sponsor + - Team + - Widget + - WidgetWorkspace + id: + type: string + maxLength: 8191 + version: + type: integer + minimum: 0 + maximum: 9223372036854775807 + record: + type: object + latency_timestamps: + type: object + additionalProperties: false + patternProperties: + "^\\w+_at$": + type: string + format: date-time + json_schema_version: + const: 1 + message_id: + type: string + description: The optional ID of the message containing this event from whatever + messaging system is being used between the publisher and the ElasticGraph + indexer. + additionalProperties: false + required: + - op + - type + - id + - version + - json_schema_version + if: + properties: + op: + const: upsert + then: + required: + - record + Address: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + full_address: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: String! + nameInIndex: full_address + timestamps: + anyOf: + - "$ref": "#/$defs/AddressTimestamps" + - type: 'null' + ElasticGraph: + type: AddressTimestamps + nameInIndex: timestamps + geo_location: + anyOf: + - "$ref": "#/$defs/GeoLocation" + - type: 'null' + ElasticGraph: + type: GeoLocation + nameInIndex: geo_location + shapes: + type: array + items: + "$ref": "#/$defs/GeoShape" + ElasticGraph: + type: "[GeoShape!]!" + nameInIndex: shapes + manufacturer_id: + anyOf: + - allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: ID + nameInIndex: manufacturer_id + __typename: + type: string + const: Address + default: Address + required: + - id + - full_address + - timestamps + - geo_location + - shapes + - manufacturer_id + AddressTimestamps: + type: object + properties: + created_at: + anyOf: + - "$ref": "#/$defs/DateTime" + - type: 'null' + ElasticGraph: + type: DateTime + nameInIndex: created_at + __typename: + type: string + const: AddressTimestamps + default: AddressTimestamps + required: + - created_at + Affiliations: + type: object + properties: + sponsorships_nested: + type: array + items: + "$ref": "#/$defs/Sponsorship" + ElasticGraph: + type: "[Sponsorship!]!" + nameInIndex: sponsorships_nested + sponsorships_object: + type: array + items: + "$ref": "#/$defs/Sponsorship" + ElasticGraph: + type: "[Sponsorship!]!" + nameInIndex: sponsorships_object + __typename: + type: string + const: Affiliations + default: Affiliations + required: + - sponsorships_nested + - sponsorships_object + Color: + type: string + enum: + - RED + - BLUE + - GREEN + Company: + type: object + properties: + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + stock_ticker: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: stock_ticker + __typename: + type: string + const: Company + default: Company + required: + - name + - stock_ticker + Component: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + created_at: + "$ref": "#/$defs/DateTime" + ElasticGraph: + type: DateTime! + nameInIndex: created_at + position: + "$ref": "#/$defs/Position" + ElasticGraph: + type: Position! + nameInIndex: position + tags: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: "[String!]!" + nameInIndex: tags + part_ids: + type: array + items: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: "[ID!]!" + nameInIndex: part_ids + __typename: + type: string + const: Component + default: Component + required: + - id + - name + - created_at + - position + - tags + - part_ids + Date: + type: string + format: date + DateTime: + type: string + format: date-time + ElectricalPart: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + created_at: + "$ref": "#/$defs/DateTime" + ElasticGraph: + type: DateTime! + nameInIndex: created_at + voltage: + "$ref": "#/$defs/Int" + ElasticGraph: + type: Int! + nameInIndex: voltage + manufacturer_id: + anyOf: + - allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: ID + nameInIndex: manufacturer_id + __typename: + type: string + const: ElectricalPart + default: ElectricalPart + required: + - id + - name + - created_at + - voltage + - manufacturer_id + Float: + type: number + GeoLocation: + type: object + properties: + latitude: + allOf: + - "$ref": "#/$defs/Float" + - minimum: -90 + maximum: 90 + ElasticGraph: + type: Float! + nameInIndex: lat + longitude: + allOf: + - "$ref": "#/$defs/Float" + - minimum: -180 + maximum: 180 + ElasticGraph: + type: Float! + nameInIndex: lon + __typename: + type: string + const: GeoLocation + default: GeoLocation + required: + - latitude + - longitude + GeoShape: + type: object + properties: + type: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: type + coordinates: + type: array + items: + "$ref": "#/$defs/Float" + ElasticGraph: + type: "[Float!]!" + nameInIndex: coordinates + __typename: + type: string + const: GeoShape + default: GeoShape + required: + - type + - coordinates + ID: + type: string + Int: + type: integer + minimum: -2147483648 + maximum: 2147483647 + Inventor: + required: + - __typename + oneOf: + - "$ref": "#/$defs/Person" + - "$ref": "#/$defs/Company" + JsonSafeLong: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + LocalTime: + type: string + pattern: "^(([0-1][0-9])|(2[0-3])):[0-5][0-9]:[0-5][0-9](\\.[0-9]{1,3})?$" + LongString: + type: integer + minimum: -9223372036854775808 + maximum: 9223372036854775807 + Manufacturer: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + created_at: + "$ref": "#/$defs/DateTime" + ElasticGraph: + type: DateTime! + nameInIndex: created_at + __typename: + type: string + const: Manufacturer + default: Manufacturer + required: + - id + - name + - created_at + Material: + type: string + enum: + - ALLOY + - CARBON_FIBER + MechanicalPart: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + created_at: + "$ref": "#/$defs/DateTime" + ElasticGraph: + type: DateTime! + nameInIndex: created_at + material: + anyOf: + - "$ref": "#/$defs/Material" + - type: 'null' + ElasticGraph: + type: Material + nameInIndex: material + manufacturer_id: + anyOf: + - allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: ID + nameInIndex: manufacturer_id + __typename: + type: string + const: MechanicalPart + default: MechanicalPart + required: + - id + - name + - created_at + - material + - manufacturer_id + Money: + type: object + properties: + currency: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: String! + nameInIndex: currency + amount_cents: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + ElasticGraph: + type: Int + nameInIndex: amount_cents + __typename: + type: string + const: Money + default: Money + required: + - currency + - amount_cents + NamedInventor: + required: + - __typename + oneOf: + - "$ref": "#/$defs/Person" + - "$ref": "#/$defs/Company" + Person: + type: object + properties: + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + nationality: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: nationality + __typename: + type: string + const: Person + default: Person + required: + - name + - nationality + Player: + type: object + properties: + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + nicknames: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: "[String!]!" + nameInIndex: nicknames + affiliations: + "$ref": "#/$defs/Affiliations" + ElasticGraph: + type: Affiliations! + nameInIndex: affiliations + seasons_nested: + type: array + items: + "$ref": "#/$defs/PlayerSeason" + ElasticGraph: + type: "[PlayerSeason!]!" + nameInIndex: seasons_nested + seasons_object: + type: array + items: + "$ref": "#/$defs/PlayerSeason" + ElasticGraph: + type: "[PlayerSeason!]!" + nameInIndex: seasons_object + __typename: + type: string + const: Player + default: Player + required: + - name + - nicknames + - affiliations + - seasons_nested + - seasons_object + PlayerSeason: + type: object + properties: + year: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + ElasticGraph: + type: Int + nameInIndex: year + games_played: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + ElasticGraph: + type: Int + nameInIndex: games_played + awards: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: "[String!]!" + nameInIndex: awards + __typename: + type: string + const: PlayerSeason + default: PlayerSeason + required: + - year + - games_played + - awards + Position: + type: object + properties: + x: + "$ref": "#/$defs/Float" + ElasticGraph: + type: Float! + nameInIndex: x + "y": + "$ref": "#/$defs/Float" + ElasticGraph: + type: Float! + nameInIndex: "y" + __typename: + type: string + const: Position + default: Position + required: + - x + - "y" + Size: + type: string + enum: + - SMALL + - MEDIUM + - LARGE + Sponsor: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + __typename: + type: string + const: Sponsor + default: Sponsor + required: + - id + - name + Sponsorship: + type: object + properties: + sponsor_id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: sponsor_id + annual_total: + "$ref": "#/$defs/Money" + ElasticGraph: + type: Money! + nameInIndex: annual_total + __typename: + type: string + const: Sponsorship + default: Sponsorship + required: + - sponsor_id + - annual_total + String: + type: string + Team: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + league: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + pattern: "[^ \t\n]+" + ElasticGraph: + type: String! + nameInIndex: league + country_code: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: country_code + formed_on: + "$ref": "#/$defs/Date" + ElasticGraph: + type: Date! + nameInIndex: formed_on + current_name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: current_name + past_names: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: "[String!]!" + nameInIndex: past_names + won_championships_at: + type: array + items: + "$ref": "#/$defs/DateTime" + ElasticGraph: + type: "[DateTime!]!" + nameInIndex: won_championships_at + details: + anyOf: + - "$ref": "#/$defs/TeamDetails" + - type: 'null' + ElasticGraph: + type: TeamDetails + nameInIndex: details + stadium_location: + anyOf: + - "$ref": "#/$defs/GeoLocation" + - type: 'null' + ElasticGraph: + type: GeoLocation + nameInIndex: stadium_location + forbes_valuations: + type: array + items: + "$ref": "#/$defs/JsonSafeLong" + ElasticGraph: + type: "[JsonSafeLong!]!" + nameInIndex: forbes_valuations + forbes_valuation_moneys_nested: + type: array + items: + "$ref": "#/$defs/Money" + ElasticGraph: + type: "[Money!]!" + nameInIndex: forbes_valuation_moneys_nested + forbes_valuation_moneys_object: + type: array + items: + "$ref": "#/$defs/Money" + ElasticGraph: + type: "[Money!]!" + nameInIndex: forbes_valuation_moneys_object + current_players_nested: + type: array + items: + "$ref": "#/$defs/Player" + ElasticGraph: + type: "[Player!]!" + nameInIndex: current_players_nested + current_players_object: + type: array + items: + "$ref": "#/$defs/Player" + ElasticGraph: + type: "[Player!]!" + nameInIndex: current_players_object + seasons_nested: + type: array + items: + "$ref": "#/$defs/TeamSeason" + ElasticGraph: + type: "[TeamSeason!]!" + nameInIndex: seasons_nested + seasons_object: + type: array + items: + "$ref": "#/$defs/TeamSeason" + ElasticGraph: + type: "[TeamSeason!]!" + nameInIndex: seasons_object + nested_fields: + anyOf: + - "$ref": "#/$defs/TeamNestedFields" + - type: 'null' + ElasticGraph: + type: TeamNestedFields + nameInIndex: the_nested_fields + nested_fields2: + anyOf: + - "$ref": "#/$defs/TeamNestedFields" + - type: 'null' + ElasticGraph: + type: TeamNestedFields + nameInIndex: nested_fields2 + __typename: + type: string + const: Team + default: Team + required: + - id + - league + - country_code + - formed_on + - current_name + - past_names + - won_championships_at + - details + - stadium_location + - forbes_valuations + - forbes_valuation_moneys_nested + - forbes_valuation_moneys_object + - current_players_nested + - current_players_object + - seasons_nested + - seasons_object + - nested_fields + - nested_fields2 + TeamDetails: + type: object + properties: + uniform_colors: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: "[String!]!" + nameInIndex: uniform_colors + count: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + ElasticGraph: + type: Int + nameInIndex: count + __typename: + type: string + const: TeamDetails + default: TeamDetails + required: + - uniform_colors + - count + TeamNestedFields: + type: object + properties: + forbes_valuation_moneys: + type: array + items: + "$ref": "#/$defs/Money" + ElasticGraph: + type: "[Money!]!" + nameInIndex: forbes_valuation_moneys + current_players: + type: array + items: + "$ref": "#/$defs/Player" + ElasticGraph: + type: "[Player!]!" + nameInIndex: current_players + seasons: + type: array + items: + "$ref": "#/$defs/TeamSeason" + ElasticGraph: + type: "[TeamSeason!]!" + nameInIndex: the_seasons + __typename: + type: string + const: TeamNestedFields + default: TeamNestedFields + required: + - forbes_valuation_moneys + - current_players + - seasons + TeamRecord: + type: object + properties: + wins: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + ElasticGraph: + type: Int + nameInIndex: win_count + losses: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + ElasticGraph: + type: Int + nameInIndex: loss_count + last_win_on: + anyOf: + - "$ref": "#/$defs/Date" + - type: 'null' + ElasticGraph: + type: Date + nameInIndex: last_win_date + first_win_on: + anyOf: + - "$ref": "#/$defs/Date" + - type: 'null' + ElasticGraph: + type: Date + nameInIndex: first_win_on + __typename: + type: string + const: TeamRecord + default: TeamRecord + required: + - wins + - losses + - last_win_on + - first_win_on + TeamSeason: + type: object + properties: + record: + anyOf: + - "$ref": "#/$defs/TeamRecord" + - type: 'null' + ElasticGraph: + type: TeamRecord + nameInIndex: the_record + year: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + ElasticGraph: + type: Int + nameInIndex: year + notes: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: "[String!]!" + nameInIndex: notes + count: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + ElasticGraph: + type: Int + nameInIndex: count + started_at: + anyOf: + - "$ref": "#/$defs/DateTime" + - type: 'null' + ElasticGraph: + type: DateTime + nameInIndex: started_at + won_games_at: + type: array + items: + "$ref": "#/$defs/DateTime" + ElasticGraph: + type: "[DateTime!]!" + nameInIndex: won_games_at + players_nested: + type: array + items: + "$ref": "#/$defs/Player" + ElasticGraph: + type: "[Player!]!" + nameInIndex: players_nested + players_object: + type: array + items: + "$ref": "#/$defs/Player" + ElasticGraph: + type: "[Player!]!" + nameInIndex: players_object + __typename: + type: string + const: TeamSeason + default: TeamSeason + required: + - record + - year + - notes + - count + - started_at + - won_games_at + - players_nested + - players_object + Untyped: + type: + - array + - boolean + - integer + - number + - object + - string + Widget: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + workspace_id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + pattern: "[^ \t\n]+" + ElasticGraph: + type: ID! + nameInIndex: workspace_id2 + amount_cents: + "$ref": "#/$defs/Int" + ElasticGraph: + type: Int! + nameInIndex: amount_cents + cost: + anyOf: + - "$ref": "#/$defs/Money" + - type: 'null' + ElasticGraph: + type: Money + nameInIndex: cost + cost_currency_unit: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: cost_currency_unit + cost_currency_name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: cost_currency_name + cost_currency_symbol: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: cost_currency_symbol + cost_currency_primary_continent: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: cost_currency_primary_continent + cost_currency_introduced_on: + anyOf: + - "$ref": "#/$defs/Date" + - type: 'null' + ElasticGraph: + type: Date + nameInIndex: cost_currency_introduced_on + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + name_text: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 104857600 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name_text + created_at: + "$ref": "#/$defs/DateTime" + ElasticGraph: + type: DateTime! + nameInIndex: created_at + created_at_time_of_day: + anyOf: + - "$ref": "#/$defs/LocalTime" + - type: 'null' + ElasticGraph: + type: LocalTime + nameInIndex: created_at_time_of_day + created_on: + anyOf: + - "$ref": "#/$defs/Date" + - type: 'null' + ElasticGraph: + type: Date + nameInIndex: created_on + release_timestamps: + type: array + items: + "$ref": "#/$defs/DateTime" + ElasticGraph: + type: "[DateTime!]!" + nameInIndex: release_timestamps + release_dates: + type: array + items: + "$ref": "#/$defs/Date" + ElasticGraph: + type: "[Date!]!" + nameInIndex: release_dates + component_ids: + type: array + items: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: "[ID!]!" + nameInIndex: component_ids + options: + anyOf: + - "$ref": "#/$defs/WidgetOptions" + - type: 'null' + ElasticGraph: + type: WidgetOptions + nameInIndex: options + the_options: + anyOf: + - "$ref": "#/$defs/WidgetOptions" + - type: 'null' + ElasticGraph: + type: WidgetOptions + nameInIndex: the_opts + inventor: + anyOf: + - "$ref": "#/$defs/Inventor" + - type: 'null' + ElasticGraph: + type: Inventor + nameInIndex: inventor + named_inventor: + anyOf: + - "$ref": "#/$defs/NamedInventor" + - type: 'null' + ElasticGraph: + type: NamedInventor + nameInIndex: named_inventor + weight_in_ng_str: + "$ref": "#/$defs/LongString" + ElasticGraph: + type: LongString! + nameInIndex: weight_in_ng_str + weight_in_ng: + "$ref": "#/$defs/JsonSafeLong" + ElasticGraph: + type: JsonSafeLong! + nameInIndex: weight_in_ng + tags: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: "[String!]!" + nameInIndex: tags + amounts: + type: array + items: + "$ref": "#/$defs/Int" + ElasticGraph: + type: "[Int!]!" + nameInIndex: amounts + fees: + type: array + items: + "$ref": "#/$defs/Money" + ElasticGraph: + type: "[Money!]!" + nameInIndex: fees + metadata: + anyOf: + - allOf: + - "$ref": "#/$defs/Untyped" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: Untyped + nameInIndex: metadata + __typename: + type: string + const: Widget + default: Widget + required: + - id + - workspace_id + - amount_cents + - cost + - cost_currency_unit + - cost_currency_name + - cost_currency_symbol + - cost_currency_primary_continent + - cost_currency_introduced_on + - name + - name_text + - created_at + - created_at_time_of_day + - created_on + - release_timestamps + - release_dates + - component_ids + - options + - the_options + - inventor + - named_inventor + - weight_in_ng_str + - weight_in_ng + - tags + - amounts + - fees + - metadata + WidgetOptions: + type: object + properties: + size: + anyOf: + - "$ref": "#/$defs/Size" + - type: 'null' + ElasticGraph: + type: Size + nameInIndex: size + the_size: + anyOf: + - "$ref": "#/$defs/Size" + - type: 'null' + ElasticGraph: + type: Size + nameInIndex: the_sighs + color: + anyOf: + - "$ref": "#/$defs/Color" + - type: 'null' + ElasticGraph: + type: Color + nameInIndex: color + __typename: + type: string + const: WidgetOptions + default: WidgetOptions + required: + - size + - the_size + - color + WidgetWorkspace: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + widget: + anyOf: + - "$ref": "#/$defs/WorkspaceWidget" + - type: 'null' + ElasticGraph: + type: WorkspaceWidget + nameInIndex: widget + __typename: + type: string + const: WidgetWorkspace + default: WidgetWorkspace + required: + - id + - name + - widget + WorkspaceWidget: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + created_at: + anyOf: + - "$ref": "#/$defs/DateTime" + - type: 'null' + ElasticGraph: + type: DateTime + nameInIndex: created_at + __typename: + type: string + const: WorkspaceWidget + default: WorkspaceWidget + required: + - id + - created_at diff --git a/config/schema/artifacts/runtime_metadata.yaml b/config/schema/artifacts/runtime_metadata.yaml new file mode 100644 index 00000000..7b32a357 --- /dev/null +++ b/config/schema/artifacts/runtime_metadata.yaml @@ -0,0 +1,4246 @@ +# Generated by `rake schema_artifacts:dump`. +# DO NOT EDIT BY HAND. Any edits will be lost the next time the rake task is run. +--- +enum_types_by_name: + AddressSortOrderInput: + values_by_name: + full_address_ASC: + sort_field: + direction: asc + field_path: full_address + full_address_DESC: + sort_field: + direction: desc + field_path: full_address + timestamps_created_at_ASC: + sort_field: + direction: asc + field_path: timestamps.created_at + timestamps_created_at_DESC: + sort_field: + direction: desc + field_path: timestamps.created_at + ComponentSortOrderInput: + values_by_name: + created_at_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_DESC: + sort_field: + direction: desc + field_path: created_at + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + position_x_ASC: + sort_field: + direction: asc + field_path: position.x + position_x_DESC: + sort_field: + direction: desc + field_path: position.x + position_y_ASC: + sort_field: + direction: asc + field_path: position.y + position_y_DESC: + sort_field: + direction: desc + field_path: position.y + widget_cost_amount_cents_ASC: + sort_field: + direction: asc + field_path: widget_cost.amount_cents + widget_cost_amount_cents_DESC: + sort_field: + direction: desc + field_path: widget_cost.amount_cents + widget_cost_currency_ASC: + sort_field: + direction: asc + field_path: widget_cost.currency + widget_cost_currency_DESC: + sort_field: + direction: desc + field_path: widget_cost.currency + widget_name_ASC: + sort_field: + direction: asc + field_path: widget_name + widget_name_DESC: + sort_field: + direction: desc + field_path: widget_name + widget_size_ASC: + sort_field: + direction: asc + field_path: widget_size + widget_size_DESC: + sort_field: + direction: desc + field_path: widget_size + widget_workspace_id_ASC: + sort_field: + direction: asc + field_path: widget_workspace_id3 + widget_workspace_id_DESC: + sort_field: + direction: desc + field_path: widget_workspace_id3 + DateGroupingGranularityInput: + values_by_name: + DAY: + datastore_value: day + MONTH: + datastore_value: month + QUARTER: + datastore_value: quarter + WEEK: + datastore_value: week + YEAR: + datastore_value: year + DateGroupingTruncationUnitInput: + values_by_name: + DAY: + datastore_value: day + MONTH: + datastore_value: month + QUARTER: + datastore_value: quarter + WEEK: + datastore_value: week + YEAR: + datastore_value: year + DateTimeGroupingGranularityInput: + values_by_name: + DAY: + datastore_value: day + HOUR: + datastore_value: hour + MINUTE: + datastore_value: minute + MONTH: + datastore_value: month + QUARTER: + datastore_value: quarter + SECOND: + datastore_value: second + WEEK: + datastore_value: week + YEAR: + datastore_value: year + DateTimeGroupingTruncationUnitInput: + values_by_name: + DAY: + datastore_value: day + HOUR: + datastore_value: hour + MINUTE: + datastore_value: minute + MONTH: + datastore_value: month + QUARTER: + datastore_value: quarter + SECOND: + datastore_value: second + WEEK: + datastore_value: week + YEAR: + datastore_value: year + DateTimeUnitInput: + values_by_name: + DAY: + datastore_abbreviation: d + datastore_value: 86400000 + HOUR: + datastore_abbreviation: h + datastore_value: 3600000 + MILLISECOND: + datastore_abbreviation: ms + datastore_value: 1 + MINUTE: + datastore_abbreviation: m + datastore_value: 60000 + SECOND: + datastore_abbreviation: s + datastore_value: 1000 + DateUnitInput: + values_by_name: + DAY: + datastore_abbreviation: d + datastore_value: 86400000 + DistanceUnitInput: + values_by_name: + CENTIMETER: + datastore_abbreviation: cm + FOOT: + datastore_abbreviation: ft + INCH: + datastore_abbreviation: in + KILOMETER: + datastore_abbreviation: km + METER: + datastore_abbreviation: m + MILE: + datastore_abbreviation: mi + MILLIMETER: + datastore_abbreviation: mm + NAUTICAL_MILE: + datastore_abbreviation: nmi + YARD: + datastore_abbreviation: yd + ElectricalPartSortOrderInput: + values_by_name: + created_at_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_DESC: + sort_field: + direction: desc + field_path: created_at + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + voltage_ASC: + sort_field: + direction: asc + field_path: voltage + voltage_DESC: + sort_field: + direction: desc + field_path: voltage + LocalTimeGroupingTruncationUnitInput: + values_by_name: + HOUR: + datastore_value: hour + MINUTE: + datastore_value: minute + SECOND: + datastore_value: second + LocalTimeUnitInput: + values_by_name: + HOUR: + datastore_abbreviation: h + datastore_value: 3600000 + MILLISECOND: + datastore_abbreviation: ms + datastore_value: 1 + MINUTE: + datastore_abbreviation: m + datastore_value: 60000 + SECOND: + datastore_abbreviation: s + datastore_value: 1000 + ManufacturerSortOrderInput: + values_by_name: + created_at_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_DESC: + sort_field: + direction: desc + field_path: created_at + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + MatchesQueryAllowedEditsPerTermInput: + values_by_name: + DYNAMIC: + datastore_abbreviation: AUTO + NONE: + datastore_abbreviation: '0' + ONE: + datastore_abbreviation: '1' + TWO: + datastore_abbreviation: '2' + MechanicalPartSortOrderInput: + values_by_name: + created_at_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_DESC: + sort_field: + direction: desc + field_path: created_at + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + material_ASC: + sort_field: + direction: asc + field_path: material + material_DESC: + sort_field: + direction: desc + field_path: material + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + NamedEntitySortOrderInput: + values_by_name: + amount_cents2_ASC: + sort_field: + direction: asc + field_path: amount_cents + amount_cents2_DESC: + sort_field: + direction: desc + field_path: amount_cents + amount_cents_ASC: + sort_field: + direction: asc + field_path: amount_cents + amount_cents_DESC: + sort_field: + direction: desc + field_path: amount_cents + cost_amount_cents_ASC: + sort_field: + direction: asc + field_path: cost.amount_cents + cost_amount_cents_DESC: + sort_field: + direction: desc + field_path: cost.amount_cents + cost_currency_ASC: + sort_field: + direction: asc + field_path: cost.currency + cost_currency_DESC: + sort_field: + direction: desc + field_path: cost.currency + cost_currency_introduced_on_ASC: + sort_field: + direction: asc + field_path: cost_currency_introduced_on + cost_currency_introduced_on_DESC: + sort_field: + direction: desc + field_path: cost_currency_introduced_on + cost_currency_name_ASC: + sort_field: + direction: asc + field_path: cost_currency_name + cost_currency_name_DESC: + sort_field: + direction: desc + field_path: cost_currency_name + cost_currency_primary_continent_ASC: + sort_field: + direction: asc + field_path: cost_currency_primary_continent + cost_currency_primary_continent_DESC: + sort_field: + direction: desc + field_path: cost_currency_primary_continent + cost_currency_symbol_ASC: + sort_field: + direction: asc + field_path: cost_currency_symbol + cost_currency_symbol_DESC: + sort_field: + direction: desc + field_path: cost_currency_symbol + cost_currency_unit_ASC: + sort_field: + direction: asc + field_path: cost_currency_unit + cost_currency_unit_DESC: + sort_field: + direction: desc + field_path: cost_currency_unit + created_at2_ASC: + sort_field: + direction: asc + field_path: created_at + created_at2_DESC: + sort_field: + direction: desc + field_path: created_at + created_at2_legacy_ASC: + sort_field: + direction: asc + field_path: created_at + created_at2_legacy_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_legacy_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_legacy_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_time_of_day_ASC: + sort_field: + direction: asc + field_path: created_at_time_of_day + created_at_time_of_day_DESC: + sort_field: + direction: desc + field_path: created_at_time_of_day + created_on_ASC: + sort_field: + direction: asc + field_path: created_on + created_on_DESC: + sort_field: + direction: desc + field_path: created_on + created_on_legacy_ASC: + sort_field: + direction: asc + field_path: created_on + created_on_legacy_DESC: + sort_field: + direction: desc + field_path: created_on + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + inventor_name_ASC: + sort_field: + direction: asc + field_path: inventor.name + inventor_name_DESC: + sort_field: + direction: desc + field_path: inventor.name + inventor_nationality_ASC: + sort_field: + direction: asc + field_path: inventor.nationality + inventor_nationality_DESC: + sort_field: + direction: desc + field_path: inventor.nationality + inventor_stock_ticker_ASC: + sort_field: + direction: asc + field_path: inventor.stock_ticker + inventor_stock_ticker_DESC: + sort_field: + direction: desc + field_path: inventor.stock_ticker + material_ASC: + sort_field: + direction: asc + field_path: material + material_DESC: + sort_field: + direction: desc + field_path: material + metadata_ASC: + sort_field: + direction: asc + field_path: metadata + metadata_DESC: + sort_field: + direction: desc + field_path: metadata + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + named_inventor_name_ASC: + sort_field: + direction: asc + field_path: named_inventor.name + named_inventor_name_DESC: + sort_field: + direction: desc + field_path: named_inventor.name + named_inventor_nationality_ASC: + sort_field: + direction: asc + field_path: named_inventor.nationality + named_inventor_nationality_DESC: + sort_field: + direction: desc + field_path: named_inventor.nationality + named_inventor_stock_ticker_ASC: + sort_field: + direction: asc + field_path: named_inventor.stock_ticker + named_inventor_stock_ticker_DESC: + sort_field: + direction: desc + field_path: named_inventor.stock_ticker + options_color_ASC: + sort_field: + direction: asc + field_path: options.color + options_color_DESC: + sort_field: + direction: desc + field_path: options.color + options_size_ASC: + sort_field: + direction: asc + field_path: options.size + options_size_DESC: + sort_field: + direction: desc + field_path: options.size + options_the_size_ASC: + sort_field: + direction: asc + field_path: options.the_sighs + options_the_size_DESC: + sort_field: + direction: desc + field_path: options.the_sighs + position_x_ASC: + sort_field: + direction: asc + field_path: position.x + position_x_DESC: + sort_field: + direction: desc + field_path: position.x + position_y_ASC: + sort_field: + direction: asc + field_path: position.y + position_y_DESC: + sort_field: + direction: desc + field_path: position.y + size_ASC: + sort_field: + direction: asc + field_path: options.size + size_DESC: + sort_field: + direction: desc + field_path: options.size + the_options_color_ASC: + sort_field: + direction: asc + field_path: the_opts.color + the_options_color_DESC: + sort_field: + direction: desc + field_path: the_opts.color + the_options_size_ASC: + sort_field: + direction: asc + field_path: the_opts.size + the_options_size_DESC: + sort_field: + direction: desc + field_path: the_opts.size + the_options_the_size_ASC: + sort_field: + direction: asc + field_path: the_opts.the_sighs + the_options_the_size_DESC: + sort_field: + direction: desc + field_path: the_opts.the_sighs + voltage_ASC: + sort_field: + direction: asc + field_path: voltage + voltage_DESC: + sort_field: + direction: desc + field_path: voltage + weight_in_ng_ASC: + sort_field: + direction: asc + field_path: weight_in_ng + weight_in_ng_DESC: + sort_field: + direction: desc + field_path: weight_in_ng + weight_in_ng_str_ASC: + sort_field: + direction: asc + field_path: weight_in_ng_str + weight_in_ng_str_DESC: + sort_field: + direction: desc + field_path: weight_in_ng_str + widget_cost_amount_cents_ASC: + sort_field: + direction: asc + field_path: widget_cost.amount_cents + widget_cost_amount_cents_DESC: + sort_field: + direction: desc + field_path: widget_cost.amount_cents + widget_cost_currency_ASC: + sort_field: + direction: asc + field_path: widget_cost.currency + widget_cost_currency_DESC: + sort_field: + direction: desc + field_path: widget_cost.currency + widget_name_ASC: + sort_field: + direction: asc + field_path: widget_name + widget_name_DESC: + sort_field: + direction: desc + field_path: widget_name + widget_size_ASC: + sort_field: + direction: asc + field_path: widget_size + widget_size_DESC: + sort_field: + direction: desc + field_path: widget_size + widget_workspace_id_ASC: + sort_field: + direction: asc + field_path: widget_workspace_id3 + widget_workspace_id_DESC: + sort_field: + direction: desc + field_path: widget_workspace_id3 + workspace_id_ASC: + sort_field: + direction: asc + field_path: workspace_id2 + workspace_id_DESC: + sort_field: + direction: desc + field_path: workspace_id2 + workspace_name_ASC: + sort_field: + direction: asc + field_path: workspace_name + workspace_name_DESC: + sort_field: + direction: desc + field_path: workspace_name + PartSortOrderInput: + values_by_name: + created_at_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_DESC: + sort_field: + direction: desc + field_path: created_at + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + material_ASC: + sort_field: + direction: asc + field_path: material + material_DESC: + sort_field: + direction: desc + field_path: material + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + voltage_ASC: + sort_field: + direction: asc + field_path: voltage + voltage_DESC: + sort_field: + direction: desc + field_path: voltage + SponsorSortOrderInput: + values_by_name: + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + TeamSortOrderInput: + values_by_name: + country_code_ASC: + sort_field: + direction: asc + field_path: country_code + country_code_DESC: + sort_field: + direction: desc + field_path: country_code + current_name_ASC: + sort_field: + direction: asc + field_path: current_name + current_name_DESC: + sort_field: + direction: desc + field_path: current_name + details_count_ASC: + sort_field: + direction: asc + field_path: details.count + details_count_DESC: + sort_field: + direction: desc + field_path: details.count + formed_on_ASC: + sort_field: + direction: asc + field_path: formed_on + formed_on_DESC: + sort_field: + direction: desc + field_path: formed_on + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + league_ASC: + sort_field: + direction: asc + field_path: league + league_DESC: + sort_field: + direction: desc + field_path: league + WidgetCurrencySortOrderInput: + values_by_name: + details_symbol_ASC: + sort_field: + direction: asc + field_path: details.symbol + details_symbol_DESC: + sort_field: + direction: desc + field_path: details.symbol + details_unit_ASC: + sort_field: + direction: asc + field_path: details.unit + details_unit_DESC: + sort_field: + direction: desc + field_path: details.unit + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + introduced_on_ASC: + sort_field: + direction: asc + field_path: introduced_on + introduced_on_DESC: + sort_field: + direction: desc + field_path: introduced_on + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + nested_fields_max_widget_cost_ASC: + sort_field: + direction: asc + field_path: nested_fields.max_widget_cost + nested_fields_max_widget_cost_DESC: + sort_field: + direction: desc + field_path: nested_fields.max_widget_cost + oldest_widget_created_at_ASC: + sort_field: + direction: asc + field_path: oldest_widget_created_at + oldest_widget_created_at_DESC: + sort_field: + direction: desc + field_path: oldest_widget_created_at + primary_continent_ASC: + sort_field: + direction: asc + field_path: primary_continent + primary_continent_DESC: + sort_field: + direction: desc + field_path: primary_continent + WidgetOrAddressSortOrderInput: + values_by_name: + amount_cents2_ASC: + sort_field: + direction: asc + field_path: amount_cents + amount_cents2_DESC: + sort_field: + direction: desc + field_path: amount_cents + amount_cents_ASC: + sort_field: + direction: asc + field_path: amount_cents + amount_cents_DESC: + sort_field: + direction: desc + field_path: amount_cents + cost_amount_cents_ASC: + sort_field: + direction: asc + field_path: cost.amount_cents + cost_amount_cents_DESC: + sort_field: + direction: desc + field_path: cost.amount_cents + cost_currency_ASC: + sort_field: + direction: asc + field_path: cost.currency + cost_currency_DESC: + sort_field: + direction: desc + field_path: cost.currency + cost_currency_introduced_on_ASC: + sort_field: + direction: asc + field_path: cost_currency_introduced_on + cost_currency_introduced_on_DESC: + sort_field: + direction: desc + field_path: cost_currency_introduced_on + cost_currency_name_ASC: + sort_field: + direction: asc + field_path: cost_currency_name + cost_currency_name_DESC: + sort_field: + direction: desc + field_path: cost_currency_name + cost_currency_primary_continent_ASC: + sort_field: + direction: asc + field_path: cost_currency_primary_continent + cost_currency_primary_continent_DESC: + sort_field: + direction: desc + field_path: cost_currency_primary_continent + cost_currency_symbol_ASC: + sort_field: + direction: asc + field_path: cost_currency_symbol + cost_currency_symbol_DESC: + sort_field: + direction: desc + field_path: cost_currency_symbol + cost_currency_unit_ASC: + sort_field: + direction: asc + field_path: cost_currency_unit + cost_currency_unit_DESC: + sort_field: + direction: desc + field_path: cost_currency_unit + created_at2_ASC: + sort_field: + direction: asc + field_path: created_at + created_at2_DESC: + sort_field: + direction: desc + field_path: created_at + created_at2_legacy_ASC: + sort_field: + direction: asc + field_path: created_at + created_at2_legacy_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_legacy_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_legacy_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_time_of_day_ASC: + sort_field: + direction: asc + field_path: created_at_time_of_day + created_at_time_of_day_DESC: + sort_field: + direction: desc + field_path: created_at_time_of_day + created_on_ASC: + sort_field: + direction: asc + field_path: created_on + created_on_DESC: + sort_field: + direction: desc + field_path: created_on + created_on_legacy_ASC: + sort_field: + direction: asc + field_path: created_on + created_on_legacy_DESC: + sort_field: + direction: desc + field_path: created_on + full_address_ASC: + sort_field: + direction: asc + field_path: full_address + full_address_DESC: + sort_field: + direction: desc + field_path: full_address + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + inventor_name_ASC: + sort_field: + direction: asc + field_path: inventor.name + inventor_name_DESC: + sort_field: + direction: desc + field_path: inventor.name + inventor_nationality_ASC: + sort_field: + direction: asc + field_path: inventor.nationality + inventor_nationality_DESC: + sort_field: + direction: desc + field_path: inventor.nationality + inventor_stock_ticker_ASC: + sort_field: + direction: asc + field_path: inventor.stock_ticker + inventor_stock_ticker_DESC: + sort_field: + direction: desc + field_path: inventor.stock_ticker + metadata_ASC: + sort_field: + direction: asc + field_path: metadata + metadata_DESC: + sort_field: + direction: desc + field_path: metadata + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + named_inventor_name_ASC: + sort_field: + direction: asc + field_path: named_inventor.name + named_inventor_name_DESC: + sort_field: + direction: desc + field_path: named_inventor.name + named_inventor_nationality_ASC: + sort_field: + direction: asc + field_path: named_inventor.nationality + named_inventor_nationality_DESC: + sort_field: + direction: desc + field_path: named_inventor.nationality + named_inventor_stock_ticker_ASC: + sort_field: + direction: asc + field_path: named_inventor.stock_ticker + named_inventor_stock_ticker_DESC: + sort_field: + direction: desc + field_path: named_inventor.stock_ticker + options_color_ASC: + sort_field: + direction: asc + field_path: options.color + options_color_DESC: + sort_field: + direction: desc + field_path: options.color + options_size_ASC: + sort_field: + direction: asc + field_path: options.size + options_size_DESC: + sort_field: + direction: desc + field_path: options.size + options_the_size_ASC: + sort_field: + direction: asc + field_path: options.the_sighs + options_the_size_DESC: + sort_field: + direction: desc + field_path: options.the_sighs + size_ASC: + sort_field: + direction: asc + field_path: options.size + size_DESC: + sort_field: + direction: desc + field_path: options.size + the_options_color_ASC: + sort_field: + direction: asc + field_path: the_opts.color + the_options_color_DESC: + sort_field: + direction: desc + field_path: the_opts.color + the_options_size_ASC: + sort_field: + direction: asc + field_path: the_opts.size + the_options_size_DESC: + sort_field: + direction: desc + field_path: the_opts.size + the_options_the_size_ASC: + sort_field: + direction: asc + field_path: the_opts.the_sighs + the_options_the_size_DESC: + sort_field: + direction: desc + field_path: the_opts.the_sighs + timestamps_created_at_ASC: + sort_field: + direction: asc + field_path: timestamps.created_at + timestamps_created_at_DESC: + sort_field: + direction: desc + field_path: timestamps.created_at + weight_in_ng_ASC: + sort_field: + direction: asc + field_path: weight_in_ng + weight_in_ng_DESC: + sort_field: + direction: desc + field_path: weight_in_ng + weight_in_ng_str_ASC: + sort_field: + direction: asc + field_path: weight_in_ng_str + weight_in_ng_str_DESC: + sort_field: + direction: desc + field_path: weight_in_ng_str + workspace_id_ASC: + sort_field: + direction: asc + field_path: workspace_id2 + workspace_id_DESC: + sort_field: + direction: desc + field_path: workspace_id2 + workspace_name_ASC: + sort_field: + direction: asc + field_path: workspace_name + workspace_name_DESC: + sort_field: + direction: desc + field_path: workspace_name + WidgetSortOrderInput: + values_by_name: + amount_cents2_ASC: + sort_field: + direction: asc + field_path: amount_cents + amount_cents2_DESC: + sort_field: + direction: desc + field_path: amount_cents + amount_cents_ASC: + sort_field: + direction: asc + field_path: amount_cents + amount_cents_DESC: + sort_field: + direction: desc + field_path: amount_cents + cost_amount_cents_ASC: + sort_field: + direction: asc + field_path: cost.amount_cents + cost_amount_cents_DESC: + sort_field: + direction: desc + field_path: cost.amount_cents + cost_currency_ASC: + sort_field: + direction: asc + field_path: cost.currency + cost_currency_DESC: + sort_field: + direction: desc + field_path: cost.currency + cost_currency_introduced_on_ASC: + sort_field: + direction: asc + field_path: cost_currency_introduced_on + cost_currency_introduced_on_DESC: + sort_field: + direction: desc + field_path: cost_currency_introduced_on + cost_currency_name_ASC: + sort_field: + direction: asc + field_path: cost_currency_name + cost_currency_name_DESC: + sort_field: + direction: desc + field_path: cost_currency_name + cost_currency_primary_continent_ASC: + sort_field: + direction: asc + field_path: cost_currency_primary_continent + cost_currency_primary_continent_DESC: + sort_field: + direction: desc + field_path: cost_currency_primary_continent + cost_currency_symbol_ASC: + sort_field: + direction: asc + field_path: cost_currency_symbol + cost_currency_symbol_DESC: + sort_field: + direction: desc + field_path: cost_currency_symbol + cost_currency_unit_ASC: + sort_field: + direction: asc + field_path: cost_currency_unit + cost_currency_unit_DESC: + sort_field: + direction: desc + field_path: cost_currency_unit + created_at2_ASC: + sort_field: + direction: asc + field_path: created_at + created_at2_DESC: + sort_field: + direction: desc + field_path: created_at + created_at2_legacy_ASC: + sort_field: + direction: asc + field_path: created_at + created_at2_legacy_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_legacy_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_legacy_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_time_of_day_ASC: + sort_field: + direction: asc + field_path: created_at_time_of_day + created_at_time_of_day_DESC: + sort_field: + direction: desc + field_path: created_at_time_of_day + created_on_ASC: + sort_field: + direction: asc + field_path: created_on + created_on_DESC: + sort_field: + direction: desc + field_path: created_on + created_on_legacy_ASC: + sort_field: + direction: asc + field_path: created_on + created_on_legacy_DESC: + sort_field: + direction: desc + field_path: created_on + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + inventor_name_ASC: + sort_field: + direction: asc + field_path: inventor.name + inventor_name_DESC: + sort_field: + direction: desc + field_path: inventor.name + inventor_nationality_ASC: + sort_field: + direction: asc + field_path: inventor.nationality + inventor_nationality_DESC: + sort_field: + direction: desc + field_path: inventor.nationality + inventor_stock_ticker_ASC: + sort_field: + direction: asc + field_path: inventor.stock_ticker + inventor_stock_ticker_DESC: + sort_field: + direction: desc + field_path: inventor.stock_ticker + metadata_ASC: + sort_field: + direction: asc + field_path: metadata + metadata_DESC: + sort_field: + direction: desc + field_path: metadata + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + named_inventor_name_ASC: + sort_field: + direction: asc + field_path: named_inventor.name + named_inventor_name_DESC: + sort_field: + direction: desc + field_path: named_inventor.name + named_inventor_nationality_ASC: + sort_field: + direction: asc + field_path: named_inventor.nationality + named_inventor_nationality_DESC: + sort_field: + direction: desc + field_path: named_inventor.nationality + named_inventor_stock_ticker_ASC: + sort_field: + direction: asc + field_path: named_inventor.stock_ticker + named_inventor_stock_ticker_DESC: + sort_field: + direction: desc + field_path: named_inventor.stock_ticker + options_color_ASC: + sort_field: + direction: asc + field_path: options.color + options_color_DESC: + sort_field: + direction: desc + field_path: options.color + options_size_ASC: + sort_field: + direction: asc + field_path: options.size + options_size_DESC: + sort_field: + direction: desc + field_path: options.size + options_the_size_ASC: + sort_field: + direction: asc + field_path: options.the_sighs + options_the_size_DESC: + sort_field: + direction: desc + field_path: options.the_sighs + size_ASC: + sort_field: + direction: asc + field_path: options.size + size_DESC: + sort_field: + direction: desc + field_path: options.size + the_options_color_ASC: + sort_field: + direction: asc + field_path: the_opts.color + the_options_color_DESC: + sort_field: + direction: desc + field_path: the_opts.color + the_options_size_ASC: + sort_field: + direction: asc + field_path: the_opts.size + the_options_size_DESC: + sort_field: + direction: desc + field_path: the_opts.size + the_options_the_size_ASC: + sort_field: + direction: asc + field_path: the_opts.the_sighs + the_options_the_size_DESC: + sort_field: + direction: desc + field_path: the_opts.the_sighs + weight_in_ng_ASC: + sort_field: + direction: asc + field_path: weight_in_ng + weight_in_ng_DESC: + sort_field: + direction: desc + field_path: weight_in_ng + weight_in_ng_str_ASC: + sort_field: + direction: asc + field_path: weight_in_ng_str + weight_in_ng_str_DESC: + sort_field: + direction: desc + field_path: weight_in_ng_str + workspace_id_ASC: + sort_field: + direction: asc + field_path: workspace_id2 + workspace_id_DESC: + sort_field: + direction: desc + field_path: workspace_id2 + workspace_name_ASC: + sort_field: + direction: asc + field_path: workspace_name + workspace_name_DESC: + sort_field: + direction: desc + field_path: workspace_name + WidgetWorkspaceSortOrderInput: + values_by_name: + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + widget_created_at_ASC: + sort_field: + direction: asc + field_path: widget.created_at + widget_created_at_DESC: + sort_field: + direction: desc + field_path: widget.created_at + widget_id_ASC: + sort_field: + direction: asc + field_path: widget.id + widget_id_DESC: + sort_field: + direction: desc + field_path: widget.id +index_definitions_by_name: + addresses: + current_sources: + - __self + default_sort_fields: + - direction: desc + field_path: id + fields_by_path: + __counts.shapes: + source: __self + __counts.shapes|coordinates: + source: __self + __counts.shapes|type: + source: __self + full_address: + source: __self + geo_location.lat: + source: __self + geo_location.lon: + source: __self + id: + source: __self + manufacturer_id: + source: __self + shapes.coordinates: + source: __self + shapes.type: + source: __self + timestamps.created_at: + source: __self + route_with: id + components: + current_sources: + - __self + - widget + default_sort_fields: + - direction: desc + field_path: created_at + fields_by_path: + __counts.part_ids: + source: __self + __counts.tags: + source: __self + __counts.widget_tags: + source: widget + created_at: + source: __self + id: + source: __self + name: + source: __self + part_ids: + source: __self + position.x: + source: __self + position.y: + source: __self + tags: + source: __self + widget_cost.amount_cents: + source: widget + widget_cost.currency: + source: widget + widget_name: + source: widget + widget_size: + source: widget + widget_tags: + source: widget + widget_workspace_id3: + source: widget + route_with: id + electrical_parts: + current_sources: + - __self + default_sort_fields: + - direction: desc + field_path: created_at + fields_by_path: + created_at: + source: __self + id: + source: __self + manufacturer_id: + source: __self + name: + source: __self + voltage: + source: __self + route_with: id + manufacturers: + current_sources: + - __self + default_sort_fields: + - direction: desc + field_path: created_at + fields_by_path: + created_at: + source: __self + id: + source: __self + name: + source: __self + route_with: id + mechanical_parts: + current_sources: + - __self + default_sort_fields: + - direction: desc + field_path: created_at + fields_by_path: + created_at: + source: __self + id: + source: __self + manufacturer_id: + source: __self + material: + source: __self + name: + source: __self + route_with: id + sponsors: + current_sources: + - __self + fields_by_path: + id: + source: __self + name: + source: __self + route_with: id + teams: + current_sources: + - __self + fields_by_path: + __counts.current_players_nested: + source: __self + __counts.current_players_object: + source: __self + __counts.current_players_object|affiliations: + source: __self + __counts.current_players_object|affiliations|sponsorships_nested: + source: __self + __counts.current_players_object|affiliations|sponsorships_object: + source: __self + __counts.current_players_object|affiliations|sponsorships_object|annual_total: + source: __self + __counts.current_players_object|affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + __counts.current_players_object|affiliations|sponsorships_object|annual_total|currency: + source: __self + __counts.current_players_object|affiliations|sponsorships_object|sponsor_id: + source: __self + __counts.current_players_object|name: + source: __self + __counts.current_players_object|nicknames: + source: __self + __counts.current_players_object|seasons_nested: + source: __self + __counts.current_players_object|seasons_object: + source: __self + __counts.current_players_object|seasons_object|awards: + source: __self + __counts.current_players_object|seasons_object|games_played: + source: __self + __counts.current_players_object|seasons_object|year: + source: __self + __counts.details|uniform_colors: + source: __self + __counts.forbes_valuation_moneys_nested: + source: __self + __counts.forbes_valuation_moneys_object: + source: __self + __counts.forbes_valuation_moneys_object|amount_cents: + source: __self + __counts.forbes_valuation_moneys_object|currency: + source: __self + __counts.forbes_valuations: + source: __self + __counts.nested_fields2|current_players: + source: __self + __counts.nested_fields2|forbes_valuation_moneys: + source: __self + __counts.nested_fields2|the_seasons: + source: __self + __counts.past_names: + source: __self + __counts.seasons_nested: + source: __self + __counts.seasons_object: + source: __self + __counts.seasons_object|count: + source: __self + __counts.seasons_object|notes: + source: __self + __counts.seasons_object|players_nested: + source: __self + __counts.seasons_object|players_object: + source: __self + __counts.seasons_object|players_object|affiliations: + source: __self + __counts.seasons_object|players_object|affiliations|sponsorships_nested: + source: __self + __counts.seasons_object|players_object|affiliations|sponsorships_object: + source: __self + __counts.seasons_object|players_object|affiliations|sponsorships_object|annual_total: + source: __self + __counts.seasons_object|players_object|affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + __counts.seasons_object|players_object|affiliations|sponsorships_object|annual_total|currency: + source: __self + __counts.seasons_object|players_object|affiliations|sponsorships_object|sponsor_id: + source: __self + __counts.seasons_object|players_object|name: + source: __self + __counts.seasons_object|players_object|nicknames: + source: __self + __counts.seasons_object|players_object|seasons_nested: + source: __self + __counts.seasons_object|players_object|seasons_object: + source: __self + __counts.seasons_object|players_object|seasons_object|awards: + source: __self + __counts.seasons_object|players_object|seasons_object|games_played: + source: __self + __counts.seasons_object|players_object|seasons_object|year: + source: __self + __counts.seasons_object|started_at: + source: __self + __counts.seasons_object|the_record: + source: __self + __counts.seasons_object|the_record|first_win_on: + source: __self + __counts.seasons_object|the_record|last_win_date: + source: __self + __counts.seasons_object|the_record|loss_count: + source: __self + __counts.seasons_object|the_record|win_count: + source: __self + __counts.seasons_object|won_games_at: + source: __self + __counts.seasons_object|year: + source: __self + __counts.the_nested_fields|current_players: + source: __self + __counts.the_nested_fields|forbes_valuation_moneys: + source: __self + __counts.the_nested_fields|the_seasons: + source: __self + __counts.won_championships_at: + source: __self + country_code: + source: __self + current_name: + source: __self + current_players_nested.__counts.affiliations|sponsorships_nested: + source: __self + current_players_nested.__counts.affiliations|sponsorships_object: + source: __self + current_players_nested.__counts.affiliations|sponsorships_object|annual_total: + source: __self + current_players_nested.__counts.affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + current_players_nested.__counts.affiliations|sponsorships_object|annual_total|currency: + source: __self + current_players_nested.__counts.affiliations|sponsorships_object|sponsor_id: + source: __self + current_players_nested.__counts.nicknames: + source: __self + current_players_nested.__counts.seasons_nested: + source: __self + current_players_nested.__counts.seasons_object: + source: __self + current_players_nested.__counts.seasons_object|awards: + source: __self + current_players_nested.__counts.seasons_object|games_played: + source: __self + current_players_nested.__counts.seasons_object|year: + source: __self + current_players_nested.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + current_players_nested.affiliations.sponsorships_nested.annual_total.currency: + source: __self + current_players_nested.affiliations.sponsorships_nested.sponsor_id: + source: __self + current_players_nested.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + current_players_nested.affiliations.sponsorships_object.annual_total.currency: + source: __self + current_players_nested.affiliations.sponsorships_object.sponsor_id: + source: __self + current_players_nested.name: + source: __self + current_players_nested.nicknames: + source: __self + current_players_nested.seasons_nested.__counts.awards: + source: __self + current_players_nested.seasons_nested.awards: + source: __self + current_players_nested.seasons_nested.games_played: + source: __self + current_players_nested.seasons_nested.year: + source: __self + current_players_nested.seasons_object.awards: + source: __self + current_players_nested.seasons_object.games_played: + source: __self + current_players_nested.seasons_object.year: + source: __self + current_players_object.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + current_players_object.affiliations.sponsorships_nested.annual_total.currency: + source: __self + current_players_object.affiliations.sponsorships_nested.sponsor_id: + source: __self + current_players_object.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + current_players_object.affiliations.sponsorships_object.annual_total.currency: + source: __self + current_players_object.affiliations.sponsorships_object.sponsor_id: + source: __self + current_players_object.name: + source: __self + current_players_object.nicknames: + source: __self + current_players_object.seasons_nested.__counts.awards: + source: __self + current_players_object.seasons_nested.awards: + source: __self + current_players_object.seasons_nested.games_played: + source: __self + current_players_object.seasons_nested.year: + source: __self + current_players_object.seasons_object.awards: + source: __self + current_players_object.seasons_object.games_played: + source: __self + current_players_object.seasons_object.year: + source: __self + details.count: + source: __self + details.uniform_colors: + source: __self + forbes_valuation_moneys_nested.amount_cents: + source: __self + forbes_valuation_moneys_nested.currency: + source: __self + forbes_valuation_moneys_object.amount_cents: + source: __self + forbes_valuation_moneys_object.currency: + source: __self + forbes_valuations: + source: __self + formed_on: + source: __self + id: + source: __self + league: + source: __self + nested_fields2.current_players.__counts.affiliations|sponsorships_nested: + source: __self + nested_fields2.current_players.__counts.affiliations|sponsorships_object: + source: __self + nested_fields2.current_players.__counts.affiliations|sponsorships_object|annual_total: + source: __self + nested_fields2.current_players.__counts.affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + nested_fields2.current_players.__counts.affiliations|sponsorships_object|annual_total|currency: + source: __self + nested_fields2.current_players.__counts.affiliations|sponsorships_object|sponsor_id: + source: __self + nested_fields2.current_players.__counts.nicknames: + source: __self + nested_fields2.current_players.__counts.seasons_nested: + source: __self + nested_fields2.current_players.__counts.seasons_object: + source: __self + nested_fields2.current_players.__counts.seasons_object|awards: + source: __self + nested_fields2.current_players.__counts.seasons_object|games_played: + source: __self + nested_fields2.current_players.__counts.seasons_object|year: + source: __self + nested_fields2.current_players.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + nested_fields2.current_players.affiliations.sponsorships_nested.annual_total.currency: + source: __self + nested_fields2.current_players.affiliations.sponsorships_nested.sponsor_id: + source: __self + nested_fields2.current_players.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + nested_fields2.current_players.affiliations.sponsorships_object.annual_total.currency: + source: __self + nested_fields2.current_players.affiliations.sponsorships_object.sponsor_id: + source: __self + nested_fields2.current_players.name: + source: __self + nested_fields2.current_players.nicknames: + source: __self + nested_fields2.current_players.seasons_nested.__counts.awards: + source: __self + nested_fields2.current_players.seasons_nested.awards: + source: __self + nested_fields2.current_players.seasons_nested.games_played: + source: __self + nested_fields2.current_players.seasons_nested.year: + source: __self + nested_fields2.current_players.seasons_object.awards: + source: __self + nested_fields2.current_players.seasons_object.games_played: + source: __self + nested_fields2.current_players.seasons_object.year: + source: __self + nested_fields2.forbes_valuation_moneys.amount_cents: + source: __self + nested_fields2.forbes_valuation_moneys.currency: + source: __self + nested_fields2.the_seasons.__counts.notes: + source: __self + nested_fields2.the_seasons.__counts.players_nested: + source: __self + nested_fields2.the_seasons.__counts.players_object: + source: __self + nested_fields2.the_seasons.__counts.players_object|affiliations: + source: __self + nested_fields2.the_seasons.__counts.players_object|affiliations|sponsorships_nested: + source: __self + nested_fields2.the_seasons.__counts.players_object|affiliations|sponsorships_object: + source: __self + nested_fields2.the_seasons.__counts.players_object|affiliations|sponsorships_object|annual_total: + source: __self + nested_fields2.the_seasons.__counts.players_object|affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + nested_fields2.the_seasons.__counts.players_object|affiliations|sponsorships_object|annual_total|currency: + source: __self + nested_fields2.the_seasons.__counts.players_object|affiliations|sponsorships_object|sponsor_id: + source: __self + nested_fields2.the_seasons.__counts.players_object|name: + source: __self + nested_fields2.the_seasons.__counts.players_object|nicknames: + source: __self + nested_fields2.the_seasons.__counts.players_object|seasons_nested: + source: __self + nested_fields2.the_seasons.__counts.players_object|seasons_object: + source: __self + nested_fields2.the_seasons.__counts.players_object|seasons_object|awards: + source: __self + nested_fields2.the_seasons.__counts.players_object|seasons_object|games_played: + source: __self + nested_fields2.the_seasons.__counts.players_object|seasons_object|year: + source: __self + nested_fields2.the_seasons.__counts.won_games_at: + source: __self + nested_fields2.the_seasons.count: + source: __self + nested_fields2.the_seasons.notes: + source: __self + nested_fields2.the_seasons.players_nested.__counts.affiliations|sponsorships_nested: + source: __self + nested_fields2.the_seasons.players_nested.__counts.affiliations|sponsorships_object: + source: __self + nested_fields2.the_seasons.players_nested.__counts.affiliations|sponsorships_object|annual_total: + source: __self + nested_fields2.the_seasons.players_nested.__counts.affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + nested_fields2.the_seasons.players_nested.__counts.affiliations|sponsorships_object|annual_total|currency: + source: __self + nested_fields2.the_seasons.players_nested.__counts.affiliations|sponsorships_object|sponsor_id: + source: __self + nested_fields2.the_seasons.players_nested.__counts.nicknames: + source: __self + nested_fields2.the_seasons.players_nested.__counts.seasons_nested: + source: __self + nested_fields2.the_seasons.players_nested.__counts.seasons_object: + source: __self + nested_fields2.the_seasons.players_nested.__counts.seasons_object|awards: + source: __self + nested_fields2.the_seasons.players_nested.__counts.seasons_object|games_played: + source: __self + nested_fields2.the_seasons.players_nested.__counts.seasons_object|year: + source: __self + nested_fields2.the_seasons.players_nested.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + nested_fields2.the_seasons.players_nested.affiliations.sponsorships_nested.annual_total.currency: + source: __self + nested_fields2.the_seasons.players_nested.affiliations.sponsorships_nested.sponsor_id: + source: __self + nested_fields2.the_seasons.players_nested.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + nested_fields2.the_seasons.players_nested.affiliations.sponsorships_object.annual_total.currency: + source: __self + nested_fields2.the_seasons.players_nested.affiliations.sponsorships_object.sponsor_id: + source: __self + nested_fields2.the_seasons.players_nested.name: + source: __self + nested_fields2.the_seasons.players_nested.nicknames: + source: __self + nested_fields2.the_seasons.players_nested.seasons_nested.__counts.awards: + source: __self + nested_fields2.the_seasons.players_nested.seasons_nested.awards: + source: __self + nested_fields2.the_seasons.players_nested.seasons_nested.games_played: + source: __self + nested_fields2.the_seasons.players_nested.seasons_nested.year: + source: __self + nested_fields2.the_seasons.players_nested.seasons_object.awards: + source: __self + nested_fields2.the_seasons.players_nested.seasons_object.games_played: + source: __self + nested_fields2.the_seasons.players_nested.seasons_object.year: + source: __self + nested_fields2.the_seasons.players_object.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + nested_fields2.the_seasons.players_object.affiliations.sponsorships_nested.annual_total.currency: + source: __self + nested_fields2.the_seasons.players_object.affiliations.sponsorships_nested.sponsor_id: + source: __self + nested_fields2.the_seasons.players_object.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + nested_fields2.the_seasons.players_object.affiliations.sponsorships_object.annual_total.currency: + source: __self + nested_fields2.the_seasons.players_object.affiliations.sponsorships_object.sponsor_id: + source: __self + nested_fields2.the_seasons.players_object.name: + source: __self + nested_fields2.the_seasons.players_object.nicknames: + source: __self + nested_fields2.the_seasons.players_object.seasons_nested.__counts.awards: + source: __self + nested_fields2.the_seasons.players_object.seasons_nested.awards: + source: __self + nested_fields2.the_seasons.players_object.seasons_nested.games_played: + source: __self + nested_fields2.the_seasons.players_object.seasons_nested.year: + source: __self + nested_fields2.the_seasons.players_object.seasons_object.awards: + source: __self + nested_fields2.the_seasons.players_object.seasons_object.games_played: + source: __self + nested_fields2.the_seasons.players_object.seasons_object.year: + source: __self + nested_fields2.the_seasons.started_at: + source: __self + nested_fields2.the_seasons.the_record.first_win_on: + source: __self + nested_fields2.the_seasons.the_record.last_win_date: + source: __self + nested_fields2.the_seasons.the_record.loss_count: + source: __self + nested_fields2.the_seasons.the_record.win_count: + source: __self + nested_fields2.the_seasons.won_games_at: + source: __self + nested_fields2.the_seasons.year: + source: __self + past_names: + source: __self + seasons_nested.__counts.notes: + source: __self + seasons_nested.__counts.players_nested: + source: __self + seasons_nested.__counts.players_object: + source: __self + seasons_nested.__counts.players_object|affiliations: + source: __self + seasons_nested.__counts.players_object|affiliations|sponsorships_nested: + source: __self + seasons_nested.__counts.players_object|affiliations|sponsorships_object: + source: __self + seasons_nested.__counts.players_object|affiliations|sponsorships_object|annual_total: + source: __self + seasons_nested.__counts.players_object|affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + seasons_nested.__counts.players_object|affiliations|sponsorships_object|annual_total|currency: + source: __self + seasons_nested.__counts.players_object|affiliations|sponsorships_object|sponsor_id: + source: __self + seasons_nested.__counts.players_object|name: + source: __self + seasons_nested.__counts.players_object|nicknames: + source: __self + seasons_nested.__counts.players_object|seasons_nested: + source: __self + seasons_nested.__counts.players_object|seasons_object: + source: __self + seasons_nested.__counts.players_object|seasons_object|awards: + source: __self + seasons_nested.__counts.players_object|seasons_object|games_played: + source: __self + seasons_nested.__counts.players_object|seasons_object|year: + source: __self + seasons_nested.__counts.won_games_at: + source: __self + seasons_nested.count: + source: __self + seasons_nested.notes: + source: __self + seasons_nested.players_nested.__counts.affiliations|sponsorships_nested: + source: __self + seasons_nested.players_nested.__counts.affiliations|sponsorships_object: + source: __self + seasons_nested.players_nested.__counts.affiliations|sponsorships_object|annual_total: + source: __self + seasons_nested.players_nested.__counts.affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + seasons_nested.players_nested.__counts.affiliations|sponsorships_object|annual_total|currency: + source: __self + seasons_nested.players_nested.__counts.affiliations|sponsorships_object|sponsor_id: + source: __self + seasons_nested.players_nested.__counts.nicknames: + source: __self + seasons_nested.players_nested.__counts.seasons_nested: + source: __self + seasons_nested.players_nested.__counts.seasons_object: + source: __self + seasons_nested.players_nested.__counts.seasons_object|awards: + source: __self + seasons_nested.players_nested.__counts.seasons_object|games_played: + source: __self + seasons_nested.players_nested.__counts.seasons_object|year: + source: __self + seasons_nested.players_nested.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + seasons_nested.players_nested.affiliations.sponsorships_nested.annual_total.currency: + source: __self + seasons_nested.players_nested.affiliations.sponsorships_nested.sponsor_id: + source: __self + seasons_nested.players_nested.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + seasons_nested.players_nested.affiliations.sponsorships_object.annual_total.currency: + source: __self + seasons_nested.players_nested.affiliations.sponsorships_object.sponsor_id: + source: __self + seasons_nested.players_nested.name: + source: __self + seasons_nested.players_nested.nicknames: + source: __self + seasons_nested.players_nested.seasons_nested.__counts.awards: + source: __self + seasons_nested.players_nested.seasons_nested.awards: + source: __self + seasons_nested.players_nested.seasons_nested.games_played: + source: __self + seasons_nested.players_nested.seasons_nested.year: + source: __self + seasons_nested.players_nested.seasons_object.awards: + source: __self + seasons_nested.players_nested.seasons_object.games_played: + source: __self + seasons_nested.players_nested.seasons_object.year: + source: __self + seasons_nested.players_object.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + seasons_nested.players_object.affiliations.sponsorships_nested.annual_total.currency: + source: __self + seasons_nested.players_object.affiliations.sponsorships_nested.sponsor_id: + source: __self + seasons_nested.players_object.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + seasons_nested.players_object.affiliations.sponsorships_object.annual_total.currency: + source: __self + seasons_nested.players_object.affiliations.sponsorships_object.sponsor_id: + source: __self + seasons_nested.players_object.name: + source: __self + seasons_nested.players_object.nicknames: + source: __self + seasons_nested.players_object.seasons_nested.__counts.awards: + source: __self + seasons_nested.players_object.seasons_nested.awards: + source: __self + seasons_nested.players_object.seasons_nested.games_played: + source: __self + seasons_nested.players_object.seasons_nested.year: + source: __self + seasons_nested.players_object.seasons_object.awards: + source: __self + seasons_nested.players_object.seasons_object.games_played: + source: __self + seasons_nested.players_object.seasons_object.year: + source: __self + seasons_nested.started_at: + source: __self + seasons_nested.the_record.first_win_on: + source: __self + seasons_nested.the_record.last_win_date: + source: __self + seasons_nested.the_record.loss_count: + source: __self + seasons_nested.the_record.win_count: + source: __self + seasons_nested.won_games_at: + source: __self + seasons_nested.year: + source: __self + seasons_object.count: + source: __self + seasons_object.notes: + source: __self + seasons_object.players_nested.__counts.affiliations|sponsorships_nested: + source: __self + seasons_object.players_nested.__counts.affiliations|sponsorships_object: + source: __self + seasons_object.players_nested.__counts.affiliations|sponsorships_object|annual_total: + source: __self + seasons_object.players_nested.__counts.affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + seasons_object.players_nested.__counts.affiliations|sponsorships_object|annual_total|currency: + source: __self + seasons_object.players_nested.__counts.affiliations|sponsorships_object|sponsor_id: + source: __self + seasons_object.players_nested.__counts.nicknames: + source: __self + seasons_object.players_nested.__counts.seasons_nested: + source: __self + seasons_object.players_nested.__counts.seasons_object: + source: __self + seasons_object.players_nested.__counts.seasons_object|awards: + source: __self + seasons_object.players_nested.__counts.seasons_object|games_played: + source: __self + seasons_object.players_nested.__counts.seasons_object|year: + source: __self + seasons_object.players_nested.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + seasons_object.players_nested.affiliations.sponsorships_nested.annual_total.currency: + source: __self + seasons_object.players_nested.affiliations.sponsorships_nested.sponsor_id: + source: __self + seasons_object.players_nested.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + seasons_object.players_nested.affiliations.sponsorships_object.annual_total.currency: + source: __self + seasons_object.players_nested.affiliations.sponsorships_object.sponsor_id: + source: __self + seasons_object.players_nested.name: + source: __self + seasons_object.players_nested.nicknames: + source: __self + seasons_object.players_nested.seasons_nested.__counts.awards: + source: __self + seasons_object.players_nested.seasons_nested.awards: + source: __self + seasons_object.players_nested.seasons_nested.games_played: + source: __self + seasons_object.players_nested.seasons_nested.year: + source: __self + seasons_object.players_nested.seasons_object.awards: + source: __self + seasons_object.players_nested.seasons_object.games_played: + source: __self + seasons_object.players_nested.seasons_object.year: + source: __self + seasons_object.players_object.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + seasons_object.players_object.affiliations.sponsorships_nested.annual_total.currency: + source: __self + seasons_object.players_object.affiliations.sponsorships_nested.sponsor_id: + source: __self + seasons_object.players_object.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + seasons_object.players_object.affiliations.sponsorships_object.annual_total.currency: + source: __self + seasons_object.players_object.affiliations.sponsorships_object.sponsor_id: + source: __self + seasons_object.players_object.name: + source: __self + seasons_object.players_object.nicknames: + source: __self + seasons_object.players_object.seasons_nested.__counts.awards: + source: __self + seasons_object.players_object.seasons_nested.awards: + source: __self + seasons_object.players_object.seasons_nested.games_played: + source: __self + seasons_object.players_object.seasons_nested.year: + source: __self + seasons_object.players_object.seasons_object.awards: + source: __self + seasons_object.players_object.seasons_object.games_played: + source: __self + seasons_object.players_object.seasons_object.year: + source: __self + seasons_object.started_at: + source: __self + seasons_object.the_record.first_win_on: + source: __self + seasons_object.the_record.last_win_date: + source: __self + seasons_object.the_record.loss_count: + source: __self + seasons_object.the_record.win_count: + source: __self + seasons_object.won_games_at: + source: __self + seasons_object.year: + source: __self + stadium_location.lat: + source: __self + stadium_location.lon: + source: __self + the_nested_fields.current_players.__counts.affiliations|sponsorships_nested: + source: __self + the_nested_fields.current_players.__counts.affiliations|sponsorships_object: + source: __self + the_nested_fields.current_players.__counts.affiliations|sponsorships_object|annual_total: + source: __self + the_nested_fields.current_players.__counts.affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + the_nested_fields.current_players.__counts.affiliations|sponsorships_object|annual_total|currency: + source: __self + the_nested_fields.current_players.__counts.affiliations|sponsorships_object|sponsor_id: + source: __self + the_nested_fields.current_players.__counts.nicknames: + source: __self + the_nested_fields.current_players.__counts.seasons_nested: + source: __self + the_nested_fields.current_players.__counts.seasons_object: + source: __self + the_nested_fields.current_players.__counts.seasons_object|awards: + source: __self + the_nested_fields.current_players.__counts.seasons_object|games_played: + source: __self + the_nested_fields.current_players.__counts.seasons_object|year: + source: __self + the_nested_fields.current_players.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + the_nested_fields.current_players.affiliations.sponsorships_nested.annual_total.currency: + source: __self + the_nested_fields.current_players.affiliations.sponsorships_nested.sponsor_id: + source: __self + the_nested_fields.current_players.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + the_nested_fields.current_players.affiliations.sponsorships_object.annual_total.currency: + source: __self + the_nested_fields.current_players.affiliations.sponsorships_object.sponsor_id: + source: __self + the_nested_fields.current_players.name: + source: __self + the_nested_fields.current_players.nicknames: + source: __self + the_nested_fields.current_players.seasons_nested.__counts.awards: + source: __self + the_nested_fields.current_players.seasons_nested.awards: + source: __self + the_nested_fields.current_players.seasons_nested.games_played: + source: __self + the_nested_fields.current_players.seasons_nested.year: + source: __self + the_nested_fields.current_players.seasons_object.awards: + source: __self + the_nested_fields.current_players.seasons_object.games_played: + source: __self + the_nested_fields.current_players.seasons_object.year: + source: __self + the_nested_fields.forbes_valuation_moneys.amount_cents: + source: __self + the_nested_fields.forbes_valuation_moneys.currency: + source: __self + the_nested_fields.the_seasons.__counts.notes: + source: __self + the_nested_fields.the_seasons.__counts.players_nested: + source: __self + the_nested_fields.the_seasons.__counts.players_object: + source: __self + the_nested_fields.the_seasons.__counts.players_object|affiliations: + source: __self + the_nested_fields.the_seasons.__counts.players_object|affiliations|sponsorships_nested: + source: __self + the_nested_fields.the_seasons.__counts.players_object|affiliations|sponsorships_object: + source: __self + the_nested_fields.the_seasons.__counts.players_object|affiliations|sponsorships_object|annual_total: + source: __self + the_nested_fields.the_seasons.__counts.players_object|affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + the_nested_fields.the_seasons.__counts.players_object|affiliations|sponsorships_object|annual_total|currency: + source: __self + the_nested_fields.the_seasons.__counts.players_object|affiliations|sponsorships_object|sponsor_id: + source: __self + the_nested_fields.the_seasons.__counts.players_object|name: + source: __self + the_nested_fields.the_seasons.__counts.players_object|nicknames: + source: __self + the_nested_fields.the_seasons.__counts.players_object|seasons_nested: + source: __self + the_nested_fields.the_seasons.__counts.players_object|seasons_object: + source: __self + the_nested_fields.the_seasons.__counts.players_object|seasons_object|awards: + source: __self + the_nested_fields.the_seasons.__counts.players_object|seasons_object|games_played: + source: __self + the_nested_fields.the_seasons.__counts.players_object|seasons_object|year: + source: __self + the_nested_fields.the_seasons.__counts.won_games_at: + source: __self + the_nested_fields.the_seasons.count: + source: __self + the_nested_fields.the_seasons.notes: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.affiliations|sponsorships_nested: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.affiliations|sponsorships_object: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.affiliations|sponsorships_object|annual_total: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.affiliations|sponsorships_object|annual_total|currency: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.affiliations|sponsorships_object|sponsor_id: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.nicknames: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.seasons_nested: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.seasons_object: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.seasons_object|awards: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.seasons_object|games_played: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.seasons_object|year: + source: __self + the_nested_fields.the_seasons.players_nested.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + the_nested_fields.the_seasons.players_nested.affiliations.sponsorships_nested.annual_total.currency: + source: __self + the_nested_fields.the_seasons.players_nested.affiliations.sponsorships_nested.sponsor_id: + source: __self + the_nested_fields.the_seasons.players_nested.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + the_nested_fields.the_seasons.players_nested.affiliations.sponsorships_object.annual_total.currency: + source: __self + the_nested_fields.the_seasons.players_nested.affiliations.sponsorships_object.sponsor_id: + source: __self + the_nested_fields.the_seasons.players_nested.name: + source: __self + the_nested_fields.the_seasons.players_nested.nicknames: + source: __self + the_nested_fields.the_seasons.players_nested.seasons_nested.__counts.awards: + source: __self + the_nested_fields.the_seasons.players_nested.seasons_nested.awards: + source: __self + the_nested_fields.the_seasons.players_nested.seasons_nested.games_played: + source: __self + the_nested_fields.the_seasons.players_nested.seasons_nested.year: + source: __self + the_nested_fields.the_seasons.players_nested.seasons_object.awards: + source: __self + the_nested_fields.the_seasons.players_nested.seasons_object.games_played: + source: __self + the_nested_fields.the_seasons.players_nested.seasons_object.year: + source: __self + the_nested_fields.the_seasons.players_object.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + the_nested_fields.the_seasons.players_object.affiliations.sponsorships_nested.annual_total.currency: + source: __self + the_nested_fields.the_seasons.players_object.affiliations.sponsorships_nested.sponsor_id: + source: __self + the_nested_fields.the_seasons.players_object.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + the_nested_fields.the_seasons.players_object.affiliations.sponsorships_object.annual_total.currency: + source: __self + the_nested_fields.the_seasons.players_object.affiliations.sponsorships_object.sponsor_id: + source: __self + the_nested_fields.the_seasons.players_object.name: + source: __self + the_nested_fields.the_seasons.players_object.nicknames: + source: __self + the_nested_fields.the_seasons.players_object.seasons_nested.__counts.awards: + source: __self + the_nested_fields.the_seasons.players_object.seasons_nested.awards: + source: __self + the_nested_fields.the_seasons.players_object.seasons_nested.games_played: + source: __self + the_nested_fields.the_seasons.players_object.seasons_nested.year: + source: __self + the_nested_fields.the_seasons.players_object.seasons_object.awards: + source: __self + the_nested_fields.the_seasons.players_object.seasons_object.games_played: + source: __self + the_nested_fields.the_seasons.players_object.seasons_object.year: + source: __self + the_nested_fields.the_seasons.started_at: + source: __self + the_nested_fields.the_seasons.the_record.first_win_on: + source: __self + the_nested_fields.the_seasons.the_record.last_win_date: + source: __self + the_nested_fields.the_seasons.the_record.loss_count: + source: __self + the_nested_fields.the_seasons.the_record.win_count: + source: __self + the_nested_fields.the_seasons.won_games_at: + source: __self + the_nested_fields.the_seasons.year: + source: __self + won_championships_at: + source: __self + rollover: + frequency: yearly + timestamp_field_path: formed_on + route_with: league + widget_currencies: + current_sources: + - __self + fields_by_path: + __counts.widget_fee_currencies: + source: __self + __counts.widget_names2: + source: __self + __counts.widget_options|colors: + source: __self + __counts.widget_options|sizes: + source: __self + __counts.widget_tags: + source: __self + details.symbol: + source: __self + details.unit: + source: __self + id: + source: __self + introduced_on: + source: __self + name: + source: __self + nested_fields.max_widget_cost: + source: __self + oldest_widget_created_at: + source: __self + primary_continent: + source: __self + widget_fee_currencies: + source: __self + widget_names2: + source: __self + widget_options.colors: + source: __self + widget_options.sizes: + source: __self + widget_tags: + source: __self + rollover: + frequency: yearly + timestamp_field_path: introduced_on + route_with: primary_continent + widget_workspaces: + current_sources: + - __self + fields_by_path: + id: + source: __self + name: + source: __self + widget.created_at: + source: __self + widget.id: + source: __self + route_with: id + widgets: + current_sources: + - __self + - workspace + default_sort_fields: + - direction: desc + field_path: created_at + fields_by_path: + __counts.amounts: + source: __self + __counts.component_ids: + source: __self + __counts.fees: + source: __self + __counts.fees|amount_cents: + source: __self + __counts.fees|currency: + source: __self + __counts.release_dates: + source: __self + __counts.release_timestamps: + source: __self + __counts.tags: + source: __self + amount_cents: + source: __self + amounts: + source: __self + component_ids: + source: __self + cost.amount_cents: + source: __self + cost.currency: + source: __self + cost_currency_introduced_on: + source: __self + cost_currency_name: + source: __self + cost_currency_primary_continent: + source: __self + cost_currency_symbol: + source: __self + cost_currency_unit: + source: __self + created_at: + source: __self + created_at_time_of_day: + source: __self + created_on: + source: __self + fees.amount_cents: + source: __self + fees.currency: + source: __self + id: + source: __self + inventor.name: + source: __self + inventor.nationality: + source: __self + inventor.stock_ticker: + source: __self + metadata: + source: __self + name: + source: __self + name_text: + source: __self + named_inventor.name: + source: __self + named_inventor.nationality: + source: __self + named_inventor.stock_ticker: + source: __self + options.color: + source: __self + options.size: + source: __self + options.the_sighs: + source: __self + release_dates: + source: __self + release_timestamps: + source: __self + tags: + source: __self + the_opts.color: + source: __self + the_opts.size: + source: __self + the_opts.the_sighs: + source: __self + weight_in_ng: + source: __self + weight_in_ng_str: + source: __self + workspace_id2: + source: __self + workspace_name: + source: workspace + rollover: + frequency: yearly + timestamp_field_path: created_at + route_with: workspace_id2 +object_types_by_name: + Address: + graphql_fields_by_name: + manufacturer: + relation: + direction: out + foreign_key: manufacturer_id + index_definition_names: + - addresses + update_targets: + - data_params: + full_address: + cardinality: one + geo_location: + cardinality: one + manufacturer_id: + cardinality: one + shapes: + cardinality: one + timestamps: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + routing_value_source: id + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: Address + AddressAggregation: + elasticgraph_category: indexed_aggregation + source_type: Address + AddressAggregationConnection: + elasticgraph_category: relay_connection + AddressAggregationEdge: + elasticgraph_category: relay_edge + AddressConnection: + elasticgraph_category: relay_connection + AddressEdge: + elasticgraph_category: relay_edge + AffiliationsFieldsListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + AggregationCountDetail: + graphql_only_return_type: true + ColorListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + Component: + graphql_fields_by_name: + dollar_widget: + relation: + additional_filter: + cost: + amount_cents: + equal_to_any_of: + - 100 + direction: in + foreign_key: component_ids + part_aggregations: + relation: + direction: out + foreign_key: part_ids + parts: + relation: + direction: out + foreign_key: part_ids + widget: + relation: + direction: in + foreign_key: component_ids + widget_aggregations: + relation: + direction: in + foreign_key: component_ids + widget_workspace_id: + name_in_index: widget_workspace_id3 + widgets: + relation: + direction: in + foreign_key: component_ids + index_definition_names: + - components + update_targets: + - data_params: + created_at: + cardinality: one + name: + cardinality: one + part_ids: + cardinality: one + position: + cardinality: one + tags: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + routing_value_source: id + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: Component + ComponentAggregatedValues: + graphql_fields_by_name: + widget_workspace_id: + name_in_index: widget_workspace_id3 + ComponentAggregation: + elasticgraph_category: indexed_aggregation + source_type: Component + ComponentAggregationConnection: + elasticgraph_category: relay_connection + ComponentAggregationEdge: + elasticgraph_category: relay_edge + ComponentConnection: + elasticgraph_category: relay_connection + ComponentEdge: + elasticgraph_category: relay_edge + ComponentFilterInput: + graphql_fields_by_name: + widget_workspace_id: + name_in_index: widget_workspace_id3 + ComponentGroupedBy: + graphql_fields_by_name: + widget_workspace_id: + name_in_index: widget_workspace_id3 + DateAggregatedValues: + elasticgraph_category: scalar_aggregated_values + graphql_fields_by_name: + approximate_avg: + computation_detail: + function: avg + approximate_distinct_value_count: + computation_detail: + empty_bucket_value: 0 + function: cardinality + exact_max: + computation_detail: + function: max + exact_min: + computation_detail: + function: min + graphql_only_return_type: true + DateGroupedBy: + elasticgraph_category: date_grouped_by_object + graphql_only_return_type: true + DateListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + DateTimeAggregatedValues: + elasticgraph_category: scalar_aggregated_values + graphql_fields_by_name: + approximate_avg: + computation_detail: + function: avg + approximate_distinct_value_count: + computation_detail: + empty_bucket_value: 0 + function: cardinality + exact_max: + computation_detail: + function: max + exact_min: + computation_detail: + function: min + graphql_only_return_type: true + DateTimeGroupedBy: + elasticgraph_category: date_grouped_by_object + graphql_only_return_type: true + DateTimeListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + ElectricalPart: + graphql_fields_by_name: + component_aggregations: + relation: + direction: in + foreign_key: part_ids + components: + relation: + direction: in + foreign_key: part_ids + manufacturer: + relation: + direction: out + foreign_key: manufacturer_id + index_definition_names: + - electrical_parts + update_targets: + - data_params: + created_at: + cardinality: one + manufacturer_id: + cardinality: one + name: + cardinality: one + voltage: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + routing_value_source: id + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: ElectricalPart + ElectricalPartAggregation: + elasticgraph_category: indexed_aggregation + source_type: ElectricalPart + ElectricalPartAggregationConnection: + elasticgraph_category: relay_connection + ElectricalPartAggregationEdge: + elasticgraph_category: relay_edge + ElectricalPartConnection: + elasticgraph_category: relay_connection + ElectricalPartEdge: + elasticgraph_category: relay_edge + FloatAggregatedValues: + elasticgraph_category: scalar_aggregated_values + graphql_fields_by_name: + approximate_avg: + computation_detail: + function: avg + approximate_distinct_value_count: + computation_detail: + empty_bucket_value: 0 + function: cardinality + approximate_sum: + computation_detail: + empty_bucket_value: 0 + function: sum + exact_max: + computation_detail: + function: max + exact_min: + computation_detail: + function: min + graphql_only_return_type: true + GeoLocation: + graphql_fields_by_name: + latitude: + name_in_index: lat + longitude: + name_in_index: lon + IDListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + IntAggregatedValues: + elasticgraph_category: scalar_aggregated_values + graphql_fields_by_name: + approximate_avg: + computation_detail: + function: avg + approximate_distinct_value_count: + computation_detail: + empty_bucket_value: 0 + function: cardinality + approximate_sum: + computation_detail: + empty_bucket_value: 0 + function: sum + exact_max: + computation_detail: + function: max + exact_min: + computation_detail: + function: min + exact_sum: + computation_detail: + empty_bucket_value: 0 + function: sum + graphql_only_return_type: true + IntListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + JsonSafeLongAggregatedValues: + elasticgraph_category: scalar_aggregated_values + graphql_fields_by_name: + approximate_avg: + computation_detail: + function: avg + approximate_distinct_value_count: + computation_detail: + empty_bucket_value: 0 + function: cardinality + approximate_sum: + computation_detail: + empty_bucket_value: 0 + function: sum + exact_max: + computation_detail: + function: max + exact_min: + computation_detail: + function: min + exact_sum: + computation_detail: + empty_bucket_value: 0 + function: sum + graphql_only_return_type: true + JsonSafeLongListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + LocalTimeAggregatedValues: + elasticgraph_category: scalar_aggregated_values + graphql_fields_by_name: + approximate_avg: + computation_detail: + function: avg + approximate_distinct_value_count: + computation_detail: + empty_bucket_value: 0 + function: cardinality + exact_max: + computation_detail: + function: max + exact_min: + computation_detail: + function: min + graphql_only_return_type: true + LongStringAggregatedValues: + elasticgraph_category: scalar_aggregated_values + graphql_fields_by_name: + approximate_avg: + computation_detail: + function: avg + approximate_distinct_value_count: + computation_detail: + empty_bucket_value: 0 + function: cardinality + approximate_max: + computation_detail: + function: max + approximate_min: + computation_detail: + function: min + approximate_sum: + computation_detail: + empty_bucket_value: 0 + function: sum + exact_max: + computation_detail: + function: max + exact_min: + computation_detail: + function: min + exact_sum: + computation_detail: + empty_bucket_value: 0 + function: sum + graphql_only_return_type: true + Manufacturer: + graphql_fields_by_name: + address: + relation: + direction: in + foreign_key: manufacturer_id + manufactured_part_aggregations: + relation: + direction: in + foreign_key: manufacturer_id + manufactured_parts: + relation: + direction: in + foreign_key: manufacturer_id + index_definition_names: + - manufacturers + update_targets: + - data_params: + created_at: + cardinality: one + name: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + routing_value_source: id + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: Manufacturer + ManufacturerAggregation: + elasticgraph_category: indexed_aggregation + source_type: Manufacturer + ManufacturerAggregationConnection: + elasticgraph_category: relay_connection + ManufacturerAggregationEdge: + elasticgraph_category: relay_edge + ManufacturerConnection: + elasticgraph_category: relay_connection + ManufacturerEdge: + elasticgraph_category: relay_edge + MechanicalPart: + graphql_fields_by_name: + component_aggregations: + relation: + direction: in + foreign_key: part_ids + components: + relation: + direction: in + foreign_key: part_ids + manufacturer: + relation: + direction: out + foreign_key: manufacturer_id + index_definition_names: + - mechanical_parts + update_targets: + - data_params: + created_at: + cardinality: one + manufacturer_id: + cardinality: one + material: + cardinality: one + name: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + routing_value_source: id + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: MechanicalPart + MechanicalPartAggregation: + elasticgraph_category: indexed_aggregation + source_type: MechanicalPart + MechanicalPartAggregationConnection: + elasticgraph_category: relay_connection + MechanicalPartAggregationEdge: + elasticgraph_category: relay_edge + MechanicalPartConnection: + elasticgraph_category: relay_connection + MechanicalPartEdge: + elasticgraph_category: relay_edge + MoneyFieldsListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + MoneyListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + NamedEntity: + graphql_fields_by_name: + address: + relation: + direction: in + foreign_key: manufacturer_id + amount_cents2: + name_in_index: amount_cents + component_aggregations: + relation: + direction: out + foreign_key: component_ids + components: + relation: + direction: out + foreign_key: component_ids + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + dollar_widget: + relation: + additional_filter: + cost: + amount_cents: + equal_to_any_of: + - 100 + direction: in + foreign_key: component_ids + manufactured_part_aggregations: + relation: + direction: in + foreign_key: manufacturer_id + manufactured_parts: + relation: + direction: in + foreign_key: manufacturer_id + manufacturer: + relation: + direction: out + foreign_key: manufacturer_id + part_aggregations: + relation: + direction: out + foreign_key: part_ids + parts: + relation: + direction: out + foreign_key: part_ids + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + widget: + relation: + direction: in + foreign_key: component_ids + widget_aggregations: + relation: + direction: in + foreign_key: component_ids + widget_workspace_id: + name_in_index: widget_workspace_id3 + widgets: + relation: + direction: in + foreign_key: component_ids + workspace: + relation: + direction: in + foreign_key: widget.id + workspace_id: + name_in_index: workspace_id2 + NamedEntityAggregatedValues: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + widget_workspace_id: + name_in_index: widget_workspace_id3 + workspace_id: + name_in_index: workspace_id2 + NamedEntityAggregation: + elasticgraph_category: indexed_aggregation + source_type: NamedEntity + NamedEntityAggregationConnection: + elasticgraph_category: relay_connection + NamedEntityAggregationEdge: + elasticgraph_category: relay_edge + NamedEntityConnection: + elasticgraph_category: relay_connection + NamedEntityEdge: + elasticgraph_category: relay_edge + NamedEntityFilterInput: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + widget_workspace_id: + name_in_index: widget_workspace_id3 + workspace_id: + name_in_index: workspace_id2 + NamedEntityGroupedBy: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + release_date: + name_in_index: release_dates + release_timestamp: + name_in_index: release_timestamps + size: + name_in_index: options.size + tag: + name_in_index: tags + the_options: + name_in_index: the_opts + widget_workspace_id: + name_in_index: widget_workspace_id3 + workspace_id: + name_in_index: workspace_id2 + NonNumericAggregatedValues: + elasticgraph_category: scalar_aggregated_values + graphql_fields_by_name: + approximate_distinct_value_count: + computation_detail: + empty_bucket_value: 0 + function: cardinality + graphql_only_return_type: true + PageInfo: + graphql_only_return_type: true + Part: + graphql_fields_by_name: + component_aggregations: + relation: + direction: in + foreign_key: part_ids + components: + relation: + direction: in + foreign_key: part_ids + manufacturer: + relation: + direction: out + foreign_key: manufacturer_id + PartAggregation: + elasticgraph_category: indexed_aggregation + source_type: Part + PartAggregationConnection: + elasticgraph_category: relay_connection + PartAggregationEdge: + elasticgraph_category: relay_edge + PartConnection: + elasticgraph_category: relay_connection + PartEdge: + elasticgraph_category: relay_edge + PlayerFieldsListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + PlayerListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + PlayerSeasonFieldsListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + PlayerSeasonListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + SizeListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + Sponsor: + graphql_fields_by_name: + affiliated_team_from_nested_aggregations: + relation: + direction: in + foreign_key: current_players_nested.affiliations.sponsorships_nested.sponsor_id + foreign_key_nested_paths: + - current_players_nested + - current_players_nested.affiliations.sponsorships_nested + affiliated_team_from_object_aggregations: + relation: + direction: in + foreign_key: current_players_object.affiliations.sponsorships_object.sponsor_id + affiliated_teams_from_nested: + relation: + direction: in + foreign_key: current_players_nested.affiliations.sponsorships_nested.sponsor_id + foreign_key_nested_paths: + - current_players_nested + - current_players_nested.affiliations.sponsorships_nested + affiliated_teams_from_object: + relation: + direction: in + foreign_key: current_players_object.affiliations.sponsorships_object.sponsor_id + index_definition_names: + - sponsors + update_targets: + - data_params: + name: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + routing_value_source: id + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: Sponsor + SponsorAggregation: + elasticgraph_category: indexed_aggregation + source_type: Sponsor + SponsorAggregationConnection: + elasticgraph_category: relay_connection + SponsorAggregationEdge: + elasticgraph_category: relay_edge + SponsorConnection: + elasticgraph_category: relay_connection + SponsorEdge: + elasticgraph_category: relay_edge + SponsorshipFieldsListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + SponsorshipListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + StringConnection: + elasticgraph_category: relay_connection + StringEdge: + elasticgraph_category: relay_edge + StringListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + Team: + graphql_fields_by_name: + nested_fields: + name_in_index: the_nested_fields + index_definition_names: + - teams + update_targets: + - data_params: + country_code: + cardinality: one + current_name: + cardinality: one + current_players_nested: + cardinality: one + current_players_object: + cardinality: one + details: + cardinality: one + forbes_valuation_moneys_nested: + cardinality: one + forbes_valuation_moneys_object: + cardinality: one + forbes_valuations: + cardinality: one + formed_on: + cardinality: one + league: + cardinality: one + nested_fields2: + cardinality: one + past_names: + cardinality: one + seasons_nested: + cardinality: one + seasons_object: + cardinality: one + stadium_location: + cardinality: one + the_nested_fields: + cardinality: one + won_championships_at: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + rollover_timestamp_value_source: formed_on + routing_value_source: league + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: Team + TeamAggregation: + elasticgraph_category: indexed_aggregation + source_type: Team + TeamAggregationConnection: + elasticgraph_category: relay_connection + TeamAggregationEdge: + elasticgraph_category: relay_edge + TeamAggregationNestedFields2SubAggregations: + graphql_fields_by_name: + seasons: + name_in_index: the_seasons + TeamAggregationNestedFieldsSubAggregations: + graphql_fields_by_name: + seasons: + name_in_index: the_seasons + TeamAggregationSubAggregations: + graphql_fields_by_name: + nested_fields: + name_in_index: the_nested_fields + TeamConnection: + elasticgraph_category: relay_connection + TeamEdge: + elasticgraph_category: relay_edge + TeamFilterInput: + graphql_fields_by_name: + nested_fields: + name_in_index: the_nested_fields + TeamMoneySubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamNestedFields: + graphql_fields_by_name: + seasons: + name_in_index: the_seasons + TeamNestedFieldsFilterInput: + graphql_fields_by_name: + seasons: + name_in_index: the_seasons + TeamPlayerPlayerSeasonSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamPlayerSeasonSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamPlayerSponsorshipSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamPlayerSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamRecord: + graphql_fields_by_name: + first_win_on_legacy: + name_in_index: first_win_on + last_win_on: + name_in_index: last_win_date + last_win_on_legacy: + name_in_index: last_win_date + losses: + name_in_index: loss_count + wins: + name_in_index: win_count + TeamRecordAggregatedValues: + graphql_fields_by_name: + first_win_on_legacy: + name_in_index: first_win_on + last_win_on: + name_in_index: last_win_date + last_win_on_legacy: + name_in_index: last_win_date + losses: + name_in_index: loss_count + wins: + name_in_index: win_count + TeamRecordFieldsListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + first_win_on_legacy: + name_in_index: first_win_on + last_win_on: + name_in_index: last_win_date + last_win_on_legacy: + name_in_index: last_win_date + losses: + name_in_index: loss_count + wins: + name_in_index: win_count + TeamRecordFilterInput: + graphql_fields_by_name: + first_win_on_legacy: + name_in_index: first_win_on + last_win_on: + name_in_index: last_win_date + last_win_on_legacy: + name_in_index: last_win_date + losses: + name_in_index: loss_count + wins: + name_in_index: win_count + TeamRecordGroupedBy: + graphql_fields_by_name: + first_win_on_legacy: + name_in_index: first_win_on + last_win_on: + name_in_index: last_win_date + last_win_on_legacy: + name_in_index: last_win_date + losses: + name_in_index: loss_count + wins: + name_in_index: win_count + TeamSeason: + graphql_fields_by_name: + record: + name_in_index: the_record + started_at_legacy: + name_in_index: started_at + won_games_at_legacy: + name_in_index: won_games_at + TeamSeasonAggregatedValues: + graphql_fields_by_name: + record: + name_in_index: the_record + started_at_legacy: + name_in_index: started_at + won_games_at_legacy: + name_in_index: won_games_at + TeamSeasonFieldsListFilterInput: + graphql_fields_by_name: + record: + name_in_index: the_record + started_at_legacy: + name_in_index: started_at + won_games_at_legacy: + name_in_index: won_games_at + TeamSeasonFilterInput: + graphql_fields_by_name: + record: + name_in_index: the_record + started_at_legacy: + name_in_index: started_at + won_games_at_legacy: + name_in_index: won_games_at + TeamSeasonGroupedBy: + graphql_fields_by_name: + note: + name_in_index: notes + record: + name_in_index: the_record + started_at_legacy: + name_in_index: started_at + won_game_at: + name_in_index: won_games_at + won_game_at_legacy: + name_in_index: won_games_at + TeamSeasonListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + TeamSponsorshipSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamTeamSeasonPlayerPlayerSeasonSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamTeamSeasonPlayerSeasonSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamTeamSeasonPlayerSponsorshipSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamTeamSeasonPlayerSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamTeamSeasonSponsorshipSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamTeamSeasonSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + Widget: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + component_aggregations: + relation: + direction: out + foreign_key: component_ids + components: + relation: + direction: out + foreign_key: component_ids + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + workspace: + relation: + direction: in + foreign_key: widget.id + workspace_id: + name_in_index: workspace_id2 + index_definition_names: + - widgets + update_targets: + - data_params: + cost.amount_cents: + cardinality: many + cost_currency_introduced_on: + cardinality: many + cost_currency_name: + cardinality: many + cost_currency_primary_continent: + cardinality: many + cost_currency_symbol: + cardinality: many + cost_currency_unit: + cardinality: many + created_at: + cardinality: many + fees.currency: + cardinality: many + name: + cardinality: many + options.color: + cardinality: many + options.size: + cardinality: many + tags: + cardinality: many + id_source: cost.currency + rollover_timestamp_value_source: cost_currency_introduced_on + routing_value_source: cost_currency_primary_continent + script_id: update_WidgetCurrency_from_Widget_0f26b3e9ea093af29e5cef02a25e75ca + type: WidgetCurrency + - data_params: + amount_cents: + cardinality: one + amounts: + cardinality: one + component_ids: + cardinality: one + cost: + cardinality: one + cost_currency_introduced_on: + cardinality: one + cost_currency_name: + cardinality: one + cost_currency_primary_continent: + cardinality: one + cost_currency_symbol: + cardinality: one + cost_currency_unit: + cardinality: one + created_at: + cardinality: one + created_at_time_of_day: + cardinality: one + created_on: + cardinality: one + fees: + cardinality: one + inventor: + cardinality: one + metadata: + cardinality: one + name: + cardinality: one + name_text: + cardinality: one + named_inventor: + cardinality: one + options: + cardinality: one + release_dates: + cardinality: one + release_timestamps: + cardinality: one + tags: + cardinality: one + the_opts: + cardinality: one + weight_in_ng: + cardinality: one + weight_in_ng_str: + cardinality: one + workspace_id2: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + rollover_timestamp_value_source: created_at + routing_value_source: workspace_id2 + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: Widget + - data_params: + widget_cost: + cardinality: one + source_path: cost + widget_name: + cardinality: one + source_path: name + widget_size: + cardinality: one + source_path: the_opts.the_sighs + widget_tags: + cardinality: one + source_path: tags + widget_workspace_id3: + cardinality: one + source_path: workspace_id2 + id_source: component_ids + metadata_params: + relationship: + value: widget + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: widget + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: Component + WidgetAggregatedValues: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + workspace_id: + name_in_index: workspace_id2 + WidgetAggregation: + elasticgraph_category: indexed_aggregation + source_type: Widget + WidgetAggregationConnection: + elasticgraph_category: relay_connection + WidgetAggregationEdge: + elasticgraph_category: relay_edge + WidgetConnection: + elasticgraph_category: relay_connection + WidgetCurrency: + graphql_fields_by_name: + widget_names: + name_in_index: widget_names2 + index_definition_names: + - widget_currencies + update_targets: + - data_params: + details: + cardinality: one + introduced_on: + cardinality: one + name: + cardinality: one + nested_fields: + cardinality: one + oldest_widget_created_at: + cardinality: one + primary_continent: + cardinality: one + widget_fee_currencies: + cardinality: one + widget_names2: + cardinality: one + widget_options: + cardinality: one + widget_tags: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + rollover_timestamp_value_source: introduced_on + routing_value_source: primary_continent + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: WidgetCurrency + WidgetCurrencyAggregatedValues: + graphql_fields_by_name: + widget_names: + name_in_index: widget_names2 + WidgetCurrencyAggregation: + elasticgraph_category: indexed_aggregation + source_type: WidgetCurrency + WidgetCurrencyAggregationConnection: + elasticgraph_category: relay_connection + WidgetCurrencyAggregationEdge: + elasticgraph_category: relay_edge + WidgetCurrencyConnection: + elasticgraph_category: relay_connection + WidgetCurrencyEdge: + elasticgraph_category: relay_edge + WidgetCurrencyFilterInput: + graphql_fields_by_name: + widget_names: + name_in_index: widget_names2 + WidgetCurrencyGroupedBy: + graphql_fields_by_name: + widget_name: + name_in_index: widget_names2 + WidgetEdge: + elasticgraph_category: relay_edge + WidgetFilterInput: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + workspace_id: + name_in_index: workspace_id2 + WidgetGroupedBy: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + release_date: + name_in_index: release_dates + release_timestamp: + name_in_index: release_timestamps + size: + name_in_index: options.size + tag: + name_in_index: tags + the_options: + name_in_index: the_opts + workspace_id: + name_in_index: workspace_id2 + WidgetOptions: + graphql_fields_by_name: + the_size: + name_in_index: the_sighs + WidgetOptionsAggregatedValues: + graphql_fields_by_name: + the_size: + name_in_index: the_sighs + WidgetOptionsFilterInput: + graphql_fields_by_name: + the_size: + name_in_index: the_sighs + WidgetOptionsGroupedBy: + graphql_fields_by_name: + the_size: + name_in_index: the_sighs + WidgetOrAddress: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + component_aggregations: + relation: + direction: out + foreign_key: component_ids + components: + relation: + direction: out + foreign_key: component_ids + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + manufacturer: + relation: + direction: out + foreign_key: manufacturer_id + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + workspace: + relation: + direction: in + foreign_key: widget.id + workspace_id: + name_in_index: workspace_id2 + WidgetOrAddressAggregatedValues: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + workspace_id: + name_in_index: workspace_id2 + WidgetOrAddressAggregation: + elasticgraph_category: indexed_aggregation + source_type: WidgetOrAddress + WidgetOrAddressAggregationConnection: + elasticgraph_category: relay_connection + WidgetOrAddressAggregationEdge: + elasticgraph_category: relay_edge + WidgetOrAddressConnection: + elasticgraph_category: relay_connection + WidgetOrAddressEdge: + elasticgraph_category: relay_edge + WidgetOrAddressFilterInput: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + workspace_id: + name_in_index: workspace_id2 + WidgetOrAddressGroupedBy: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + release_date: + name_in_index: release_dates + release_timestamp: + name_in_index: release_timestamps + size: + name_in_index: options.size + tag: + name_in_index: tags + the_options: + name_in_index: the_opts + workspace_id: + name_in_index: workspace_id2 + WidgetWorkspace: + index_definition_names: + - widget_workspaces + update_targets: + - data_params: + name: + cardinality: one + widget: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + routing_value_source: id + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: WidgetWorkspace + - data_params: + workspace_name: + cardinality: one + source_path: name + id_source: widget.id + metadata_params: + relationship: + value: workspace + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: workspace + rollover_timestamp_value_source: widget.created_at + routing_value_source: id + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: Widget + WidgetWorkspaceAggregation: + elasticgraph_category: indexed_aggregation + source_type: WidgetWorkspace + WidgetWorkspaceAggregationConnection: + elasticgraph_category: relay_connection + WidgetWorkspaceAggregationEdge: + elasticgraph_category: relay_edge + WidgetWorkspaceConnection: + elasticgraph_category: relay_connection + WidgetWorkspaceEdge: + elasticgraph_category: relay_edge +scalar_types_by_name: + Boolean: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::NoOp + require_path: elastic_graph/graphql/scalar_coercion_adapters/no_op + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + Cursor: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::Cursor + require_path: elastic_graph/graphql/scalar_coercion_adapters/cursor + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + Date: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::Date + require_path: elastic_graph/graphql/scalar_coercion_adapters/date + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + DateTime: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::DateTime + require_path: elastic_graph/graphql/scalar_coercion_adapters/date_time + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + Float: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::NoOp + require_path: elastic_graph/graphql/scalar_coercion_adapters/no_op + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + ID: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::NoOp + require_path: elastic_graph/graphql/scalar_coercion_adapters/no_op + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + Int: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::NoOp + require_path: elastic_graph/graphql/scalar_coercion_adapters/no_op + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::Integer + require_path: elastic_graph/indexer/indexing_preparers/integer + JsonSafeLong: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::JsonSafeLong + require_path: elastic_graph/graphql/scalar_coercion_adapters/longs + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::Integer + require_path: elastic_graph/indexer/indexing_preparers/integer + LocalTime: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::LocalTime + require_path: elastic_graph/graphql/scalar_coercion_adapters/local_time + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + LongString: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::LongString + require_path: elastic_graph/graphql/scalar_coercion_adapters/longs + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::Integer + require_path: elastic_graph/indexer/indexing_preparers/integer + String: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::NoOp + require_path: elastic_graph/graphql/scalar_coercion_adapters/no_op + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + TimeZone: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::TimeZone + require_path: elastic_graph/graphql/scalar_coercion_adapters/time_zone + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + Untyped: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::Untyped + require_path: elastic_graph/graphql/scalar_coercion_adapters/untyped + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::Untyped + require_path: elastic_graph/indexer/indexing_preparers/untyped +schema_element_names: + form: snake_case +static_script_ids_by_scoped_name: + field/as_day_of_week: field_as_day_of_week_f2b5c7d9e8f75bf2457b52412bfb6537 + field/as_time_of_day: field_as_time_of_day_ed82aba44fc66bff5635bec4305c1c66 + filter/by_time_of_day: filter_by_time_of_day_ea12d0561b24961789ab68ed38435612 + update/index_data: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 diff --git a/config/schema/artifacts/schema.graphql b/config/schema/artifacts/schema.graphql new file mode 100644 index 00000000..5647249f --- /dev/null +++ b/config/schema/artifacts/schema.graphql @@ -0,0 +1,14893 @@ +# Generated by `rake schema_artifacts:dump`. +# DO NOT EDIT BY HAND. Any edits will be lost the next time the rake task is run. + +""" +Indicates an upper bound on how quickly a query must respond to meet the service-level objective. +ElasticGraph will log a "good event" message if the query latency is less than or equal to this value, +and a "bad event" message if the query latency is greater than this value. These messages can be used +to drive an SLO dashboard. + +Note that the latency compared against this only contains processing time within ElasticGraph itself. +Any time spent on sending the request or response over the network is not included in the comparison. +""" +directive @eg_latency_slo(ms: Int!) on QUERY + +type Address { + full_address: String! + geo_location: GeoLocation + manufacturer: Manufacturer + shapes: [GeoShape!]! + timestamps: AddressTimestamps +} + +""" +Type used to perform aggregation computations on `Address` fields. +""" +type AddressAggregatedValues { + """ + Computed aggregate values for the `full_address` field. + """ + full_address: NonNumericAggregatedValues + + """ + Computed aggregate values for the `geo_location` field. + """ + geo_location: NonNumericAggregatedValues + + """ + Computed aggregate values for the `shapes` field. + """ + shapes: NonNumericAggregatedValues + + """ + Computed aggregate values for the `timestamps` field. + """ + timestamps: AddressTimestampsAggregatedValues +} + +""" +Return type representing a bucket of `Address` documents for an aggregations query. +""" +type AddressAggregation { + """ + Provides computed aggregated values over all `Address` documents in an aggregation bucket. + """ + aggregated_values: AddressAggregatedValues + + """ + The count of `Address` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `Address` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: AddressGroupedBy +} + +""" +Represents a paginated collection of `AddressAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type AddressAggregationConnection { + """ + Wraps a specific `AddressAggregation` to pair it with its pagination cursor. + """ + edges: [AddressAggregationEdge!]! + + """ + The list of `AddressAggregation` results. + """ + nodes: [AddressAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `AddressAggregation` in the context of a `AddressAggregationConnection`, +providing access to both the `AddressAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type AddressAggregationEdge { + """ + The `Cursor` of this `AddressAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `AddressAggregation`. + """ + cursor: Cursor + + """ + The `AddressAggregation` of this edge. + """ + node: AddressAggregation +} + +""" +Represents a paginated collection of `Address` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type AddressConnection { + """ + Wraps a specific `Address` to pair it with its pagination cursor. + """ + edges: [AddressEdge!]! + + """ + The list of `Address` results. + """ + nodes: [Address!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `Address` in the context of a `AddressConnection`, +providing access to both the `Address` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type AddressEdge { + """ + The `Cursor` of this `Address`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `Address`. + """ + cursor: Cursor + + """ + The `Address` of this edge. + """ + node: Address +} + +""" +Input type used to specify filters on `Address` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input AddressFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [AddressFilterInput!] + + """ + Used to filter on the `full_address` field. + + Will be ignored if `null` or an empty object is passed. + """ + full_address: StringFilterInput + + """ + Used to filter on the `geo_location` field. + + Will be ignored if `null` or an empty object is passed. + """ + geo_location: GeoLocationFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: AddressFilterInput + + """ + Used to filter on the `timestamps` field. + + Will be ignored if `null` or an empty object is passed. + """ + timestamps: AddressTimestampsFilterInput +} + +""" +Type used to specify the `Address` fields to group by for aggregations. +""" +type AddressGroupedBy { + """ + The `full_address` field value for this group. + """ + full_address: String + + """ + The `timestamps` field value for this group. + """ + timestamps: AddressTimestampsGroupedBy +} + +""" +Enumerates the ways `Address`s can be sorted. +""" +enum AddressSortOrderInput { + """ + Sorts ascending by the `full_address` field. + """ + full_address_ASC + + """ + Sorts descending by the `full_address` field. + """ + full_address_DESC + + """ + Sorts ascending by the `timestamps.created_at` field. + """ + timestamps_created_at_ASC + + """ + Sorts descending by the `timestamps.created_at` field. + """ + timestamps_created_at_DESC +} + +type AddressTimestamps { + created_at: DateTime +} + +""" +Type used to perform aggregation computations on `AddressTimestamps` fields. +""" +type AddressTimestampsAggregatedValues { + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues +} + +""" +Input type used to specify filters on `AddressTimestamps` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input AddressTimestampsFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [AddressTimestampsFilterInput!] + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: AddressTimestampsFilterInput +} + +""" +Type used to specify the `AddressTimestamps` fields to group by for aggregations. +""" +type AddressTimestampsGroupedBy { + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy +} + +type Affiliations { + sponsorships_nested: [Sponsorship!]! + sponsorships_object: [Sponsorship!]! +} + +""" +Type used to perform aggregation computations on `Affiliations` fields. +""" +type AffiliationsAggregatedValues { + """ + Computed aggregate values for the `sponsorships_object` field. + """ + sponsorships_object: SponsorshipAggregatedValues +} + +""" +Input type used to specify filters on a `Affiliations` object referenced directly +or transitively from a list field that has been configured to index each leaf field as +its own flattened list of values. + +Will be ignored if passed as an empty object (or as `null`). +""" +input AffiliationsFieldsListFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [AffiliationsFieldsListFilterInput!] + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: AffiliationsFieldsListFilterInput + + """ + Used to filter on the `sponsorships_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + sponsorships_nested: SponsorshipListFilterInput + + """ + Used to filter on the `sponsorships_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + sponsorships_object: SponsorshipFieldsListFilterInput +} + +""" +Input type used to specify filters on `Affiliations` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input AffiliationsFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [AffiliationsFilterInput!] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: AffiliationsFilterInput + + """ + Used to filter on the `sponsorships_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + sponsorships_nested: SponsorshipListFilterInput + + """ + Used to filter on the `sponsorships_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + sponsorships_object: SponsorshipFieldsListFilterInput +} + +""" +Type used to specify the `Affiliations` fields to group by for aggregations. +""" +type AffiliationsGroupedBy { + """ + The `sponsorships_object` field value for this group. + + Note: `sponsorships_object` is a collection field, but selecting this field + will group on individual values of the selected subfields of + `sponsorships_object`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `sponsorships_object` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `sponsorships_object` multiple times for a single document, that document will only be included in the group + once. + """ + sponsorships_object: SponsorshipGroupedBy +} + +""" +Provides detail about an aggregation `count`. +""" +type AggregationCountDetail { + """ + The (approximate) count of documents in this aggregation bucket. + + When documents in an aggregation bucket are sourced from multiple shards, the count may be only + approximate. The `upper_bound` indicates the maximum value of the true count, but usually + the true count is much closer to this approximate value (which also provides a lower bound on the + true count). + + When this approximation is known to be exact, the same value will be available from `exact_value` + and `upper_bound`. + """ + approximate_value: JsonSafeLong! + + """ + The exact count of documents in this aggregation bucket, if an exact value can be determined. + + When documents in an aggregation bucket are sourced from multiple shards, it may not be possible to + efficiently determine an exact value. When no exact value can be determined, this field will be `null`. + The `approximate_value` field--which will never be `null`--can be used to get an approximation + for the count. + """ + exact_value: JsonSafeLong + + """ + An upper bound on how large the true count of documents in this aggregation bucket could be. + + When documents in an aggregation bucket are sourced from multiple shards, it may not be possible to + efficiently determine an exact value. The `approximate_value` field provides an approximation, + and this field puts an upper bound on the true count. + """ + upper_bound: JsonSafeLong! +} + +enum Color { + BLUE + GREEN + RED +} + +""" +Input type used to specify filters on `Color` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input ColorFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [ColorFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [ColorInput] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: ColorFilterInput +} + +enum ColorInput { + BLUE + GREEN + RED +} + +""" +Input type used to specify filters on elements of a `[Color]` field. + +Will be ignored if passed as an empty object (or as `null`). +""" +input ColorListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [ColorListElementFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [ColorInput!] +} + +""" +Input type used to specify filters on `[Color]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input ColorListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `ColorListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [ColorListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [ColorListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: ColorListElementFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: ColorListFilterInput +} + +type Company implements NamedInventor { + name: String + stock_ticker: String +} + +type Component implements NamedEntity { + created_at: DateTime! + dollar_widget: Widget + id: ID! + name: String + + """ + Aggregations over the `parts` data. + """ + part_aggregations( + """ + Used to forward-paginate through the `part_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `part_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Part` documents that get aggregated over based on the provided criteria. + """ + filter: PartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `part_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `part_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `part_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `part_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): PartAggregationConnection + parts( + """ + Used to forward-paginate through the `parts`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `parts`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `parts` based on the provided criteria. + """ + filter: PartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `parts`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `parts`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `parts`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `parts`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `parts` should be sorted. + """ + order_by: [PartSortOrderInput!] + ): PartConnection + position: Position! + tags: [String!]! + widget: Widget + + """ + Aggregations over the `widgets` data. + """ + widget_aggregations( + """ + Used to forward-paginate through the `widget_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widget_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Widget` documents that get aggregated over based on the provided criteria. + """ + filter: WidgetFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widget_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widget_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widget_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widget_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): WidgetAggregationConnection + widget_cost: Money + widget_name: String + widget_size: Size + widget_tags: [String!] + widget_workspace_id: ID + widgets( + """ + Used to forward-paginate through the `widgets`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widgets`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `widgets` based on the provided criteria. + """ + filter: WidgetFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widgets`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widgets`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widgets`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widgets`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `widgets` should be sorted. + """ + order_by: [WidgetSortOrderInput!] + ): WidgetConnection +} + +""" +Type used to perform aggregation computations on `Component` fields. +""" +type ComponentAggregatedValues { + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `position` field. + """ + position: PositionAggregatedValues + + """ + Computed aggregate values for the `tags` field. + """ + tags: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_cost` field. + """ + widget_cost: MoneyAggregatedValues + + """ + Computed aggregate values for the `widget_name` field. + """ + widget_name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_size` field. + """ + widget_size: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_tags` field. + """ + widget_tags: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_workspace_id` field. + """ + widget_workspace_id: NonNumericAggregatedValues +} + +""" +Return type representing a bucket of `Component` documents for an aggregations query. +""" +type ComponentAggregation { + """ + Provides computed aggregated values over all `Component` documents in an aggregation bucket. + """ + aggregated_values: ComponentAggregatedValues + + """ + The count of `Component` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `Component` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: ComponentGroupedBy +} + +""" +Represents a paginated collection of `ComponentAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type ComponentAggregationConnection { + """ + Wraps a specific `ComponentAggregation` to pair it with its pagination cursor. + """ + edges: [ComponentAggregationEdge!]! + + """ + The list of `ComponentAggregation` results. + """ + nodes: [ComponentAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `ComponentAggregation` in the context of a `ComponentAggregationConnection`, +providing access to both the `ComponentAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type ComponentAggregationEdge { + """ + The `Cursor` of this `ComponentAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `ComponentAggregation`. + """ + cursor: Cursor + + """ + The `ComponentAggregation` of this edge. + """ + node: ComponentAggregation +} + +""" +Represents a paginated collection of `Component` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type ComponentConnection { + """ + Wraps a specific `Component` to pair it with its pagination cursor. + """ + edges: [ComponentEdge!]! + + """ + The list of `Component` results. + """ + nodes: [Component!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `Component` in the context of a `ComponentConnection`, +providing access to both the `Component` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type ComponentEdge { + """ + The `Cursor` of this `Component`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `Component`. + """ + cursor: Cursor + + """ + The `Component` of this edge. + """ + node: Component +} + +""" +Input type used to specify filters on `Component` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input ComponentFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [ComponentFilterInput!] + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: ComponentFilterInput + + """ + Used to filter on the `position` field. + + Will be ignored if `null` or an empty object is passed. + """ + position: PositionFilterInput + + """ + Used to filter on the `tags` field. + + Will be ignored if `null` or an empty object is passed. + """ + tags: StringListFilterInput + + """ + Used to filter on the `widget_cost` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_cost: MoneyFilterInput + + """ + Used to filter on the `widget_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_name: StringFilterInput + + """ + Used to filter on the `widget_size` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_size: SizeFilterInput + + """ + Used to filter on the `widget_tags` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_tags: StringListFilterInput + + """ + Used to filter on the `widget_workspace_id` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_workspace_id: IDFilterInput +} + +""" +Type used to specify the `Component` fields to group by for aggregations. +""" +type ComponentGroupedBy { + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + The `name` field value for this group. + """ + name: String + + """ + The `position` field value for this group. + """ + position: PositionGroupedBy + + """ + The `widget_cost` field value for this group. + """ + widget_cost: MoneyGroupedBy + + """ + The `widget_name` field value for this group. + """ + widget_name: String + + """ + The `widget_size` field value for this group. + """ + widget_size: Size + + """ + The `widget_workspace_id` field value for this group. + """ + widget_workspace_id: ID +} + +""" +Enumerates the ways `Component`s can be sorted. +""" +enum ComponentSortOrderInput { + """ + Sorts ascending by the `created_at` field. + """ + created_at_ASC + + """ + Sorts descending by the `created_at` field. + """ + created_at_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC + + """ + Sorts ascending by the `position.x` field. + """ + position_x_ASC + + """ + Sorts descending by the `position.x` field. + """ + position_x_DESC + + """ + Sorts ascending by the `position.y` field. + """ + position_y_ASC + + """ + Sorts descending by the `position.y` field. + """ + position_y_DESC + + """ + Sorts ascending by the `widget_cost.amount_cents` field. + """ + widget_cost_amount_cents_ASC + + """ + Sorts descending by the `widget_cost.amount_cents` field. + """ + widget_cost_amount_cents_DESC + + """ + Sorts ascending by the `widget_cost.currency` field. + """ + widget_cost_currency_ASC + + """ + Sorts descending by the `widget_cost.currency` field. + """ + widget_cost_currency_DESC + + """ + Sorts ascending by the `widget_name` field. + """ + widget_name_ASC + + """ + Sorts descending by the `widget_name` field. + """ + widget_name_DESC + + """ + Sorts ascending by the `widget_size` field. + """ + widget_size_ASC + + """ + Sorts descending by the `widget_size` field. + """ + widget_size_DESC + + """ + Sorts ascending by the `widget_workspace_id` field. + """ + widget_workspace_id_ASC + + """ + Sorts descending by the `widget_workspace_id` field. + """ + widget_workspace_id_DESC +} + +type CurrencyDetails { + symbol: String + unit: String +} + +""" +Type used to perform aggregation computations on `CurrencyDetails` fields. +""" +type CurrencyDetailsAggregatedValues { + """ + Computed aggregate values for the `symbol` field. + """ + symbol: NonNumericAggregatedValues + + """ + Computed aggregate values for the `unit` field. + """ + unit: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on `CurrencyDetails` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input CurrencyDetailsFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [CurrencyDetailsFilterInput!] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: CurrencyDetailsFilterInput + + """ + Used to filter on the `symbol` field. + + Will be ignored if `null` or an empty object is passed. + """ + symbol: StringFilterInput + + """ + Used to filter on the `unit` field. + + Will be ignored if `null` or an empty object is passed. + """ + unit: StringFilterInput +} + +""" +Type used to specify the `CurrencyDetails` fields to group by for aggregations. +""" +type CurrencyDetailsGroupedBy { + """ + The `symbol` field value for this group. + """ + symbol: String + + """ + The `unit` field value for this group. + """ + unit: String +} + +""" +An opaque string value representing a specific location in a paginated connection type. +Returned cursors can be passed back in the next query via the `before` or `after` +arguments to continue paginating from that point. +""" +scalar Cursor + +""" +A date, represented as an [ISO 8601 date string](https://en.wikipedia.org/wiki/ISO_8601). +""" +scalar Date + +""" +A return type used from aggregations to provided aggregated values over `Date` fields. +""" +type DateAggregatedValues { + """ + The average (mean) of the field values within this grouping. + The returned value will be rounded to the nearest `Date` value. + """ + approximate_avg: Date + + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in + Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + approximate_distinct_value_count: JsonSafeLong + + """ + The maximum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_max: Date + + """ + The minimum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_min: Date +} + +""" +Input type used to specify filters on `Date` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input DateFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [DateFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [Date] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: Date + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: Date + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: Date + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: Date + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: DateFilterInput +} + +""" +Allows for grouping `Date` values based on the desired return type. +""" +type DateGroupedBy { + """ + Used when grouping on the full `Date` value. + """ + as_date( + """ + Amount of offset (positive or negative) to shift the `Date` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 1 day to change what + day-of-week weeks are considered to start on. + """ + offset: DateGroupingOffsetInput + + """ + Determines the grouping truncation unit for this field. + """ + truncation_unit: DateGroupingTruncationUnitInput! + ): Date + + """ + An alternative to `as_date` for when grouping on the day-of-week is desired. + """ + as_day_of_week( + """ + Amount of offset (positive or negative) to shift the `DayOfWeek` boundaries of each grouping bucket. + + For example, you can apply an offset of -2 hours to shift `DateTime` values to the prior `DayOfWeek` + when they fall between midnight and 2 AM. + """ + offset: DayOfWeekGroupingOffsetInput + ): DayOfWeek +} + +""" +Enumerates the supported granularities of a `Date`. +""" +enum DateGroupingGranularityInput { + """ + The exact day of a `Date`. + """ + DAY + + """ + The month a `Date` falls in. + """ + MONTH + + """ + The quarter a `Date` falls in. + """ + QUARTER + + """ + The week, beginning on Monday, a `Date` falls in. + """ + WEEK + + """ + The year a `Date` falls in. + """ + YEAR +} + +""" +Input type offered when grouping on `Date` fields, representing the amount of offset +(positive or negative) to shift the `Date` boundaries of each grouping bucket. + +For example, when grouping by `WEEK`, you can shift by 1 day to change +what day-of-week weeks are considered to start on. +""" +input DateGroupingOffsetInput { + """ + Number (positive or negative) of the given `unit` to offset the boundaries of the `Date` groupings. + """ + amount: Int! + + """ + Unit of offsetting to apply to the boundaries of the `Date` groupings. + """ + unit: DateUnitInput! +} + +""" +Enumerates the supported truncation units of a `Date`. +""" +enum DateGroupingTruncationUnitInput { + """ + The exact day of a `Date`. + """ + DAY + + """ + The month a `Date` falls in. + """ + MONTH + + """ + The quarter a `Date` falls in. + """ + QUARTER + + """ + The week, beginning on Monday, a `Date` falls in. + """ + WEEK + + """ + The year a `Date` falls in. + """ + YEAR +} + +""" +Input type used to specify filters on elements of a `[Date]` field. + +Will be ignored if passed as an empty object (or as `null`). +""" +input DateListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [DateListElementFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [Date!] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: Date + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: Date + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: Date + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: Date +} + +""" +Input type used to specify filters on `[Date]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input DateListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `DateListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [DateListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [DateListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: DateListElementFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: DateListFilterInput +} + +""" +A timestamp, represented as an [ISO 8601 time string](https://en.wikipedia.org/wiki/ISO_8601). +""" +scalar DateTime + +""" +A return type used from aggregations to provided aggregated values over `DateTime` fields. +""" +type DateTimeAggregatedValues { + """ + The average (mean) of the field values within this grouping. + The returned value will be rounded to the nearest `DateTime` value. + """ + approximate_avg: DateTime + + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in + Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + approximate_distinct_value_count: JsonSafeLong + + """ + The maximum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_max: DateTime + + """ + The minimum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_min: DateTime +} + +""" +Input type used to specify filters on `DateTime` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input DateTimeFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [DateTimeFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [DateTime] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: DateTime + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: DateTime + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: DateTime + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: DateTime + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: DateTimeFilterInput + + """ + Matches records based on the time-of-day of the `DateTime` values. + + Will be ignored when `null` or an empty list is passed. + """ + time_of_day: DateTimeTimeOfDayFilterInput +} + +""" +Allows for grouping `DateTime` values based on the desired return type. +""" +type DateTimeGroupedBy { + """ + An alternative to `as_date_time` for when grouping on just the date is desired. + """ + as_date( + """ + Amount of offset (positive or negative) to shift the `Date` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 1 day to change what + day-of-week weeks are considered to start on. + """ + offset: DateGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `Date` value falls in. + """ + time_zone: TimeZone! = "UTC" + + """ + Determines the grouping truncation unit for this field. + """ + truncation_unit: DateGroupingTruncationUnitInput! + ): Date + + """ + Used when grouping on the full `DateTime` value. + """ + as_date_time( + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 1 day to change what + day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone! = "UTC" + + """ + Determines the grouping truncation unit for this field. + """ + truncation_unit: DateTimeGroupingTruncationUnitInput! + ): DateTime + + """ + An alternative to `as_date_time` for when grouping on the day-of-week is desired. + """ + as_day_of_week( + """ + Amount of offset (positive or negative) to shift the `DayOfWeek` boundaries of each grouping bucket. + + For example, you can apply an offset of -2 hours to shift `DateTime` values to the prior `DayOfWeek` + when they fall between midnight and 2 AM. + """ + offset: DayOfWeekGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DayOfWeek` value falls in. + """ + time_zone: TimeZone! = "UTC" + ): DayOfWeek + + """ + An alternative to `as_date_time` for when grouping on just the time-of-day is desired. + """ + as_time_of_day( + """ + Amount of offset (positive or negative) to shift the `LocalTime` boundaries of each grouping bucket. + + For example, when grouping by `HOUR`, you can apply an offset of -5 minutes to shift `LocalTime` + values to the prior hour when they fall between the the top of an hour and 5 after. + """ + offset: LocalTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `LocalTime` value falls in. + """ + time_zone: TimeZone! = "UTC" + + """ + Determines the grouping truncation unit for this field. + """ + truncation_unit: LocalTimeGroupingTruncationUnitInput! + ): LocalTime +} + +""" +Enumerates the supported granularities of a `DateTime`. +""" +enum DateTimeGroupingGranularityInput { + """ + The day a `DateTime` falls in. + """ + DAY + + """ + The hour a `DateTime` falls in. + """ + HOUR + + """ + The minute a `DateTime` falls in. + """ + MINUTE + + """ + The month a `DateTime` falls in. + """ + MONTH + + """ + The quarter a `DateTime` falls in. + """ + QUARTER + + """ + The second a `DateTime` falls in. + """ + SECOND + + """ + The week, beginning on Monday, a `DateTime` falls in. + """ + WEEK + + """ + The year a `DateTime` falls in. + """ + YEAR +} + +""" +Input type offered when grouping on `DateTime` fields, representing the amount of offset +(positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + +For example, when grouping by `WEEK`, you can shift by 1 day to change +what day-of-week weeks are considered to start on. +""" +input DateTimeGroupingOffsetInput { + """ + Number (positive or negative) of the given `unit` to offset the boundaries of the `DateTime` groupings. + """ + amount: Int! + + """ + Unit of offsetting to apply to the boundaries of the `DateTime` groupings. + """ + unit: DateTimeUnitInput! +} + +""" +Enumerates the supported truncation units of a `DateTime`. +""" +enum DateTimeGroupingTruncationUnitInput { + """ + The day a `DateTime` falls in. + """ + DAY + + """ + The hour a `DateTime` falls in. + """ + HOUR + + """ + The minute a `DateTime` falls in. + """ + MINUTE + + """ + The month a `DateTime` falls in. + """ + MONTH + + """ + The quarter a `DateTime` falls in. + """ + QUARTER + + """ + The second a `DateTime` falls in. + """ + SECOND + + """ + The week, beginning on Monday, a `DateTime` falls in. + """ + WEEK + + """ + The year a `DateTime` falls in. + """ + YEAR +} + +""" +Input type used to specify filters on elements of a `[DateTime]` field. + +Will be ignored if passed as an empty object (or as `null`). +""" +input DateTimeListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [DateTimeListElementFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [DateTime!] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: DateTime + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: DateTime + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: DateTime + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: DateTime + + """ + Matches records based on the time-of-day of the `DateTime` values. + + Will be ignored when `null` or an empty list is passed. + """ + time_of_day: DateTimeTimeOfDayFilterInput +} + +""" +Input type used to specify filters on `[DateTime]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input DateTimeListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `DateTimeListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [DateTimeListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [DateTimeListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: DateTimeListElementFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: DateTimeListFilterInput +} + +""" +Input type used to specify filters on the time-of-day of `DateTime` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input DateTimeTimeOfDayFilterInput { + """ + Matches records where the time of day of the `DateTime` field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [LocalTime!] + + """ + Matches records where the time of day of the `DateTime` field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: LocalTime + + """ + Matches records where the time of day of the `DateTime` field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: LocalTime + + """ + Matches records where the time of day of the `DateTime` field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: LocalTime + + """ + Matches records where the time of day of the `DateTime` field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: LocalTime + + """ + TimeZone to use when comparing the `DateTime` values against the provided `LocalTime` values. + """ + time_zone: TimeZone! = "UTC" +} + +""" +Enumeration of `DateTime` units. +""" +enum DateTimeUnitInput { + """ + The time period of a full rotation of the Earth with respect to the Sun. + """ + DAY + + """ + 1/24th of a day. + """ + HOUR + + """ + 1/1000th of a second. + """ + MILLISECOND + + """ + 1/60th of an hour. + """ + MINUTE + + """ + 1/60th of a minute. + """ + SECOND +} + +""" +Enumeration of `Date` units. +""" +enum DateUnitInput { + """ + The time period of a full rotation of the Earth with respect to the Sun. + """ + DAY +} + +""" +Indicates the specific day of the week. +""" +enum DayOfWeek { + """ + Friday. + """ + FRIDAY + + """ + Monday. + """ + MONDAY + + """ + Saturday. + """ + SATURDAY + + """ + Sunday. + """ + SUNDAY + + """ + Thursday. + """ + THURSDAY + + """ + Tuesday. + """ + TUESDAY + + """ + Wednesday. + """ + WEDNESDAY +} + +""" +Input type offered when grouping on `DayOfWeek` fields, representing the amount of offset +(positive or negative) to shift the `DayOfWeek` boundaries of each grouping bucket. + +For example, you can apply an offset of -2 hours to shift `DateTime` values to the prior `DayOfWeek` +when they fall between midnight and 2 AM. +""" +input DayOfWeekGroupingOffsetInput { + """ + Number (positive or negative) of the given `unit` to offset the boundaries of the `DayOfWeek` groupings. + """ + amount: Int! + + """ + Unit of offsetting to apply to the boundaries of the `DayOfWeek` groupings. + """ + unit: DateTimeUnitInput! +} + +""" +Enumerates the supported distance units. +""" +enum DistanceUnitInput { + """ + A metric system unit equal to 1/100th of a meter. + """ + CENTIMETER + + """ + A United States customary unit of 12 inches. + """ + FOOT + + """ + A United States customary unit equal to 1/12th of a foot. + """ + INCH + + """ + A metric system unit equal to 1,000 meters. + """ + KILOMETER + + """ + The base unit of length in the metric system. + """ + METER + + """ + A United States customary unit of 5,280 feet. + """ + MILE + + """ + A metric system unit equal to 1/1,000th of a meter. + """ + MILLIMETER + + """ + An international unit of length used for air, marine, and space navigation. Equivalent to 1,852 meters. + """ + NAUTICAL_MILE + + """ + A United States customary unit of 3 feet. + """ + YARD +} + +type ElectricalPart implements NamedEntity { + """ + Aggregations over the `components` data. + """ + component_aggregations( + """ + Used to forward-paginate through the `component_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `component_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Component` documents that get aggregated over based on the provided criteria. + """ + filter: ComponentFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `component_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `component_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `component_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `component_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): ComponentAggregationConnection + components( + """ + Used to forward-paginate through the `components`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `components`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `components` based on the provided criteria. + """ + filter: ComponentFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `components`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `components`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `components`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `components`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `components` should be sorted. + """ + order_by: [ComponentSortOrderInput!] + ): ComponentConnection + created_at: DateTime! + id: ID! + manufacturer: Manufacturer + name: String + voltage: Int! +} + +""" +Type used to perform aggregation computations on `ElectricalPart` fields. +""" +type ElectricalPartAggregatedValues { + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `voltage` field. + """ + voltage: IntAggregatedValues +} + +""" +Return type representing a bucket of `ElectricalPart` documents for an aggregations query. +""" +type ElectricalPartAggregation { + """ + Provides computed aggregated values over all `ElectricalPart` documents in an aggregation bucket. + """ + aggregated_values: ElectricalPartAggregatedValues + + """ + The count of `ElectricalPart` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `ElectricalPart` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: ElectricalPartGroupedBy +} + +""" +Represents a paginated collection of `ElectricalPartAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type ElectricalPartAggregationConnection { + """ + Wraps a specific `ElectricalPartAggregation` to pair it with its pagination cursor. + """ + edges: [ElectricalPartAggregationEdge!]! + + """ + The list of `ElectricalPartAggregation` results. + """ + nodes: [ElectricalPartAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `ElectricalPartAggregation` in the context of a `ElectricalPartAggregationConnection`, +providing access to both the `ElectricalPartAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type ElectricalPartAggregationEdge { + """ + The `Cursor` of this `ElectricalPartAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `ElectricalPartAggregation`. + """ + cursor: Cursor + + """ + The `ElectricalPartAggregation` of this edge. + """ + node: ElectricalPartAggregation +} + +""" +Represents a paginated collection of `ElectricalPart` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type ElectricalPartConnection { + """ + Wraps a specific `ElectricalPart` to pair it with its pagination cursor. + """ + edges: [ElectricalPartEdge!]! + + """ + The list of `ElectricalPart` results. + """ + nodes: [ElectricalPart!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `ElectricalPart` in the context of a `ElectricalPartConnection`, +providing access to both the `ElectricalPart` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type ElectricalPartEdge { + """ + The `Cursor` of this `ElectricalPart`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `ElectricalPart`. + """ + cursor: Cursor + + """ + The `ElectricalPart` of this edge. + """ + node: ElectricalPart +} + +""" +Input type used to specify filters on `ElectricalPart` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input ElectricalPartFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [ElectricalPartFilterInput!] + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: ElectricalPartFilterInput + + """ + Used to filter on the `voltage` field. + + Will be ignored if `null` or an empty object is passed. + """ + voltage: IntFilterInput +} + +""" +Type used to specify the `ElectricalPart` fields to group by for aggregations. +""" +type ElectricalPartGroupedBy { + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + The `name` field value for this group. + """ + name: String + + """ + The `voltage` field value for this group. + """ + voltage: Int +} + +""" +Enumerates the ways `ElectricalPart`s can be sorted. +""" +enum ElectricalPartSortOrderInput { + """ + Sorts ascending by the `created_at` field. + """ + created_at_ASC + + """ + Sorts descending by the `created_at` field. + """ + created_at_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC + + """ + Sorts ascending by the `voltage` field. + """ + voltage_ASC + + """ + Sorts descending by the `voltage` field. + """ + voltage_DESC +} + +""" +A return type used from aggregations to provided aggregated values over `Float` fields. +""" +type FloatAggregatedValues { + """ + The average (mean) of the field values within this grouping. + + The computation of this value may introduce additional imprecision (on top of the + natural imprecision of floats) when it deals with intermediary values that are + outside the `JsonSafeLong` range (-9,007,199,254,740,991 to 9,007,199,254,740,991). + """ + approximate_avg: Float + + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in + Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + approximate_distinct_value_count: JsonSafeLong + + """ + The sum of the field values within this grouping. + + As with all double-precision `Float` values, operations are subject to floating-point loss + of precision, so the value may be approximate. + """ + approximate_sum: Float! + + """ + The maximum of the field values within this grouping. + + The value will be "exact" in that the aggregation computation will return + the exact value of the largest float that has been indexed, without + introducing any new imprecision. However, floats by their nature are + naturally imprecise since they cannot precisely represent all real numbers. + """ + exact_max: Float + + """ + The minimum of the field values within this grouping. + + The value will be "exact" in that the aggregation computation will return + the exact value of the smallest float that has been indexed, without + introducing any new imprecision. However, floats by their nature are + naturally imprecise since they cannot precisely represent all real numbers. + """ + exact_min: Float +} + +""" +Input type used to specify filters on `Float` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input FloatFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [FloatFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [Float] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: Float + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: Float + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: Float + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: Float + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: FloatFilterInput +} + +""" +Geographic coordinates representing a location on the Earth's surface. +""" +type GeoLocation { + """ + Angular distance north or south of the Earth's equator, measured in degrees from -90 to +90. + """ + latitude: Float + + """ + Angular distance east or west of the Prime Meridian at Greenwich, UK, measured in degrees from -180 to +180. + """ + longitude: Float +} + +""" +Input type used to specify distance filtering parameters on `GeoLocation` fields. +""" +input GeoLocationDistanceFilterInput { + """ + Angular distance north or south of the Earth's equator, measured in degrees from -90 to +90. + """ + latitude: Float! + + """ + Angular distance east or west of the Prime Meridian at Greenwich, UK, measured in degrees from -180 to +180. + """ + longitude: Float! + + """ + Maximum distance (of the provided `unit`) to consider "near" the location identified + by `latitude` and `longitude`. + """ + max_distance: Float! + + """ + Determines the unit of the specified `max_distance`. + """ + unit: DistanceUnitInput! +} + +""" +Input type used to specify filters on `GeoLocation` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input GeoLocationFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [GeoLocationFilterInput!] + + """ + Matches records where the field's geographic location is within a specified distance from the + location identified by `latitude` and `longitude`. + + Will be ignored when `null` or an empty object is passed. + """ + near: GeoLocationDistanceFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: GeoLocationFilterInput +} + +type GeoShape { + coordinates: [Float!]! + type: String +} + +""" +Input type used to specify filters on `ID` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input IDFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [IDFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [ID] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: IDFilterInput +} + +""" +Input type used to specify filters on elements of a `[ID]` field. + +Will be ignored if passed as an empty object (or as `null`). +""" +input IDListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [IDListElementFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [ID!] +} + +""" +Input type used to specify filters on `[ID]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input IDListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `IDListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [IDListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [IDListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: IDListElementFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: IDListFilterInput +} + +""" +A return type used from aggregations to provided aggregated values over `Int` fields. +""" +type IntAggregatedValues { + """ + The average (mean) of the field values within this grouping. + + Note that the returned value is approximate. Imprecision can be introduced by the computation if + any intermediary values fall outside the `JsonSafeLong` range (-9,007,199,254,740,991 + to 9,007,199,254,740,991). + """ + approximate_avg: Float + + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in + Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + approximate_distinct_value_count: JsonSafeLong + + """ + The (approximate) sum of the field values within this grouping. + + Sums of large `Int` values can result in overflow, where the exact sum cannot + fit in a `JsonSafeLong` return value. This field, as a double-precision `Float`, can + represent larger sums, but the value may only be approximate. + """ + approximate_sum: Float! + + """ + The maximum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_max: Int + + """ + The minimum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_min: Int + + """ + The exact sum of the field values within this grouping, if it fits in a `JsonSafeLong`. + + Sums of large `Int` values can result in overflow, where the exact sum cannot + fit in a `JsonSafeLong`. In that case, `null` will be returned, and `approximate_sum` + can be used to get an approximate value. + """ + exact_sum: JsonSafeLong +} + +""" +Input type used to specify filters on `Int` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input IntFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [IntFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [Int] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: Int + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: Int + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: Int + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: Int + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: IntFilterInput +} + +""" +Input type used to specify filters on elements of a `[Int]` field. + +Will be ignored if passed as an empty object (or as `null`). +""" +input IntListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [IntListElementFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [Int!] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: Int + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: Int + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: Int + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: Int +} + +""" +Input type used to specify filters on `[Int]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input IntListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `IntListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [IntListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [IntListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: IntListElementFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: IntListFilterInput +} + +union Inventor = Company | Person + +""" +Type used to perform aggregation computations on `Inventor` fields. +""" +type InventorAggregatedValues { + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `nationality` field. + """ + nationality: NonNumericAggregatedValues + + """ + Computed aggregate values for the `stock_ticker` field. + """ + stock_ticker: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on `Inventor` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input InventorFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [InventorFilterInput!] + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Used to filter on the `nationality` field. + + Will be ignored if `null` or an empty object is passed. + """ + nationality: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: InventorFilterInput + + """ + Used to filter on the `stock_ticker` field. + + Will be ignored if `null` or an empty object is passed. + """ + stock_ticker: StringFilterInput +} + +""" +Type used to specify the `Inventor` fields to group by for aggregations. +""" +type InventorGroupedBy { + """ + The `name` field value for this group. + """ + name: String + + """ + The `nationality` field value for this group. + """ + nationality: String + + """ + The `stock_ticker` field value for this group. + """ + stock_ticker: String +} + +""" +A numeric type for large integer values that can serialize safely as JSON. + +While JSON itself has no hard limit on the size of integers, the RFC-7159 spec +mentions that values outside of the range -9,007,199,254,740,991 (-(2^53) + 1) +to 9,007,199,254,740,991 (2^53 - 1) may not be interopable with all JSON +implementations. As it turns out, the number implementation used by JavaScript +has this issue. When you parse a JSON string that contains a numeric value like +`4693522397653681111`, the parsed result will contain a rounded value like +`4693522397653681000`. + +While this is entirely a client-side problem, we want to preserve maximum compatibility +with common client languages. Given the ubiquity of GraphiQL as a GraphQL client, +we want to avoid this problem. + +Our solution is to support two separate types: + +- This type (`JsonSafeLong`) is serialized as a number, but limits values to the safely + serializable range. +- The `LongString` type supports long values that use all 64 bits, but serializes as a + string rather than a number, avoiding the JavaScript compatibility problems. + +For more background, see the [JavaScript `Number.MAX_SAFE_INTEGER` +docs](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER). +""" +scalar JsonSafeLong + +""" +A return type used from aggregations to provided aggregated values over `JsonSafeLong` fields. +""" +type JsonSafeLongAggregatedValues { + """ + The average (mean) of the field values within this grouping. + + Note that the returned value is approximate. Imprecision can be introduced by the computation if + any intermediary values fall outside the `JsonSafeLong` range (-9,007,199,254,740,991 + to 9,007,199,254,740,991). + """ + approximate_avg: Float + + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in + Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + approximate_distinct_value_count: JsonSafeLong + + """ + The (approximate) sum of the field values within this grouping. + + Sums of large `JsonSafeLong` values can result in overflow, where the exact sum cannot + fit in a `JsonSafeLong` return value. This field, as a double-precision `Float`, can + represent larger sums, but the value may only be approximate. + """ + approximate_sum: Float! + + """ + The maximum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_max: JsonSafeLong + + """ + The minimum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_min: JsonSafeLong + + """ + The exact sum of the field values within this grouping, if it fits in a `JsonSafeLong`. + + Sums of large `JsonSafeLong` values can result in overflow, where the exact sum cannot + fit in a `JsonSafeLong`. In that case, `null` will be returned, and `approximate_sum` + can be used to get an approximate value. + """ + exact_sum: JsonSafeLong +} + +""" +Input type used to specify filters on `JsonSafeLong` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input JsonSafeLongFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [JsonSafeLongFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [JsonSafeLong] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: JsonSafeLong + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: JsonSafeLong + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: JsonSafeLong + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: JsonSafeLong + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: JsonSafeLongFilterInput +} + +""" +Input type used to specify filters on elements of a `[JsonSafeLong]` field. + +Will be ignored if passed as an empty object (or as `null`). +""" +input JsonSafeLongListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [JsonSafeLongListElementFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [JsonSafeLong!] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: JsonSafeLong + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: JsonSafeLong + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: JsonSafeLong + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: JsonSafeLong +} + +""" +Input type used to specify filters on `[JsonSafeLong]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input JsonSafeLongListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `JsonSafeLongListFilterInput` input because of + collisions between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [JsonSafeLongListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [JsonSafeLongListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: JsonSafeLongListElementFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: JsonSafeLongListFilterInput +} + +""" +A local time such as `"23:59:33"` or `"07:20:47.454"` without a time zone or offset, formatted based on the +[partial-time portion of RFC3339](https://datatracker.ietf.org/doc/html/rfc3339#section-5.6). +""" +scalar LocalTime + +""" +A return type used from aggregations to provided aggregated values over `LocalTime` fields. +""" +type LocalTimeAggregatedValues { + """ + The average (mean) of the field values within this grouping. + The returned value will be rounded to the nearest `LocalTime` value. + """ + approximate_avg: LocalTime + + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in + Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + approximate_distinct_value_count: JsonSafeLong + + """ + The maximum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_max: LocalTime + + """ + The minimum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_min: LocalTime +} + +""" +Input type used to specify filters on `LocalTime` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input LocalTimeFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [LocalTimeFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [LocalTime] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: LocalTime + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: LocalTime + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: LocalTime + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: LocalTime + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: LocalTimeFilterInput +} + +""" +Input type offered when grouping on `LocalTime` fields, representing the amount of offset +(positive or negative) to shift the `LocalTime` boundaries of each grouping bucket. + +For example, when grouping by `HOUR`, you can shift by 30 minutes to change +what minute-of-hour hours are considered to start on. +""" +input LocalTimeGroupingOffsetInput { + """ + Number (positive or negative) of the given `unit` to offset the boundaries of the `LocalTime` groupings. + """ + amount: Int! + + """ + Unit of offsetting to apply to the boundaries of the `LocalTime` groupings. + """ + unit: LocalTimeUnitInput! +} + +""" +Enumerates the supported truncation units of a `LocalTime`. +""" +enum LocalTimeGroupingTruncationUnitInput { + """ + The hour a `LocalTime` falls in. + """ + HOUR + + """ + The minute a `LocalTime` falls in. + """ + MINUTE + + """ + The second a `LocalTime` falls in. + """ + SECOND +} + +""" +Enumeration of `LocalTime` units. +""" +enum LocalTimeUnitInput { + """ + 1/24th of a day. + """ + HOUR + + """ + 1/1000th of a second. + """ + MILLISECOND + + """ + 1/60th of an hour. + """ + MINUTE + + """ + 1/60th of a minute. + """ + SECOND +} + +""" +A numeric type for large integer values in the inclusive range -2^63 +(-9,223,372,036,854,775,808) to (2^63 - 1) (9,223,372,036,854,775,807). + +Note that `LongString` values are serialized as strings within JSON, to avoid +interopability problems with JavaScript. If you want a large integer type that +serializes within JSON as a number, use `JsonSafeLong`. +""" +scalar LongString + +""" +A return type used from aggregations to provided aggregated values over `LongString` fields. +""" +type LongStringAggregatedValues { + """ + The average (mean) of the field values within this grouping. + + Note that the returned value is approximate. Imprecision can be introduced by the computation if + any intermediary values fall outside the `JsonSafeLong` range (-9,007,199,254,740,991 + to 9,007,199,254,740,991). + """ + approximate_avg: Float + + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in + Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + approximate_distinct_value_count: JsonSafeLong + + """ + The maximum of the field values within this grouping. + + The aggregation computation performed to identify the largest value is not able + to maintain exact precision when dealing with values that are outside the `JsonSafeLong` + range (-9,007,199,254,740,991 to 9,007,199,254,740,991). + In that case, the `exact_max` field will return `null`, but this field will provide + a value which may be approximate. + """ + approximate_max: LongString + + """ + The minimum of the field values within this grouping. + + The aggregation computation performed to identify the smallest value is not able + to maintain exact precision when dealing with values that are outside the `JsonSafeLong` + range (-9,007,199,254,740,991 to 9,007,199,254,740,991). + In that case, the `exact_min` field will return `null`, but this field will provide + a value which may be approximate. + """ + approximate_min: LongString + + """ + The (approximate) sum of the field values within this grouping. + + Sums of large `LongString` values can result in overflow, where the exact sum cannot + fit in a `LongString` return value. This field, as a double-precision `Float`, can + represent larger sums, but the value may only be approximate. + """ + approximate_sum: Float! + + """ + The maximum of the field values within this grouping. + + So long as the grouping contains at least one non-null value, and no values exceed the + `JsonSafeLong` range in the underlying indexed field, this will return an exact non-null value. + + If no non-null values are available, or if the maximum value is outside the `JsonSafeLong` + range, `null` will be returned. `approximate_max` can be used to differentiate between these + cases and to get an approximate value. + """ + exact_max: JsonSafeLong + + """ + The minimum of the field values within this grouping. + + So long as the grouping contains at least one non-null value, and no values exceed the + `JsonSafeLong` range in the underlying indexed field, this will return an exact non-null value. + + If no non-null values are available, or if the minimum value is outside the `JsonSafeLong` + range, `null` will be returned. `approximate_min` can be used to differentiate between these + cases and to get an approximate value. + """ + exact_min: JsonSafeLong + + """ + The exact sum of the field values within this grouping, if it fits in a `JsonSafeLong`. + + Sums of large `LongString` values can result in overflow, where the exact sum cannot + fit in a `JsonSafeLong`. In that case, `null` will be returned, and `approximate_sum` + can be used to get an approximate value. + """ + exact_sum: JsonSafeLong +} + +""" +Input type used to specify filters on `LongString` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input LongStringFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [LongStringFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [LongString] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: LongString + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: LongString + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: LongString + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: LongString + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: LongStringFilterInput +} + +type Manufacturer implements NamedEntity { + address: Address + created_at: DateTime! + id: ID! + + """ + Aggregations over the `manufactured_parts` data. + """ + manufactured_part_aggregations( + """ + Used to forward-paginate through the `manufactured_part_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `manufactured_part_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Part` documents that get aggregated over based on the provided criteria. + """ + filter: PartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `manufactured_part_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `manufactured_part_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `manufactured_part_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `manufactured_part_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): PartAggregationConnection + manufactured_parts( + """ + Used to forward-paginate through the `manufactured_parts`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `manufactured_parts`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `manufactured_parts` based on the provided criteria. + """ + filter: PartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `manufactured_parts`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `manufactured_parts`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `manufactured_parts`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `manufactured_parts`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `manufactured_parts` should be sorted. + """ + order_by: [PartSortOrderInput!] + ): PartConnection + name: String +} + +""" +Type used to perform aggregation computations on `Manufacturer` fields. +""" +type ManufacturerAggregatedValues { + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues +} + +""" +Return type representing a bucket of `Manufacturer` documents for an aggregations query. +""" +type ManufacturerAggregation { + """ + Provides computed aggregated values over all `Manufacturer` documents in an aggregation bucket. + """ + aggregated_values: ManufacturerAggregatedValues + + """ + The count of `Manufacturer` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `Manufacturer` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: ManufacturerGroupedBy +} + +""" +Represents a paginated collection of `ManufacturerAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type ManufacturerAggregationConnection { + """ + Wraps a specific `ManufacturerAggregation` to pair it with its pagination cursor. + """ + edges: [ManufacturerAggregationEdge!]! + + """ + The list of `ManufacturerAggregation` results. + """ + nodes: [ManufacturerAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `ManufacturerAggregation` in the context of a `ManufacturerAggregationConnection`, +providing access to both the `ManufacturerAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type ManufacturerAggregationEdge { + """ + The `Cursor` of this `ManufacturerAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `ManufacturerAggregation`. + """ + cursor: Cursor + + """ + The `ManufacturerAggregation` of this edge. + """ + node: ManufacturerAggregation +} + +""" +Represents a paginated collection of `Manufacturer` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type ManufacturerConnection { + """ + Wraps a specific `Manufacturer` to pair it with its pagination cursor. + """ + edges: [ManufacturerEdge!]! + + """ + The list of `Manufacturer` results. + """ + nodes: [Manufacturer!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `Manufacturer` in the context of a `ManufacturerConnection`, +providing access to both the `Manufacturer` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type ManufacturerEdge { + """ + The `Cursor` of this `Manufacturer`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `Manufacturer`. + """ + cursor: Cursor + + """ + The `Manufacturer` of this edge. + """ + node: Manufacturer +} + +""" +Input type used to specify filters on `Manufacturer` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input ManufacturerFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [ManufacturerFilterInput!] + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: ManufacturerFilterInput +} + +""" +Type used to specify the `Manufacturer` fields to group by for aggregations. +""" +type ManufacturerGroupedBy { + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + The `name` field value for this group. + """ + name: String +} + +""" +Enumerates the ways `Manufacturer`s can be sorted. +""" +enum ManufacturerSortOrderInput { + """ + Sorts ascending by the `created_at` field. + """ + created_at_ASC + + """ + Sorts descending by the `created_at` field. + """ + created_at_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC +} + +""" +Input type used to specify parameters for the `matches_phrase` filtering operator. + +Will be ignored if passed as `null`. +""" +input MatchesPhraseFilterInput { + """ + The input phrase to search for. + """ + phrase: String! +} + +""" +Enumeration of allowed values for the `matches_query: {allowed_edits_per_term: ...}` filter option. +""" +enum MatchesQueryAllowedEditsPerTermInput { + """ + Allowed edits per term is dynamically chosen based on the length of the term. + """ + DYNAMIC + + """ + No allowed edits per term. + """ + NONE + + """ + One allowed edit per term. + """ + ONE + + """ + Two allowed edits per term. + """ + TWO +} + +""" +Input type used to specify parameters for the `matches_query` filtering operator. + +Will be ignored if passed as `null`. +""" +input MatchesQueryFilterInput { + """ + Number of allowed modifications per term to arrive at a match. For example, if set to 'ONE', the input + term 'glue' would match 'blue' but not 'clued', since the latter requires two modifications. + """ + allowed_edits_per_term: MatchesQueryAllowedEditsPerTermInput! = DYNAMIC + + """ + The input query to search for. + """ + query: String! + + """ + Set to `true` to match only if all terms in `query` are found, or + `false` to only require one term to be found. + """ + require_all_terms: Boolean! = false +} + +enum Material { + ALLOY + CARBON_FIBER +} + +""" +Input type used to specify filters on `Material` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input MaterialFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [MaterialFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [MaterialInput] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: MaterialFilterInput +} + +enum MaterialInput { + ALLOY + CARBON_FIBER +} + +type MechanicalPart implements NamedEntity { + """ + Aggregations over the `components` data. + """ + component_aggregations( + """ + Used to forward-paginate through the `component_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `component_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Component` documents that get aggregated over based on the provided criteria. + """ + filter: ComponentFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `component_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `component_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `component_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `component_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): ComponentAggregationConnection + components( + """ + Used to forward-paginate through the `components`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `components`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `components` based on the provided criteria. + """ + filter: ComponentFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `components`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `components`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `components`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `components`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `components` should be sorted. + """ + order_by: [ComponentSortOrderInput!] + ): ComponentConnection + created_at: DateTime! + id: ID! + manufacturer: Manufacturer + material: Material + name: String +} + +""" +Type used to perform aggregation computations on `MechanicalPart` fields. +""" +type MechanicalPartAggregatedValues { + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `material` field. + """ + material: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues +} + +""" +Return type representing a bucket of `MechanicalPart` documents for an aggregations query. +""" +type MechanicalPartAggregation { + """ + Provides computed aggregated values over all `MechanicalPart` documents in an aggregation bucket. + """ + aggregated_values: MechanicalPartAggregatedValues + + """ + The count of `MechanicalPart` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `MechanicalPart` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: MechanicalPartGroupedBy +} + +""" +Represents a paginated collection of `MechanicalPartAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type MechanicalPartAggregationConnection { + """ + Wraps a specific `MechanicalPartAggregation` to pair it with its pagination cursor. + """ + edges: [MechanicalPartAggregationEdge!]! + + """ + The list of `MechanicalPartAggregation` results. + """ + nodes: [MechanicalPartAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `MechanicalPartAggregation` in the context of a `MechanicalPartAggregationConnection`, +providing access to both the `MechanicalPartAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type MechanicalPartAggregationEdge { + """ + The `Cursor` of this `MechanicalPartAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `MechanicalPartAggregation`. + """ + cursor: Cursor + + """ + The `MechanicalPartAggregation` of this edge. + """ + node: MechanicalPartAggregation +} + +""" +Represents a paginated collection of `MechanicalPart` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type MechanicalPartConnection { + """ + Wraps a specific `MechanicalPart` to pair it with its pagination cursor. + """ + edges: [MechanicalPartEdge!]! + + """ + The list of `MechanicalPart` results. + """ + nodes: [MechanicalPart!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `MechanicalPart` in the context of a `MechanicalPartConnection`, +providing access to both the `MechanicalPart` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type MechanicalPartEdge { + """ + The `Cursor` of this `MechanicalPart`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `MechanicalPart`. + """ + cursor: Cursor + + """ + The `MechanicalPart` of this edge. + """ + node: MechanicalPart +} + +""" +Input type used to specify filters on `MechanicalPart` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input MechanicalPartFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [MechanicalPartFilterInput!] + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `material` field. + + Will be ignored if `null` or an empty object is passed. + """ + material: MaterialFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: MechanicalPartFilterInput +} + +""" +Type used to specify the `MechanicalPart` fields to group by for aggregations. +""" +type MechanicalPartGroupedBy { + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + The `material` field value for this group. + """ + material: Material + + """ + The `name` field value for this group. + """ + name: String +} + +""" +Enumerates the ways `MechanicalPart`s can be sorted. +""" +enum MechanicalPartSortOrderInput { + """ + Sorts ascending by the `created_at` field. + """ + created_at_ASC + + """ + Sorts descending by the `created_at` field. + """ + created_at_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `material` field. + """ + material_ASC + + """ + Sorts descending by the `material` field. + """ + material_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC +} + +type Money { + amount_cents: Int + currency: String! +} + +""" +Type used to perform aggregation computations on `Money` fields. +""" +type MoneyAggregatedValues { + """ + Computed aggregate values for the `amount_cents` field. + """ + amount_cents: IntAggregatedValues + + """ + Computed aggregate values for the `currency` field. + """ + currency: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on a `Money` object referenced directly +or transitively from a list field that has been configured to index each leaf field as +its own flattened list of values. + +Will be ignored if passed as an empty object (or as `null`). +""" +input MoneyFieldsListFilterInput { + """ + Used to filter on the `amount_cents` field. + + Will be ignored if `null` or an empty object is passed. + """ + amount_cents: IntListFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [MoneyFieldsListFilterInput!] + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Used to filter on the `currency` field. + + Will be ignored if `null` or an empty object is passed. + """ + currency: StringListFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: MoneyFieldsListFilterInput +} + +""" +Input type used to specify filters on `Money` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input MoneyFilterInput { + """ + Used to filter on the `amount_cents` field. + + Will be ignored if `null` or an empty object is passed. + """ + amount_cents: IntFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [MoneyFilterInput!] + + """ + Used to filter on the `currency` field. + + Will be ignored if `null` or an empty object is passed. + """ + currency: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: MoneyFilterInput +} + +""" +Type used to specify the `Money` fields to group by for aggregations. +""" +type MoneyGroupedBy { + """ + The `amount_cents` field value for this group. + """ + amount_cents: Int + + """ + The `currency` field value for this group. + """ + currency: String +} + +""" +Input type used to specify filters on `[Money]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input MoneyListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `MoneyListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [MoneyListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [MoneyListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: MoneyFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: MoneyListFilterInput +} + +interface NamedEntity { + id: ID! + name: String +} + +""" +Type used to perform aggregation computations on `NamedEntity` fields. +""" +type NamedEntityAggregatedValues { + """ + Computed aggregate values for the `amount_cents` field. + """ + amount_cents: IntAggregatedValues + + """ + Computed aggregate values for the `amount_cents2` field. + """ + amount_cents2: IntAggregatedValues + + """ + Computed aggregate values for the `amounts` field. + """ + amounts: IntAggregatedValues + + """ + Computed aggregate values for the `cost` field. + """ + cost: MoneyAggregatedValues + + """ + Computed aggregate values for the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on: DateAggregatedValues + + """ + Computed aggregate values for the `cost_currency_name` field. + """ + cost_currency_name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_symbol` field. + """ + cost_currency_symbol: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_unit` field. + """ + cost_currency_unit: NonNumericAggregatedValues + + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at2` field. + """ + created_at2: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at2_legacy` field. + """ + created_at2_legacy: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at_legacy` field. + """ + created_at_legacy: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at_time_of_day` field. + """ + created_at_time_of_day: LocalTimeAggregatedValues + + """ + Computed aggregate values for the `created_on` field. + """ + created_on: DateAggregatedValues + + """ + Computed aggregate values for the `created_on_legacy` field. + """ + created_on_legacy: DateAggregatedValues + + """ + Computed aggregate values for the `fees` field. + """ + fees: MoneyAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `inventor` field. + """ + inventor: InventorAggregatedValues + + """ + Computed aggregate values for the `material` field. + """ + material: NonNumericAggregatedValues + + """ + Computed aggregate values for the `metadata` field. + """ + metadata: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `named_inventor` field. + """ + named_inventor: NamedInventorAggregatedValues + + """ + Computed aggregate values for the `options` field. + """ + options: WidgetOptionsAggregatedValues + + """ + Computed aggregate values for the `position` field. + """ + position: PositionAggregatedValues + + """ + Computed aggregate values for the `release_dates` field. + """ + release_dates: DateAggregatedValues + + """ + Computed aggregate values for the `release_timestamps` field. + """ + release_timestamps: DateTimeAggregatedValues + + """ + Computed aggregate values for the `size` field. + """ + size: NonNumericAggregatedValues + + """ + Computed aggregate values for the `tags` field. + """ + tags: NonNumericAggregatedValues + + """ + Computed aggregate values for the `the_options` field. + """ + the_options: WidgetOptionsAggregatedValues + + """ + Computed aggregate values for the `voltage` field. + """ + voltage: IntAggregatedValues + + """ + Computed aggregate values for the `weight_in_ng` field. + """ + weight_in_ng: JsonSafeLongAggregatedValues + + """ + Computed aggregate values for the `weight_in_ng_str` field. + """ + weight_in_ng_str: LongStringAggregatedValues + + """ + Computed aggregate values for the `widget_cost` field. + """ + widget_cost: MoneyAggregatedValues + + """ + Computed aggregate values for the `widget_name` field. + """ + widget_name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_size` field. + """ + widget_size: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_tags` field. + """ + widget_tags: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_workspace_id` field. + """ + widget_workspace_id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `workspace_id` field. + """ + workspace_id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `workspace_name` field. + """ + workspace_name: NonNumericAggregatedValues +} + +""" +Return type representing a bucket of `NamedEntity` documents for an aggregations query. +""" +type NamedEntityAggregation { + """ + Provides computed aggregated values over all `NamedEntity` documents in an aggregation bucket. + """ + aggregated_values: NamedEntityAggregatedValues + + """ + The count of `NamedEntity` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `NamedEntity` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: NamedEntityGroupedBy +} + +""" +Represents a paginated collection of `NamedEntityAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type NamedEntityAggregationConnection { + """ + Wraps a specific `NamedEntityAggregation` to pair it with its pagination cursor. + """ + edges: [NamedEntityAggregationEdge!]! + + """ + The list of `NamedEntityAggregation` results. + """ + nodes: [NamedEntityAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `NamedEntityAggregation` in the context of a `NamedEntityAggregationConnection`, +providing access to both the `NamedEntityAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type NamedEntityAggregationEdge { + """ + The `Cursor` of this `NamedEntityAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `NamedEntityAggregation`. + """ + cursor: Cursor + + """ + The `NamedEntityAggregation` of this edge. + """ + node: NamedEntityAggregation +} + +""" +Represents a paginated collection of `NamedEntity` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type NamedEntityConnection { + """ + Wraps a specific `NamedEntity` to pair it with its pagination cursor. + """ + edges: [NamedEntityEdge!]! + + """ + The list of `NamedEntity` results. + """ + nodes: [NamedEntity!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `NamedEntity` in the context of a `NamedEntityConnection`, +providing access to both the `NamedEntity` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type NamedEntityEdge { + """ + The `Cursor` of this `NamedEntity`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `NamedEntity`. + """ + cursor: Cursor + + """ + The `NamedEntity` of this edge. + """ + node: NamedEntity +} + +""" +Input type used to specify filters on `NamedEntity` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input NamedEntityFilterInput { + """ + Used to filter on the `amount_cents` field. + + Will be ignored if `null` or an empty object is passed. + """ + amount_cents: IntFilterInput + + """ + Used to filter on the `amount_cents2` field. + + Will be ignored if `null` or an empty object is passed. + """ + amount_cents2: IntFilterInput + + """ + Used to filter on the `amounts` field. + + Will be ignored if `null` or an empty object is passed. + """ + amounts: IntListFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [NamedEntityFilterInput!] + + """ + Used to filter on the `cost` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost: MoneyFilterInput + + """ + Used to filter on the `cost_currency_introduced_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_introduced_on: DateFilterInput + + """ + Used to filter on the `cost_currency_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_name: StringFilterInput + + """ + Used to filter on the `cost_currency_primary_continent` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_primary_continent: StringFilterInput + + """ + Used to filter on the `cost_currency_symbol` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_symbol: StringFilterInput + + """ + Used to filter on the `cost_currency_unit` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_unit: StringFilterInput + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `created_at2` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at2: DateTimeFilterInput + + """ + Used to filter on the `created_at2_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at2_legacy: DateTimeFilterInput + + """ + Used to filter on the `created_at_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at_legacy: DateTimeFilterInput + + """ + Used to filter on the `created_at_time_of_day` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at_time_of_day: LocalTimeFilterInput + + """ + Used to filter on the `created_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_on: DateFilterInput + + """ + Used to filter on the `created_on_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_on_legacy: DateFilterInput + + """ + Used to filter on the `fees` field. + + Will be ignored if `null` or an empty object is passed. + """ + fees: MoneyFieldsListFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `inventor` field. + + Will be ignored if `null` or an empty object is passed. + """ + inventor: InventorFilterInput + + """ + Used to filter on the `material` field. + + Will be ignored if `null` or an empty object is passed. + """ + material: MaterialFilterInput + + """ + Used to filter on the `metadata` field. + + Will be ignored if `null` or an empty object is passed. + """ + metadata: UntypedFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Used to filter on the `name_text` field. + + Will be ignored if `null` or an empty object is passed. + """ + name_text: TextFilterInput + + """ + Used to filter on the `named_inventor` field. + + Will be ignored if `null` or an empty object is passed. + """ + named_inventor: NamedInventorFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: NamedEntityFilterInput + + """ + Used to filter on the `options` field. + + Will be ignored if `null` or an empty object is passed. + """ + options: WidgetOptionsFilterInput + + """ + Used to filter on the `position` field. + + Will be ignored if `null` or an empty object is passed. + """ + position: PositionFilterInput + + """ + Used to filter on the `release_dates` field. + + Will be ignored if `null` or an empty object is passed. + """ + release_dates: DateListFilterInput + + """ + Used to filter on the `release_timestamps` field. + + Will be ignored if `null` or an empty object is passed. + """ + release_timestamps: DateTimeListFilterInput + + """ + Used to filter on the `size` field. + + Will be ignored if `null` or an empty object is passed. + """ + size: SizeFilterInput + + """ + Used to filter on the `tags` field. + + Will be ignored if `null` or an empty object is passed. + """ + tags: StringListFilterInput + + """ + Used to filter on the `the_options` field. + + Will be ignored if `null` or an empty object is passed. + """ + the_options: WidgetOptionsFilterInput + + """ + Used to filter on the `voltage` field. + + Will be ignored if `null` or an empty object is passed. + """ + voltage: IntFilterInput + + """ + Used to filter on the `weight_in_ng` field. + + Will be ignored if `null` or an empty object is passed. + """ + weight_in_ng: JsonSafeLongFilterInput + + """ + Used to filter on the `weight_in_ng_str` field. + + Will be ignored if `null` or an empty object is passed. + """ + weight_in_ng_str: LongStringFilterInput + + """ + Used to filter on the `widget_cost` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_cost: MoneyFilterInput + + """ + Used to filter on the `widget_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_name: StringFilterInput + + """ + Used to filter on the `widget_size` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_size: SizeFilterInput + + """ + Used to filter on the `widget_tags` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_tags: StringListFilterInput + + """ + Used to filter on the `widget_workspace_id` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_workspace_id: IDFilterInput + + """ + Used to filter on the `workspace_id` field. + + Will be ignored if `null` or an empty object is passed. + """ + workspace_id: IDFilterInput + + """ + Used to filter on the `workspace_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + workspace_name: StringFilterInput +} + +""" +Type used to specify the `NamedEntity` fields to group by for aggregations. +""" +type NamedEntityGroupedBy { + """ + The `amount_cents` field value for this group. + """ + amount_cents: Int + + """ + The `amount_cents2` field value for this group. + """ + amount_cents2: Int + + """ + The `cost` field value for this group. + """ + cost: MoneyGroupedBy + + """ + Offers the different grouping options for the `cost_currency_introduced_on` value within this group. + """ + cost_currency_introduced_on: DateGroupedBy + + """ + The `cost_currency_name` field value for this group. + """ + cost_currency_name: String + + """ + The `cost_currency_primary_continent` field value for this group. + """ + cost_currency_primary_continent: String + + """ + The `cost_currency_symbol` field value for this group. + """ + cost_currency_symbol: String + + """ + The `cost_currency_unit` field value for this group. + """ + cost_currency_unit: String + + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + Offers the different grouping options for the `created_at2` value within this group. + """ + created_at2: DateTimeGroupedBy + + """ + The `created_at2_legacy` field value for this group. + """ + created_at2_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateTimeGroupingGranularityInput! + + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change + what day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone = "UTC" + ): DateTime + + """ + The `created_at_legacy` field value for this group. + """ + created_at_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateTimeGroupingGranularityInput! + + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change + what day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone = "UTC" + ): DateTime + + """ + The `created_at_time_of_day` field value for this group. + """ + created_at_time_of_day: LocalTime + + """ + Offers the different grouping options for the `created_on` value within this group. + """ + created_on: DateGroupedBy + + """ + The `created_on_legacy` field value for this group. + """ + created_on_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateGroupingGranularityInput! + + """ + Number of days (positive or negative) to shift the `Date` boundaries of each date grouping bucket. + + For example, when grouping by `YEAR`, this can be used to align the buckets + with fiscal or school years instead of calendar years. + """ + offset_days: Int + ): Date + + """ + The `fees` field value for this group. + + Note: `fees` is a collection field, but selecting this field will group on + individual values of the selected subfields of `fees`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `fees` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `fees` multiple times for a single document, that document will only be included in the group + once. + """ + fees: MoneyGroupedBy + + """ + The `inventor` field value for this group. + """ + inventor: InventorGroupedBy + + """ + The `material` field value for this group. + """ + material: Material + + """ + The `metadata` field value for this group. + """ + metadata: Untyped + + """ + The `name` field value for this group. + """ + name: String + + """ + The `named_inventor` field value for this group. + """ + named_inventor: NamedInventorGroupedBy + + """ + The `options` field value for this group. + """ + options: WidgetOptionsGroupedBy + + """ + The `position` field value for this group. + """ + position: PositionGroupedBy + + """ + The individual value from `release_dates` for this group. + + Note: `release_dates` is a collection field, but selecting this field will group on individual values of `release_dates`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `release_dates` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `release_dates` multiple times for a single document, that document will only be included in the group + once. + """ + release_date: DateGroupedBy + + """ + The individual value from `release_timestamps` for this group. + + Note: `release_timestamps` is a collection field, but selecting this field + will group on individual values of `release_timestamps`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `release_timestamps` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `release_timestamps` multiple times for a single document, that document will only be included in the group + once. + """ + release_timestamp: DateTimeGroupedBy + + """ + The `size` field value for this group. + """ + size: Size + + """ + The individual value from `tags` for this group. + + Note: `tags` is a collection field, but selecting this field will group on individual values of `tags`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `tags` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `tags` multiple times for a single document, that document will only be included in the group + once. + """ + tag: String + + """ + The `the_options` field value for this group. + """ + the_options: WidgetOptionsGroupedBy + + """ + The `voltage` field value for this group. + """ + voltage: Int + + """ + The `weight_in_ng` field value for this group. + """ + weight_in_ng: JsonSafeLong + + """ + The `weight_in_ng_str` field value for this group. + """ + weight_in_ng_str: LongString + + """ + The `widget_cost` field value for this group. + """ + widget_cost: MoneyGroupedBy + + """ + The `widget_name` field value for this group. + """ + widget_name: String + + """ + The `widget_size` field value for this group. + """ + widget_size: Size + + """ + The `widget_workspace_id` field value for this group. + """ + widget_workspace_id: ID + + """ + The `workspace_id` field value for this group. + """ + workspace_id: ID + + """ + The `workspace_name` field value for this group. + """ + workspace_name: String +} + +""" +Enumerates the ways `NamedEntity`s can be sorted. +""" +enum NamedEntitySortOrderInput { + """ + Sorts ascending by the `amount_cents2` field. + """ + amount_cents2_ASC + + """ + Sorts descending by the `amount_cents2` field. + """ + amount_cents2_DESC + + """ + Sorts ascending by the `amount_cents` field. + """ + amount_cents_ASC + + """ + Sorts descending by the `amount_cents` field. + """ + amount_cents_DESC + + """ + Sorts ascending by the `cost.amount_cents` field. + """ + cost_amount_cents_ASC + + """ + Sorts descending by the `cost.amount_cents` field. + """ + cost_amount_cents_DESC + + """ + Sorts ascending by the `cost.currency` field. + """ + cost_currency_ASC + + """ + Sorts descending by the `cost.currency` field. + """ + cost_currency_DESC + + """ + Sorts ascending by the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on_ASC + + """ + Sorts descending by the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on_DESC + + """ + Sorts ascending by the `cost_currency_name` field. + """ + cost_currency_name_ASC + + """ + Sorts descending by the `cost_currency_name` field. + """ + cost_currency_name_DESC + + """ + Sorts ascending by the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent_ASC + + """ + Sorts descending by the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent_DESC + + """ + Sorts ascending by the `cost_currency_symbol` field. + """ + cost_currency_symbol_ASC + + """ + Sorts descending by the `cost_currency_symbol` field. + """ + cost_currency_symbol_DESC + + """ + Sorts ascending by the `cost_currency_unit` field. + """ + cost_currency_unit_ASC + + """ + Sorts descending by the `cost_currency_unit` field. + """ + cost_currency_unit_DESC + + """ + Sorts ascending by the `created_at2` field. + """ + created_at2_ASC + + """ + Sorts descending by the `created_at2` field. + """ + created_at2_DESC + + """ + Sorts ascending by the `created_at2_legacy` field. + """ + created_at2_legacy_ASC + + """ + Sorts descending by the `created_at2_legacy` field. + """ + created_at2_legacy_DESC + + """ + Sorts ascending by the `created_at` field. + """ + created_at_ASC + + """ + Sorts descending by the `created_at` field. + """ + created_at_DESC + + """ + Sorts ascending by the `created_at_legacy` field. + """ + created_at_legacy_ASC + + """ + Sorts descending by the `created_at_legacy` field. + """ + created_at_legacy_DESC + + """ + Sorts ascending by the `created_at_time_of_day` field. + """ + created_at_time_of_day_ASC + + """ + Sorts descending by the `created_at_time_of_day` field. + """ + created_at_time_of_day_DESC + + """ + Sorts ascending by the `created_on` field. + """ + created_on_ASC + + """ + Sorts descending by the `created_on` field. + """ + created_on_DESC + + """ + Sorts ascending by the `created_on_legacy` field. + """ + created_on_legacy_ASC + + """ + Sorts descending by the `created_on_legacy` field. + """ + created_on_legacy_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `inventor.name` field. + """ + inventor_name_ASC + + """ + Sorts descending by the `inventor.name` field. + """ + inventor_name_DESC + + """ + Sorts ascending by the `inventor.nationality` field. + """ + inventor_nationality_ASC + + """ + Sorts descending by the `inventor.nationality` field. + """ + inventor_nationality_DESC + + """ + Sorts ascending by the `inventor.stock_ticker` field. + """ + inventor_stock_ticker_ASC + + """ + Sorts descending by the `inventor.stock_ticker` field. + """ + inventor_stock_ticker_DESC + + """ + Sorts ascending by the `material` field. + """ + material_ASC + + """ + Sorts descending by the `material` field. + """ + material_DESC + + """ + Sorts ascending by the `metadata` field. + """ + metadata_ASC + + """ + Sorts descending by the `metadata` field. + """ + metadata_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC + + """ + Sorts ascending by the `named_inventor.name` field. + """ + named_inventor_name_ASC + + """ + Sorts descending by the `named_inventor.name` field. + """ + named_inventor_name_DESC + + """ + Sorts ascending by the `named_inventor.nationality` field. + """ + named_inventor_nationality_ASC + + """ + Sorts descending by the `named_inventor.nationality` field. + """ + named_inventor_nationality_DESC + + """ + Sorts ascending by the `named_inventor.stock_ticker` field. + """ + named_inventor_stock_ticker_ASC + + """ + Sorts descending by the `named_inventor.stock_ticker` field. + """ + named_inventor_stock_ticker_DESC + + """ + Sorts ascending by the `options.color` field. + """ + options_color_ASC + + """ + Sorts descending by the `options.color` field. + """ + options_color_DESC + + """ + Sorts ascending by the `options.size` field. + """ + options_size_ASC + + """ + Sorts descending by the `options.size` field. + """ + options_size_DESC + + """ + Sorts ascending by the `options.the_size` field. + """ + options_the_size_ASC + + """ + Sorts descending by the `options.the_size` field. + """ + options_the_size_DESC + + """ + Sorts ascending by the `position.x` field. + """ + position_x_ASC + + """ + Sorts descending by the `position.x` field. + """ + position_x_DESC + + """ + Sorts ascending by the `position.y` field. + """ + position_y_ASC + + """ + Sorts descending by the `position.y` field. + """ + position_y_DESC + + """ + Sorts ascending by the `size` field. + """ + size_ASC + + """ + Sorts descending by the `size` field. + """ + size_DESC + + """ + Sorts ascending by the `the_options.color` field. + """ + the_options_color_ASC + + """ + Sorts descending by the `the_options.color` field. + """ + the_options_color_DESC + + """ + Sorts ascending by the `the_options.size` field. + """ + the_options_size_ASC + + """ + Sorts descending by the `the_options.size` field. + """ + the_options_size_DESC + + """ + Sorts ascending by the `the_options.the_size` field. + """ + the_options_the_size_ASC + + """ + Sorts descending by the `the_options.the_size` field. + """ + the_options_the_size_DESC + + """ + Sorts ascending by the `voltage` field. + """ + voltage_ASC + + """ + Sorts descending by the `voltage` field. + """ + voltage_DESC + + """ + Sorts ascending by the `weight_in_ng` field. + """ + weight_in_ng_ASC + + """ + Sorts descending by the `weight_in_ng` field. + """ + weight_in_ng_DESC + + """ + Sorts ascending by the `weight_in_ng_str` field. + """ + weight_in_ng_str_ASC + + """ + Sorts descending by the `weight_in_ng_str` field. + """ + weight_in_ng_str_DESC + + """ + Sorts ascending by the `widget_cost.amount_cents` field. + """ + widget_cost_amount_cents_ASC + + """ + Sorts descending by the `widget_cost.amount_cents` field. + """ + widget_cost_amount_cents_DESC + + """ + Sorts ascending by the `widget_cost.currency` field. + """ + widget_cost_currency_ASC + + """ + Sorts descending by the `widget_cost.currency` field. + """ + widget_cost_currency_DESC + + """ + Sorts ascending by the `widget_name` field. + """ + widget_name_ASC + + """ + Sorts descending by the `widget_name` field. + """ + widget_name_DESC + + """ + Sorts ascending by the `widget_size` field. + """ + widget_size_ASC + + """ + Sorts descending by the `widget_size` field. + """ + widget_size_DESC + + """ + Sorts ascending by the `widget_workspace_id` field. + """ + widget_workspace_id_ASC + + """ + Sorts descending by the `widget_workspace_id` field. + """ + widget_workspace_id_DESC + + """ + Sorts ascending by the `workspace_id` field. + """ + workspace_id_ASC + + """ + Sorts descending by the `workspace_id` field. + """ + workspace_id_DESC + + """ + Sorts ascending by the `workspace_name` field. + """ + workspace_name_ASC + + """ + Sorts descending by the `workspace_name` field. + """ + workspace_name_DESC +} + +interface NamedInventor { + name: String +} + +""" +Type used to perform aggregation computations on `NamedInventor` fields. +""" +type NamedInventorAggregatedValues { + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `nationality` field. + """ + nationality: NonNumericAggregatedValues + + """ + Computed aggregate values for the `stock_ticker` field. + """ + stock_ticker: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on `NamedInventor` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input NamedInventorFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [NamedInventorFilterInput!] + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Used to filter on the `nationality` field. + + Will be ignored if `null` or an empty object is passed. + """ + nationality: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: NamedInventorFilterInput + + """ + Used to filter on the `stock_ticker` field. + + Will be ignored if `null` or an empty object is passed. + """ + stock_ticker: StringFilterInput +} + +""" +Type used to specify the `NamedInventor` fields to group by for aggregations. +""" +type NamedInventorGroupedBy { + """ + The `name` field value for this group. + """ + name: String + + """ + The `nationality` field value for this group. + """ + nationality: String + + """ + The `stock_ticker` field value for this group. + """ + stock_ticker: String +} + +""" +A return type used from aggregations to provided aggregated values over non-numeric fields. +""" +type NonNumericAggregatedValues { + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in + Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + approximate_distinct_value_count: JsonSafeLong +} + +""" +Provides information about the specific fetched page. This implements the `PageInfo` +specification from the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-undefined.PageInfo). +""" +type PageInfo { + """ + The `Cursor` of the last edge of the current page. This can be passed in the next query as + a `after` argument to paginate forwards. + """ + end_cursor: Cursor + + """ + Indicates if there is another page of results available after the current one. + """ + has_next_page: Boolean! + + """ + Indicates if there is another page of results available before the current one. + """ + has_previous_page: Boolean! + + """ + The `Cursor` of the first edge of the current page. This can be passed in the next query as + a `before` argument to paginate backwards. + """ + start_cursor: Cursor +} + +union Part = ElectricalPart | MechanicalPart + +""" +Type used to perform aggregation computations on `Part` fields. +""" +type PartAggregatedValues { + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `material` field. + """ + material: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `voltage` field. + """ + voltage: IntAggregatedValues +} + +""" +Return type representing a bucket of `Part` documents for an aggregations query. +""" +type PartAggregation { + """ + Provides computed aggregated values over all `Part` documents in an aggregation bucket. + """ + aggregated_values: PartAggregatedValues + + """ + The count of `Part` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `Part` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: PartGroupedBy +} + +""" +Represents a paginated collection of `PartAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type PartAggregationConnection { + """ + Wraps a specific `PartAggregation` to pair it with its pagination cursor. + """ + edges: [PartAggregationEdge!]! + + """ + The list of `PartAggregation` results. + """ + nodes: [PartAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `PartAggregation` in the context of a `PartAggregationConnection`, +providing access to both the `PartAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type PartAggregationEdge { + """ + The `Cursor` of this `PartAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `PartAggregation`. + """ + cursor: Cursor + + """ + The `PartAggregation` of this edge. + """ + node: PartAggregation +} + +""" +Represents a paginated collection of `Part` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type PartConnection { + """ + Wraps a specific `Part` to pair it with its pagination cursor. + """ + edges: [PartEdge!]! + + """ + The list of `Part` results. + """ + nodes: [Part!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `Part` in the context of a `PartConnection`, +providing access to both the `Part` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type PartEdge { + """ + The `Cursor` of this `Part`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `Part`. + """ + cursor: Cursor + + """ + The `Part` of this edge. + """ + node: Part +} + +""" +Input type used to specify filters on `Part` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input PartFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [PartFilterInput!] + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `material` field. + + Will be ignored if `null` or an empty object is passed. + """ + material: MaterialFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: PartFilterInput + + """ + Used to filter on the `voltage` field. + + Will be ignored if `null` or an empty object is passed. + """ + voltage: IntFilterInput +} + +""" +Type used to specify the `Part` fields to group by for aggregations. +""" +type PartGroupedBy { + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + The `material` field value for this group. + """ + material: Material + + """ + The `name` field value for this group. + """ + name: String + + """ + The `voltage` field value for this group. + """ + voltage: Int +} + +""" +Enumerates the ways `Part`s can be sorted. +""" +enum PartSortOrderInput { + """ + Sorts ascending by the `created_at` field. + """ + created_at_ASC + + """ + Sorts descending by the `created_at` field. + """ + created_at_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `material` field. + """ + material_ASC + + """ + Sorts descending by the `material` field. + """ + material_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC + + """ + Sorts ascending by the `voltage` field. + """ + voltage_ASC + + """ + Sorts descending by the `voltage` field. + """ + voltage_DESC +} + +type Person implements NamedInventor { + name: String + nationality: String +} + +type Player { + affiliations: Affiliations! + name: String + nicknames: [String!]! + seasons_nested: [PlayerSeason!]! + seasons_object: [PlayerSeason!]! +} + +""" +Type used to perform aggregation computations on `Player` fields. +""" +type PlayerAggregatedValues { + """ + Computed aggregate values for the `affiliations` field. + """ + affiliations: AffiliationsAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `nicknames` field. + """ + nicknames: NonNumericAggregatedValues + + """ + Computed aggregate values for the `seasons_object` field. + """ + seasons_object: PlayerSeasonAggregatedValues +} + +""" +Input type used to specify filters on a `Player` object referenced directly +or transitively from a list field that has been configured to index each leaf field as +its own flattened list of values. + +Will be ignored if passed as an empty object (or as `null`). +""" +input PlayerFieldsListFilterInput { + """ + Used to filter on the `affiliations` field. + + Will be ignored if `null` or an empty object is passed. + """ + affiliations: AffiliationsFieldsListFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [PlayerFieldsListFilterInput!] + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringListFilterInput + + """ + Used to filter on the `nicknames` field. + + Will be ignored if `null` or an empty object is passed. + """ + nicknames: StringListFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: PlayerFieldsListFilterInput + + """ + Used to filter on the `seasons_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + seasons_nested: PlayerSeasonListFilterInput + + """ + Used to filter on the `seasons_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + seasons_object: PlayerSeasonFieldsListFilterInput +} + +""" +Input type used to specify filters on `Player` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input PlayerFilterInput { + """ + Used to filter on the `affiliations` field. + + Will be ignored if `null` or an empty object is passed. + """ + affiliations: AffiliationsFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [PlayerFilterInput!] + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Used to filter on the `nicknames` field. + + Will be ignored if `null` or an empty object is passed. + """ + nicknames: StringListFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: PlayerFilterInput + + """ + Used to filter on the `seasons_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + seasons_nested: PlayerSeasonListFilterInput + + """ + Used to filter on the `seasons_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + seasons_object: PlayerSeasonFieldsListFilterInput +} + +""" +Type used to specify the `Player` fields to group by for aggregations. +""" +type PlayerGroupedBy { + """ + The `affiliations` field value for this group. + """ + affiliations: AffiliationsGroupedBy + + """ + The `name` field value for this group. + """ + name: String + + """ + The `seasons_object` field value for this group. + + Note: `seasons_object` is a collection field, but selecting this field will + group on individual values of the selected subfields of `seasons_object`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `seasons_object` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `seasons_object` multiple times for a single document, that document will only be included in the group + once. + """ + seasons_object: PlayerSeasonGroupedBy +} + +""" +Input type used to specify filters on `[Player]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input PlayerListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `PlayerListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [PlayerListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [PlayerListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: PlayerFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: PlayerListFilterInput +} + +type PlayerSeason { + awards( + """ + Used to forward-paginate through the `awards`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `awards`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used in conjunction with the `after` argument to forward-paginate through the `awards`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `awards`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `awards`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `awards`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): StringConnection + games_played: Int + year: Int +} + +""" +Type used to perform aggregation computations on `PlayerSeason` fields. +""" +type PlayerSeasonAggregatedValues { + """ + Computed aggregate values for the `awards` field. + """ + awards: NonNumericAggregatedValues + + """ + Computed aggregate values for the `games_played` field. + """ + games_played: IntAggregatedValues + + """ + Computed aggregate values for the `year` field. + """ + year: IntAggregatedValues +} + +""" +Input type used to specify filters on a `PlayerSeason` object referenced directly +or transitively from a list field that has been configured to index each leaf field as +its own flattened list of values. + +Will be ignored if passed as an empty object (or as `null`). +""" +input PlayerSeasonFieldsListFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [PlayerSeasonFieldsListFilterInput!] + + """ + Used to filter on the `awards` field. + + Will be ignored if `null` or an empty object is passed. + """ + awards: StringListFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Used to filter on the `games_played` field. + + Will be ignored if `null` or an empty object is passed. + """ + games_played: IntListFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: PlayerSeasonFieldsListFilterInput + + """ + Used to filter on the `year` field. + + Will be ignored if `null` or an empty object is passed. + """ + year: IntListFilterInput +} + +""" +Input type used to specify filters on `PlayerSeason` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input PlayerSeasonFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [PlayerSeasonFilterInput!] + + """ + Used to filter on the `awards` field. + + Will be ignored if `null` or an empty object is passed. + """ + awards: StringListFilterInput + + """ + Used to filter on the `games_played` field. + + Will be ignored if `null` or an empty object is passed. + """ + games_played: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: PlayerSeasonFilterInput + + """ + Used to filter on the `year` field. + + Will be ignored if `null` or an empty object is passed. + """ + year: IntFilterInput +} + +""" +Type used to specify the `PlayerSeason` fields to group by for aggregations. +""" +type PlayerSeasonGroupedBy { + """ + The `games_played` field value for this group. + """ + games_played: Int + + """ + The `year` field value for this group. + """ + year: Int +} + +""" +Input type used to specify filters on `[PlayerSeason]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input PlayerSeasonListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `PlayerSeasonListFilterInput` input because of + collisions between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [PlayerSeasonListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [PlayerSeasonListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: PlayerSeasonFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: PlayerSeasonListFilterInput +} + +type Position { + x: Float! + y: Float! +} + +""" +Type used to perform aggregation computations on `Position` fields. +""" +type PositionAggregatedValues { + """ + Computed aggregate values for the `x` field. + """ + x: FloatAggregatedValues + + """ + Computed aggregate values for the `y` field. + """ + y: FloatAggregatedValues +} + +""" +Input type used to specify filters on `Position` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input PositionFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [PositionFilterInput!] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: PositionFilterInput + + """ + Used to filter on the `x` field. + + Will be ignored if `null` or an empty object is passed. + """ + x: FloatFilterInput + + """ + Used to filter on the `y` field. + + Will be ignored if `null` or an empty object is passed. + """ + y: FloatFilterInput +} + +""" +Type used to specify the `Position` fields to group by for aggregations. +""" +type PositionGroupedBy { + """ + The `x` field value for this group. + """ + x: Float + + """ + The `y` field value for this group. + """ + y: Float +} + +""" +The query entry point for the entire schema. +""" +type Query { + """ + Aggregations over the `addresses` data: + + > Fetches `Address`s based on the provided arguments. + """ + address_aggregations( + """ + Used to forward-paginate through the `address_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `address_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Address` documents that get aggregated over based on the provided criteria. + """ + filter: AddressFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `address_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `address_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `address_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `address_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): AddressAggregationConnection + + """ + Fetches `Address`s based on the provided arguments. + """ + addresses( + """ + Used to forward-paginate through the `addresses`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `addresses`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `addresses` based on the provided criteria. + """ + filter: AddressFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `addresses`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `addresses`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `addresses`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `addresses`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `addresses` should be sorted. + """ + order_by: [AddressSortOrderInput!] + ): AddressConnection + + """ + Aggregations over the `components` data: + + > Fetches `Component`s based on the provided arguments. + """ + component_aggregations( + """ + Used to forward-paginate through the `component_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `component_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Component` documents that get aggregated over based on the provided criteria. + """ + filter: ComponentFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `component_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `component_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `component_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `component_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): ComponentAggregationConnection + + """ + Fetches `Component`s based on the provided arguments. + """ + components( + """ + Used to forward-paginate through the `components`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `components`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `components` based on the provided criteria. + """ + filter: ComponentFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `components`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `components`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `components`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `components`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `components` should be sorted. + """ + order_by: [ComponentSortOrderInput!] + ): ComponentConnection + + """ + Aggregations over the `electrical_parts` data: + + > Fetches `ElectricalPart`s based on the provided arguments. + """ + electrical_part_aggregations( + """ + Used to forward-paginate through the `electrical_part_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `electrical_part_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `ElectricalPart` documents that get aggregated over based on the provided criteria. + """ + filter: ElectricalPartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `electrical_part_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `electrical_part_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `electrical_part_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `electrical_part_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): ElectricalPartAggregationConnection + + """ + Fetches `ElectricalPart`s based on the provided arguments. + """ + electrical_parts( + """ + Used to forward-paginate through the `electrical_parts`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `electrical_parts`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `electrical_parts` based on the provided criteria. + """ + filter: ElectricalPartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `electrical_parts`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `electrical_parts`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `electrical_parts`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `electrical_parts`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `electrical_parts` should be sorted. + """ + order_by: [ElectricalPartSortOrderInput!] + ): ElectricalPartConnection + + """ + Aggregations over the `manufacturers` data: + + > Fetches `Manufacturer`s based on the provided arguments. + """ + manufacturer_aggregations( + """ + Used to forward-paginate through the `manufacturer_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `manufacturer_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Manufacturer` documents that get aggregated over based on the provided criteria. + """ + filter: ManufacturerFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `manufacturer_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `manufacturer_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `manufacturer_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `manufacturer_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): ManufacturerAggregationConnection + + """ + Fetches `Manufacturer`s based on the provided arguments. + """ + manufacturers( + """ + Used to forward-paginate through the `manufacturers`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `manufacturers`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `manufacturers` based on the provided criteria. + """ + filter: ManufacturerFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `manufacturers`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `manufacturers`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `manufacturers`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `manufacturers`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `manufacturers` should be sorted. + """ + order_by: [ManufacturerSortOrderInput!] + ): ManufacturerConnection + + """ + Aggregations over the `mechanical_parts` data: + + > Fetches `MechanicalPart`s based on the provided arguments. + """ + mechanical_part_aggregations( + """ + Used to forward-paginate through the `mechanical_part_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `mechanical_part_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `MechanicalPart` documents that get aggregated over based on the provided criteria. + """ + filter: MechanicalPartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `mechanical_part_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `mechanical_part_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `mechanical_part_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `mechanical_part_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): MechanicalPartAggregationConnection + + """ + Fetches `MechanicalPart`s based on the provided arguments. + """ + mechanical_parts( + """ + Used to forward-paginate through the `mechanical_parts`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `mechanical_parts`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `mechanical_parts` based on the provided criteria. + """ + filter: MechanicalPartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `mechanical_parts`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `mechanical_parts`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `mechanical_parts`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `mechanical_parts`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `mechanical_parts` should be sorted. + """ + order_by: [MechanicalPartSortOrderInput!] + ): MechanicalPartConnection + + """ + Fetches `NamedEntity`s based on the provided arguments. + """ + named_entities( + """ + Used to forward-paginate through the `named_entities`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `named_entities`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `named_entities` based on the provided criteria. + """ + filter: NamedEntityFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `named_entities`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `named_entities`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `named_entities`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `named_entities`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `named_entities` should be sorted. + """ + order_by: [NamedEntitySortOrderInput!] + ): NamedEntityConnection + + """ + Aggregations over the `named_entities` data: + + > Fetches `NamedEntity`s based on the provided arguments. + """ + named_entity_aggregations( + """ + Used to forward-paginate through the `named_entity_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `named_entity_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `NamedEntity` documents that get aggregated over based on the provided criteria. + """ + filter: NamedEntityFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `named_entity_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `named_entity_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `named_entity_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `named_entity_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): NamedEntityAggregationConnection + + """ + Aggregations over the `parts` data: + + > Fetches `Part`s based on the provided arguments. + """ + part_aggregations( + """ + Used to forward-paginate through the `part_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `part_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Part` documents that get aggregated over based on the provided criteria. + """ + filter: PartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `part_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `part_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `part_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `part_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): PartAggregationConnection + + """ + Fetches `Part`s based on the provided arguments. + """ + parts( + """ + Used to forward-paginate through the `parts`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `parts`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `parts` based on the provided criteria. + """ + filter: PartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `parts`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `parts`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `parts`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `parts`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `parts` should be sorted. + """ + order_by: [PartSortOrderInput!] + ): PartConnection + + """ + Aggregations over the `sponsors` data: + + > Fetches `Sponsor`s based on the provided arguments. + """ + sponsor_aggregations( + """ + Used to forward-paginate through the `sponsor_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `sponsor_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Sponsor` documents that get aggregated over based on the provided criteria. + """ + filter: SponsorFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `sponsor_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `sponsor_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `sponsor_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `sponsor_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): SponsorAggregationConnection + + """ + Fetches `Sponsor`s based on the provided arguments. + """ + sponsors( + """ + Used to forward-paginate through the `sponsors`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `sponsors`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `sponsors` based on the provided criteria. + """ + filter: SponsorFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `sponsors`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `sponsors`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `sponsors`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `sponsors`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `sponsors` should be sorted. + """ + order_by: [SponsorSortOrderInput!] + ): SponsorConnection + + """ + Aggregations over the `teams` data: + + > Fetches `Team`s based on the provided arguments. + """ + team_aggregations( + """ + Used to forward-paginate through the `team_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `team_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Team` documents that get aggregated over based on the provided criteria. + """ + filter: TeamFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `team_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `team_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `team_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `team_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): TeamAggregationConnection + + """ + Fetches `Team`s based on the provided arguments. + """ + teams( + """ + Used to forward-paginate through the `teams`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `teams`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `teams` based on the provided criteria. + """ + filter: TeamFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `teams`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `teams`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `teams`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `teams`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `teams` should be sorted. + """ + order_by: [TeamSortOrderInput!] + ): TeamConnection + + """ + Aggregations over the `widgets` data: + + > Fetches `Widget`s based on the provided arguments. + + Note: aggregation queries are relatively expensive, and some fields have been pre-aggregated to allow + more efficient queries for some common aggregation cases: + + - The root `widget_currencies` field groups by `cost.currency` + """ + widget_aggregations( + """ + Used to forward-paginate through the `widget_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widget_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Widget` documents that get aggregated over based on the provided criteria. + """ + filter: WidgetFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widget_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widget_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widget_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widget_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): WidgetAggregationConnection + + """ + Fetches `WidgetCurrency`s based on the provided arguments. + """ + widget_currencies( + """ + Used to forward-paginate through the `widget_currencies`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widget_currencies`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `widget_currencies` based on the provided criteria. + """ + filter: WidgetCurrencyFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widget_currencies`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widget_currencies`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widget_currencies`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widget_currencies`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `widget_currencies` should be sorted. + """ + order_by: [WidgetCurrencySortOrderInput!] + ): WidgetCurrencyConnection + + """ + Aggregations over the `widget_currencies` data: + + > Fetches `WidgetCurrency`s based on the provided arguments. + """ + widget_currency_aggregations( + """ + Used to forward-paginate through the `widget_currency_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widget_currency_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `WidgetCurrency` documents that get aggregated over based on the provided criteria. + """ + filter: WidgetCurrencyFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widget_currency_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widget_currency_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widget_currency_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widget_currency_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): WidgetCurrencyAggregationConnection + + """ + Aggregations over the `widgets_or_addresses` data: + + > Fetches `WidgetOrAddress`s based on the provided arguments. + """ + widget_or_address_aggregations( + """ + Used to forward-paginate through the `widget_or_address_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widget_or_address_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `WidgetOrAddress` documents that get aggregated over based on the provided criteria. + """ + filter: WidgetOrAddressFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widget_or_address_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widget_or_address_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widget_or_address_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widget_or_address_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): WidgetOrAddressAggregationConnection + + """ + Aggregations over the `widget_workspaces` data: + + > Fetches `WidgetWorkspace`s based on the provided arguments. + """ + widget_workspace_aggregations( + """ + Used to forward-paginate through the `widget_workspace_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widget_workspace_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `WidgetWorkspace` documents that get aggregated over based on the provided criteria. + """ + filter: WidgetWorkspaceFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widget_workspace_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widget_workspace_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widget_workspace_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widget_workspace_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): WidgetWorkspaceAggregationConnection + + """ + Fetches `WidgetWorkspace`s based on the provided arguments. + """ + widget_workspaces( + """ + Used to forward-paginate through the `widget_workspaces`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widget_workspaces`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `widget_workspaces` based on the provided criteria. + """ + filter: WidgetWorkspaceFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widget_workspaces`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widget_workspaces`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widget_workspaces`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widget_workspaces`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `widget_workspaces` should be sorted. + """ + order_by: [WidgetWorkspaceSortOrderInput!] + ): WidgetWorkspaceConnection + + """ + Fetches `Widget`s based on the provided arguments. + """ + widgets( + """ + Used to forward-paginate through the `widgets`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widgets`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `widgets` based on the provided criteria. + """ + filter: WidgetFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widgets`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widgets`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widgets`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widgets`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `widgets` should be sorted. + """ + order_by: [WidgetSortOrderInput!] + ): WidgetConnection + + """ + Fetches `WidgetOrAddress`s based on the provided arguments. + """ + widgets_or_addresses( + """ + Used to forward-paginate through the `widgets_or_addresses`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widgets_or_addresses`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `widgets_or_addresses` based on the provided criteria. + """ + filter: WidgetOrAddressFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widgets_or_addresses`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widgets_or_addresses`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widgets_or_addresses`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widgets_or_addresses`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `widgets_or_addresses` should be sorted. + """ + order_by: [WidgetOrAddressSortOrderInput!] + ): WidgetOrAddressConnection +} + +enum Size { + LARGE + MEDIUM + SMALL +} + +""" +Input type used to specify filters on `Size` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input SizeFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [SizeFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [SizeInput] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: SizeFilterInput +} + +enum SizeInput { + LARGE + MEDIUM + SMALL +} + +""" +Input type used to specify filters on elements of a `[Size]` field. + +Will be ignored if passed as an empty object (or as `null`). +""" +input SizeListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [SizeListElementFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [SizeInput!] +} + +""" +Input type used to specify filters on `[Size]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input SizeListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `SizeListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [SizeListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [SizeListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: SizeListElementFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: SizeListFilterInput +} + +type Sponsor { + """ + Aggregations over the `affiliated_teams_from_nested` data. + """ + affiliated_team_from_nested_aggregations( + """ + Used to forward-paginate through the `affiliated_team_from_nested_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the + `affiliated_team_from_nested_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Team` documents that get aggregated over based on the provided criteria. + """ + filter: TeamFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through + the `affiliated_team_from_nested_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `affiliated_team_from_nested_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through + the `affiliated_team_from_nested_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `affiliated_team_from_nested_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): TeamAggregationConnection + + """ + Aggregations over the `affiliated_teams_from_object` data. + """ + affiliated_team_from_object_aggregations( + """ + Used to forward-paginate through the `affiliated_team_from_object_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the + `affiliated_team_from_object_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Team` documents that get aggregated over based on the provided criteria. + """ + filter: TeamFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through + the `affiliated_team_from_object_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `affiliated_team_from_object_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through + the `affiliated_team_from_object_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `affiliated_team_from_object_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): TeamAggregationConnection + affiliated_teams_from_nested( + """ + Used to forward-paginate through the `affiliated_teams_from_nested`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `affiliated_teams_from_nested`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `affiliated_teams_from_nested` based on the provided criteria. + """ + filter: TeamFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `affiliated_teams_from_nested`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `affiliated_teams_from_nested`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `affiliated_teams_from_nested`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `affiliated_teams_from_nested`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `affiliated_teams_from_nested` should be sorted. + """ + order_by: [TeamSortOrderInput!] + ): TeamConnection + affiliated_teams_from_object( + """ + Used to forward-paginate through the `affiliated_teams_from_object`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `affiliated_teams_from_object`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `affiliated_teams_from_object` based on the provided criteria. + """ + filter: TeamFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `affiliated_teams_from_object`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `affiliated_teams_from_object`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `affiliated_teams_from_object`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `affiliated_teams_from_object`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `affiliated_teams_from_object` should be sorted. + """ + order_by: [TeamSortOrderInput!] + ): TeamConnection + id: ID! + name: String +} + +""" +Type used to perform aggregation computations on `Sponsor` fields. +""" +type SponsorAggregatedValues { + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues +} + +""" +Return type representing a bucket of `Sponsor` documents for an aggregations query. +""" +type SponsorAggregation { + """ + Provides computed aggregated values over all `Sponsor` documents in an aggregation bucket. + """ + aggregated_values: SponsorAggregatedValues + + """ + The count of `Sponsor` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `Sponsor` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: SponsorGroupedBy +} + +""" +Represents a paginated collection of `SponsorAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type SponsorAggregationConnection { + """ + Wraps a specific `SponsorAggregation` to pair it with its pagination cursor. + """ + edges: [SponsorAggregationEdge!]! + + """ + The list of `SponsorAggregation` results. + """ + nodes: [SponsorAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `SponsorAggregation` in the context of a `SponsorAggregationConnection`, +providing access to both the `SponsorAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type SponsorAggregationEdge { + """ + The `Cursor` of this `SponsorAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `SponsorAggregation`. + """ + cursor: Cursor + + """ + The `SponsorAggregation` of this edge. + """ + node: SponsorAggregation +} + +""" +Represents a paginated collection of `Sponsor` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type SponsorConnection { + """ + Wraps a specific `Sponsor` to pair it with its pagination cursor. + """ + edges: [SponsorEdge!]! + + """ + The list of `Sponsor` results. + """ + nodes: [Sponsor!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `Sponsor` in the context of a `SponsorConnection`, +providing access to both the `Sponsor` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type SponsorEdge { + """ + The `Cursor` of this `Sponsor`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `Sponsor`. + """ + cursor: Cursor + + """ + The `Sponsor` of this edge. + """ + node: Sponsor +} + +""" +Input type used to specify filters on `Sponsor` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input SponsorFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [SponsorFilterInput!] + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: SponsorFilterInput +} + +""" +Type used to specify the `Sponsor` fields to group by for aggregations. +""" +type SponsorGroupedBy { + """ + The `name` field value for this group. + """ + name: String +} + +""" +Enumerates the ways `Sponsor`s can be sorted. +""" +enum SponsorSortOrderInput { + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC +} + +type Sponsorship { + annual_total: Money! + sponsor_id: ID! +} + +""" +Type used to perform aggregation computations on `Sponsorship` fields. +""" +type SponsorshipAggregatedValues { + """ + Computed aggregate values for the `annual_total` field. + """ + annual_total: MoneyAggregatedValues + + """ + Computed aggregate values for the `sponsor_id` field. + """ + sponsor_id: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on a `Sponsorship` object referenced directly +or transitively from a list field that has been configured to index each leaf field as +its own flattened list of values. + +Will be ignored if passed as an empty object (or as `null`). +""" +input SponsorshipFieldsListFilterInput { + """ + Used to filter on the `annual_total` field. + + Will be ignored if `null` or an empty object is passed. + """ + annual_total: MoneyFieldsListFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [SponsorshipFieldsListFilterInput!] + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: SponsorshipFieldsListFilterInput + + """ + Used to filter on the `sponsor_id` field. + + Will be ignored if `null` or an empty object is passed. + """ + sponsor_id: IDListFilterInput +} + +""" +Input type used to specify filters on `Sponsorship` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input SponsorshipFilterInput { + """ + Used to filter on the `annual_total` field. + + Will be ignored if `null` or an empty object is passed. + """ + annual_total: MoneyFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [SponsorshipFilterInput!] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: SponsorshipFilterInput + + """ + Used to filter on the `sponsor_id` field. + + Will be ignored if `null` or an empty object is passed. + """ + sponsor_id: IDFilterInput +} + +""" +Type used to specify the `Sponsorship` fields to group by for aggregations. +""" +type SponsorshipGroupedBy { + """ + The `annual_total` field value for this group. + """ + annual_total: MoneyGroupedBy + + """ + The `sponsor_id` field value for this group. + """ + sponsor_id: ID +} + +""" +Input type used to specify filters on `[Sponsorship]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input SponsorshipListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `SponsorshipListFilterInput` input because of + collisions between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [SponsorshipListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [SponsorshipListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: SponsorshipFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: SponsorshipListFilterInput +} + +""" +Represents a paginated collection of `String` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type StringConnection { + """ + Wraps a specific `String` to pair it with its pagination cursor. + """ + edges: [StringEdge!]! + + """ + The list of `String` results. + """ + nodes: [String!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `String` in the context of a `StringConnection`, +providing access to both the `String` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type StringEdge { + """ + The `Cursor` of this `String`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `String`. + """ + cursor: Cursor + + """ + The `String` of this edge. + """ + node: String +} + +""" +Input type used to specify filters on `String` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input StringFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [StringFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [String] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: StringFilterInput +} + +""" +Input type used to specify filters on elements of a `[String]` field. + +Will be ignored if passed as an empty object (or as `null`). +""" +input StringListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [StringListElementFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [String!] +} + +""" +Input type used to specify filters on `[String]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input StringListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `StringListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [StringListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [StringListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: StringListElementFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: StringListFilterInput +} + +""" +For more performant queries on this type, please filter on `league` if possible. +""" +type Team { + country_code: ID! + current_name: String + current_players_nested: [Player!]! + current_players_object: [Player!]! + details: TeamDetails + forbes_valuation_moneys_nested: [Money!]! + forbes_valuation_moneys_object: [Money!]! + forbes_valuations: [JsonSafeLong!]! + formed_on: Date + id: ID! + league: String + nested_fields: TeamNestedFields + nested_fields2: TeamNestedFields + past_names: [String!]! + seasons_nested: [TeamSeason!]! + seasons_object: [TeamSeason!]! + stadium_location: GeoLocation + won_championships_at: [DateTime!]! +} + +""" +Type used to perform aggregation computations on `Team` fields. +""" +type TeamAggregatedValues { + """ + Computed aggregate values for the `country_code` field. + """ + country_code: NonNumericAggregatedValues + + """ + Computed aggregate values for the `current_name` field. + """ + current_name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `current_players_object` field. + """ + current_players_object: PlayerAggregatedValues + + """ + Computed aggregate values for the `details` field. + """ + details: TeamDetailsAggregatedValues + + """ + Computed aggregate values for the `forbes_valuation_moneys_object` field. + """ + forbes_valuation_moneys_object: MoneyAggregatedValues + + """ + Computed aggregate values for the `forbes_valuations` field. + """ + forbes_valuations: JsonSafeLongAggregatedValues + + """ + Computed aggregate values for the `formed_on` field. + """ + formed_on: DateAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `league` field. + """ + league: NonNumericAggregatedValues + + """ + Computed aggregate values for the `past_names` field. + """ + past_names: NonNumericAggregatedValues + + """ + Computed aggregate values for the `seasons_object` field. + """ + seasons_object: TeamSeasonAggregatedValues + + """ + Computed aggregate values for the `stadium_location` field. + """ + stadium_location: NonNumericAggregatedValues + + """ + Computed aggregate values for the `won_championships_at` field. + """ + won_championships_at: DateTimeAggregatedValues +} + +""" +Return type representing a bucket of `Team` documents for an aggregations query. +""" +type TeamAggregation { + """ + Provides computed aggregated values over all `Team` documents in an aggregation bucket. + """ + aggregated_values: TeamAggregatedValues + + """ + The count of `Team` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `Team` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: TeamGroupedBy + + """ + Used to perform sub-aggregations of `TeamAggregation` data. + """ + sub_aggregations: TeamAggregationSubAggregations +} + +""" +Represents a paginated collection of `TeamAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type TeamAggregationConnection { + """ + Wraps a specific `TeamAggregation` to pair it with its pagination cursor. + """ + edges: [TeamAggregationEdge!]! + + """ + The list of `TeamAggregation` results. + """ + nodes: [TeamAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Provides access to the `sub_aggregations` under `current_players_object.affiliations` within each `TeamAggregation`. +""" +type TeamAggregationCurrentPlayersObjectAffiliationsSubAggregations { + """ + Used to perform a sub-aggregation of `sponsorships_nested`. + """ + sponsorships_nested( + """ + Used to filter the `Sponsorship` documents included in this sub-aggregation based on the provided criteria. + """ + filter: SponsorshipFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamSponsorshipSubAggregationConnection +} + +""" +Provides access to the `sub_aggregations` under `current_players_object` within each `TeamAggregation`. +""" +type TeamAggregationCurrentPlayersObjectSubAggregations { + """ + Used to perform a sub-aggregation of `affiliations`. + """ + affiliations: TeamAggregationCurrentPlayersObjectAffiliationsSubAggregations + + """ + Used to perform a sub-aggregation of `seasons_nested`. + """ + seasons_nested( + """ + Used to filter the `PlayerSeason` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerSeasonFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamPlayerSeasonSubAggregationConnection +} + +""" +Represents a specific `TeamAggregation` in the context of a `TeamAggregationConnection`, +providing access to both the `TeamAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type TeamAggregationEdge { + """ + The `Cursor` of this `TeamAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `TeamAggregation`. + """ + cursor: Cursor + + """ + The `TeamAggregation` of this edge. + """ + node: TeamAggregation +} + +""" +Provides access to the `sub_aggregations` under `nested_fields2` within each `TeamAggregation`. +""" +type TeamAggregationNestedFields2SubAggregations { + """ + Used to perform a sub-aggregation of `current_players`. + """ + current_players( + """ + Used to filter the `Player` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamPlayerSubAggregationConnection + + """ + Used to perform a sub-aggregation of `forbes_valuation_moneys`. + """ + forbes_valuation_moneys( + """ + Used to filter the `Money` documents included in this sub-aggregation based on the provided criteria. + """ + filter: MoneyFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamMoneySubAggregationConnection + + """ + Used to perform a sub-aggregation of `seasons`. + """ + seasons( + """ + Used to filter the `TeamSeason` documents included in this sub-aggregation based on the provided criteria. + """ + filter: TeamSeasonFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamTeamSeasonSubAggregationConnection +} + +""" +Provides access to the `sub_aggregations` under `nested_fields` within each `TeamAggregation`. +""" +type TeamAggregationNestedFieldsSubAggregations { + """ + Used to perform a sub-aggregation of `current_players`. + """ + current_players( + """ + Used to filter the `Player` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamPlayerSubAggregationConnection + + """ + Used to perform a sub-aggregation of `forbes_valuation_moneys`. + """ + forbes_valuation_moneys( + """ + Used to filter the `Money` documents included in this sub-aggregation based on the provided criteria. + """ + filter: MoneyFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamMoneySubAggregationConnection + + """ + Used to perform a sub-aggregation of `seasons`. + """ + seasons( + """ + Used to filter the `TeamSeason` documents included in this sub-aggregation based on the provided criteria. + """ + filter: TeamSeasonFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamTeamSeasonSubAggregationConnection +} + +""" +Provides access to the `sub_aggregations` under `seasons_object.players_object.affiliations` within each `TeamAggregation`. +""" +type TeamAggregationSeasonsObjectPlayersObjectAffiliationsSubAggregations { + """ + Used to perform a sub-aggregation of `sponsorships_nested`. + """ + sponsorships_nested( + """ + Used to filter the `Sponsorship` documents included in this sub-aggregation based on the provided criteria. + """ + filter: SponsorshipFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamSponsorshipSubAggregationConnection +} + +""" +Provides access to the `sub_aggregations` under `seasons_object.players_object` within each `TeamAggregation`. +""" +type TeamAggregationSeasonsObjectPlayersObjectSubAggregations { + """ + Used to perform a sub-aggregation of `affiliations`. + """ + affiliations: TeamAggregationSeasonsObjectPlayersObjectAffiliationsSubAggregations + + """ + Used to perform a sub-aggregation of `seasons_nested`. + """ + seasons_nested( + """ + Used to filter the `PlayerSeason` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerSeasonFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamPlayerSeasonSubAggregationConnection +} + +""" +Provides access to the `sub_aggregations` under `seasons_object` within each `TeamAggregation`. +""" +type TeamAggregationSeasonsObjectSubAggregations { + """ + Used to perform a sub-aggregation of `players_nested`. + """ + players_nested( + """ + Used to filter the `Player` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamPlayerSubAggregationConnection + + """ + Used to perform a sub-aggregation of `players_object`. + """ + players_object: TeamAggregationSeasonsObjectPlayersObjectSubAggregations +} + +""" +Provides access to the `sub_aggregations` within each `TeamAggregation`. +""" +type TeamAggregationSubAggregations { + """ + Used to perform a sub-aggregation of `current_players_nested`. + """ + current_players_nested( + """ + Used to filter the `Player` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamPlayerSubAggregationConnection + + """ + Used to perform a sub-aggregation of `current_players_object`. + """ + current_players_object: TeamAggregationCurrentPlayersObjectSubAggregations + + """ + Used to perform a sub-aggregation of `forbes_valuation_moneys_nested`. + """ + forbes_valuation_moneys_nested( + """ + Used to filter the `Money` documents included in this sub-aggregation based on the provided criteria. + """ + filter: MoneyFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamMoneySubAggregationConnection + + """ + Used to perform a sub-aggregation of `nested_fields`. + """ + nested_fields: TeamAggregationNestedFieldsSubAggregations + + """ + Used to perform a sub-aggregation of `nested_fields2`. + """ + nested_fields2: TeamAggregationNestedFields2SubAggregations + + """ + Used to perform a sub-aggregation of `seasons_nested`. + """ + seasons_nested( + """ + Used to filter the `TeamSeason` documents included in this sub-aggregation based on the provided criteria. + """ + filter: TeamSeasonFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamTeamSeasonSubAggregationConnection + + """ + Used to perform a sub-aggregation of `seasons_object`. + """ + seasons_object: TeamAggregationSeasonsObjectSubAggregations +} + +""" +Represents a paginated collection of `Team` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type TeamConnection { + """ + Wraps a specific `Team` to pair it with its pagination cursor. + """ + edges: [TeamEdge!]! + + """ + The list of `Team` results. + """ + nodes: [Team!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +type TeamDetails { + count: Int + uniform_colors: [String!]! +} + +""" +Type used to perform aggregation computations on `TeamDetails` fields. +""" +type TeamDetailsAggregatedValues { + """ + Computed aggregate values for the `count` field. + """ + count: IntAggregatedValues + + """ + Computed aggregate values for the `uniform_colors` field. + """ + uniform_colors: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on `TeamDetails` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TeamDetailsFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TeamDetailsFilterInput!] + + """ + Used to filter on the `count` field. + + Will be ignored if `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TeamDetailsFilterInput + + """ + Used to filter on the `uniform_colors` field. + + Will be ignored if `null` or an empty object is passed. + """ + uniform_colors: StringListFilterInput +} + +""" +Type used to specify the `TeamDetails` fields to group by for aggregations. +""" +type TeamDetailsGroupedBy { + """ + The `count` field value for this group. + """ + count: Int +} + +""" +Represents a specific `Team` in the context of a `TeamConnection`, +providing access to both the `Team` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type TeamEdge { + """ + The `Cursor` of this `Team`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `Team`. + """ + cursor: Cursor + + """ + The `Team` of this edge. + """ + node: Team +} + +""" +Input type used to specify filters on `Team` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TeamFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TeamFilterInput!] + + """ + Used to filter on the `country_code` field. + + Will be ignored if `null` or an empty object is passed. + """ + country_code: IDFilterInput + + """ + Used to filter on the `current_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + current_name: StringFilterInput + + """ + Used to filter on the `current_players_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + current_players_nested: PlayerListFilterInput + + """ + Used to filter on the `current_players_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + current_players_object: PlayerFieldsListFilterInput + + """ + Used to filter on the `details` field. + + Will be ignored if `null` or an empty object is passed. + """ + details: TeamDetailsFilterInput + + """ + Used to filter on the `forbes_valuation_moneys_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + forbes_valuation_moneys_nested: MoneyListFilterInput + + """ + Used to filter on the `forbes_valuation_moneys_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + forbes_valuation_moneys_object: MoneyFieldsListFilterInput + + """ + Used to filter on the `forbes_valuations` field. + + Will be ignored if `null` or an empty object is passed. + """ + forbes_valuations: JsonSafeLongListFilterInput + + """ + Used to filter on the `formed_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + formed_on: DateFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `league` field. + + Will be ignored if `null` or an empty object is passed. + """ + league: StringFilterInput + + """ + Used to filter on the `nested_fields` field. + + Will be ignored if `null` or an empty object is passed. + """ + nested_fields: TeamNestedFieldsFilterInput + + """ + Used to filter on the `nested_fields2` field. + + Will be ignored if `null` or an empty object is passed. + """ + nested_fields2: TeamNestedFieldsFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TeamFilterInput + + """ + Used to filter on the `past_names` field. + + Will be ignored if `null` or an empty object is passed. + """ + past_names: StringListFilterInput + + """ + Used to filter on the `seasons_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + seasons_nested: TeamSeasonListFilterInput + + """ + Used to filter on the `seasons_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + seasons_object: TeamSeasonFieldsListFilterInput + + """ + Used to filter on the `stadium_location` field. + + Will be ignored if `null` or an empty object is passed. + """ + stadium_location: GeoLocationFilterInput + + """ + Used to filter on the `won_championships_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + won_championships_at: DateTimeListFilterInput +} + +""" +Type used to specify the `Team` fields to group by for aggregations. +""" +type TeamGroupedBy { + """ + The `country_code` field value for this group. + """ + country_code: ID + + """ + The `current_name` field value for this group. + """ + current_name: String + + """ + The `current_players_object` field value for this group. + + Note: `current_players_object` is a collection field, but selecting this field + will group on individual values of the selected subfields of + `current_players_object`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `current_players_object` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `current_players_object` multiple times for a single document, that document will only be included in the group + once. + """ + current_players_object: PlayerGroupedBy + + """ + The `details` field value for this group. + """ + details: TeamDetailsGroupedBy + + """ + The `forbes_valuation_moneys_object` field value for this group. + + Note: `forbes_valuation_moneys_object` is a collection field, but selecting + this field will group on individual values of the selected subfields of + `forbes_valuation_moneys_object`. + That means that a document may be grouped into multiple aggregation groupings + (i.e. when its `forbes_valuation_moneys_object` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `forbes_valuation_moneys_object` multiple times for a single document, + that document will only be included in the group + once. + """ + forbes_valuation_moneys_object: MoneyGroupedBy + + """ + Offers the different grouping options for the `formed_on` value within this group. + """ + formed_on: DateGroupedBy + + """ + The `league` field value for this group. + """ + league: String + + """ + The `seasons_object` field value for this group. + + Note: `seasons_object` is a collection field, but selecting this field will + group on individual values of the selected subfields of `seasons_object`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `seasons_object` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `seasons_object` multiple times for a single document, that document will only be included in the group + once. + """ + seasons_object: TeamSeasonGroupedBy +} + +""" +Return type representing a bucket of `Money` objects for a sub-aggregation within each `TeamAggregation`. +""" +type TeamMoneySubAggregation { + """ + Provides computed aggregated values over all `Money` documents in a sub-aggregation bucket. + """ + aggregated_values: MoneyAggregatedValues + + """ + Details of the count of `Money` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `Money` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: MoneyGroupedBy +} + +""" +Represents a collection of `TeamMoneySubAggregation` results. +""" +type TeamMoneySubAggregationConnection { + """ + The list of `TeamMoneySubAggregation` results. + """ + nodes: [TeamMoneySubAggregation!]! +} + +type TeamNestedFields { + current_players: [Player!]! + forbes_valuation_moneys: [Money!]! + seasons: [TeamSeason!]! +} + +""" +Input type used to specify filters on `TeamNestedFields` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TeamNestedFieldsFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TeamNestedFieldsFilterInput!] + + """ + Used to filter on the `current_players` field. + + Will be ignored if `null` or an empty object is passed. + """ + current_players: PlayerListFilterInput + + """ + Used to filter on the `forbes_valuation_moneys` field. + + Will be ignored if `null` or an empty object is passed. + """ + forbes_valuation_moneys: MoneyListFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TeamNestedFieldsFilterInput + + """ + Used to filter on the `seasons` field. + + Will be ignored if `null` or an empty object is passed. + """ + seasons: TeamSeasonListFilterInput +} + +""" +Return type representing a bucket of `PlayerSeason` objects for a sub-aggregation within each `TeamPlayerSubAggregation`. +""" +type TeamPlayerPlayerSeasonSubAggregation { + """ + Provides computed aggregated values over all `PlayerSeason` documents in a sub-aggregation bucket. + """ + aggregated_values: PlayerSeasonAggregatedValues + + """ + Details of the count of `PlayerSeason` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `PlayerSeason` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: PlayerSeasonGroupedBy +} + +""" +Represents a collection of `TeamPlayerPlayerSeasonSubAggregation` results. +""" +type TeamPlayerPlayerSeasonSubAggregationConnection { + """ + The list of `TeamPlayerPlayerSeasonSubAggregation` results. + """ + nodes: [TeamPlayerPlayerSeasonSubAggregation!]! +} + +""" +Return type representing a bucket of `PlayerSeason` objects for a sub-aggregation within each `TeamAggregation`. +""" +type TeamPlayerSeasonSubAggregation { + """ + Provides computed aggregated values over all `PlayerSeason` documents in a sub-aggregation bucket. + """ + aggregated_values: PlayerSeasonAggregatedValues + + """ + Details of the count of `PlayerSeason` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `PlayerSeason` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: PlayerSeasonGroupedBy +} + +""" +Represents a collection of `TeamPlayerSeasonSubAggregation` results. +""" +type TeamPlayerSeasonSubAggregationConnection { + """ + The list of `TeamPlayerSeasonSubAggregation` results. + """ + nodes: [TeamPlayerSeasonSubAggregation!]! +} + +""" +Return type representing a bucket of `Sponsorship` objects for a sub-aggregation within each `TeamPlayerSubAggregation`. +""" +type TeamPlayerSponsorshipSubAggregation { + """ + Provides computed aggregated values over all `Sponsorship` documents in a sub-aggregation bucket. + """ + aggregated_values: SponsorshipAggregatedValues + + """ + Details of the count of `Sponsorship` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `Sponsorship` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: SponsorshipGroupedBy +} + +""" +Represents a collection of `TeamPlayerSponsorshipSubAggregation` results. +""" +type TeamPlayerSponsorshipSubAggregationConnection { + """ + The list of `TeamPlayerSponsorshipSubAggregation` results. + """ + nodes: [TeamPlayerSponsorshipSubAggregation!]! +} + +""" +Return type representing a bucket of `Player` objects for a sub-aggregation within each `TeamAggregation`. +""" +type TeamPlayerSubAggregation { + """ + Provides computed aggregated values over all `Player` documents in a sub-aggregation bucket. + """ + aggregated_values: PlayerAggregatedValues + + """ + Details of the count of `Player` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `Player` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: PlayerGroupedBy + + """ + Used to perform sub-aggregations of `TeamPlayerSubAggregation` data. + """ + sub_aggregations: TeamPlayerSubAggregationSubAggregations +} + +""" +Provides access to the `sub_aggregations` under `affiliations` within each `TeamPlayerSubAggregation`. +""" +type TeamPlayerSubAggregationAffiliationsSubAggregations { + """ + Used to perform a sub-aggregation of `sponsorships_nested`. + """ + sponsorships_nested( + """ + Used to filter the `Sponsorship` documents included in this sub-aggregation based on the provided criteria. + """ + filter: SponsorshipFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamPlayerSponsorshipSubAggregationConnection +} + +""" +Represents a collection of `TeamPlayerSubAggregation` results. +""" +type TeamPlayerSubAggregationConnection { + """ + The list of `TeamPlayerSubAggregation` results. + """ + nodes: [TeamPlayerSubAggregation!]! +} + +""" +Provides access to the `sub_aggregations` within each `TeamPlayerSubAggregation`. +""" +type TeamPlayerSubAggregationSubAggregations { + """ + Used to perform a sub-aggregation of `affiliations`. + """ + affiliations: TeamPlayerSubAggregationAffiliationsSubAggregations + + """ + Used to perform a sub-aggregation of `seasons_nested`. + """ + seasons_nested( + """ + Used to filter the `PlayerSeason` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerSeasonFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamPlayerPlayerSeasonSubAggregationConnection +} + +type TeamRecord { + first_win_on: Date + first_win_on_legacy: Date + last_win_on: Date + last_win_on_legacy: Date + losses: Int + wins: Int +} + +""" +Type used to perform aggregation computations on `TeamRecord` fields. +""" +type TeamRecordAggregatedValues { + """ + Computed aggregate values for the `first_win_on` field. + """ + first_win_on: DateAggregatedValues + + """ + Computed aggregate values for the `first_win_on_legacy` field. + """ + first_win_on_legacy: DateAggregatedValues + + """ + Computed aggregate values for the `last_win_on` field. + """ + last_win_on: DateAggregatedValues + + """ + Computed aggregate values for the `last_win_on_legacy` field. + """ + last_win_on_legacy: DateAggregatedValues + + """ + Computed aggregate values for the `losses` field. + """ + losses: IntAggregatedValues + + """ + Computed aggregate values for the `wins` field. + """ + wins: IntAggregatedValues +} + +""" +Input type used to specify filters on a `TeamRecord` object referenced directly +or transitively from a list field that has been configured to index each leaf field as +its own flattened list of values. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TeamRecordFieldsListFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TeamRecordFieldsListFilterInput!] + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Used to filter on the `first_win_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + first_win_on: DateListFilterInput + + """ + Used to filter on the `first_win_on_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + first_win_on_legacy: DateListFilterInput + + """ + Used to filter on the `last_win_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + last_win_on: DateListFilterInput + + """ + Used to filter on the `last_win_on_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + last_win_on_legacy: DateListFilterInput + + """ + Used to filter on the `losses` field. + + Will be ignored if `null` or an empty object is passed. + """ + losses: IntListFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TeamRecordFieldsListFilterInput + + """ + Used to filter on the `wins` field. + + Will be ignored if `null` or an empty object is passed. + """ + wins: IntListFilterInput +} + +""" +Input type used to specify filters on `TeamRecord` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TeamRecordFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TeamRecordFilterInput!] + + """ + Used to filter on the `first_win_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + first_win_on: DateFilterInput + + """ + Used to filter on the `first_win_on_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + first_win_on_legacy: DateFilterInput + + """ + Used to filter on the `last_win_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + last_win_on: DateFilterInput + + """ + Used to filter on the `last_win_on_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + last_win_on_legacy: DateFilterInput + + """ + Used to filter on the `losses` field. + + Will be ignored if `null` or an empty object is passed. + """ + losses: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TeamRecordFilterInput + + """ + Used to filter on the `wins` field. + + Will be ignored if `null` or an empty object is passed. + """ + wins: IntFilterInput +} + +""" +Type used to specify the `TeamRecord` fields to group by for aggregations. +""" +type TeamRecordGroupedBy { + """ + Offers the different grouping options for the `first_win_on` value within this group. + """ + first_win_on: DateGroupedBy + + """ + The `first_win_on_legacy` field value for this group. + """ + first_win_on_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateGroupingGranularityInput! + + """ + Number of days (positive or negative) to shift the `Date` boundaries of each date grouping bucket. + + For example, when grouping by `YEAR`, this can be used to align the buckets + with fiscal or school years instead of calendar years. + """ + offset_days: Int + ): Date + + """ + Offers the different grouping options for the `last_win_on` value within this group. + """ + last_win_on: DateGroupedBy + + """ + The `last_win_on_legacy` field value for this group. + """ + last_win_on_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateGroupingGranularityInput! + + """ + Number of days (positive or negative) to shift the `Date` boundaries of each date grouping bucket. + + For example, when grouping by `YEAR`, this can be used to align the buckets + with fiscal or school years instead of calendar years. + """ + offset_days: Int + ): Date + + """ + The `losses` field value for this group. + """ + losses: Int + + """ + The `wins` field value for this group. + """ + wins: Int +} + +type TeamSeason { + count: Int + notes: [String!]! + players_nested: [Player!]! + players_object: [Player!]! + record: TeamRecord + started_at: DateTime + started_at_legacy: DateTime + won_games_at: [DateTime!]! + won_games_at_legacy: [DateTime!]! + year: Int +} + +""" +Type used to perform aggregation computations on `TeamSeason` fields. +""" +type TeamSeasonAggregatedValues { + """ + Computed aggregate values for the `count` field. + """ + count: IntAggregatedValues + + """ + Computed aggregate values for the `notes` field. + """ + notes: NonNumericAggregatedValues + + """ + Computed aggregate values for the `players_object` field. + """ + players_object: PlayerAggregatedValues + + """ + Computed aggregate values for the `record` field. + """ + record: TeamRecordAggregatedValues + + """ + Computed aggregate values for the `started_at` field. + """ + started_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `started_at_legacy` field. + """ + started_at_legacy: DateTimeAggregatedValues + + """ + Computed aggregate values for the `won_games_at` field. + """ + won_games_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `won_games_at_legacy` field. + """ + won_games_at_legacy: DateTimeAggregatedValues + + """ + Computed aggregate values for the `year` field. + """ + year: IntAggregatedValues +} + +""" +Input type used to specify filters on a `TeamSeason` object referenced directly +or transitively from a list field that has been configured to index each leaf field as +its own flattened list of values. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TeamSeasonFieldsListFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TeamSeasonFieldsListFilterInput!] + + """ + Used to filter on the `count` field. + + Will be ignored if `null` or an empty object is passed. + """ + count: IntListFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TeamSeasonFieldsListFilterInput + + """ + Used to filter on the `notes` field. + + Will be ignored if `null` or an empty object is passed. + """ + notes: StringListFilterInput + + """ + Used to filter on the `players_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + players_nested: PlayerListFilterInput + + """ + Used to filter on the `players_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + players_object: PlayerFieldsListFilterInput + + """ + Used to filter on the `record` field. + + Will be ignored if `null` or an empty object is passed. + """ + record: TeamRecordFieldsListFilterInput + + """ + Used to filter on the `started_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + started_at: DateTimeListFilterInput + + """ + Used to filter on the `started_at_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + started_at_legacy: DateTimeListFilterInput + + """ + Used to filter on the `won_games_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + won_games_at: DateTimeListFilterInput + + """ + Used to filter on the `won_games_at_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + won_games_at_legacy: DateTimeListFilterInput + + """ + Used to filter on the `year` field. + + Will be ignored if `null` or an empty object is passed. + """ + year: IntListFilterInput +} + +""" +Input type used to specify filters on `TeamSeason` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TeamSeasonFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TeamSeasonFilterInput!] + + """ + Used to filter on the `count` field. + + Will be ignored if `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TeamSeasonFilterInput + + """ + Used to filter on the `notes` field. + + Will be ignored if `null` or an empty object is passed. + """ + notes: StringListFilterInput + + """ + Used to filter on the `players_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + players_nested: PlayerListFilterInput + + """ + Used to filter on the `players_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + players_object: PlayerFieldsListFilterInput + + """ + Used to filter on the `record` field. + + Will be ignored if `null` or an empty object is passed. + """ + record: TeamRecordFilterInput + + """ + Used to filter on the `started_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + started_at: DateTimeFilterInput + + """ + Used to filter on the `started_at_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + started_at_legacy: DateTimeFilterInput + + """ + Used to filter on the `won_games_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + won_games_at: DateTimeListFilterInput + + """ + Used to filter on the `won_games_at_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + won_games_at_legacy: DateTimeListFilterInput + + """ + Used to filter on the `year` field. + + Will be ignored if `null` or an empty object is passed. + """ + year: IntFilterInput +} + +""" +Type used to specify the `TeamSeason` fields to group by for aggregations. +""" +type TeamSeasonGroupedBy { + """ + The `count` field value for this group. + """ + count: Int + + """ + The individual value from `notes` for this group. + + Note: `notes` is a collection field, but selecting this field will group on individual values of `notes`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `notes` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `notes` multiple times for a single document, that document will only be included in the group + once. + """ + note: String + + """ + The `players_object` field value for this group. + + Note: `players_object` is a collection field, but selecting this field will + group on individual values of the selected subfields of `players_object`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `players_object` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `players_object` multiple times for a single document, that document will only be included in the group + once. + """ + players_object: PlayerGroupedBy + + """ + The `record` field value for this group. + """ + record: TeamRecordGroupedBy + + """ + Offers the different grouping options for the `started_at` value within this group. + """ + started_at: DateTimeGroupedBy + + """ + The `started_at_legacy` field value for this group. + """ + started_at_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateTimeGroupingGranularityInput! + + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change + what day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone = "UTC" + ): DateTime + + """ + The individual value from `won_games_at` for this group. + + Note: `won_games_at` is a collection field, but selecting this field will group on individual values of `won_games_at`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `won_games_at` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `won_games_at` multiple times for a single document, that document will only be included in the group + once. + """ + won_game_at: DateTimeGroupedBy + + """ + The individual value from `won_games_at_legacy` for this group. + + Note: `won_games_at_legacy` is a collection field, but selecting this field + will group on individual values of `won_games_at_legacy`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `won_games_at_legacy` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `won_games_at_legacy` multiple times for a single document, that document will only be included in the group + once. + """ + won_game_at_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateTimeGroupingGranularityInput! + + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change + what day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone = "UTC" + ): DateTime + + """ + The `year` field value for this group. + """ + year: Int +} + +""" +Input type used to specify filters on `[TeamSeason]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TeamSeasonListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `TeamSeasonListFilterInput` input because of + collisions between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [TeamSeasonListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TeamSeasonListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: TeamSeasonFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TeamSeasonListFilterInput +} + +""" +Enumerates the ways `Team`s can be sorted. +""" +enum TeamSortOrderInput { + """ + Sorts ascending by the `country_code` field. + """ + country_code_ASC + + """ + Sorts descending by the `country_code` field. + """ + country_code_DESC + + """ + Sorts ascending by the `current_name` field. + """ + current_name_ASC + + """ + Sorts descending by the `current_name` field. + """ + current_name_DESC + + """ + Sorts ascending by the `details.count` field. + """ + details_count_ASC + + """ + Sorts descending by the `details.count` field. + """ + details_count_DESC + + """ + Sorts ascending by the `formed_on` field. + """ + formed_on_ASC + + """ + Sorts descending by the `formed_on` field. + """ + formed_on_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `league` field. + """ + league_ASC + + """ + Sorts descending by the `league` field. + """ + league_DESC +} + +""" +Return type representing a bucket of `Sponsorship` objects for a sub-aggregation within each `TeamAggregation`. +""" +type TeamSponsorshipSubAggregation { + """ + Provides computed aggregated values over all `Sponsorship` documents in a sub-aggregation bucket. + """ + aggregated_values: SponsorshipAggregatedValues + + """ + Details of the count of `Sponsorship` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `Sponsorship` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: SponsorshipGroupedBy +} + +""" +Represents a collection of `TeamSponsorshipSubAggregation` results. +""" +type TeamSponsorshipSubAggregationConnection { + """ + The list of `TeamSponsorshipSubAggregation` results. + """ + nodes: [TeamSponsorshipSubAggregation!]! +} + +""" +Return type representing a bucket of `PlayerSeason` objects for a +sub-aggregation within each `TeamTeamSeasonPlayerSubAggregation`. +""" +type TeamTeamSeasonPlayerPlayerSeasonSubAggregation { + """ + Provides computed aggregated values over all `PlayerSeason` documents in a sub-aggregation bucket. + """ + aggregated_values: PlayerSeasonAggregatedValues + + """ + Details of the count of `PlayerSeason` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `PlayerSeason` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: PlayerSeasonGroupedBy +} + +""" +Represents a collection of `TeamTeamSeasonPlayerPlayerSeasonSubAggregation` results. +""" +type TeamTeamSeasonPlayerPlayerSeasonSubAggregationConnection { + """ + The list of `TeamTeamSeasonPlayerPlayerSeasonSubAggregation` results. + """ + nodes: [TeamTeamSeasonPlayerPlayerSeasonSubAggregation!]! +} + +""" +Return type representing a bucket of `PlayerSeason` objects for a +sub-aggregation within each `TeamTeamSeasonSubAggregation`. +""" +type TeamTeamSeasonPlayerSeasonSubAggregation { + """ + Provides computed aggregated values over all `PlayerSeason` documents in a sub-aggregation bucket. + """ + aggregated_values: PlayerSeasonAggregatedValues + + """ + Details of the count of `PlayerSeason` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `PlayerSeason` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: PlayerSeasonGroupedBy +} + +""" +Represents a collection of `TeamTeamSeasonPlayerSeasonSubAggregation` results. +""" +type TeamTeamSeasonPlayerSeasonSubAggregationConnection { + """ + The list of `TeamTeamSeasonPlayerSeasonSubAggregation` results. + """ + nodes: [TeamTeamSeasonPlayerSeasonSubAggregation!]! +} + +""" +Return type representing a bucket of `Sponsorship` objects for a sub-aggregation +within each `TeamTeamSeasonPlayerSubAggregation`. +""" +type TeamTeamSeasonPlayerSponsorshipSubAggregation { + """ + Provides computed aggregated values over all `Sponsorship` documents in a sub-aggregation bucket. + """ + aggregated_values: SponsorshipAggregatedValues + + """ + Details of the count of `Sponsorship` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `Sponsorship` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: SponsorshipGroupedBy +} + +""" +Represents a collection of `TeamTeamSeasonPlayerSponsorshipSubAggregation` results. +""" +type TeamTeamSeasonPlayerSponsorshipSubAggregationConnection { + """ + The list of `TeamTeamSeasonPlayerSponsorshipSubAggregation` results. + """ + nodes: [TeamTeamSeasonPlayerSponsorshipSubAggregation!]! +} + +""" +Return type representing a bucket of `Player` objects for a sub-aggregation within each `TeamTeamSeasonSubAggregation`. +""" +type TeamTeamSeasonPlayerSubAggregation { + """ + Provides computed aggregated values over all `Player` documents in a sub-aggregation bucket. + """ + aggregated_values: PlayerAggregatedValues + + """ + Details of the count of `Player` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `Player` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: PlayerGroupedBy + + """ + Used to perform sub-aggregations of `TeamTeamSeasonPlayerSubAggregation` data. + """ + sub_aggregations: TeamTeamSeasonPlayerSubAggregationSubAggregations +} + +""" +Provides access to the `sub_aggregations` under `affiliations` within each `TeamTeamSeasonPlayerSubAggregation`. +""" +type TeamTeamSeasonPlayerSubAggregationAffiliationsSubAggregations { + """ + Used to perform a sub-aggregation of `sponsorships_nested`. + """ + sponsorships_nested( + """ + Used to filter the `Sponsorship` documents included in this sub-aggregation based on the provided criteria. + """ + filter: SponsorshipFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamTeamSeasonPlayerSponsorshipSubAggregationConnection +} + +""" +Represents a collection of `TeamTeamSeasonPlayerSubAggregation` results. +""" +type TeamTeamSeasonPlayerSubAggregationConnection { + """ + The list of `TeamTeamSeasonPlayerSubAggregation` results. + """ + nodes: [TeamTeamSeasonPlayerSubAggregation!]! +} + +""" +Provides access to the `sub_aggregations` within each `TeamTeamSeasonPlayerSubAggregation`. +""" +type TeamTeamSeasonPlayerSubAggregationSubAggregations { + """ + Used to perform a sub-aggregation of `affiliations`. + """ + affiliations: TeamTeamSeasonPlayerSubAggregationAffiliationsSubAggregations + + """ + Used to perform a sub-aggregation of `seasons_nested`. + """ + seasons_nested( + """ + Used to filter the `PlayerSeason` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerSeasonFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamTeamSeasonPlayerPlayerSeasonSubAggregationConnection +} + +""" +Return type representing a bucket of `Sponsorship` objects for a sub-aggregation within each `TeamTeamSeasonSubAggregation`. +""" +type TeamTeamSeasonSponsorshipSubAggregation { + """ + Provides computed aggregated values over all `Sponsorship` documents in a sub-aggregation bucket. + """ + aggregated_values: SponsorshipAggregatedValues + + """ + Details of the count of `Sponsorship` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `Sponsorship` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: SponsorshipGroupedBy +} + +""" +Represents a collection of `TeamTeamSeasonSponsorshipSubAggregation` results. +""" +type TeamTeamSeasonSponsorshipSubAggregationConnection { + """ + The list of `TeamTeamSeasonSponsorshipSubAggregation` results. + """ + nodes: [TeamTeamSeasonSponsorshipSubAggregation!]! +} + +""" +Return type representing a bucket of `TeamSeason` objects for a sub-aggregation within each `TeamAggregation`. +""" +type TeamTeamSeasonSubAggregation { + """ + Provides computed aggregated values over all `TeamSeason` documents in a sub-aggregation bucket. + """ + aggregated_values: TeamSeasonAggregatedValues + + """ + Details of the count of `TeamSeason` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `TeamSeason` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: TeamSeasonGroupedBy + + """ + Used to perform sub-aggregations of `TeamTeamSeasonSubAggregation` data. + """ + sub_aggregations: TeamTeamSeasonSubAggregationSubAggregations +} + +""" +Represents a collection of `TeamTeamSeasonSubAggregation` results. +""" +type TeamTeamSeasonSubAggregationConnection { + """ + The list of `TeamTeamSeasonSubAggregation` results. + """ + nodes: [TeamTeamSeasonSubAggregation!]! +} + +""" +Provides access to the `sub_aggregations` under `players_object.affiliations` within each `TeamTeamSeasonSubAggregation`. +""" +type TeamTeamSeasonSubAggregationPlayersObjectAffiliationsSubAggregations { + """ + Used to perform a sub-aggregation of `sponsorships_nested`. + """ + sponsorships_nested( + """ + Used to filter the `Sponsorship` documents included in this sub-aggregation based on the provided criteria. + """ + filter: SponsorshipFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamTeamSeasonSponsorshipSubAggregationConnection +} + +""" +Provides access to the `sub_aggregations` under `players_object` within each `TeamTeamSeasonSubAggregation`. +""" +type TeamTeamSeasonSubAggregationPlayersObjectSubAggregations { + """ + Used to perform a sub-aggregation of `affiliations`. + """ + affiliations: TeamTeamSeasonSubAggregationPlayersObjectAffiliationsSubAggregations + + """ + Used to perform a sub-aggregation of `seasons_nested`. + """ + seasons_nested( + """ + Used to filter the `PlayerSeason` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerSeasonFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamTeamSeasonPlayerSeasonSubAggregationConnection +} + +""" +Provides access to the `sub_aggregations` within each `TeamTeamSeasonSubAggregation`. +""" +type TeamTeamSeasonSubAggregationSubAggregations { + """ + Used to perform a sub-aggregation of `players_nested`. + """ + players_nested( + """ + Used to filter the `Player` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamTeamSeasonPlayerSubAggregationConnection + + """ + Used to perform a sub-aggregation of `players_object`. + """ + players_object: TeamTeamSeasonSubAggregationPlayersObjectSubAggregations +} + +""" +Input type used to specify filters on `String` fields that have been indexed for full text search. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TextFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TextFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [String] + + """ + Matches records where the field value matches the provided value using full text search. + + Will be ignored when `null` is passed. + """ + matches: String @deprecated(reason: "Use `matches_query` instead.") + + """ + Matches records where the field value has a phrase matching the provided phrase using + full text search. This is stricter than `matches_query`: all terms must match + and be in the same order as the provided phrase. + + Will be ignored when `null` is passed. + """ + matches_phrase: MatchesPhraseFilterInput + + """ + Matches records where the field value matches the provided query using full text search. + This is more lenient than `matches_phrase`: the order of terms is ignored, and, + by default, only one search term is required to be in the field value. + + Will be ignored when `null` is passed. + """ + matches_query: MatchesQueryFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TextFilterInput +} + +""" +An [IANA time zone identifier](https://www.iana.org/time-zones), such as `America/Los_Angeles` or `UTC`. + +For a full list of valid identifiers, see the [wikipedia +article](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List). +""" +scalar TimeZone + +""" +A custom scalar type that allows any type of data, including: + +- strings +- numbers +- objects and arrays (nested as deeply as you like) +- booleans + +Note: fields of this type are effectively untyped. We recommend it only be used for +parts of your schema that can't be statically typed. +""" +scalar Untyped + +""" +Input type used to specify filters on `Untyped` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input UntypedFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [UntypedFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [Untyped] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: UntypedFilterInput +} + +""" +For more performant queries on this type, please filter on `workspace_id` if possible. +""" +type Widget implements NamedEntity { + amount_cents: Int! + amount_cents2: Int! + amounts: [Int!]! + + """ + Aggregations over the `components` data. + """ + component_aggregations( + """ + Used to forward-paginate through the `component_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `component_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Component` documents that get aggregated over based on the provided criteria. + """ + filter: ComponentFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `component_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `component_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `component_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `component_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): ComponentAggregationConnection + components( + """ + Used to forward-paginate through the `components`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `components`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `components` based on the provided criteria. + """ + filter: ComponentFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `components`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `components`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `components`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `components`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `components` should be sorted. + """ + order_by: [ComponentSortOrderInput!] + ): ComponentConnection + cost: Money + cost_currency_introduced_on: Date + cost_currency_name: String + cost_currency_primary_continent: String + cost_currency_symbol: String + cost_currency_unit: String + created_at: DateTime! + created_at2: DateTime! + created_at2_legacy: DateTime! + created_at_legacy: DateTime! + created_at_time_of_day: LocalTime + created_on: Date + created_on_legacy: Date + fees: [Money!]! + id: ID! + inventor: Inventor + metadata: Untyped + name: String + name_text: String + named_inventor: NamedInventor + options: WidgetOptions + release_dates: [Date!]! + release_timestamps: [DateTime!]! + size: Size + tags: [String!]! + the_options: WidgetOptions + weight_in_ng: JsonSafeLong! + weight_in_ng_str: LongString! + workspace: WidgetWorkspace + workspace_id: ID + workspace_name: String +} + +""" +Type used to perform aggregation computations on `Widget` fields. +""" +type WidgetAggregatedValues { + """ + Computed aggregate values for the `amount_cents` field. + """ + amount_cents: IntAggregatedValues + + """ + Computed aggregate values for the `amount_cents2` field. + """ + amount_cents2: IntAggregatedValues + + """ + Computed aggregate values for the `amounts` field. + """ + amounts: IntAggregatedValues + + """ + Computed aggregate values for the `cost` field. + """ + cost: MoneyAggregatedValues + + """ + Computed aggregate values for the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on: DateAggregatedValues + + """ + Computed aggregate values for the `cost_currency_name` field. + """ + cost_currency_name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_symbol` field. + """ + cost_currency_symbol: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_unit` field. + """ + cost_currency_unit: NonNumericAggregatedValues + + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at2` field. + """ + created_at2: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at2_legacy` field. + """ + created_at2_legacy: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at_legacy` field. + """ + created_at_legacy: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at_time_of_day` field. + """ + created_at_time_of_day: LocalTimeAggregatedValues + + """ + Computed aggregate values for the `created_on` field. + """ + created_on: DateAggregatedValues + + """ + Computed aggregate values for the `created_on_legacy` field. + """ + created_on_legacy: DateAggregatedValues + + """ + Computed aggregate values for the `fees` field. + """ + fees: MoneyAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `inventor` field. + """ + inventor: InventorAggregatedValues + + """ + Computed aggregate values for the `metadata` field. + """ + metadata: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `named_inventor` field. + """ + named_inventor: NamedInventorAggregatedValues + + """ + Computed aggregate values for the `options` field. + """ + options: WidgetOptionsAggregatedValues + + """ + Computed aggregate values for the `release_dates` field. + """ + release_dates: DateAggregatedValues + + """ + Computed aggregate values for the `release_timestamps` field. + """ + release_timestamps: DateTimeAggregatedValues + + """ + Computed aggregate values for the `size` field. + """ + size: NonNumericAggregatedValues + + """ + Computed aggregate values for the `tags` field. + """ + tags: NonNumericAggregatedValues + + """ + Computed aggregate values for the `the_options` field. + """ + the_options: WidgetOptionsAggregatedValues + + """ + Computed aggregate values for the `weight_in_ng` field. + """ + weight_in_ng: JsonSafeLongAggregatedValues + + """ + Computed aggregate values for the `weight_in_ng_str` field. + """ + weight_in_ng_str: LongStringAggregatedValues + + """ + Computed aggregate values for the `workspace_id` field. + """ + workspace_id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `workspace_name` field. + """ + workspace_name: NonNumericAggregatedValues +} + +""" +Return type representing a bucket of `Widget` documents for an aggregations query. +""" +type WidgetAggregation { + """ + Provides computed aggregated values over all `Widget` documents in an aggregation bucket. + """ + aggregated_values: WidgetAggregatedValues + + """ + The count of `Widget` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `Widget` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: WidgetGroupedBy +} + +""" +Represents a paginated collection of `WidgetAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type WidgetAggregationConnection { + """ + Wraps a specific `WidgetAggregation` to pair it with its pagination cursor. + """ + edges: [WidgetAggregationEdge!]! + + """ + The list of `WidgetAggregation` results. + """ + nodes: [WidgetAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `WidgetAggregation` in the context of a `WidgetAggregationConnection`, +providing access to both the `WidgetAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type WidgetAggregationEdge { + """ + The `Cursor` of this `WidgetAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `WidgetAggregation`. + """ + cursor: Cursor + + """ + The `WidgetAggregation` of this edge. + """ + node: WidgetAggregation +} + +""" +Represents a paginated collection of `Widget` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type WidgetConnection { + """ + Wraps a specific `Widget` to pair it with its pagination cursor. + """ + edges: [WidgetEdge!]! + + """ + The list of `Widget` results. + """ + nodes: [Widget!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +For more performant queries on this type, please filter on `primary_continent` if possible. +""" +type WidgetCurrency { + details: CurrencyDetails + id: ID! + introduced_on: Date + name: String + nested_fields: WidgetCurrencyNestedFields + oldest_widget_created_at: DateTime + primary_continent: String + widget_fee_currencies: [String!]! + widget_names( + """ + Used to forward-paginate through the `widget_names`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widget_names`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widget_names`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widget_names`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widget_names`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widget_names`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): StringConnection + widget_options: WidgetOptionSets + widget_tags: [String!]! +} + +""" +Type used to perform aggregation computations on `WidgetCurrency` fields. +""" +type WidgetCurrencyAggregatedValues { + """ + Computed aggregate values for the `details` field. + """ + details: CurrencyDetailsAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `introduced_on` field. + """ + introduced_on: DateAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `nested_fields` field. + """ + nested_fields: WidgetCurrencyNestedFieldsAggregatedValues + + """ + Computed aggregate values for the `oldest_widget_created_at` field. + """ + oldest_widget_created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `primary_continent` field. + """ + primary_continent: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_fee_currencies` field. + """ + widget_fee_currencies: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_names` field. + """ + widget_names: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_options` field. + """ + widget_options: WidgetOptionSetsAggregatedValues + + """ + Computed aggregate values for the `widget_tags` field. + """ + widget_tags: NonNumericAggregatedValues +} + +""" +Return type representing a bucket of `WidgetCurrency` documents for an aggregations query. +""" +type WidgetCurrencyAggregation { + """ + Provides computed aggregated values over all `WidgetCurrency` documents in an aggregation bucket. + """ + aggregated_values: WidgetCurrencyAggregatedValues + + """ + The count of `WidgetCurrency` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `WidgetCurrency` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: WidgetCurrencyGroupedBy +} + +""" +Represents a paginated collection of `WidgetCurrencyAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type WidgetCurrencyAggregationConnection { + """ + Wraps a specific `WidgetCurrencyAggregation` to pair it with its pagination cursor. + """ + edges: [WidgetCurrencyAggregationEdge!]! + + """ + The list of `WidgetCurrencyAggregation` results. + """ + nodes: [WidgetCurrencyAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `WidgetCurrencyAggregation` in the context of a `WidgetCurrencyAggregationConnection`, +providing access to both the `WidgetCurrencyAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type WidgetCurrencyAggregationEdge { + """ + The `Cursor` of this `WidgetCurrencyAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `WidgetCurrencyAggregation`. + """ + cursor: Cursor + + """ + The `WidgetCurrencyAggregation` of this edge. + """ + node: WidgetCurrencyAggregation +} + +""" +Represents a paginated collection of `WidgetCurrency` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type WidgetCurrencyConnection { + """ + Wraps a specific `WidgetCurrency` to pair it with its pagination cursor. + """ + edges: [WidgetCurrencyEdge!]! + + """ + The list of `WidgetCurrency` results. + """ + nodes: [WidgetCurrency!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `WidgetCurrency` in the context of a `WidgetCurrencyConnection`, +providing access to both the `WidgetCurrency` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type WidgetCurrencyEdge { + """ + The `Cursor` of this `WidgetCurrency`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `WidgetCurrency`. + """ + cursor: Cursor + + """ + The `WidgetCurrency` of this edge. + """ + node: WidgetCurrency +} + +""" +Input type used to specify filters on `WidgetCurrency` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input WidgetCurrencyFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [WidgetCurrencyFilterInput!] + + """ + Used to filter on the `details` field. + + Will be ignored if `null` or an empty object is passed. + """ + details: CurrencyDetailsFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `introduced_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + introduced_on: DateFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Used to filter on the `nested_fields` field. + + Will be ignored if `null` or an empty object is passed. + """ + nested_fields: WidgetCurrencyNestedFieldsFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: WidgetCurrencyFilterInput + + """ + Used to filter on the `oldest_widget_created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + oldest_widget_created_at: DateTimeFilterInput + + """ + Used to filter on the `primary_continent` field. + + Will be ignored if `null` or an empty object is passed. + """ + primary_continent: StringFilterInput + + """ + Used to filter on the `widget_fee_currencies` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_fee_currencies: StringListFilterInput + + """ + Used to filter on the `widget_names` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_names: StringListFilterInput + + """ + Used to filter on the `widget_options` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_options: WidgetOptionSetsFilterInput + + """ + Used to filter on the `widget_tags` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_tags: StringListFilterInput +} + +""" +Type used to specify the `WidgetCurrency` fields to group by for aggregations. +""" +type WidgetCurrencyGroupedBy { + """ + The `details` field value for this group. + """ + details: CurrencyDetailsGroupedBy + + """ + Offers the different grouping options for the `introduced_on` value within this group. + """ + introduced_on: DateGroupedBy + + """ + The `name` field value for this group. + """ + name: String + + """ + The `nested_fields` field value for this group. + """ + nested_fields: WidgetCurrencyNestedFieldsGroupedBy + + """ + Offers the different grouping options for the `oldest_widget_created_at` value within this group. + """ + oldest_widget_created_at: DateTimeGroupedBy + + """ + The `primary_continent` field value for this group. + """ + primary_continent: String + + """ + The individual value from `widget_names` for this group. + + Note: `widget_names` is a collection field, but selecting this field will group on individual values of `widget_names`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `widget_names` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `widget_names` multiple times for a single document, that document will only be included in the group + once. + """ + widget_name: String +} + +type WidgetCurrencyNestedFields { + max_widget_cost: Int +} + +""" +Type used to perform aggregation computations on `WidgetCurrencyNestedFields` fields. +""" +type WidgetCurrencyNestedFieldsAggregatedValues { + """ + Computed aggregate values for the `max_widget_cost` field. + """ + max_widget_cost: IntAggregatedValues +} + +""" +Input type used to specify filters on `WidgetCurrencyNestedFields` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input WidgetCurrencyNestedFieldsFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [WidgetCurrencyNestedFieldsFilterInput!] + + """ + Used to filter on the `max_widget_cost` field. + + Will be ignored if `null` or an empty object is passed. + """ + max_widget_cost: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: WidgetCurrencyNestedFieldsFilterInput +} + +""" +Type used to specify the `WidgetCurrencyNestedFields` fields to group by for aggregations. +""" +type WidgetCurrencyNestedFieldsGroupedBy { + """ + The `max_widget_cost` field value for this group. + """ + max_widget_cost: Int +} + +""" +Enumerates the ways `WidgetCurrency`s can be sorted. +""" +enum WidgetCurrencySortOrderInput { + """ + Sorts ascending by the `details.symbol` field. + """ + details_symbol_ASC + + """ + Sorts descending by the `details.symbol` field. + """ + details_symbol_DESC + + """ + Sorts ascending by the `details.unit` field. + """ + details_unit_ASC + + """ + Sorts descending by the `details.unit` field. + """ + details_unit_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `introduced_on` field. + """ + introduced_on_ASC + + """ + Sorts descending by the `introduced_on` field. + """ + introduced_on_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC + + """ + Sorts ascending by the `nested_fields.max_widget_cost` field. + """ + nested_fields_max_widget_cost_ASC + + """ + Sorts descending by the `nested_fields.max_widget_cost` field. + """ + nested_fields_max_widget_cost_DESC + + """ + Sorts ascending by the `oldest_widget_created_at` field. + """ + oldest_widget_created_at_ASC + + """ + Sorts descending by the `oldest_widget_created_at` field. + """ + oldest_widget_created_at_DESC + + """ + Sorts ascending by the `primary_continent` field. + """ + primary_continent_ASC + + """ + Sorts descending by the `primary_continent` field. + """ + primary_continent_DESC +} + +""" +Represents a specific `Widget` in the context of a `WidgetConnection`, +providing access to both the `Widget` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type WidgetEdge { + """ + The `Cursor` of this `Widget`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `Widget`. + """ + cursor: Cursor + + """ + The `Widget` of this edge. + """ + node: Widget +} + +""" +Input type used to specify filters on `Widget` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input WidgetFilterInput { + """ + Used to filter on the `amount_cents` field. + + Will be ignored if `null` or an empty object is passed. + """ + amount_cents: IntFilterInput + + """ + Used to filter on the `amount_cents2` field. + + Will be ignored if `null` or an empty object is passed. + """ + amount_cents2: IntFilterInput + + """ + Used to filter on the `amounts` field. + + Will be ignored if `null` or an empty object is passed. + """ + amounts: IntListFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [WidgetFilterInput!] + + """ + Used to filter on the `cost` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost: MoneyFilterInput + + """ + Used to filter on the `cost_currency_introduced_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_introduced_on: DateFilterInput + + """ + Used to filter on the `cost_currency_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_name: StringFilterInput + + """ + Used to filter on the `cost_currency_primary_continent` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_primary_continent: StringFilterInput + + """ + Used to filter on the `cost_currency_symbol` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_symbol: StringFilterInput + + """ + Used to filter on the `cost_currency_unit` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_unit: StringFilterInput + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `created_at2` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at2: DateTimeFilterInput + + """ + Used to filter on the `created_at2_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at2_legacy: DateTimeFilterInput + + """ + Used to filter on the `created_at_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at_legacy: DateTimeFilterInput + + """ + Used to filter on the `created_at_time_of_day` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at_time_of_day: LocalTimeFilterInput + + """ + Used to filter on the `created_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_on: DateFilterInput + + """ + Used to filter on the `created_on_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_on_legacy: DateFilterInput + + """ + Used to filter on the `fees` field. + + Will be ignored if `null` or an empty object is passed. + """ + fees: MoneyFieldsListFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `inventor` field. + + Will be ignored if `null` or an empty object is passed. + """ + inventor: InventorFilterInput + + """ + Used to filter on the `metadata` field. + + Will be ignored if `null` or an empty object is passed. + """ + metadata: UntypedFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Used to filter on the `name_text` field. + + Will be ignored if `null` or an empty object is passed. + """ + name_text: TextFilterInput + + """ + Used to filter on the `named_inventor` field. + + Will be ignored if `null` or an empty object is passed. + """ + named_inventor: NamedInventorFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: WidgetFilterInput + + """ + Used to filter on the `options` field. + + Will be ignored if `null` or an empty object is passed. + """ + options: WidgetOptionsFilterInput + + """ + Used to filter on the `release_dates` field. + + Will be ignored if `null` or an empty object is passed. + """ + release_dates: DateListFilterInput + + """ + Used to filter on the `release_timestamps` field. + + Will be ignored if `null` or an empty object is passed. + """ + release_timestamps: DateTimeListFilterInput + + """ + Used to filter on the `size` field. + + Will be ignored if `null` or an empty object is passed. + """ + size: SizeFilterInput + + """ + Used to filter on the `tags` field. + + Will be ignored if `null` or an empty object is passed. + """ + tags: StringListFilterInput + + """ + Used to filter on the `the_options` field. + + Will be ignored if `null` or an empty object is passed. + """ + the_options: WidgetOptionsFilterInput + + """ + Used to filter on the `weight_in_ng` field. + + Will be ignored if `null` or an empty object is passed. + """ + weight_in_ng: JsonSafeLongFilterInput + + """ + Used to filter on the `weight_in_ng_str` field. + + Will be ignored if `null` or an empty object is passed. + """ + weight_in_ng_str: LongStringFilterInput + + """ + Used to filter on the `workspace_id` field. + + Will be ignored if `null` or an empty object is passed. + """ + workspace_id: IDFilterInput + + """ + Used to filter on the `workspace_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + workspace_name: StringFilterInput +} + +""" +Type used to specify the `Widget` fields to group by for aggregations. +""" +type WidgetGroupedBy { + """ + The `amount_cents` field value for this group. + """ + amount_cents: Int + + """ + The `amount_cents2` field value for this group. + """ + amount_cents2: Int + + """ + The `cost` field value for this group. + """ + cost: MoneyGroupedBy + + """ + Offers the different grouping options for the `cost_currency_introduced_on` value within this group. + """ + cost_currency_introduced_on: DateGroupedBy + + """ + The `cost_currency_name` field value for this group. + """ + cost_currency_name: String + + """ + The `cost_currency_primary_continent` field value for this group. + """ + cost_currency_primary_continent: String + + """ + The `cost_currency_symbol` field value for this group. + """ + cost_currency_symbol: String + + """ + The `cost_currency_unit` field value for this group. + """ + cost_currency_unit: String + + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + Offers the different grouping options for the `created_at2` value within this group. + """ + created_at2: DateTimeGroupedBy + + """ + The `created_at2_legacy` field value for this group. + """ + created_at2_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateTimeGroupingGranularityInput! + + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change + what day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone = "UTC" + ): DateTime + + """ + The `created_at_legacy` field value for this group. + """ + created_at_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateTimeGroupingGranularityInput! + + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change + what day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone = "UTC" + ): DateTime + + """ + The `created_at_time_of_day` field value for this group. + """ + created_at_time_of_day: LocalTime + + """ + Offers the different grouping options for the `created_on` value within this group. + """ + created_on: DateGroupedBy + + """ + The `created_on_legacy` field value for this group. + """ + created_on_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateGroupingGranularityInput! + + """ + Number of days (positive or negative) to shift the `Date` boundaries of each date grouping bucket. + + For example, when grouping by `YEAR`, this can be used to align the buckets + with fiscal or school years instead of calendar years. + """ + offset_days: Int + ): Date + + """ + The `fees` field value for this group. + + Note: `fees` is a collection field, but selecting this field will group on + individual values of the selected subfields of `fees`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `fees` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `fees` multiple times for a single document, that document will only be included in the group + once. + """ + fees: MoneyGroupedBy + + """ + The `inventor` field value for this group. + """ + inventor: InventorGroupedBy + + """ + The `metadata` field value for this group. + """ + metadata: Untyped + + """ + The `name` field value for this group. + """ + name: String + + """ + The `named_inventor` field value for this group. + """ + named_inventor: NamedInventorGroupedBy + + """ + The `options` field value for this group. + """ + options: WidgetOptionsGroupedBy + + """ + The individual value from `release_dates` for this group. + + Note: `release_dates` is a collection field, but selecting this field will group on individual values of `release_dates`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `release_dates` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `release_dates` multiple times for a single document, that document will only be included in the group + once. + """ + release_date: DateGroupedBy + + """ + The individual value from `release_timestamps` for this group. + + Note: `release_timestamps` is a collection field, but selecting this field + will group on individual values of `release_timestamps`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `release_timestamps` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `release_timestamps` multiple times for a single document, that document will only be included in the group + once. + """ + release_timestamp: DateTimeGroupedBy + + """ + The `size` field value for this group. + """ + size: Size + + """ + The individual value from `tags` for this group. + + Note: `tags` is a collection field, but selecting this field will group on individual values of `tags`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `tags` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `tags` multiple times for a single document, that document will only be included in the group + once. + """ + tag: String + + """ + The `the_options` field value for this group. + """ + the_options: WidgetOptionsGroupedBy + + """ + The `weight_in_ng` field value for this group. + """ + weight_in_ng: JsonSafeLong + + """ + The `weight_in_ng_str` field value for this group. + """ + weight_in_ng_str: LongString + + """ + The `workspace_id` field value for this group. + """ + workspace_id: ID + + """ + The `workspace_name` field value for this group. + """ + workspace_name: String +} + +type WidgetOptionSets { + colors: [Color!]! + sizes: [Size!]! +} + +""" +Type used to perform aggregation computations on `WidgetOptionSets` fields. +""" +type WidgetOptionSetsAggregatedValues { + """ + Computed aggregate values for the `colors` field. + """ + colors: NonNumericAggregatedValues + + """ + Computed aggregate values for the `sizes` field. + """ + sizes: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on `WidgetOptionSets` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input WidgetOptionSetsFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [WidgetOptionSetsFilterInput!] + + """ + Used to filter on the `colors` field. + + Will be ignored if `null` or an empty object is passed. + """ + colors: ColorListFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: WidgetOptionSetsFilterInput + + """ + Used to filter on the `sizes` field. + + Will be ignored if `null` or an empty object is passed. + """ + sizes: SizeListFilterInput +} + +type WidgetOptions { + color: Color + size: Size + the_size: Size +} + +""" +Type used to perform aggregation computations on `WidgetOptions` fields. +""" +type WidgetOptionsAggregatedValues { + """ + Computed aggregate values for the `color` field. + """ + color: NonNumericAggregatedValues + + """ + Computed aggregate values for the `size` field. + """ + size: NonNumericAggregatedValues + + """ + Computed aggregate values for the `the_size` field. + """ + the_size: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on `WidgetOptions` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input WidgetOptionsFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [WidgetOptionsFilterInput!] + + """ + Used to filter on the `color` field. + + Will be ignored if `null` or an empty object is passed. + """ + color: ColorFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: WidgetOptionsFilterInput + + """ + Used to filter on the `size` field. + + Will be ignored if `null` or an empty object is passed. + """ + size: SizeFilterInput + + """ + Used to filter on the `the_size` field. + + Will be ignored if `null` or an empty object is passed. + """ + the_size: SizeFilterInput +} + +""" +Type used to specify the `WidgetOptions` fields to group by for aggregations. +""" +type WidgetOptionsGroupedBy { + """ + The `color` field value for this group. + """ + color: Color + + """ + The `size` field value for this group. + """ + size: Size + + """ + The `the_size` field value for this group. + """ + the_size: Size +} + +union WidgetOrAddress = Address | Widget + +""" +Type used to perform aggregation computations on `WidgetOrAddress` fields. +""" +type WidgetOrAddressAggregatedValues { + """ + Computed aggregate values for the `amount_cents` field. + """ + amount_cents: IntAggregatedValues + + """ + Computed aggregate values for the `amount_cents2` field. + """ + amount_cents2: IntAggregatedValues + + """ + Computed aggregate values for the `amounts` field. + """ + amounts: IntAggregatedValues + + """ + Computed aggregate values for the `cost` field. + """ + cost: MoneyAggregatedValues + + """ + Computed aggregate values for the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on: DateAggregatedValues + + """ + Computed aggregate values for the `cost_currency_name` field. + """ + cost_currency_name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_symbol` field. + """ + cost_currency_symbol: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_unit` field. + """ + cost_currency_unit: NonNumericAggregatedValues + + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at2` field. + """ + created_at2: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at2_legacy` field. + """ + created_at2_legacy: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at_legacy` field. + """ + created_at_legacy: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at_time_of_day` field. + """ + created_at_time_of_day: LocalTimeAggregatedValues + + """ + Computed aggregate values for the `created_on` field. + """ + created_on: DateAggregatedValues + + """ + Computed aggregate values for the `created_on_legacy` field. + """ + created_on_legacy: DateAggregatedValues + + """ + Computed aggregate values for the `fees` field. + """ + fees: MoneyAggregatedValues + + """ + Computed aggregate values for the `full_address` field. + """ + full_address: NonNumericAggregatedValues + + """ + Computed aggregate values for the `geo_location` field. + """ + geo_location: NonNumericAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `inventor` field. + """ + inventor: InventorAggregatedValues + + """ + Computed aggregate values for the `metadata` field. + """ + metadata: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `named_inventor` field. + """ + named_inventor: NamedInventorAggregatedValues + + """ + Computed aggregate values for the `options` field. + """ + options: WidgetOptionsAggregatedValues + + """ + Computed aggregate values for the `release_dates` field. + """ + release_dates: DateAggregatedValues + + """ + Computed aggregate values for the `release_timestamps` field. + """ + release_timestamps: DateTimeAggregatedValues + + """ + Computed aggregate values for the `shapes` field. + """ + shapes: NonNumericAggregatedValues + + """ + Computed aggregate values for the `size` field. + """ + size: NonNumericAggregatedValues + + """ + Computed aggregate values for the `tags` field. + """ + tags: NonNumericAggregatedValues + + """ + Computed aggregate values for the `the_options` field. + """ + the_options: WidgetOptionsAggregatedValues + + """ + Computed aggregate values for the `timestamps` field. + """ + timestamps: AddressTimestampsAggregatedValues + + """ + Computed aggregate values for the `weight_in_ng` field. + """ + weight_in_ng: JsonSafeLongAggregatedValues + + """ + Computed aggregate values for the `weight_in_ng_str` field. + """ + weight_in_ng_str: LongStringAggregatedValues + + """ + Computed aggregate values for the `workspace_id` field. + """ + workspace_id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `workspace_name` field. + """ + workspace_name: NonNumericAggregatedValues +} + +""" +Return type representing a bucket of `WidgetOrAddress` documents for an aggregations query. +""" +type WidgetOrAddressAggregation { + """ + Provides computed aggregated values over all `WidgetOrAddress` documents in an aggregation bucket. + """ + aggregated_values: WidgetOrAddressAggregatedValues + + """ + The count of `WidgetOrAddress` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `WidgetOrAddress` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: WidgetOrAddressGroupedBy +} + +""" +Represents a paginated collection of `WidgetOrAddressAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type WidgetOrAddressAggregationConnection { + """ + Wraps a specific `WidgetOrAddressAggregation` to pair it with its pagination cursor. + """ + edges: [WidgetOrAddressAggregationEdge!]! + + """ + The list of `WidgetOrAddressAggregation` results. + """ + nodes: [WidgetOrAddressAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `WidgetOrAddressAggregation` in the context of a `WidgetOrAddressAggregationConnection`, +providing access to both the `WidgetOrAddressAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type WidgetOrAddressAggregationEdge { + """ + The `Cursor` of this `WidgetOrAddressAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `WidgetOrAddressAggregation`. + """ + cursor: Cursor + + """ + The `WidgetOrAddressAggregation` of this edge. + """ + node: WidgetOrAddressAggregation +} + +""" +Represents a paginated collection of `WidgetOrAddress` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type WidgetOrAddressConnection { + """ + Wraps a specific `WidgetOrAddress` to pair it with its pagination cursor. + """ + edges: [WidgetOrAddressEdge!]! + + """ + The list of `WidgetOrAddress` results. + """ + nodes: [WidgetOrAddress!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `WidgetOrAddress` in the context of a `WidgetOrAddressConnection`, +providing access to both the `WidgetOrAddress` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type WidgetOrAddressEdge { + """ + The `Cursor` of this `WidgetOrAddress`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `WidgetOrAddress`. + """ + cursor: Cursor + + """ + The `WidgetOrAddress` of this edge. + """ + node: WidgetOrAddress +} + +""" +Input type used to specify filters on `WidgetOrAddress` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input WidgetOrAddressFilterInput { + """ + Used to filter on the `amount_cents` field. + + Will be ignored if `null` or an empty object is passed. + """ + amount_cents: IntFilterInput + + """ + Used to filter on the `amount_cents2` field. + + Will be ignored if `null` or an empty object is passed. + """ + amount_cents2: IntFilterInput + + """ + Used to filter on the `amounts` field. + + Will be ignored if `null` or an empty object is passed. + """ + amounts: IntListFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [WidgetOrAddressFilterInput!] + + """ + Used to filter on the `cost` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost: MoneyFilterInput + + """ + Used to filter on the `cost_currency_introduced_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_introduced_on: DateFilterInput + + """ + Used to filter on the `cost_currency_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_name: StringFilterInput + + """ + Used to filter on the `cost_currency_primary_continent` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_primary_continent: StringFilterInput + + """ + Used to filter on the `cost_currency_symbol` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_symbol: StringFilterInput + + """ + Used to filter on the `cost_currency_unit` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_unit: StringFilterInput + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `created_at2` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at2: DateTimeFilterInput + + """ + Used to filter on the `created_at2_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at2_legacy: DateTimeFilterInput + + """ + Used to filter on the `created_at_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at_legacy: DateTimeFilterInput + + """ + Used to filter on the `created_at_time_of_day` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at_time_of_day: LocalTimeFilterInput + + """ + Used to filter on the `created_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_on: DateFilterInput + + """ + Used to filter on the `created_on_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_on_legacy: DateFilterInput + + """ + Used to filter on the `fees` field. + + Will be ignored if `null` or an empty object is passed. + """ + fees: MoneyFieldsListFilterInput + + """ + Used to filter on the `full_address` field. + + Will be ignored if `null` or an empty object is passed. + """ + full_address: StringFilterInput + + """ + Used to filter on the `geo_location` field. + + Will be ignored if `null` or an empty object is passed. + """ + geo_location: GeoLocationFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `inventor` field. + + Will be ignored if `null` or an empty object is passed. + """ + inventor: InventorFilterInput + + """ + Used to filter on the `metadata` field. + + Will be ignored if `null` or an empty object is passed. + """ + metadata: UntypedFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Used to filter on the `name_text` field. + + Will be ignored if `null` or an empty object is passed. + """ + name_text: TextFilterInput + + """ + Used to filter on the `named_inventor` field. + + Will be ignored if `null` or an empty object is passed. + """ + named_inventor: NamedInventorFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: WidgetOrAddressFilterInput + + """ + Used to filter on the `options` field. + + Will be ignored if `null` or an empty object is passed. + """ + options: WidgetOptionsFilterInput + + """ + Used to filter on the `release_dates` field. + + Will be ignored if `null` or an empty object is passed. + """ + release_dates: DateListFilterInput + + """ + Used to filter on the `release_timestamps` field. + + Will be ignored if `null` or an empty object is passed. + """ + release_timestamps: DateTimeListFilterInput + + """ + Used to filter on the `size` field. + + Will be ignored if `null` or an empty object is passed. + """ + size: SizeFilterInput + + """ + Used to filter on the `tags` field. + + Will be ignored if `null` or an empty object is passed. + """ + tags: StringListFilterInput + + """ + Used to filter on the `the_options` field. + + Will be ignored if `null` or an empty object is passed. + """ + the_options: WidgetOptionsFilterInput + + """ + Used to filter on the `timestamps` field. + + Will be ignored if `null` or an empty object is passed. + """ + timestamps: AddressTimestampsFilterInput + + """ + Used to filter on the `weight_in_ng` field. + + Will be ignored if `null` or an empty object is passed. + """ + weight_in_ng: JsonSafeLongFilterInput + + """ + Used to filter on the `weight_in_ng_str` field. + + Will be ignored if `null` or an empty object is passed. + """ + weight_in_ng_str: LongStringFilterInput + + """ + Used to filter on the `workspace_id` field. + + Will be ignored if `null` or an empty object is passed. + """ + workspace_id: IDFilterInput + + """ + Used to filter on the `workspace_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + workspace_name: StringFilterInput +} + +""" +Type used to specify the `WidgetOrAddress` fields to group by for aggregations. +""" +type WidgetOrAddressGroupedBy { + """ + The `amount_cents` field value for this group. + """ + amount_cents: Int + + """ + The `amount_cents2` field value for this group. + """ + amount_cents2: Int + + """ + The `cost` field value for this group. + """ + cost: MoneyGroupedBy + + """ + Offers the different grouping options for the `cost_currency_introduced_on` value within this group. + """ + cost_currency_introduced_on: DateGroupedBy + + """ + The `cost_currency_name` field value for this group. + """ + cost_currency_name: String + + """ + The `cost_currency_primary_continent` field value for this group. + """ + cost_currency_primary_continent: String + + """ + The `cost_currency_symbol` field value for this group. + """ + cost_currency_symbol: String + + """ + The `cost_currency_unit` field value for this group. + """ + cost_currency_unit: String + + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + Offers the different grouping options for the `created_at2` value within this group. + """ + created_at2: DateTimeGroupedBy + + """ + The `created_at2_legacy` field value for this group. + """ + created_at2_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateTimeGroupingGranularityInput! + + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change + what day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone = "UTC" + ): DateTime + + """ + The `created_at_legacy` field value for this group. + """ + created_at_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateTimeGroupingGranularityInput! + + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change + what day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone = "UTC" + ): DateTime + + """ + The `created_at_time_of_day` field value for this group. + """ + created_at_time_of_day: LocalTime + + """ + Offers the different grouping options for the `created_on` value within this group. + """ + created_on: DateGroupedBy + + """ + The `created_on_legacy` field value for this group. + """ + created_on_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateGroupingGranularityInput! + + """ + Number of days (positive or negative) to shift the `Date` boundaries of each date grouping bucket. + + For example, when grouping by `YEAR`, this can be used to align the buckets + with fiscal or school years instead of calendar years. + """ + offset_days: Int + ): Date + + """ + The `fees` field value for this group. + + Note: `fees` is a collection field, but selecting this field will group on + individual values of the selected subfields of `fees`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `fees` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `fees` multiple times for a single document, that document will only be included in the group + once. + """ + fees: MoneyGroupedBy + + """ + The `full_address` field value for this group. + """ + full_address: String + + """ + The `inventor` field value for this group. + """ + inventor: InventorGroupedBy + + """ + The `metadata` field value for this group. + """ + metadata: Untyped + + """ + The `name` field value for this group. + """ + name: String + + """ + The `named_inventor` field value for this group. + """ + named_inventor: NamedInventorGroupedBy + + """ + The `options` field value for this group. + """ + options: WidgetOptionsGroupedBy + + """ + The individual value from `release_dates` for this group. + + Note: `release_dates` is a collection field, but selecting this field will group on individual values of `release_dates`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `release_dates` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `release_dates` multiple times for a single document, that document will only be included in the group + once. + """ + release_date: DateGroupedBy + + """ + The individual value from `release_timestamps` for this group. + + Note: `release_timestamps` is a collection field, but selecting this field + will group on individual values of `release_timestamps`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `release_timestamps` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `release_timestamps` multiple times for a single document, that document will only be included in the group + once. + """ + release_timestamp: DateTimeGroupedBy + + """ + The `size` field value for this group. + """ + size: Size + + """ + The individual value from `tags` for this group. + + Note: `tags` is a collection field, but selecting this field will group on individual values of `tags`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `tags` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `tags` multiple times for a single document, that document will only be included in the group + once. + """ + tag: String + + """ + The `the_options` field value for this group. + """ + the_options: WidgetOptionsGroupedBy + + """ + The `timestamps` field value for this group. + """ + timestamps: AddressTimestampsGroupedBy + + """ + The `weight_in_ng` field value for this group. + """ + weight_in_ng: JsonSafeLong + + """ + The `weight_in_ng_str` field value for this group. + """ + weight_in_ng_str: LongString + + """ + The `workspace_id` field value for this group. + """ + workspace_id: ID + + """ + The `workspace_name` field value for this group. + """ + workspace_name: String +} + +""" +Enumerates the ways `WidgetOrAddress`s can be sorted. +""" +enum WidgetOrAddressSortOrderInput { + """ + Sorts ascending by the `amount_cents2` field. + """ + amount_cents2_ASC + + """ + Sorts descending by the `amount_cents2` field. + """ + amount_cents2_DESC + + """ + Sorts ascending by the `amount_cents` field. + """ + amount_cents_ASC + + """ + Sorts descending by the `amount_cents` field. + """ + amount_cents_DESC + + """ + Sorts ascending by the `cost.amount_cents` field. + """ + cost_amount_cents_ASC + + """ + Sorts descending by the `cost.amount_cents` field. + """ + cost_amount_cents_DESC + + """ + Sorts ascending by the `cost.currency` field. + """ + cost_currency_ASC + + """ + Sorts descending by the `cost.currency` field. + """ + cost_currency_DESC + + """ + Sorts ascending by the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on_ASC + + """ + Sorts descending by the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on_DESC + + """ + Sorts ascending by the `cost_currency_name` field. + """ + cost_currency_name_ASC + + """ + Sorts descending by the `cost_currency_name` field. + """ + cost_currency_name_DESC + + """ + Sorts ascending by the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent_ASC + + """ + Sorts descending by the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent_DESC + + """ + Sorts ascending by the `cost_currency_symbol` field. + """ + cost_currency_symbol_ASC + + """ + Sorts descending by the `cost_currency_symbol` field. + """ + cost_currency_symbol_DESC + + """ + Sorts ascending by the `cost_currency_unit` field. + """ + cost_currency_unit_ASC + + """ + Sorts descending by the `cost_currency_unit` field. + """ + cost_currency_unit_DESC + + """ + Sorts ascending by the `created_at2` field. + """ + created_at2_ASC + + """ + Sorts descending by the `created_at2` field. + """ + created_at2_DESC + + """ + Sorts ascending by the `created_at2_legacy` field. + """ + created_at2_legacy_ASC + + """ + Sorts descending by the `created_at2_legacy` field. + """ + created_at2_legacy_DESC + + """ + Sorts ascending by the `created_at` field. + """ + created_at_ASC + + """ + Sorts descending by the `created_at` field. + """ + created_at_DESC + + """ + Sorts ascending by the `created_at_legacy` field. + """ + created_at_legacy_ASC + + """ + Sorts descending by the `created_at_legacy` field. + """ + created_at_legacy_DESC + + """ + Sorts ascending by the `created_at_time_of_day` field. + """ + created_at_time_of_day_ASC + + """ + Sorts descending by the `created_at_time_of_day` field. + """ + created_at_time_of_day_DESC + + """ + Sorts ascending by the `created_on` field. + """ + created_on_ASC + + """ + Sorts descending by the `created_on` field. + """ + created_on_DESC + + """ + Sorts ascending by the `created_on_legacy` field. + """ + created_on_legacy_ASC + + """ + Sorts descending by the `created_on_legacy` field. + """ + created_on_legacy_DESC + + """ + Sorts ascending by the `full_address` field. + """ + full_address_ASC + + """ + Sorts descending by the `full_address` field. + """ + full_address_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `inventor.name` field. + """ + inventor_name_ASC + + """ + Sorts descending by the `inventor.name` field. + """ + inventor_name_DESC + + """ + Sorts ascending by the `inventor.nationality` field. + """ + inventor_nationality_ASC + + """ + Sorts descending by the `inventor.nationality` field. + """ + inventor_nationality_DESC + + """ + Sorts ascending by the `inventor.stock_ticker` field. + """ + inventor_stock_ticker_ASC + + """ + Sorts descending by the `inventor.stock_ticker` field. + """ + inventor_stock_ticker_DESC + + """ + Sorts ascending by the `metadata` field. + """ + metadata_ASC + + """ + Sorts descending by the `metadata` field. + """ + metadata_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC + + """ + Sorts ascending by the `named_inventor.name` field. + """ + named_inventor_name_ASC + + """ + Sorts descending by the `named_inventor.name` field. + """ + named_inventor_name_DESC + + """ + Sorts ascending by the `named_inventor.nationality` field. + """ + named_inventor_nationality_ASC + + """ + Sorts descending by the `named_inventor.nationality` field. + """ + named_inventor_nationality_DESC + + """ + Sorts ascending by the `named_inventor.stock_ticker` field. + """ + named_inventor_stock_ticker_ASC + + """ + Sorts descending by the `named_inventor.stock_ticker` field. + """ + named_inventor_stock_ticker_DESC + + """ + Sorts ascending by the `options.color` field. + """ + options_color_ASC + + """ + Sorts descending by the `options.color` field. + """ + options_color_DESC + + """ + Sorts ascending by the `options.size` field. + """ + options_size_ASC + + """ + Sorts descending by the `options.size` field. + """ + options_size_DESC + + """ + Sorts ascending by the `options.the_size` field. + """ + options_the_size_ASC + + """ + Sorts descending by the `options.the_size` field. + """ + options_the_size_DESC + + """ + Sorts ascending by the `size` field. + """ + size_ASC + + """ + Sorts descending by the `size` field. + """ + size_DESC + + """ + Sorts ascending by the `the_options.color` field. + """ + the_options_color_ASC + + """ + Sorts descending by the `the_options.color` field. + """ + the_options_color_DESC + + """ + Sorts ascending by the `the_options.size` field. + """ + the_options_size_ASC + + """ + Sorts descending by the `the_options.size` field. + """ + the_options_size_DESC + + """ + Sorts ascending by the `the_options.the_size` field. + """ + the_options_the_size_ASC + + """ + Sorts descending by the `the_options.the_size` field. + """ + the_options_the_size_DESC + + """ + Sorts ascending by the `timestamps.created_at` field. + """ + timestamps_created_at_ASC + + """ + Sorts descending by the `timestamps.created_at` field. + """ + timestamps_created_at_DESC + + """ + Sorts ascending by the `weight_in_ng` field. + """ + weight_in_ng_ASC + + """ + Sorts descending by the `weight_in_ng` field. + """ + weight_in_ng_DESC + + """ + Sorts ascending by the `weight_in_ng_str` field. + """ + weight_in_ng_str_ASC + + """ + Sorts descending by the `weight_in_ng_str` field. + """ + weight_in_ng_str_DESC + + """ + Sorts ascending by the `workspace_id` field. + """ + workspace_id_ASC + + """ + Sorts descending by the `workspace_id` field. + """ + workspace_id_DESC + + """ + Sorts ascending by the `workspace_name` field. + """ + workspace_name_ASC + + """ + Sorts descending by the `workspace_name` field. + """ + workspace_name_DESC +} + +""" +Enumerates the ways `Widget`s can be sorted. +""" +enum WidgetSortOrderInput { + """ + Sorts ascending by the `amount_cents2` field. + """ + amount_cents2_ASC + + """ + Sorts descending by the `amount_cents2` field. + """ + amount_cents2_DESC + + """ + Sorts ascending by the `amount_cents` field. + """ + amount_cents_ASC + + """ + Sorts descending by the `amount_cents` field. + """ + amount_cents_DESC + + """ + Sorts ascending by the `cost.amount_cents` field. + """ + cost_amount_cents_ASC + + """ + Sorts descending by the `cost.amount_cents` field. + """ + cost_amount_cents_DESC + + """ + Sorts ascending by the `cost.currency` field. + """ + cost_currency_ASC + + """ + Sorts descending by the `cost.currency` field. + """ + cost_currency_DESC + + """ + Sorts ascending by the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on_ASC + + """ + Sorts descending by the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on_DESC + + """ + Sorts ascending by the `cost_currency_name` field. + """ + cost_currency_name_ASC + + """ + Sorts descending by the `cost_currency_name` field. + """ + cost_currency_name_DESC + + """ + Sorts ascending by the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent_ASC + + """ + Sorts descending by the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent_DESC + + """ + Sorts ascending by the `cost_currency_symbol` field. + """ + cost_currency_symbol_ASC + + """ + Sorts descending by the `cost_currency_symbol` field. + """ + cost_currency_symbol_DESC + + """ + Sorts ascending by the `cost_currency_unit` field. + """ + cost_currency_unit_ASC + + """ + Sorts descending by the `cost_currency_unit` field. + """ + cost_currency_unit_DESC + + """ + Sorts ascending by the `created_at2` field. + """ + created_at2_ASC + + """ + Sorts descending by the `created_at2` field. + """ + created_at2_DESC + + """ + Sorts ascending by the `created_at2_legacy` field. + """ + created_at2_legacy_ASC + + """ + Sorts descending by the `created_at2_legacy` field. + """ + created_at2_legacy_DESC + + """ + Sorts ascending by the `created_at` field. + """ + created_at_ASC + + """ + Sorts descending by the `created_at` field. + """ + created_at_DESC + + """ + Sorts ascending by the `created_at_legacy` field. + """ + created_at_legacy_ASC + + """ + Sorts descending by the `created_at_legacy` field. + """ + created_at_legacy_DESC + + """ + Sorts ascending by the `created_at_time_of_day` field. + """ + created_at_time_of_day_ASC + + """ + Sorts descending by the `created_at_time_of_day` field. + """ + created_at_time_of_day_DESC + + """ + Sorts ascending by the `created_on` field. + """ + created_on_ASC + + """ + Sorts descending by the `created_on` field. + """ + created_on_DESC + + """ + Sorts ascending by the `created_on_legacy` field. + """ + created_on_legacy_ASC + + """ + Sorts descending by the `created_on_legacy` field. + """ + created_on_legacy_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `inventor.name` field. + """ + inventor_name_ASC + + """ + Sorts descending by the `inventor.name` field. + """ + inventor_name_DESC + + """ + Sorts ascending by the `inventor.nationality` field. + """ + inventor_nationality_ASC + + """ + Sorts descending by the `inventor.nationality` field. + """ + inventor_nationality_DESC + + """ + Sorts ascending by the `inventor.stock_ticker` field. + """ + inventor_stock_ticker_ASC + + """ + Sorts descending by the `inventor.stock_ticker` field. + """ + inventor_stock_ticker_DESC + + """ + Sorts ascending by the `metadata` field. + """ + metadata_ASC + + """ + Sorts descending by the `metadata` field. + """ + metadata_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC + + """ + Sorts ascending by the `named_inventor.name` field. + """ + named_inventor_name_ASC + + """ + Sorts descending by the `named_inventor.name` field. + """ + named_inventor_name_DESC + + """ + Sorts ascending by the `named_inventor.nationality` field. + """ + named_inventor_nationality_ASC + + """ + Sorts descending by the `named_inventor.nationality` field. + """ + named_inventor_nationality_DESC + + """ + Sorts ascending by the `named_inventor.stock_ticker` field. + """ + named_inventor_stock_ticker_ASC + + """ + Sorts descending by the `named_inventor.stock_ticker` field. + """ + named_inventor_stock_ticker_DESC + + """ + Sorts ascending by the `options.color` field. + """ + options_color_ASC + + """ + Sorts descending by the `options.color` field. + """ + options_color_DESC + + """ + Sorts ascending by the `options.size` field. + """ + options_size_ASC + + """ + Sorts descending by the `options.size` field. + """ + options_size_DESC + + """ + Sorts ascending by the `options.the_size` field. + """ + options_the_size_ASC + + """ + Sorts descending by the `options.the_size` field. + """ + options_the_size_DESC + + """ + Sorts ascending by the `size` field. + """ + size_ASC + + """ + Sorts descending by the `size` field. + """ + size_DESC + + """ + Sorts ascending by the `the_options.color` field. + """ + the_options_color_ASC + + """ + Sorts descending by the `the_options.color` field. + """ + the_options_color_DESC + + """ + Sorts ascending by the `the_options.size` field. + """ + the_options_size_ASC + + """ + Sorts descending by the `the_options.size` field. + """ + the_options_size_DESC + + """ + Sorts ascending by the `the_options.the_size` field. + """ + the_options_the_size_ASC + + """ + Sorts descending by the `the_options.the_size` field. + """ + the_options_the_size_DESC + + """ + Sorts ascending by the `weight_in_ng` field. + """ + weight_in_ng_ASC + + """ + Sorts descending by the `weight_in_ng` field. + """ + weight_in_ng_DESC + + """ + Sorts ascending by the `weight_in_ng_str` field. + """ + weight_in_ng_str_ASC + + """ + Sorts descending by the `weight_in_ng_str` field. + """ + weight_in_ng_str_DESC + + """ + Sorts ascending by the `workspace_id` field. + """ + workspace_id_ASC + + """ + Sorts descending by the `workspace_id` field. + """ + workspace_id_DESC + + """ + Sorts ascending by the `workspace_name` field. + """ + workspace_name_ASC + + """ + Sorts descending by the `workspace_name` field. + """ + workspace_name_DESC +} + +type WidgetWorkspace { + id: ID! + name: String + widget: WorkspaceWidget +} + +""" +Type used to perform aggregation computations on `WidgetWorkspace` fields. +""" +type WidgetWorkspaceAggregatedValues { + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget` field. + """ + widget: WorkspaceWidgetAggregatedValues +} + +""" +Return type representing a bucket of `WidgetWorkspace` documents for an aggregations query. +""" +type WidgetWorkspaceAggregation { + """ + Provides computed aggregated values over all `WidgetWorkspace` documents in an aggregation bucket. + """ + aggregated_values: WidgetWorkspaceAggregatedValues + + """ + The count of `WidgetWorkspace` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `WidgetWorkspace` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: WidgetWorkspaceGroupedBy +} + +""" +Represents a paginated collection of `WidgetWorkspaceAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type WidgetWorkspaceAggregationConnection { + """ + Wraps a specific `WidgetWorkspaceAggregation` to pair it with its pagination cursor. + """ + edges: [WidgetWorkspaceAggregationEdge!]! + + """ + The list of `WidgetWorkspaceAggregation` results. + """ + nodes: [WidgetWorkspaceAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `WidgetWorkspaceAggregation` in the context of a `WidgetWorkspaceAggregationConnection`, +providing access to both the `WidgetWorkspaceAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type WidgetWorkspaceAggregationEdge { + """ + The `Cursor` of this `WidgetWorkspaceAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `WidgetWorkspaceAggregation`. + """ + cursor: Cursor + + """ + The `WidgetWorkspaceAggregation` of this edge. + """ + node: WidgetWorkspaceAggregation +} + +""" +Represents a paginated collection of `WidgetWorkspace` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type WidgetWorkspaceConnection { + """ + Wraps a specific `WidgetWorkspace` to pair it with its pagination cursor. + """ + edges: [WidgetWorkspaceEdge!]! + + """ + The list of `WidgetWorkspace` results. + """ + nodes: [WidgetWorkspace!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `WidgetWorkspace` in the context of a `WidgetWorkspaceConnection`, +providing access to both the `WidgetWorkspace` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type WidgetWorkspaceEdge { + """ + The `Cursor` of this `WidgetWorkspace`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `WidgetWorkspace`. + """ + cursor: Cursor + + """ + The `WidgetWorkspace` of this edge. + """ + node: WidgetWorkspace +} + +""" +Input type used to specify filters on `WidgetWorkspace` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input WidgetWorkspaceFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [WidgetWorkspaceFilterInput!] + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: WidgetWorkspaceFilterInput + + """ + Used to filter on the `widget` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget: WorkspaceWidgetFilterInput +} + +""" +Type used to specify the `WidgetWorkspace` fields to group by for aggregations. +""" +type WidgetWorkspaceGroupedBy { + """ + The `name` field value for this group. + """ + name: String + + """ + The `widget` field value for this group. + """ + widget: WorkspaceWidgetGroupedBy +} + +""" +Enumerates the ways `WidgetWorkspace`s can be sorted. +""" +enum WidgetWorkspaceSortOrderInput { + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC + + """ + Sorts ascending by the `widget.created_at` field. + """ + widget_created_at_ASC + + """ + Sorts descending by the `widget.created_at` field. + """ + widget_created_at_DESC + + """ + Sorts ascending by the `widget.id` field. + """ + widget_id_ASC + + """ + Sorts descending by the `widget.id` field. + """ + widget_id_DESC +} + +type WorkspaceWidget { + created_at: DateTime + id: ID! +} + +""" +Type used to perform aggregation computations on `WorkspaceWidget` fields. +""" +type WorkspaceWidgetAggregatedValues { + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on `WorkspaceWidget` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input WorkspaceWidgetFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [WorkspaceWidgetFilterInput!] + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: WorkspaceWidgetFilterInput +} + +""" +Type used to specify the `WorkspaceWidget` fields to group by for aggregations. +""" +type WorkspaceWidgetGroupedBy { + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + The `id` field value for this group. + """ + id: ID +} \ No newline at end of file diff --git a/config/schema/artifacts_with_apollo/datastore_config.yaml b/config/schema/artifacts_with_apollo/datastore_config.yaml new file mode 100644 index 00000000..6788f952 --- /dev/null +++ b/config/schema/artifacts_with_apollo/datastore_config.yaml @@ -0,0 +1,2046 @@ +# Generated by `rake schema_artifacts:dump`. +# DO NOT EDIT BY HAND. Any edits will be lost the next time the rake task is run. +--- +index_templates: + teams: + index_patterns: + - teams_rollover__* + template: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + league: + type: keyword + country_code: + type: keyword + formed_on: + type: date + format: strict_date + current_name: + type: keyword + past_names: + type: keyword + won_championships_at: + type: date + format: strict_date_time + details: + properties: + uniform_colors: + type: keyword + count: + type: integer + stadium_location: + type: geo_point + forbes_valuations: + type: long + forbes_valuation_moneys_nested: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + forbes_valuation_moneys_object: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + current_players_nested: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + __counts: + properties: + nicknames: + type: integer + affiliations|sponsorships_nested: + type: integer + affiliations|sponsorships_object: + type: integer + affiliations|sponsorships_object|sponsor_id: + type: integer + affiliations|sponsorships_object|annual_total: + type: integer + affiliations|sponsorships_object|annual_total|currency: + type: integer + affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + seasons_nested: + type: integer + seasons_object: + type: integer + seasons_object|year: + type: integer + seasons_object|games_played: + type: integer + seasons_object|awards: + type: integer + type: nested + current_players_object: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + type: object + seasons_nested: + properties: + the_record: + properties: + win_count: + type: integer + loss_count: + type: integer + last_win_date: + type: date + format: strict_date + first_win_on: + type: date + format: strict_date + year: + type: integer + notes: + type: keyword + count: + type: integer + started_at: + type: date + format: strict_date_time + won_games_at: + type: date + format: strict_date_time + players_nested: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + __counts: + properties: + nicknames: + type: integer + affiliations|sponsorships_nested: + type: integer + affiliations|sponsorships_object: + type: integer + affiliations|sponsorships_object|sponsor_id: + type: integer + affiliations|sponsorships_object|annual_total: + type: integer + affiliations|sponsorships_object|annual_total|currency: + type: integer + affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + seasons_nested: + type: integer + seasons_object: + type: integer + seasons_object|year: + type: integer + seasons_object|games_played: + type: integer + seasons_object|awards: + type: integer + type: nested + players_object: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + type: object + __counts: + properties: + notes: + type: integer + won_games_at: + type: integer + players_nested: + type: integer + players_object: + type: integer + players_object|name: + type: integer + players_object|nicknames: + type: integer + players_object|affiliations: + type: integer + players_object|affiliations|sponsorships_nested: + type: integer + players_object|affiliations|sponsorships_object: + type: integer + players_object|affiliations|sponsorships_object|sponsor_id: + type: integer + players_object|affiliations|sponsorships_object|annual_total: + type: integer + players_object|affiliations|sponsorships_object|annual_total|currency: + type: integer + players_object|affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + players_object|seasons_nested: + type: integer + players_object|seasons_object: + type: integer + players_object|seasons_object|year: + type: integer + players_object|seasons_object|games_played: + type: integer + players_object|seasons_object|awards: + type: integer + type: nested + seasons_object: + properties: + the_record: + properties: + win_count: + type: integer + loss_count: + type: integer + last_win_date: + type: date + format: strict_date + first_win_on: + type: date + format: strict_date + year: + type: integer + notes: + type: keyword + count: + type: integer + started_at: + type: date + format: strict_date_time + won_games_at: + type: date + format: strict_date_time + players_nested: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + __counts: + properties: + nicknames: + type: integer + affiliations|sponsorships_nested: + type: integer + affiliations|sponsorships_object: + type: integer + affiliations|sponsorships_object|sponsor_id: + type: integer + affiliations|sponsorships_object|annual_total: + type: integer + affiliations|sponsorships_object|annual_total|currency: + type: integer + affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + seasons_nested: + type: integer + seasons_object: + type: integer + seasons_object|year: + type: integer + seasons_object|games_played: + type: integer + seasons_object|awards: + type: integer + type: nested + players_object: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + type: object + type: object + the_nested_fields: + properties: + forbes_valuation_moneys: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + current_players: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + __counts: + properties: + nicknames: + type: integer + affiliations|sponsorships_nested: + type: integer + affiliations|sponsorships_object: + type: integer + affiliations|sponsorships_object|sponsor_id: + type: integer + affiliations|sponsorships_object|annual_total: + type: integer + affiliations|sponsorships_object|annual_total|currency: + type: integer + affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + seasons_nested: + type: integer + seasons_object: + type: integer + seasons_object|year: + type: integer + seasons_object|games_played: + type: integer + seasons_object|awards: + type: integer + type: nested + the_seasons: + properties: + the_record: + properties: + win_count: + type: integer + loss_count: + type: integer + last_win_date: + type: date + format: strict_date + first_win_on: + type: date + format: strict_date + year: + type: integer + notes: + type: keyword + count: + type: integer + started_at: + type: date + format: strict_date_time + won_games_at: + type: date + format: strict_date_time + players_nested: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + __counts: + properties: + nicknames: + type: integer + affiliations|sponsorships_nested: + type: integer + affiliations|sponsorships_object: + type: integer + affiliations|sponsorships_object|sponsor_id: + type: integer + affiliations|sponsorships_object|annual_total: + type: integer + affiliations|sponsorships_object|annual_total|currency: + type: integer + affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + seasons_nested: + type: integer + seasons_object: + type: integer + seasons_object|year: + type: integer + seasons_object|games_played: + type: integer + seasons_object|awards: + type: integer + type: nested + players_object: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + type: object + __counts: + properties: + notes: + type: integer + won_games_at: + type: integer + players_nested: + type: integer + players_object: + type: integer + players_object|name: + type: integer + players_object|nicknames: + type: integer + players_object|affiliations: + type: integer + players_object|affiliations|sponsorships_nested: + type: integer + players_object|affiliations|sponsorships_object: + type: integer + players_object|affiliations|sponsorships_object|sponsor_id: + type: integer + players_object|affiliations|sponsorships_object|annual_total: + type: integer + players_object|affiliations|sponsorships_object|annual_total|currency: + type: integer + players_object|affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + players_object|seasons_nested: + type: integer + players_object|seasons_object: + type: integer + players_object|seasons_object|year: + type: integer + players_object|seasons_object|games_played: + type: integer + players_object|seasons_object|awards: + type: integer + type: nested + nested_fields2: + properties: + forbes_valuation_moneys: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + current_players: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + __counts: + properties: + nicknames: + type: integer + affiliations|sponsorships_nested: + type: integer + affiliations|sponsorships_object: + type: integer + affiliations|sponsorships_object|sponsor_id: + type: integer + affiliations|sponsorships_object|annual_total: + type: integer + affiliations|sponsorships_object|annual_total|currency: + type: integer + affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + seasons_nested: + type: integer + seasons_object: + type: integer + seasons_object|year: + type: integer + seasons_object|games_played: + type: integer + seasons_object|awards: + type: integer + type: nested + the_seasons: + properties: + the_record: + properties: + win_count: + type: integer + loss_count: + type: integer + last_win_date: + type: date + format: strict_date + first_win_on: + type: date + format: strict_date + year: + type: integer + notes: + type: keyword + count: + type: integer + started_at: + type: date + format: strict_date_time + won_games_at: + type: date + format: strict_date_time + players_nested: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + __counts: + properties: + nicknames: + type: integer + affiliations|sponsorships_nested: + type: integer + affiliations|sponsorships_object: + type: integer + affiliations|sponsorships_object|sponsor_id: + type: integer + affiliations|sponsorships_object|annual_total: + type: integer + affiliations|sponsorships_object|annual_total|currency: + type: integer + affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + seasons_nested: + type: integer + seasons_object: + type: integer + seasons_object|year: + type: integer + seasons_object|games_played: + type: integer + seasons_object|awards: + type: integer + type: nested + players_object: + properties: + name: + type: keyword + nicknames: + type: keyword + affiliations: + properties: + sponsorships_nested: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: nested + sponsorships_object: + properties: + sponsor_id: + type: keyword + annual_total: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + seasons_nested: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + __counts: + properties: + awards: + type: integer + type: nested + seasons_object: + properties: + year: + type: integer + games_played: + type: integer + awards: + type: keyword + type: object + type: object + __counts: + properties: + notes: + type: integer + won_games_at: + type: integer + players_nested: + type: integer + players_object: + type: integer + players_object|name: + type: integer + players_object|nicknames: + type: integer + players_object|affiliations: + type: integer + players_object|affiliations|sponsorships_nested: + type: integer + players_object|affiliations|sponsorships_object: + type: integer + players_object|affiliations|sponsorships_object|sponsor_id: + type: integer + players_object|affiliations|sponsorships_object|annual_total: + type: integer + players_object|affiliations|sponsorships_object|annual_total|currency: + type: integer + players_object|affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + players_object|seasons_nested: + type: integer + players_object|seasons_object: + type: integer + players_object|seasons_object|year: + type: integer + players_object|seasons_object|games_played: + type: integer + players_object|seasons_object|awards: + type: integer + type: nested + __counts: + properties: + past_names: + type: integer + won_championships_at: + type: integer + details|uniform_colors: + type: integer + forbes_valuations: + type: integer + forbes_valuation_moneys_nested: + type: integer + forbes_valuation_moneys_object: + type: integer + forbes_valuation_moneys_object|currency: + type: integer + forbes_valuation_moneys_object|amount_cents: + type: integer + current_players_nested: + type: integer + current_players_object: + type: integer + current_players_object|name: + type: integer + current_players_object|nicknames: + type: integer + current_players_object|affiliations: + type: integer + current_players_object|affiliations|sponsorships_nested: + type: integer + current_players_object|affiliations|sponsorships_object: + type: integer + current_players_object|affiliations|sponsorships_object|sponsor_id: + type: integer + current_players_object|affiliations|sponsorships_object|annual_total: + type: integer + current_players_object|affiliations|sponsorships_object|annual_total|currency: + type: integer + current_players_object|affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + current_players_object|seasons_nested: + type: integer + current_players_object|seasons_object: + type: integer + current_players_object|seasons_object|year: + type: integer + current_players_object|seasons_object|games_played: + type: integer + current_players_object|seasons_object|awards: + type: integer + seasons_nested: + type: integer + seasons_object: + type: integer + seasons_object|the_record: + type: integer + seasons_object|the_record|win_count: + type: integer + seasons_object|the_record|loss_count: + type: integer + seasons_object|the_record|last_win_date: + type: integer + seasons_object|the_record|first_win_on: + type: integer + seasons_object|year: + type: integer + seasons_object|notes: + type: integer + seasons_object|count: + type: integer + seasons_object|started_at: + type: integer + seasons_object|won_games_at: + type: integer + seasons_object|players_nested: + type: integer + seasons_object|players_object: + type: integer + seasons_object|players_object|name: + type: integer + seasons_object|players_object|nicknames: + type: integer + seasons_object|players_object|affiliations: + type: integer + seasons_object|players_object|affiliations|sponsorships_nested: + type: integer + seasons_object|players_object|affiliations|sponsorships_object: + type: integer + seasons_object|players_object|affiliations|sponsorships_object|sponsor_id: + type: integer + seasons_object|players_object|affiliations|sponsorships_object|annual_total: + type: integer + seasons_object|players_object|affiliations|sponsorships_object|annual_total|currency: + type: integer + seasons_object|players_object|affiliations|sponsorships_object|annual_total|amount_cents: + type: integer + seasons_object|players_object|seasons_nested: + type: integer + seasons_object|players_object|seasons_object: + type: integer + seasons_object|players_object|seasons_object|year: + type: integer + seasons_object|players_object|seasons_object|games_played: + type: integer + seasons_object|players_object|seasons_object|awards: + type: integer + the_nested_fields|forbes_valuation_moneys: + type: integer + the_nested_fields|current_players: + type: integer + the_nested_fields|the_seasons: + type: integer + nested_fields2|forbes_valuation_moneys: + type: integer + nested_fields2|current_players: + type: integer + nested_fields2|the_seasons: + type: integer + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _routing: + required: true + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 + widget_currencies: + index_patterns: + - widget_currencies_rollover__* + template: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + name: + type: keyword + introduced_on: + type: date + format: strict_date + primary_continent: + type: keyword + details: + properties: + unit: + type: keyword + symbol: + type: keyword + widget_names2: + type: keyword + widget_tags: + type: keyword + widget_fee_currencies: + type: keyword + widget_options: + properties: + sizes: + type: keyword + colors: + type: keyword + nested_fields: + properties: + max_widget_cost: + type: integer + oldest_widget_created_at: + type: date + format: strict_date_time + __counts: + properties: + widget_names2: + type: integer + widget_tags: + type: integer + widget_fee_currencies: + type: integer + widget_options|sizes: + type: integer + widget_options|colors: + type: integer + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _routing: + required: true + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 + widgets: + index_patterns: + - widgets_rollover__* + template: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + workspace_id2: + type: keyword + amount_cents: + type: integer + cost: + properties: + currency: + type: keyword + amount_cents: + type: integer + cost_currency_unit: + type: keyword + cost_currency_name: + type: keyword + cost_currency_symbol: + type: keyword + cost_currency_primary_continent: + type: keyword + cost_currency_introduced_on: + type: date + format: strict_date + name: + type: keyword + name_text: + type: text + created_at: + type: date + format: strict_date_time + created_at_time_of_day: + type: date + format: HH:mm:ss||HH:mm:ss.S||HH:mm:ss.SS||HH:mm:ss.SSS + created_on: + type: date + format: strict_date + release_timestamps: + type: date + format: strict_date_time + release_dates: + type: date + format: strict_date + component_ids: + type: keyword + options: + properties: + size: + type: keyword + the_sighs: + type: keyword + color: + type: keyword + the_opts: + properties: + size: + type: keyword + the_sighs: + type: keyword + color: + type: keyword + inventor: + properties: + name: + type: keyword + nationality: + type: keyword + stock_ticker: + type: keyword + __typename: + type: keyword + named_inventor: + properties: + name: + type: keyword + nationality: + type: keyword + stock_ticker: + type: keyword + __typename: + type: keyword + weight_in_ng_str: + type: long + weight_in_ng: + type: long + tags: + type: keyword + amounts: + type: integer + index: false + fees: + properties: + currency: + type: keyword + amount_cents: + type: integer + type: object + metadata: + type: keyword + workspace_name: + type: keyword + __counts: + properties: + release_timestamps: + type: integer + release_dates: + type: integer + component_ids: + type: integer + tags: + type: integer + amounts: + type: integer + fees: + type: integer + fees|currency: + type: integer + fees|amount_cents: + type: integer + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _routing: + required: true + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 3 +indices: + addresses: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + full_address: + type: keyword + timestamps: + properties: + created_at: + type: date + format: strict_date_time + geo_location: + type: geo_point + shapes: + type: geo_shape + manufacturer_id: + type: keyword + __counts: + properties: + shapes: + type: integer + shapes|type: + type: integer + shapes|coordinates: + type: integer + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 + components: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + name: + type: keyword + created_at: + type: date + format: strict_date_time + position: + properties: + x: + type: double + "y": + type: double + tags: + type: keyword + widget_name: + type: keyword + widget_tags: + type: keyword + widget_workspace_id3: + type: keyword + widget_size: + type: keyword + widget_cost: + properties: + currency: + type: keyword + amount_cents: + type: integer + part_ids: + type: keyword + __counts: + properties: + tags: + type: integer + widget_tags: + type: integer + part_ids: + type: integer + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 + electrical_parts: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + name: + type: keyword + created_at: + type: date + format: strict_date_time + voltage: + type: integer + manufacturer_id: + type: keyword + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 + manufacturers: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + name: + type: keyword + created_at: + type: date + format: strict_date_time + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 + mechanical_parts: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + name: + type: keyword + created_at: + type: date + format: strict_date_time + material: + type: keyword + manufacturer_id: + type: keyword + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 + sponsors: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + name: + type: keyword + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 + widget_workspaces: + aliases: {} + mappings: + dynamic: strict + properties: + id: + type: keyword + name: + type: keyword + widget: + properties: + id: + type: keyword + created_at: + type: date + format: strict_date_time + __sources: + type: keyword + __versions: + type: object + dynamic: 'false' + _size: + enabled: true + settings: + index.mapping.ignore_malformed: false + index.mapping.coerce: false + index.number_of_replicas: 1 + index.number_of_shards: 1 +scripts: + update_WidgetCurrency_from_Widget_0f26b3e9ea093af29e5cef02a25e75ca: + context: update + script: + lang: painless + source: |- + // Idempotently inserts the given value in the `sortedList`, returning `true` if the list was updated. + boolean appendOnlySet_idempotentlyInsertValue(def value, List sortedList) { + // As per https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/Collections.html#binarySearch(java.util.List,java.lang.Object): + // + // > Returns the index of the search key, if it is contained in the list; otherwise, (-(insertion point) - 1). + // > The insertion point is defined as the point at which the key would be inserted into the list: the index + // > of the first element greater than the key, or list.size() if all elements in the list are less than the + // > specified key. Note that this guarantees that the return value will be >= 0 if and only if the key is found. + int binarySearchResult = Collections.binarySearch(sortedList, value); + + if (binarySearchResult < 0) { + sortedList.add(-binarySearchResult - 1, value); + return true; + } else { + return false; + } + } + + // Wrapper around `idempotentlyInsertValue` that handles a list of values. + // Returns `true` if the list field was updated. + boolean appendOnlySet_idempotentlyInsertValues(List values, List sortedList) { + boolean listUpdated = false; + + for (def value : values) { + listUpdated = appendOnlySet_idempotentlyInsertValue(value, sortedList) || listUpdated; + } + + return listUpdated; + } + + boolean immutableValue_idempotentlyUpdateValue(List scriptErrors, List values, def parentObject, String fullPath, String fieldName, boolean nullable, boolean canChangeFromNull) { + boolean fieldAlreadySet = parentObject.containsKey(fieldName); + + // `values` is always passed to us as a `List` (the indexer normalizes to a list, wrapping single + // values in a list as needed) but we only ever expect at most 1 element. + def newValueCandidate = values.isEmpty() ? null : values[0]; + + if (fieldAlreadySet) { + def currentValue = parentObject[fieldName]; + + // Usually we do not allow `immutable_value` fields to ever change values. However, we make + // a special case for `null`, but only when `can_change_from_null: true` has been configured. + // This can be important when deriving a field that has not always existed on the source events. + // On early events, the value may be `null`, and, when this is enabled, we do not want that to + // interfere with our ability to set the value to the correct non-null value based on a different + // event which has a value for the source field. + if (canChangeFromNull) { + if (currentValue == null) { + parentObject[fieldName] = newValueCandidate; + return true; + } + + // When `can_change_from_null: true` is enabled we also need to ignore NEW `null` values that we + // see _after_ a non-null value. This is necessary because an ElasticGraph invariant is that events + // can be processed in any order. So we might process an old event (predating the existence of the + // source field) after we've already set the field to a non-null value. We must always "converge" + // on the same indexed state regardless, of the order events are seen, so here we just ignore it. + if (newValueCandidate == null) { + return false; + } + } + + // Otherwise, if the values differ, it means we are attempting to mutate the immutable value field, which we cannot allow. + if (currentValue != newValueCandidate) { + if (currentValue == null) { + scriptErrors.add("Field `" + fullPath + "` cannot be changed (" + currentValue + " => " + newValueCandidate + "). Set `can_change_from_null: true` on the `immutable_value` definition to allow this."); + } else { + scriptErrors.add("Field `" + fullPath + "` cannot be changed (" + currentValue + " => " + newValueCandidate + ")."); + } + } + + return false; + } + + if (newValueCandidate == null && !nullable) { + scriptErrors.add("Field `" + fullPath + "` cannot be set to `null`, but the source event contains no value for it. Remove `nullable: false` from the `immutable_value` definition to allow this."); + return false; + } + + parentObject[fieldName] = newValueCandidate; + return true; + } + + boolean maxValue_idempotentlyUpdateValue(List values, def parentObject, String fieldName) { + def currentFieldValue = parentObject[fieldName]; + def maxNewValue = values.isEmpty() ? null : Collections.max(values); + + if (currentFieldValue == null || (maxNewValue != null && maxNewValue.compareTo(currentFieldValue) > 0)) { + parentObject[fieldName] = maxNewValue; + return true; + } + + return false; + } + + boolean minValue_idempotentlyUpdateValue(List values, def parentObject, String fieldName) { + def currentFieldValue = parentObject[fieldName]; + def minNewValue = values.isEmpty() ? null : Collections.min(values); + + if (currentFieldValue == null || (minNewValue != null && minNewValue.compareTo(currentFieldValue) < 0)) { + parentObject[fieldName] = minNewValue; + return true; + } + + return false; + } + + Map data = params.data; + // A variable to accumulate script errors so that we can surface _all_ issues and not just the first. + List scriptErrors = new ArrayList(); + if (ctx._source.details == null) { + ctx._source.details = [:]; + } + if (ctx._source.nested_fields == null) { + ctx._source.nested_fields = [:]; + } + if (ctx._source.widget_fee_currencies == null) { + ctx._source.widget_fee_currencies = []; + } + if (ctx._source.widget_names2 == null) { + ctx._source.widget_names2 = []; + } + if (ctx._source.widget_options == null) { + ctx._source.widget_options = [:]; + } + if (ctx._source.widget_options.colors == null) { + ctx._source.widget_options.colors = []; + } + if (ctx._source.widget_options.sizes == null) { + ctx._source.widget_options.sizes = []; + } + if (ctx._source.widget_tags == null) { + ctx._source.widget_tags = []; + } + + boolean details__symbol_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_symbol"], ctx._source.details, "details.symbol", "symbol", true, true); + boolean details__unit_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_unit"], ctx._source.details, "details.unit", "unit", false, false); + boolean introduced_on_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_introduced_on"], ctx._source, "introduced_on", "introduced_on", true, false); + boolean name_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_name"], ctx._source, "name", "name", true, false); + boolean nested_fields__max_widget_cost_was_noop = !maxValue_idempotentlyUpdateValue(data["cost.amount_cents"], ctx._source.nested_fields, "max_widget_cost"); + boolean oldest_widget_created_at_was_noop = !minValue_idempotentlyUpdateValue(data["created_at"], ctx._source, "oldest_widget_created_at"); + boolean primary_continent_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_primary_continent"], ctx._source, "primary_continent", "primary_continent", true, false); + boolean widget_fee_currencies_was_noop = !appendOnlySet_idempotentlyInsertValues(data["fees.currency"], ctx._source.widget_fee_currencies); + boolean widget_names2_was_noop = !appendOnlySet_idempotentlyInsertValues(data["name"], ctx._source.widget_names2); + boolean widget_options__colors_was_noop = !appendOnlySet_idempotentlyInsertValues(data["options.color"], ctx._source.widget_options.colors); + boolean widget_options__sizes_was_noop = !appendOnlySet_idempotentlyInsertValues(data["options.size"], ctx._source.widget_options.sizes); + boolean widget_tags_was_noop = !appendOnlySet_idempotentlyInsertValues(data["tags"], ctx._source.widget_tags); + + if (!scriptErrors.isEmpty()) { + throw new IllegalArgumentException("Derived index update failed due to bad input data: " + scriptErrors.join(" ")); + } + + // For records with no new values to index, only skip the update if the document itself doesn't already exist. + // Otherwise create an (empty) document to reflect the fact that the id has been seen. + if (ctx._source.id != null && details__symbol_was_noop && details__unit_was_noop && introduced_on_was_noop && name_was_noop && nested_fields__max_widget_cost_was_noop && oldest_widget_created_at_was_noop && primary_continent_was_noop && widget_fee_currencies_was_noop && widget_names2_was_noop && widget_options__colors_was_noop && widget_options__sizes_was_noop && widget_tags_was_noop) { + ctx.op = 'none'; + } else { + // Here we set `_source.id` because if we don't, it'll never be set, making these docs subtly + // different from docs indexed the normal way. + // + // Note also that we MUST use `params.id` instead of `ctx._id`. The latter works on an update + // of an existing document, but is unavailable when we are inserting the document for the first time. + ctx._source.id = params.id; + } + field_as_day_of_week_f2b5c7d9e8f75bf2457b52412bfb6537: + context: field + script: + lang: painless + source: |- + // Check if required params are missing + if (params.offset_ms == null) { + throw new IllegalArgumentException("Missing required parameter: offset_ms"); + } + if (params.time_zone == null) { + throw new IllegalArgumentException("Missing required parameter: time_zone"); + } + + // Set variables used in the loop + ZoneId zoneId = ZoneId.of(params.time_zone); + List results = new ArrayList(); + + for (ZonedDateTime timestamp : doc[params.field]) { + // Convert the timestamp to the specified time zone + ZonedDateTime zonedTimestamp = timestamp.withZoneSameInstant(zoneId); + + // Adjust the timestamp based on the offset_ms parameter + ZonedDateTime adjustedTimestamp = zonedTimestamp.plus(params.offset_ms, ChronoUnit.MILLIS); + + // Format and add the result to the list + results.add(adjustedTimestamp.getDayOfWeek().name()); + } + + return results; + field_as_time_of_day_ed82aba44fc66bff5635bec4305c1c66: + context: field + script: + lang: painless + source: |- + // Check if required params are missing + if (params.offset_ms == null) { + throw new IllegalArgumentException("Missing required parameter: offset_ms"); + } + if (params.time_zone == null) { + throw new IllegalArgumentException("Missing required parameter: time_zone"); + } + if (params.interval == null) { + throw new IllegalArgumentException("Missing required parameter: interval"); + } + + // Set variables used in the loop + ZoneId zoneId = ZoneId.of(params.time_zone); + ChronoUnit intervalUnit; + if (params.interval == "hour") { + intervalUnit = ChronoUnit.HOURS; + } else if (params.interval == "minute") { + intervalUnit = ChronoUnit.MINUTES; + } else if (params.interval == "second") { + intervalUnit = ChronoUnit.SECONDS; + } else { + throw new IllegalArgumentException("Invalid interval value: " + params.interval); + } + DateTimeFormatter formatter = DateTimeFormatter.ISO_LOCAL_TIME; + List results = new ArrayList(); + + for (ZonedDateTime timestamp : doc[params.field]) { + // Convert the timestamp to the specified time zone + ZonedDateTime zonedTimestamp = timestamp.withZoneSameInstant(zoneId); + + // Adjust the timestamp based on the offset_ms parameter + ZonedDateTime adjustedTimestamp = zonedTimestamp.plus(params.offset_ms, ChronoUnit.MILLIS); + + // Truncate the timestamp to the specified interval + adjustedTimestamp = adjustedTimestamp.truncatedTo(intervalUnit); + + // Format and add the result to the list + results.add(adjustedTimestamp.format(formatter)); + } + + return results; + filter_by_time_of_day_ea12d0561b24961789ab68ed38435612: + context: filter + script: + lang: painless + source: |- + ZoneId zoneId = ZoneId.of(params.time_zone); + + for (ZonedDateTime timestamp : doc[params.field]) { + long docValue = timestamp + .withZoneSameInstant(zoneId) + .toLocalTime() + .toNanoOfDay(); + + // Perform comparisons based on whichever params are set. + // ElasticGraph takes care of passing us param values as nano-of-day so that we + // can directly and efficiently compare against `docValue`. + if ((params.gte == null || docValue >= params.gte) && + (params.gt == null || docValue > params.gt) && + (params.lte == null || docValue <= params.lte) && + (params.lt == null || docValue < params.lt) && + (params.equal_to_any_of == null || params.equal_to_any_of.contains(docValue))) { + return true; + } + } + + // No timestamp values matched the params, so return `false`. + return false; + update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413: + context: update + script: + lang: painless + source: |- + Map source = ctx._source; + String sourceId = params.sourceId; + String relationship = params.relationship; + + // Numbers in JSON appear to be parsed as doubles, but we want the version stored as a long, so we need to cast it here. + long eventVersion = (long) params.version; + + if (source.__sources == null) { + source.__sources = []; + } + + if (source.__versions == null) { + source.__versions = [:]; + } + + if (source.__versions[relationship] == null) { + source.__versions[relationship] = [:]; + } + + Map relationshipVersionsMap = source.__versions.get(relationship); + List previousSourceIdsForRelationship = relationshipVersionsMap.keySet().stream().filter(id -> id != sourceId).collect(Collectors.toList()); + + if (previousSourceIdsForRelationship.size() > 0) { + String previousIdDescription = previousSourceIdsForRelationship.size() == 1 ? previousSourceIdsForRelationship.get(0) : previousSourceIdsForRelationship.toString(); + throw new IllegalArgumentException( + "Cannot update document " + params.id + " " + + "with data from related " + relationship + " " + sourceId + " " + + "because the related " + relationship + " has apparently changed (was: " + previousSourceIdsForRelationship + "), " + + "but mutations of relationships used with `sourced_from` are not supported because " + + "allowing it could break ElasticGraph's out-of-order processing guarantees." + ); + } + + // While the version in `__versions` is going to be used for the doc version in the future, for now + // we need to continue getting it from `__sourceVersions`. Both our old version and this versions of this + // script keep the value in `__sourceVersions` up-to-date, whereas the old script only writes it to + // `__sourceVersions`. Until we have completely migrated off of the old script for all ElasticGraph + // clusters, we need to keep using it. + // + // Later, after the old script is no longer used by any clusters, we'll stop using `__sourceVersions`. + // TODO: switch to `__versions` when we no longer need to maintain compatibility with the old version of the script. + Number _versionForSourceType = source.get("__sourceVersions")?.get(params.sourceType)?.get(sourceId); + Number _versionForRelationship = relationshipVersionsMap.get(sourceId); + + // Our JSON schema requires event versions to be non-negative, so we can safely use Long.MIN_VALUE as a stand-in when the value is null. + long versionForSourceType = _versionForSourceType == null ? Long.MIN_VALUE : _versionForSourceType.longValue(); + long versionForRelationship = _versionForRelationship == null ? Long.MIN_VALUE : _versionForRelationship.longValue(); + + // Pick the larger of the two versions as our doc version. Note that `Math.max` didn't work for me here for + // reasons I don't understand, but a simple ternary works fine. + // + // In theory, we could just use `versionForSourceType` as the `docVersion` (and not even check `__versions` at all) + // since both the old version and this version maintain the doc version in `__sourceVersions`. However, that would + // prevent this version of the script from being forward-compatible with the planned next version of this script. + // In the next version, we plan to stop writing to `__sourceVersions`, and as we can't deploy that change atomically, + // this version of the script will continue to run after that has begun to be used. So this version of the script + // must consider which version is greater here, and not simply trust either version value. + long docVersion = versionForSourceType > versionForRelationship ? versionForSourceType : versionForRelationship; + + if (docVersion >= eventVersion) { + throw new IllegalArgumentException("ElasticGraph update was a no-op: [" + + params.id + "]: version conflict, current version [" + + docVersion + "] is higher or equal to the one provided [" + + eventVersion + "]"); + } else { + source.putAll(params.data); + Map __counts = params.__counts; + + if (__counts != null) { + if (source.__counts == null) { + source.__counts = [:]; + } + + source.__counts.putAll(__counts); + } + + source.id = params.id; + source.__versions[relationship][sourceId] = eventVersion; + + // Record the relationship in `__sources` if it's not already there. We maintain it as an append-only set using a sorted list. + // This ensures deterministic ordering of its elements regardless of event ingestion order, and lets us check membership in O(log N) time. + // + // As per https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/Collections.html#binarySearch(java.util.List,java.lang.Object): + // + // > Returns the index of the search key, if it is contained in the list; otherwise, (-(insertion point) - 1). + // > The insertion point is defined as the point at which the key would be inserted into the list: the index + // > of the first element greater than the key, or list.size() if all elements in the list are less than the + // > specified key. Note that this guarantees that the return value will be >= 0 if and only if the key is found. + int sourceBinarySearchResult = Collections.binarySearch(source.__sources, relationship); + if (sourceBinarySearchResult < 0) { + source.__sources.add(-sourceBinarySearchResult - 1, relationship); + } + } diff --git a/config/schema/artifacts_with_apollo/json_schemas.yaml b/config/schema/artifacts_with_apollo/json_schemas.yaml new file mode 100644 index 00000000..b5dbc256 --- /dev/null +++ b/config/schema/artifacts_with_apollo/json_schemas.yaml @@ -0,0 +1,985 @@ +# This is the "public" JSON schema file and is intended to be provided to publishers so that +# they can perform code generation and event validation. +# +# Generated by `rake schema_artifacts:dump`. +# DO NOT EDIT BY HAND. Any edits will be lost the next time the rake task is run. +--- +"$schema": http://json-schema.org/draft-07/schema# +json_schema_version: 1 +"$defs": + ElasticGraphEventEnvelope: + type: object + properties: + op: + type: string + enum: + - upsert + type: + type: string + enum: + - Address + - Component + - ElectricalPart + - Manufacturer + - MechanicalPart + - Sponsor + - Team + - Widget + - WidgetWorkspace + id: + type: string + maxLength: 8191 + version: + type: integer + minimum: 0 + maximum: 9223372036854775807 + record: + type: object + latency_timestamps: + type: object + additionalProperties: false + patternProperties: + "^\\w+_at$": + type: string + format: date-time + json_schema_version: + const: 1 + message_id: + type: string + description: The optional ID of the message containing this event from whatever + messaging system is being used between the publisher and the ElasticGraph + indexer. + additionalProperties: false + required: + - op + - type + - id + - version + - json_schema_version + if: + properties: + op: + const: upsert + then: + required: + - record + Address: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + full_address: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + timestamps: + anyOf: + - "$ref": "#/$defs/AddressTimestamps" + - type: 'null' + geo_location: + anyOf: + - "$ref": "#/$defs/GeoLocation" + - type: 'null' + shapes: + type: array + items: + "$ref": "#/$defs/GeoShape" + manufacturer_id: + anyOf: + - allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + - type: 'null' + __typename: + type: string + const: Address + default: Address + required: + - id + - full_address + - timestamps + - geo_location + - shapes + - manufacturer_id + AddressTimestamps: + type: object + properties: + created_at: + anyOf: + - "$ref": "#/$defs/DateTime" + - type: 'null' + __typename: + type: string + const: AddressTimestamps + default: AddressTimestamps + required: + - created_at + Affiliations: + type: object + properties: + sponsorships_nested: + type: array + items: + "$ref": "#/$defs/Sponsorship" + sponsorships_object: + type: array + items: + "$ref": "#/$defs/Sponsorship" + __typename: + type: string + const: Affiliations + default: Affiliations + required: + - sponsorships_nested + - sponsorships_object + Color: + type: string + enum: + - RED + - BLUE + - GREEN + Company: + type: object + properties: + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + stock_ticker: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + __typename: + type: string + const: Company + default: Company + required: + - name + - stock_ticker + Component: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + created_at: + "$ref": "#/$defs/DateTime" + position: + "$ref": "#/$defs/Position" + tags: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + part_ids: + type: array + items: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + __typename: + type: string + const: Component + default: Component + required: + - id + - name + - created_at + - position + - tags + - part_ids + Date: + type: string + format: date + DateTime: + type: string + format: date-time + ElectricalPart: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + created_at: + "$ref": "#/$defs/DateTime" + voltage: + "$ref": "#/$defs/Int" + manufacturer_id: + anyOf: + - allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + - type: 'null' + __typename: + type: string + const: ElectricalPart + default: ElectricalPart + required: + - id + - name + - created_at + - voltage + - manufacturer_id + Float: + type: number + GeoLocation: + type: object + properties: + latitude: + allOf: + - "$ref": "#/$defs/Float" + - minimum: -90 + maximum: 90 + longitude: + allOf: + - "$ref": "#/$defs/Float" + - minimum: -180 + maximum: 180 + __typename: + type: string + const: GeoLocation + default: GeoLocation + required: + - latitude + - longitude + GeoShape: + type: object + properties: + type: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + coordinates: + type: array + items: + "$ref": "#/$defs/Float" + __typename: + type: string + const: GeoShape + default: GeoShape + required: + - type + - coordinates + ID: + type: string + Int: + type: integer + minimum: -2147483648 + maximum: 2147483647 + Inventor: + required: + - __typename + oneOf: + - "$ref": "#/$defs/Person" + - "$ref": "#/$defs/Company" + JsonSafeLong: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + LocalTime: + type: string + pattern: "^(([0-1][0-9])|(2[0-3])):[0-5][0-9]:[0-5][0-9](\\.[0-9]{1,3})?$" + LongString: + type: integer + minimum: -9223372036854775808 + maximum: 9223372036854775807 + Manufacturer: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + created_at: + "$ref": "#/$defs/DateTime" + __typename: + type: string + const: Manufacturer + default: Manufacturer + required: + - id + - name + - created_at + Material: + type: string + enum: + - ALLOY + - CARBON_FIBER + MechanicalPart: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + created_at: + "$ref": "#/$defs/DateTime" + material: + anyOf: + - "$ref": "#/$defs/Material" + - type: 'null' + manufacturer_id: + anyOf: + - allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + - type: 'null' + __typename: + type: string + const: MechanicalPart + default: MechanicalPart + required: + - id + - name + - created_at + - material + - manufacturer_id + Money: + type: object + properties: + currency: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + amount_cents: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + __typename: + type: string + const: Money + default: Money + required: + - currency + - amount_cents + NamedInventor: + required: + - __typename + oneOf: + - "$ref": "#/$defs/Person" + - "$ref": "#/$defs/Company" + Person: + type: object + properties: + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + nationality: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + __typename: + type: string + const: Person + default: Person + required: + - name + - nationality + Player: + type: object + properties: + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + nicknames: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + affiliations: + "$ref": "#/$defs/Affiliations" + seasons_nested: + type: array + items: + "$ref": "#/$defs/PlayerSeason" + seasons_object: + type: array + items: + "$ref": "#/$defs/PlayerSeason" + __typename: + type: string + const: Player + default: Player + required: + - name + - nicknames + - affiliations + - seasons_nested + - seasons_object + PlayerSeason: + type: object + properties: + year: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + games_played: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + awards: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + __typename: + type: string + const: PlayerSeason + default: PlayerSeason + required: + - year + - games_played + - awards + Position: + type: object + properties: + x: + "$ref": "#/$defs/Float" + "y": + "$ref": "#/$defs/Float" + __typename: + type: string + const: Position + default: Position + required: + - x + - "y" + Size: + type: string + enum: + - SMALL + - MEDIUM + - LARGE + Sponsor: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + __typename: + type: string + const: Sponsor + default: Sponsor + required: + - id + - name + Sponsorship: + type: object + properties: + sponsor_id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + annual_total: + "$ref": "#/$defs/Money" + __typename: + type: string + const: Sponsorship + default: Sponsorship + required: + - sponsor_id + - annual_total + String: + type: string + Team: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + league: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + pattern: "[^ \t\n]+" + country_code: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + formed_on: + "$ref": "#/$defs/Date" + current_name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + past_names: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + won_championships_at: + type: array + items: + "$ref": "#/$defs/DateTime" + details: + anyOf: + - "$ref": "#/$defs/TeamDetails" + - type: 'null' + stadium_location: + anyOf: + - "$ref": "#/$defs/GeoLocation" + - type: 'null' + forbes_valuations: + type: array + items: + "$ref": "#/$defs/JsonSafeLong" + forbes_valuation_moneys_nested: + type: array + items: + "$ref": "#/$defs/Money" + forbes_valuation_moneys_object: + type: array + items: + "$ref": "#/$defs/Money" + current_players_nested: + type: array + items: + "$ref": "#/$defs/Player" + current_players_object: + type: array + items: + "$ref": "#/$defs/Player" + seasons_nested: + type: array + items: + "$ref": "#/$defs/TeamSeason" + seasons_object: + type: array + items: + "$ref": "#/$defs/TeamSeason" + nested_fields: + anyOf: + - "$ref": "#/$defs/TeamNestedFields" + - type: 'null' + nested_fields2: + anyOf: + - "$ref": "#/$defs/TeamNestedFields" + - type: 'null' + __typename: + type: string + const: Team + default: Team + required: + - id + - league + - country_code + - formed_on + - current_name + - past_names + - won_championships_at + - details + - stadium_location + - forbes_valuations + - forbes_valuation_moneys_nested + - forbes_valuation_moneys_object + - current_players_nested + - current_players_object + - seasons_nested + - seasons_object + - nested_fields + - nested_fields2 + TeamDetails: + type: object + properties: + uniform_colors: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + count: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + __typename: + type: string + const: TeamDetails + default: TeamDetails + required: + - uniform_colors + - count + TeamNestedFields: + type: object + properties: + forbes_valuation_moneys: + type: array + items: + "$ref": "#/$defs/Money" + current_players: + type: array + items: + "$ref": "#/$defs/Player" + seasons: + type: array + items: + "$ref": "#/$defs/TeamSeason" + __typename: + type: string + const: TeamNestedFields + default: TeamNestedFields + required: + - forbes_valuation_moneys + - current_players + - seasons + TeamRecord: + type: object + properties: + wins: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + losses: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + last_win_on: + anyOf: + - "$ref": "#/$defs/Date" + - type: 'null' + first_win_on: + anyOf: + - "$ref": "#/$defs/Date" + - type: 'null' + __typename: + type: string + const: TeamRecord + default: TeamRecord + required: + - wins + - losses + - last_win_on + - first_win_on + TeamSeason: + type: object + properties: + record: + anyOf: + - "$ref": "#/$defs/TeamRecord" + - type: 'null' + year: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + notes: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + count: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + started_at: + anyOf: + - "$ref": "#/$defs/DateTime" + - type: 'null' + won_games_at: + type: array + items: + "$ref": "#/$defs/DateTime" + players_nested: + type: array + items: + "$ref": "#/$defs/Player" + players_object: + type: array + items: + "$ref": "#/$defs/Player" + __typename: + type: string + const: TeamSeason + default: TeamSeason + required: + - record + - year + - notes + - count + - started_at + - won_games_at + - players_nested + - players_object + Untyped: + type: + - array + - boolean + - integer + - number + - object + - string + Widget: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + workspace_id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + pattern: "[^ \t\n]+" + amount_cents: + "$ref": "#/$defs/Int" + cost: + anyOf: + - "$ref": "#/$defs/Money" + - type: 'null' + cost_currency_unit: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + cost_currency_name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + cost_currency_symbol: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + cost_currency_primary_continent: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + cost_currency_introduced_on: + anyOf: + - "$ref": "#/$defs/Date" + - type: 'null' + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + name_text: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 104857600 + - type: 'null' + created_at: + "$ref": "#/$defs/DateTime" + created_at_time_of_day: + anyOf: + - "$ref": "#/$defs/LocalTime" + - type: 'null' + created_on: + anyOf: + - "$ref": "#/$defs/Date" + - type: 'null' + release_timestamps: + type: array + items: + "$ref": "#/$defs/DateTime" + release_dates: + type: array + items: + "$ref": "#/$defs/Date" + component_ids: + type: array + items: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + options: + anyOf: + - "$ref": "#/$defs/WidgetOptions" + - type: 'null' + the_options: + anyOf: + - "$ref": "#/$defs/WidgetOptions" + - type: 'null' + inventor: + anyOf: + - "$ref": "#/$defs/Inventor" + - type: 'null' + named_inventor: + anyOf: + - "$ref": "#/$defs/NamedInventor" + - type: 'null' + weight_in_ng_str: + "$ref": "#/$defs/LongString" + weight_in_ng: + "$ref": "#/$defs/JsonSafeLong" + tags: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + amounts: + type: array + items: + "$ref": "#/$defs/Int" + fees: + type: array + items: + "$ref": "#/$defs/Money" + metadata: + anyOf: + - allOf: + - "$ref": "#/$defs/Untyped" + - maxLength: 8191 + - type: 'null' + __typename: + type: string + const: Widget + default: Widget + required: + - id + - workspace_id + - amount_cents + - cost + - cost_currency_unit + - cost_currency_name + - cost_currency_symbol + - cost_currency_primary_continent + - cost_currency_introduced_on + - name + - name_text + - created_at + - created_at_time_of_day + - created_on + - release_timestamps + - release_dates + - component_ids + - options + - the_options + - inventor + - named_inventor + - weight_in_ng_str + - weight_in_ng + - tags + - amounts + - fees + - metadata + WidgetOptions: + type: object + properties: + size: + anyOf: + - "$ref": "#/$defs/Size" + - type: 'null' + the_size: + anyOf: + - "$ref": "#/$defs/Size" + - type: 'null' + color: + anyOf: + - "$ref": "#/$defs/Color" + - type: 'null' + __typename: + type: string + const: WidgetOptions + default: WidgetOptions + required: + - size + - the_size + - color + WidgetWorkspace: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + widget: + anyOf: + - "$ref": "#/$defs/WorkspaceWidget" + - type: 'null' + __typename: + type: string + const: WidgetWorkspace + default: WidgetWorkspace + required: + - id + - name + - widget + WorkspaceWidget: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + created_at: + anyOf: + - "$ref": "#/$defs/DateTime" + - type: 'null' + __typename: + type: string + const: WorkspaceWidget + default: WorkspaceWidget + required: + - id + - created_at diff --git a/config/schema/artifacts_with_apollo/json_schemas_by_version/v1.yaml b/config/schema/artifacts_with_apollo/json_schemas_by_version/v1.yaml new file mode 100644 index 00000000..a836e325 --- /dev/null +++ b/config/schema/artifacts_with_apollo/json_schemas_by_version/v1.yaml @@ -0,0 +1,1351 @@ +# This JSON schema file contains internal ElasticGraph metadata and should be considered private. +# The unversioned JSON schema file is public and intended to be provided to publishers. +# +# Generated by `rake schema_artifacts:dump`. +# DO NOT EDIT BY HAND. Any edits will be lost the next time the rake task is run. +--- +"$schema": http://json-schema.org/draft-07/schema# +json_schema_version: 1 +"$defs": + ElasticGraphEventEnvelope: + type: object + properties: + op: + type: string + enum: + - upsert + type: + type: string + enum: + - Address + - Component + - ElectricalPart + - Manufacturer + - MechanicalPart + - Sponsor + - Team + - Widget + - WidgetWorkspace + id: + type: string + maxLength: 8191 + version: + type: integer + minimum: 0 + maximum: 9223372036854775807 + record: + type: object + latency_timestamps: + type: object + additionalProperties: false + patternProperties: + "^\\w+_at$": + type: string + format: date-time + json_schema_version: + const: 1 + message_id: + type: string + description: The optional ID of the message containing this event from whatever + messaging system is being used between the publisher and the ElasticGraph + indexer. + additionalProperties: false + required: + - op + - type + - id + - version + - json_schema_version + if: + properties: + op: + const: upsert + then: + required: + - record + Address: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + full_address: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: String! + nameInIndex: full_address + timestamps: + anyOf: + - "$ref": "#/$defs/AddressTimestamps" + - type: 'null' + ElasticGraph: + type: AddressTimestamps + nameInIndex: timestamps + geo_location: + anyOf: + - "$ref": "#/$defs/GeoLocation" + - type: 'null' + ElasticGraph: + type: GeoLocation + nameInIndex: geo_location + shapes: + type: array + items: + "$ref": "#/$defs/GeoShape" + ElasticGraph: + type: "[GeoShape!]!" + nameInIndex: shapes + manufacturer_id: + anyOf: + - allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: ID + nameInIndex: manufacturer_id + __typename: + type: string + const: Address + default: Address + required: + - id + - full_address + - timestamps + - geo_location + - shapes + - manufacturer_id + AddressTimestamps: + type: object + properties: + created_at: + anyOf: + - "$ref": "#/$defs/DateTime" + - type: 'null' + ElasticGraph: + type: DateTime + nameInIndex: created_at + __typename: + type: string + const: AddressTimestamps + default: AddressTimestamps + required: + - created_at + Affiliations: + type: object + properties: + sponsorships_nested: + type: array + items: + "$ref": "#/$defs/Sponsorship" + ElasticGraph: + type: "[Sponsorship!]!" + nameInIndex: sponsorships_nested + sponsorships_object: + type: array + items: + "$ref": "#/$defs/Sponsorship" + ElasticGraph: + type: "[Sponsorship!]!" + nameInIndex: sponsorships_object + __typename: + type: string + const: Affiliations + default: Affiliations + required: + - sponsorships_nested + - sponsorships_object + Color: + type: string + enum: + - RED + - BLUE + - GREEN + Company: + type: object + properties: + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + stock_ticker: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: stock_ticker + __typename: + type: string + const: Company + default: Company + required: + - name + - stock_ticker + Component: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + created_at: + "$ref": "#/$defs/DateTime" + ElasticGraph: + type: DateTime! + nameInIndex: created_at + position: + "$ref": "#/$defs/Position" + ElasticGraph: + type: Position! + nameInIndex: position + tags: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: "[String!]!" + nameInIndex: tags + part_ids: + type: array + items: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: "[ID!]!" + nameInIndex: part_ids + __typename: + type: string + const: Component + default: Component + required: + - id + - name + - created_at + - position + - tags + - part_ids + Date: + type: string + format: date + DateTime: + type: string + format: date-time + ElectricalPart: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + created_at: + "$ref": "#/$defs/DateTime" + ElasticGraph: + type: DateTime! + nameInIndex: created_at + voltage: + "$ref": "#/$defs/Int" + ElasticGraph: + type: Int! + nameInIndex: voltage + manufacturer_id: + anyOf: + - allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: ID + nameInIndex: manufacturer_id + __typename: + type: string + const: ElectricalPart + default: ElectricalPart + required: + - id + - name + - created_at + - voltage + - manufacturer_id + Float: + type: number + GeoLocation: + type: object + properties: + latitude: + allOf: + - "$ref": "#/$defs/Float" + - minimum: -90 + maximum: 90 + ElasticGraph: + type: Float! + nameInIndex: lat + longitude: + allOf: + - "$ref": "#/$defs/Float" + - minimum: -180 + maximum: 180 + ElasticGraph: + type: Float! + nameInIndex: lon + __typename: + type: string + const: GeoLocation + default: GeoLocation + required: + - latitude + - longitude + GeoShape: + type: object + properties: + type: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: type + coordinates: + type: array + items: + "$ref": "#/$defs/Float" + ElasticGraph: + type: "[Float!]!" + nameInIndex: coordinates + __typename: + type: string + const: GeoShape + default: GeoShape + required: + - type + - coordinates + ID: + type: string + Int: + type: integer + minimum: -2147483648 + maximum: 2147483647 + Inventor: + required: + - __typename + oneOf: + - "$ref": "#/$defs/Person" + - "$ref": "#/$defs/Company" + JsonSafeLong: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + LocalTime: + type: string + pattern: "^(([0-1][0-9])|(2[0-3])):[0-5][0-9]:[0-5][0-9](\\.[0-9]{1,3})?$" + LongString: + type: integer + minimum: -9223372036854775808 + maximum: 9223372036854775807 + Manufacturer: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + created_at: + "$ref": "#/$defs/DateTime" + ElasticGraph: + type: DateTime! + nameInIndex: created_at + __typename: + type: string + const: Manufacturer + default: Manufacturer + required: + - id + - name + - created_at + Material: + type: string + enum: + - ALLOY + - CARBON_FIBER + MechanicalPart: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + created_at: + "$ref": "#/$defs/DateTime" + ElasticGraph: + type: DateTime! + nameInIndex: created_at + material: + anyOf: + - "$ref": "#/$defs/Material" + - type: 'null' + ElasticGraph: + type: Material + nameInIndex: material + manufacturer_id: + anyOf: + - allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: ID + nameInIndex: manufacturer_id + __typename: + type: string + const: MechanicalPart + default: MechanicalPart + required: + - id + - name + - created_at + - material + - manufacturer_id + Money: + type: object + properties: + currency: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: String! + nameInIndex: currency + amount_cents: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + ElasticGraph: + type: Int + nameInIndex: amount_cents + __typename: + type: string + const: Money + default: Money + required: + - currency + - amount_cents + NamedInventor: + required: + - __typename + oneOf: + - "$ref": "#/$defs/Person" + - "$ref": "#/$defs/Company" + Person: + type: object + properties: + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + nationality: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: nationality + __typename: + type: string + const: Person + default: Person + required: + - name + - nationality + Player: + type: object + properties: + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + nicknames: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: "[String!]!" + nameInIndex: nicknames + affiliations: + "$ref": "#/$defs/Affiliations" + ElasticGraph: + type: Affiliations! + nameInIndex: affiliations + seasons_nested: + type: array + items: + "$ref": "#/$defs/PlayerSeason" + ElasticGraph: + type: "[PlayerSeason!]!" + nameInIndex: seasons_nested + seasons_object: + type: array + items: + "$ref": "#/$defs/PlayerSeason" + ElasticGraph: + type: "[PlayerSeason!]!" + nameInIndex: seasons_object + __typename: + type: string + const: Player + default: Player + required: + - name + - nicknames + - affiliations + - seasons_nested + - seasons_object + PlayerSeason: + type: object + properties: + year: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + ElasticGraph: + type: Int + nameInIndex: year + games_played: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + ElasticGraph: + type: Int + nameInIndex: games_played + awards: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: "[String!]!" + nameInIndex: awards + __typename: + type: string + const: PlayerSeason + default: PlayerSeason + required: + - year + - games_played + - awards + Position: + type: object + properties: + x: + "$ref": "#/$defs/Float" + ElasticGraph: + type: Float! + nameInIndex: x + "y": + "$ref": "#/$defs/Float" + ElasticGraph: + type: Float! + nameInIndex: "y" + __typename: + type: string + const: Position + default: Position + required: + - x + - "y" + Size: + type: string + enum: + - SMALL + - MEDIUM + - LARGE + Sponsor: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + __typename: + type: string + const: Sponsor + default: Sponsor + required: + - id + - name + Sponsorship: + type: object + properties: + sponsor_id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: sponsor_id + annual_total: + "$ref": "#/$defs/Money" + ElasticGraph: + type: Money! + nameInIndex: annual_total + __typename: + type: string + const: Sponsorship + default: Sponsorship + required: + - sponsor_id + - annual_total + String: + type: string + Team: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + league: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + pattern: "[^ \t\n]+" + ElasticGraph: + type: String! + nameInIndex: league + country_code: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: country_code + formed_on: + "$ref": "#/$defs/Date" + ElasticGraph: + type: Date! + nameInIndex: formed_on + current_name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: current_name + past_names: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: "[String!]!" + nameInIndex: past_names + won_championships_at: + type: array + items: + "$ref": "#/$defs/DateTime" + ElasticGraph: + type: "[DateTime!]!" + nameInIndex: won_championships_at + details: + anyOf: + - "$ref": "#/$defs/TeamDetails" + - type: 'null' + ElasticGraph: + type: TeamDetails + nameInIndex: details + stadium_location: + anyOf: + - "$ref": "#/$defs/GeoLocation" + - type: 'null' + ElasticGraph: + type: GeoLocation + nameInIndex: stadium_location + forbes_valuations: + type: array + items: + "$ref": "#/$defs/JsonSafeLong" + ElasticGraph: + type: "[JsonSafeLong!]!" + nameInIndex: forbes_valuations + forbes_valuation_moneys_nested: + type: array + items: + "$ref": "#/$defs/Money" + ElasticGraph: + type: "[Money!]!" + nameInIndex: forbes_valuation_moneys_nested + forbes_valuation_moneys_object: + type: array + items: + "$ref": "#/$defs/Money" + ElasticGraph: + type: "[Money!]!" + nameInIndex: forbes_valuation_moneys_object + current_players_nested: + type: array + items: + "$ref": "#/$defs/Player" + ElasticGraph: + type: "[Player!]!" + nameInIndex: current_players_nested + current_players_object: + type: array + items: + "$ref": "#/$defs/Player" + ElasticGraph: + type: "[Player!]!" + nameInIndex: current_players_object + seasons_nested: + type: array + items: + "$ref": "#/$defs/TeamSeason" + ElasticGraph: + type: "[TeamSeason!]!" + nameInIndex: seasons_nested + seasons_object: + type: array + items: + "$ref": "#/$defs/TeamSeason" + ElasticGraph: + type: "[TeamSeason!]!" + nameInIndex: seasons_object + nested_fields: + anyOf: + - "$ref": "#/$defs/TeamNestedFields" + - type: 'null' + ElasticGraph: + type: TeamNestedFields + nameInIndex: the_nested_fields + nested_fields2: + anyOf: + - "$ref": "#/$defs/TeamNestedFields" + - type: 'null' + ElasticGraph: + type: TeamNestedFields + nameInIndex: nested_fields2 + __typename: + type: string + const: Team + default: Team + required: + - id + - league + - country_code + - formed_on + - current_name + - past_names + - won_championships_at + - details + - stadium_location + - forbes_valuations + - forbes_valuation_moneys_nested + - forbes_valuation_moneys_object + - current_players_nested + - current_players_object + - seasons_nested + - seasons_object + - nested_fields + - nested_fields2 + TeamDetails: + type: object + properties: + uniform_colors: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: "[String!]!" + nameInIndex: uniform_colors + count: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + ElasticGraph: + type: Int + nameInIndex: count + __typename: + type: string + const: TeamDetails + default: TeamDetails + required: + - uniform_colors + - count + TeamNestedFields: + type: object + properties: + forbes_valuation_moneys: + type: array + items: + "$ref": "#/$defs/Money" + ElasticGraph: + type: "[Money!]!" + nameInIndex: forbes_valuation_moneys + current_players: + type: array + items: + "$ref": "#/$defs/Player" + ElasticGraph: + type: "[Player!]!" + nameInIndex: current_players + seasons: + type: array + items: + "$ref": "#/$defs/TeamSeason" + ElasticGraph: + type: "[TeamSeason!]!" + nameInIndex: the_seasons + __typename: + type: string + const: TeamNestedFields + default: TeamNestedFields + required: + - forbes_valuation_moneys + - current_players + - seasons + TeamRecord: + type: object + properties: + wins: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + ElasticGraph: + type: Int + nameInIndex: win_count + losses: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + ElasticGraph: + type: Int + nameInIndex: loss_count + last_win_on: + anyOf: + - "$ref": "#/$defs/Date" + - type: 'null' + ElasticGraph: + type: Date + nameInIndex: last_win_date + first_win_on: + anyOf: + - "$ref": "#/$defs/Date" + - type: 'null' + ElasticGraph: + type: Date + nameInIndex: first_win_on + __typename: + type: string + const: TeamRecord + default: TeamRecord + required: + - wins + - losses + - last_win_on + - first_win_on + TeamSeason: + type: object + properties: + record: + anyOf: + - "$ref": "#/$defs/TeamRecord" + - type: 'null' + ElasticGraph: + type: TeamRecord + nameInIndex: the_record + year: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + ElasticGraph: + type: Int + nameInIndex: year + notes: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: "[String!]!" + nameInIndex: notes + count: + anyOf: + - "$ref": "#/$defs/Int" + - type: 'null' + ElasticGraph: + type: Int + nameInIndex: count + started_at: + anyOf: + - "$ref": "#/$defs/DateTime" + - type: 'null' + ElasticGraph: + type: DateTime + nameInIndex: started_at + won_games_at: + type: array + items: + "$ref": "#/$defs/DateTime" + ElasticGraph: + type: "[DateTime!]!" + nameInIndex: won_games_at + players_nested: + type: array + items: + "$ref": "#/$defs/Player" + ElasticGraph: + type: "[Player!]!" + nameInIndex: players_nested + players_object: + type: array + items: + "$ref": "#/$defs/Player" + ElasticGraph: + type: "[Player!]!" + nameInIndex: players_object + __typename: + type: string + const: TeamSeason + default: TeamSeason + required: + - record + - year + - notes + - count + - started_at + - won_games_at + - players_nested + - players_object + Untyped: + type: + - array + - boolean + - integer + - number + - object + - string + Widget: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + workspace_id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + pattern: "[^ \t\n]+" + ElasticGraph: + type: ID! + nameInIndex: workspace_id2 + amount_cents: + "$ref": "#/$defs/Int" + ElasticGraph: + type: Int! + nameInIndex: amount_cents + cost: + anyOf: + - "$ref": "#/$defs/Money" + - type: 'null' + ElasticGraph: + type: Money + nameInIndex: cost + cost_currency_unit: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: cost_currency_unit + cost_currency_name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: cost_currency_name + cost_currency_symbol: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: cost_currency_symbol + cost_currency_primary_continent: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: cost_currency_primary_continent + cost_currency_introduced_on: + anyOf: + - "$ref": "#/$defs/Date" + - type: 'null' + ElasticGraph: + type: Date + nameInIndex: cost_currency_introduced_on + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + name_text: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 104857600 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name_text + created_at: + "$ref": "#/$defs/DateTime" + ElasticGraph: + type: DateTime! + nameInIndex: created_at + created_at_time_of_day: + anyOf: + - "$ref": "#/$defs/LocalTime" + - type: 'null' + ElasticGraph: + type: LocalTime + nameInIndex: created_at_time_of_day + created_on: + anyOf: + - "$ref": "#/$defs/Date" + - type: 'null' + ElasticGraph: + type: Date + nameInIndex: created_on + release_timestamps: + type: array + items: + "$ref": "#/$defs/DateTime" + ElasticGraph: + type: "[DateTime!]!" + nameInIndex: release_timestamps + release_dates: + type: array + items: + "$ref": "#/$defs/Date" + ElasticGraph: + type: "[Date!]!" + nameInIndex: release_dates + component_ids: + type: array + items: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: "[ID!]!" + nameInIndex: component_ids + options: + anyOf: + - "$ref": "#/$defs/WidgetOptions" + - type: 'null' + ElasticGraph: + type: WidgetOptions + nameInIndex: options + the_options: + anyOf: + - "$ref": "#/$defs/WidgetOptions" + - type: 'null' + ElasticGraph: + type: WidgetOptions + nameInIndex: the_opts + inventor: + anyOf: + - "$ref": "#/$defs/Inventor" + - type: 'null' + ElasticGraph: + type: Inventor + nameInIndex: inventor + named_inventor: + anyOf: + - "$ref": "#/$defs/NamedInventor" + - type: 'null' + ElasticGraph: + type: NamedInventor + nameInIndex: named_inventor + weight_in_ng_str: + "$ref": "#/$defs/LongString" + ElasticGraph: + type: LongString! + nameInIndex: weight_in_ng_str + weight_in_ng: + "$ref": "#/$defs/JsonSafeLong" + ElasticGraph: + type: JsonSafeLong! + nameInIndex: weight_in_ng + tags: + type: array + items: + allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + ElasticGraph: + type: "[String!]!" + nameInIndex: tags + amounts: + type: array + items: + "$ref": "#/$defs/Int" + ElasticGraph: + type: "[Int!]!" + nameInIndex: amounts + fees: + type: array + items: + "$ref": "#/$defs/Money" + ElasticGraph: + type: "[Money!]!" + nameInIndex: fees + metadata: + anyOf: + - allOf: + - "$ref": "#/$defs/Untyped" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: Untyped + nameInIndex: metadata + __typename: + type: string + const: Widget + default: Widget + required: + - id + - workspace_id + - amount_cents + - cost + - cost_currency_unit + - cost_currency_name + - cost_currency_symbol + - cost_currency_primary_continent + - cost_currency_introduced_on + - name + - name_text + - created_at + - created_at_time_of_day + - created_on + - release_timestamps + - release_dates + - component_ids + - options + - the_options + - inventor + - named_inventor + - weight_in_ng_str + - weight_in_ng + - tags + - amounts + - fees + - metadata + WidgetOptions: + type: object + properties: + size: + anyOf: + - "$ref": "#/$defs/Size" + - type: 'null' + ElasticGraph: + type: Size + nameInIndex: size + the_size: + anyOf: + - "$ref": "#/$defs/Size" + - type: 'null' + ElasticGraph: + type: Size + nameInIndex: the_sighs + color: + anyOf: + - "$ref": "#/$defs/Color" + - type: 'null' + ElasticGraph: + type: Color + nameInIndex: color + __typename: + type: string + const: WidgetOptions + default: WidgetOptions + required: + - size + - the_size + - color + WidgetWorkspace: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + name: + anyOf: + - allOf: + - "$ref": "#/$defs/String" + - maxLength: 8191 + - type: 'null' + ElasticGraph: + type: String + nameInIndex: name + widget: + anyOf: + - "$ref": "#/$defs/WorkspaceWidget" + - type: 'null' + ElasticGraph: + type: WorkspaceWidget + nameInIndex: widget + __typename: + type: string + const: WidgetWorkspace + default: WidgetWorkspace + required: + - id + - name + - widget + WorkspaceWidget: + type: object + properties: + id: + allOf: + - "$ref": "#/$defs/ID" + - maxLength: 8191 + ElasticGraph: + type: ID! + nameInIndex: id + created_at: + anyOf: + - "$ref": "#/$defs/DateTime" + - type: 'null' + ElasticGraph: + type: DateTime + nameInIndex: created_at + __typename: + type: string + const: WorkspaceWidget + default: WorkspaceWidget + required: + - id + - created_at diff --git a/config/schema/artifacts_with_apollo/runtime_metadata.yaml b/config/schema/artifacts_with_apollo/runtime_metadata.yaml new file mode 100644 index 00000000..1a1526cd --- /dev/null +++ b/config/schema/artifacts_with_apollo/runtime_metadata.yaml @@ -0,0 +1,4298 @@ +# Generated by `rake schema_artifacts:dump`. +# DO NOT EDIT BY HAND. Any edits will be lost the next time the rake task is run. +--- +enum_types_by_name: + AddressSortOrderInput: + values_by_name: + full_address_ASC: + sort_field: + direction: asc + field_path: full_address + full_address_DESC: + sort_field: + direction: desc + field_path: full_address + timestamps_created_at_ASC: + sort_field: + direction: asc + field_path: timestamps.created_at + timestamps_created_at_DESC: + sort_field: + direction: desc + field_path: timestamps.created_at + ComponentSortOrderInput: + values_by_name: + created_at_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_DESC: + sort_field: + direction: desc + field_path: created_at + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + position_x_ASC: + sort_field: + direction: asc + field_path: position.x + position_x_DESC: + sort_field: + direction: desc + field_path: position.x + position_y_ASC: + sort_field: + direction: asc + field_path: position.y + position_y_DESC: + sort_field: + direction: desc + field_path: position.y + widget_cost_amount_cents_ASC: + sort_field: + direction: asc + field_path: widget_cost.amount_cents + widget_cost_amount_cents_DESC: + sort_field: + direction: desc + field_path: widget_cost.amount_cents + widget_cost_currency_ASC: + sort_field: + direction: asc + field_path: widget_cost.currency + widget_cost_currency_DESC: + sort_field: + direction: desc + field_path: widget_cost.currency + widget_name_ASC: + sort_field: + direction: asc + field_path: widget_name + widget_name_DESC: + sort_field: + direction: desc + field_path: widget_name + widget_size_ASC: + sort_field: + direction: asc + field_path: widget_size + widget_size_DESC: + sort_field: + direction: desc + field_path: widget_size + widget_workspace_id_ASC: + sort_field: + direction: asc + field_path: widget_workspace_id3 + widget_workspace_id_DESC: + sort_field: + direction: desc + field_path: widget_workspace_id3 + DateGroupingGranularityInput: + values_by_name: + DAY: + datastore_value: day + MONTH: + datastore_value: month + QUARTER: + datastore_value: quarter + WEEK: + datastore_value: week + YEAR: + datastore_value: year + DateGroupingTruncationUnitInput: + values_by_name: + DAY: + datastore_value: day + MONTH: + datastore_value: month + QUARTER: + datastore_value: quarter + WEEK: + datastore_value: week + YEAR: + datastore_value: year + DateTimeGroupingGranularityInput: + values_by_name: + DAY: + datastore_value: day + HOUR: + datastore_value: hour + MINUTE: + datastore_value: minute + MONTH: + datastore_value: month + QUARTER: + datastore_value: quarter + SECOND: + datastore_value: second + WEEK: + datastore_value: week + YEAR: + datastore_value: year + DateTimeGroupingTruncationUnitInput: + values_by_name: + DAY: + datastore_value: day + HOUR: + datastore_value: hour + MINUTE: + datastore_value: minute + MONTH: + datastore_value: month + QUARTER: + datastore_value: quarter + SECOND: + datastore_value: second + WEEK: + datastore_value: week + YEAR: + datastore_value: year + DateTimeUnitInput: + values_by_name: + DAY: + datastore_abbreviation: d + datastore_value: 86400000 + HOUR: + datastore_abbreviation: h + datastore_value: 3600000 + MILLISECOND: + datastore_abbreviation: ms + datastore_value: 1 + MINUTE: + datastore_abbreviation: m + datastore_value: 60000 + SECOND: + datastore_abbreviation: s + datastore_value: 1000 + DateUnitInput: + values_by_name: + DAY: + datastore_abbreviation: d + datastore_value: 86400000 + DistanceUnitInput: + values_by_name: + CENTIMETER: + datastore_abbreviation: cm + FOOT: + datastore_abbreviation: ft + INCH: + datastore_abbreviation: in + KILOMETER: + datastore_abbreviation: km + METER: + datastore_abbreviation: m + MILE: + datastore_abbreviation: mi + MILLIMETER: + datastore_abbreviation: mm + NAUTICAL_MILE: + datastore_abbreviation: nmi + YARD: + datastore_abbreviation: yd + ElectricalPartSortOrderInput: + values_by_name: + created_at_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_DESC: + sort_field: + direction: desc + field_path: created_at + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + voltage_ASC: + sort_field: + direction: asc + field_path: voltage + voltage_DESC: + sort_field: + direction: desc + field_path: voltage + LocalTimeGroupingTruncationUnitInput: + values_by_name: + HOUR: + datastore_value: hour + MINUTE: + datastore_value: minute + SECOND: + datastore_value: second + LocalTimeUnitInput: + values_by_name: + HOUR: + datastore_abbreviation: h + datastore_value: 3600000 + MILLISECOND: + datastore_abbreviation: ms + datastore_value: 1 + MINUTE: + datastore_abbreviation: m + datastore_value: 60000 + SECOND: + datastore_abbreviation: s + datastore_value: 1000 + ManufacturerSortOrderInput: + values_by_name: + created_at_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_DESC: + sort_field: + direction: desc + field_path: created_at + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + MatchesQueryAllowedEditsPerTermInput: + values_by_name: + DYNAMIC: + datastore_abbreviation: AUTO + NONE: + datastore_abbreviation: '0' + ONE: + datastore_abbreviation: '1' + TWO: + datastore_abbreviation: '2' + MechanicalPartSortOrderInput: + values_by_name: + created_at_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_DESC: + sort_field: + direction: desc + field_path: created_at + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + material_ASC: + sort_field: + direction: asc + field_path: material + material_DESC: + sort_field: + direction: desc + field_path: material + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + NamedEntitySortOrderInput: + values_by_name: + amount_cents2_ASC: + sort_field: + direction: asc + field_path: amount_cents + amount_cents2_DESC: + sort_field: + direction: desc + field_path: amount_cents + amount_cents_ASC: + sort_field: + direction: asc + field_path: amount_cents + amount_cents_DESC: + sort_field: + direction: desc + field_path: amount_cents + cost_amount_cents_ASC: + sort_field: + direction: asc + field_path: cost.amount_cents + cost_amount_cents_DESC: + sort_field: + direction: desc + field_path: cost.amount_cents + cost_currency_ASC: + sort_field: + direction: asc + field_path: cost.currency + cost_currency_DESC: + sort_field: + direction: desc + field_path: cost.currency + cost_currency_introduced_on_ASC: + sort_field: + direction: asc + field_path: cost_currency_introduced_on + cost_currency_introduced_on_DESC: + sort_field: + direction: desc + field_path: cost_currency_introduced_on + cost_currency_name_ASC: + sort_field: + direction: asc + field_path: cost_currency_name + cost_currency_name_DESC: + sort_field: + direction: desc + field_path: cost_currency_name + cost_currency_primary_continent_ASC: + sort_field: + direction: asc + field_path: cost_currency_primary_continent + cost_currency_primary_continent_DESC: + sort_field: + direction: desc + field_path: cost_currency_primary_continent + cost_currency_symbol_ASC: + sort_field: + direction: asc + field_path: cost_currency_symbol + cost_currency_symbol_DESC: + sort_field: + direction: desc + field_path: cost_currency_symbol + cost_currency_unit_ASC: + sort_field: + direction: asc + field_path: cost_currency_unit + cost_currency_unit_DESC: + sort_field: + direction: desc + field_path: cost_currency_unit + created_at2_ASC: + sort_field: + direction: asc + field_path: created_at + created_at2_DESC: + sort_field: + direction: desc + field_path: created_at + created_at2_legacy_ASC: + sort_field: + direction: asc + field_path: created_at + created_at2_legacy_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_legacy_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_legacy_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_time_of_day_ASC: + sort_field: + direction: asc + field_path: created_at_time_of_day + created_at_time_of_day_DESC: + sort_field: + direction: desc + field_path: created_at_time_of_day + created_on_ASC: + sort_field: + direction: asc + field_path: created_on + created_on_DESC: + sort_field: + direction: desc + field_path: created_on + created_on_legacy_ASC: + sort_field: + direction: asc + field_path: created_on + created_on_legacy_DESC: + sort_field: + direction: desc + field_path: created_on + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + inventor_name_ASC: + sort_field: + direction: asc + field_path: inventor.name + inventor_name_DESC: + sort_field: + direction: desc + field_path: inventor.name + inventor_nationality_ASC: + sort_field: + direction: asc + field_path: inventor.nationality + inventor_nationality_DESC: + sort_field: + direction: desc + field_path: inventor.nationality + inventor_stock_ticker_ASC: + sort_field: + direction: asc + field_path: inventor.stock_ticker + inventor_stock_ticker_DESC: + sort_field: + direction: desc + field_path: inventor.stock_ticker + material_ASC: + sort_field: + direction: asc + field_path: material + material_DESC: + sort_field: + direction: desc + field_path: material + metadata_ASC: + sort_field: + direction: asc + field_path: metadata + metadata_DESC: + sort_field: + direction: desc + field_path: metadata + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + named_inventor_name_ASC: + sort_field: + direction: asc + field_path: named_inventor.name + named_inventor_name_DESC: + sort_field: + direction: desc + field_path: named_inventor.name + named_inventor_nationality_ASC: + sort_field: + direction: asc + field_path: named_inventor.nationality + named_inventor_nationality_DESC: + sort_field: + direction: desc + field_path: named_inventor.nationality + named_inventor_stock_ticker_ASC: + sort_field: + direction: asc + field_path: named_inventor.stock_ticker + named_inventor_stock_ticker_DESC: + sort_field: + direction: desc + field_path: named_inventor.stock_ticker + options_color_ASC: + sort_field: + direction: asc + field_path: options.color + options_color_DESC: + sort_field: + direction: desc + field_path: options.color + options_size_ASC: + sort_field: + direction: asc + field_path: options.size + options_size_DESC: + sort_field: + direction: desc + field_path: options.size + options_the_size_ASC: + sort_field: + direction: asc + field_path: options.the_sighs + options_the_size_DESC: + sort_field: + direction: desc + field_path: options.the_sighs + position_x_ASC: + sort_field: + direction: asc + field_path: position.x + position_x_DESC: + sort_field: + direction: desc + field_path: position.x + position_y_ASC: + sort_field: + direction: asc + field_path: position.y + position_y_DESC: + sort_field: + direction: desc + field_path: position.y + size_ASC: + sort_field: + direction: asc + field_path: options.size + size_DESC: + sort_field: + direction: desc + field_path: options.size + the_options_color_ASC: + sort_field: + direction: asc + field_path: the_opts.color + the_options_color_DESC: + sort_field: + direction: desc + field_path: the_opts.color + the_options_size_ASC: + sort_field: + direction: asc + field_path: the_opts.size + the_options_size_DESC: + sort_field: + direction: desc + field_path: the_opts.size + the_options_the_size_ASC: + sort_field: + direction: asc + field_path: the_opts.the_sighs + the_options_the_size_DESC: + sort_field: + direction: desc + field_path: the_opts.the_sighs + voltage_ASC: + sort_field: + direction: asc + field_path: voltage + voltage_DESC: + sort_field: + direction: desc + field_path: voltage + weight_in_ng_ASC: + sort_field: + direction: asc + field_path: weight_in_ng + weight_in_ng_DESC: + sort_field: + direction: desc + field_path: weight_in_ng + weight_in_ng_str_ASC: + sort_field: + direction: asc + field_path: weight_in_ng_str + weight_in_ng_str_DESC: + sort_field: + direction: desc + field_path: weight_in_ng_str + widget_cost_amount_cents_ASC: + sort_field: + direction: asc + field_path: widget_cost.amount_cents + widget_cost_amount_cents_DESC: + sort_field: + direction: desc + field_path: widget_cost.amount_cents + widget_cost_currency_ASC: + sort_field: + direction: asc + field_path: widget_cost.currency + widget_cost_currency_DESC: + sort_field: + direction: desc + field_path: widget_cost.currency + widget_name_ASC: + sort_field: + direction: asc + field_path: widget_name + widget_name_DESC: + sort_field: + direction: desc + field_path: widget_name + widget_size_ASC: + sort_field: + direction: asc + field_path: widget_size + widget_size_DESC: + sort_field: + direction: desc + field_path: widget_size + widget_workspace_id_ASC: + sort_field: + direction: asc + field_path: widget_workspace_id3 + widget_workspace_id_DESC: + sort_field: + direction: desc + field_path: widget_workspace_id3 + workspace_id_ASC: + sort_field: + direction: asc + field_path: workspace_id2 + workspace_id_DESC: + sort_field: + direction: desc + field_path: workspace_id2 + workspace_name_ASC: + sort_field: + direction: asc + field_path: workspace_name + workspace_name_DESC: + sort_field: + direction: desc + field_path: workspace_name + PartSortOrderInput: + values_by_name: + created_at_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_DESC: + sort_field: + direction: desc + field_path: created_at + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + material_ASC: + sort_field: + direction: asc + field_path: material + material_DESC: + sort_field: + direction: desc + field_path: material + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + voltage_ASC: + sort_field: + direction: asc + field_path: voltage + voltage_DESC: + sort_field: + direction: desc + field_path: voltage + SponsorSortOrderInput: + values_by_name: + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + TeamSortOrderInput: + values_by_name: + country_code_ASC: + sort_field: + direction: asc + field_path: country_code + country_code_DESC: + sort_field: + direction: desc + field_path: country_code + current_name_ASC: + sort_field: + direction: asc + field_path: current_name + current_name_DESC: + sort_field: + direction: desc + field_path: current_name + details_count_ASC: + sort_field: + direction: asc + field_path: details.count + details_count_DESC: + sort_field: + direction: desc + field_path: details.count + formed_on_ASC: + sort_field: + direction: asc + field_path: formed_on + formed_on_DESC: + sort_field: + direction: desc + field_path: formed_on + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + league_ASC: + sort_field: + direction: asc + field_path: league + league_DESC: + sort_field: + direction: desc + field_path: league + WidgetCurrencySortOrderInput: + values_by_name: + details_symbol_ASC: + sort_field: + direction: asc + field_path: details.symbol + details_symbol_DESC: + sort_field: + direction: desc + field_path: details.symbol + details_unit_ASC: + sort_field: + direction: asc + field_path: details.unit + details_unit_DESC: + sort_field: + direction: desc + field_path: details.unit + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + introduced_on_ASC: + sort_field: + direction: asc + field_path: introduced_on + introduced_on_DESC: + sort_field: + direction: desc + field_path: introduced_on + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + nested_fields_max_widget_cost_ASC: + sort_field: + direction: asc + field_path: nested_fields.max_widget_cost + nested_fields_max_widget_cost_DESC: + sort_field: + direction: desc + field_path: nested_fields.max_widget_cost + oldest_widget_created_at_ASC: + sort_field: + direction: asc + field_path: oldest_widget_created_at + oldest_widget_created_at_DESC: + sort_field: + direction: desc + field_path: oldest_widget_created_at + primary_continent_ASC: + sort_field: + direction: asc + field_path: primary_continent + primary_continent_DESC: + sort_field: + direction: desc + field_path: primary_continent + WidgetOrAddressSortOrderInput: + values_by_name: + amount_cents2_ASC: + sort_field: + direction: asc + field_path: amount_cents + amount_cents2_DESC: + sort_field: + direction: desc + field_path: amount_cents + amount_cents_ASC: + sort_field: + direction: asc + field_path: amount_cents + amount_cents_DESC: + sort_field: + direction: desc + field_path: amount_cents + cost_amount_cents_ASC: + sort_field: + direction: asc + field_path: cost.amount_cents + cost_amount_cents_DESC: + sort_field: + direction: desc + field_path: cost.amount_cents + cost_currency_ASC: + sort_field: + direction: asc + field_path: cost.currency + cost_currency_DESC: + sort_field: + direction: desc + field_path: cost.currency + cost_currency_introduced_on_ASC: + sort_field: + direction: asc + field_path: cost_currency_introduced_on + cost_currency_introduced_on_DESC: + sort_field: + direction: desc + field_path: cost_currency_introduced_on + cost_currency_name_ASC: + sort_field: + direction: asc + field_path: cost_currency_name + cost_currency_name_DESC: + sort_field: + direction: desc + field_path: cost_currency_name + cost_currency_primary_continent_ASC: + sort_field: + direction: asc + field_path: cost_currency_primary_continent + cost_currency_primary_continent_DESC: + sort_field: + direction: desc + field_path: cost_currency_primary_continent + cost_currency_symbol_ASC: + sort_field: + direction: asc + field_path: cost_currency_symbol + cost_currency_symbol_DESC: + sort_field: + direction: desc + field_path: cost_currency_symbol + cost_currency_unit_ASC: + sort_field: + direction: asc + field_path: cost_currency_unit + cost_currency_unit_DESC: + sort_field: + direction: desc + field_path: cost_currency_unit + created_at2_ASC: + sort_field: + direction: asc + field_path: created_at + created_at2_DESC: + sort_field: + direction: desc + field_path: created_at + created_at2_legacy_ASC: + sort_field: + direction: asc + field_path: created_at + created_at2_legacy_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_legacy_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_legacy_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_time_of_day_ASC: + sort_field: + direction: asc + field_path: created_at_time_of_day + created_at_time_of_day_DESC: + sort_field: + direction: desc + field_path: created_at_time_of_day + created_on_ASC: + sort_field: + direction: asc + field_path: created_on + created_on_DESC: + sort_field: + direction: desc + field_path: created_on + created_on_legacy_ASC: + sort_field: + direction: asc + field_path: created_on + created_on_legacy_DESC: + sort_field: + direction: desc + field_path: created_on + full_address_ASC: + sort_field: + direction: asc + field_path: full_address + full_address_DESC: + sort_field: + direction: desc + field_path: full_address + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + inventor_name_ASC: + sort_field: + direction: asc + field_path: inventor.name + inventor_name_DESC: + sort_field: + direction: desc + field_path: inventor.name + inventor_nationality_ASC: + sort_field: + direction: asc + field_path: inventor.nationality + inventor_nationality_DESC: + sort_field: + direction: desc + field_path: inventor.nationality + inventor_stock_ticker_ASC: + sort_field: + direction: asc + field_path: inventor.stock_ticker + inventor_stock_ticker_DESC: + sort_field: + direction: desc + field_path: inventor.stock_ticker + metadata_ASC: + sort_field: + direction: asc + field_path: metadata + metadata_DESC: + sort_field: + direction: desc + field_path: metadata + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + named_inventor_name_ASC: + sort_field: + direction: asc + field_path: named_inventor.name + named_inventor_name_DESC: + sort_field: + direction: desc + field_path: named_inventor.name + named_inventor_nationality_ASC: + sort_field: + direction: asc + field_path: named_inventor.nationality + named_inventor_nationality_DESC: + sort_field: + direction: desc + field_path: named_inventor.nationality + named_inventor_stock_ticker_ASC: + sort_field: + direction: asc + field_path: named_inventor.stock_ticker + named_inventor_stock_ticker_DESC: + sort_field: + direction: desc + field_path: named_inventor.stock_ticker + options_color_ASC: + sort_field: + direction: asc + field_path: options.color + options_color_DESC: + sort_field: + direction: desc + field_path: options.color + options_size_ASC: + sort_field: + direction: asc + field_path: options.size + options_size_DESC: + sort_field: + direction: desc + field_path: options.size + options_the_size_ASC: + sort_field: + direction: asc + field_path: options.the_sighs + options_the_size_DESC: + sort_field: + direction: desc + field_path: options.the_sighs + size_ASC: + sort_field: + direction: asc + field_path: options.size + size_DESC: + sort_field: + direction: desc + field_path: options.size + the_options_color_ASC: + sort_field: + direction: asc + field_path: the_opts.color + the_options_color_DESC: + sort_field: + direction: desc + field_path: the_opts.color + the_options_size_ASC: + sort_field: + direction: asc + field_path: the_opts.size + the_options_size_DESC: + sort_field: + direction: desc + field_path: the_opts.size + the_options_the_size_ASC: + sort_field: + direction: asc + field_path: the_opts.the_sighs + the_options_the_size_DESC: + sort_field: + direction: desc + field_path: the_opts.the_sighs + timestamps_created_at_ASC: + sort_field: + direction: asc + field_path: timestamps.created_at + timestamps_created_at_DESC: + sort_field: + direction: desc + field_path: timestamps.created_at + weight_in_ng_ASC: + sort_field: + direction: asc + field_path: weight_in_ng + weight_in_ng_DESC: + sort_field: + direction: desc + field_path: weight_in_ng + weight_in_ng_str_ASC: + sort_field: + direction: asc + field_path: weight_in_ng_str + weight_in_ng_str_DESC: + sort_field: + direction: desc + field_path: weight_in_ng_str + workspace_id_ASC: + sort_field: + direction: asc + field_path: workspace_id2 + workspace_id_DESC: + sort_field: + direction: desc + field_path: workspace_id2 + workspace_name_ASC: + sort_field: + direction: asc + field_path: workspace_name + workspace_name_DESC: + sort_field: + direction: desc + field_path: workspace_name + WidgetSortOrderInput: + values_by_name: + amount_cents2_ASC: + sort_field: + direction: asc + field_path: amount_cents + amount_cents2_DESC: + sort_field: + direction: desc + field_path: amount_cents + amount_cents_ASC: + sort_field: + direction: asc + field_path: amount_cents + amount_cents_DESC: + sort_field: + direction: desc + field_path: amount_cents + cost_amount_cents_ASC: + sort_field: + direction: asc + field_path: cost.amount_cents + cost_amount_cents_DESC: + sort_field: + direction: desc + field_path: cost.amount_cents + cost_currency_ASC: + sort_field: + direction: asc + field_path: cost.currency + cost_currency_DESC: + sort_field: + direction: desc + field_path: cost.currency + cost_currency_introduced_on_ASC: + sort_field: + direction: asc + field_path: cost_currency_introduced_on + cost_currency_introduced_on_DESC: + sort_field: + direction: desc + field_path: cost_currency_introduced_on + cost_currency_name_ASC: + sort_field: + direction: asc + field_path: cost_currency_name + cost_currency_name_DESC: + sort_field: + direction: desc + field_path: cost_currency_name + cost_currency_primary_continent_ASC: + sort_field: + direction: asc + field_path: cost_currency_primary_continent + cost_currency_primary_continent_DESC: + sort_field: + direction: desc + field_path: cost_currency_primary_continent + cost_currency_symbol_ASC: + sort_field: + direction: asc + field_path: cost_currency_symbol + cost_currency_symbol_DESC: + sort_field: + direction: desc + field_path: cost_currency_symbol + cost_currency_unit_ASC: + sort_field: + direction: asc + field_path: cost_currency_unit + cost_currency_unit_DESC: + sort_field: + direction: desc + field_path: cost_currency_unit + created_at2_ASC: + sort_field: + direction: asc + field_path: created_at + created_at2_DESC: + sort_field: + direction: desc + field_path: created_at + created_at2_legacy_ASC: + sort_field: + direction: asc + field_path: created_at + created_at2_legacy_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_legacy_ASC: + sort_field: + direction: asc + field_path: created_at + created_at_legacy_DESC: + sort_field: + direction: desc + field_path: created_at + created_at_time_of_day_ASC: + sort_field: + direction: asc + field_path: created_at_time_of_day + created_at_time_of_day_DESC: + sort_field: + direction: desc + field_path: created_at_time_of_day + created_on_ASC: + sort_field: + direction: asc + field_path: created_on + created_on_DESC: + sort_field: + direction: desc + field_path: created_on + created_on_legacy_ASC: + sort_field: + direction: asc + field_path: created_on + created_on_legacy_DESC: + sort_field: + direction: desc + field_path: created_on + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + inventor_name_ASC: + sort_field: + direction: asc + field_path: inventor.name + inventor_name_DESC: + sort_field: + direction: desc + field_path: inventor.name + inventor_nationality_ASC: + sort_field: + direction: asc + field_path: inventor.nationality + inventor_nationality_DESC: + sort_field: + direction: desc + field_path: inventor.nationality + inventor_stock_ticker_ASC: + sort_field: + direction: asc + field_path: inventor.stock_ticker + inventor_stock_ticker_DESC: + sort_field: + direction: desc + field_path: inventor.stock_ticker + metadata_ASC: + sort_field: + direction: asc + field_path: metadata + metadata_DESC: + sort_field: + direction: desc + field_path: metadata + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + named_inventor_name_ASC: + sort_field: + direction: asc + field_path: named_inventor.name + named_inventor_name_DESC: + sort_field: + direction: desc + field_path: named_inventor.name + named_inventor_nationality_ASC: + sort_field: + direction: asc + field_path: named_inventor.nationality + named_inventor_nationality_DESC: + sort_field: + direction: desc + field_path: named_inventor.nationality + named_inventor_stock_ticker_ASC: + sort_field: + direction: asc + field_path: named_inventor.stock_ticker + named_inventor_stock_ticker_DESC: + sort_field: + direction: desc + field_path: named_inventor.stock_ticker + options_color_ASC: + sort_field: + direction: asc + field_path: options.color + options_color_DESC: + sort_field: + direction: desc + field_path: options.color + options_size_ASC: + sort_field: + direction: asc + field_path: options.size + options_size_DESC: + sort_field: + direction: desc + field_path: options.size + options_the_size_ASC: + sort_field: + direction: asc + field_path: options.the_sighs + options_the_size_DESC: + sort_field: + direction: desc + field_path: options.the_sighs + size_ASC: + sort_field: + direction: asc + field_path: options.size + size_DESC: + sort_field: + direction: desc + field_path: options.size + the_options_color_ASC: + sort_field: + direction: asc + field_path: the_opts.color + the_options_color_DESC: + sort_field: + direction: desc + field_path: the_opts.color + the_options_size_ASC: + sort_field: + direction: asc + field_path: the_opts.size + the_options_size_DESC: + sort_field: + direction: desc + field_path: the_opts.size + the_options_the_size_ASC: + sort_field: + direction: asc + field_path: the_opts.the_sighs + the_options_the_size_DESC: + sort_field: + direction: desc + field_path: the_opts.the_sighs + weight_in_ng_ASC: + sort_field: + direction: asc + field_path: weight_in_ng + weight_in_ng_DESC: + sort_field: + direction: desc + field_path: weight_in_ng + weight_in_ng_str_ASC: + sort_field: + direction: asc + field_path: weight_in_ng_str + weight_in_ng_str_DESC: + sort_field: + direction: desc + field_path: weight_in_ng_str + workspace_id_ASC: + sort_field: + direction: asc + field_path: workspace_id2 + workspace_id_DESC: + sort_field: + direction: desc + field_path: workspace_id2 + workspace_name_ASC: + sort_field: + direction: asc + field_path: workspace_name + workspace_name_DESC: + sort_field: + direction: desc + field_path: workspace_name + WidgetWorkspaceSortOrderInput: + values_by_name: + id_ASC: + sort_field: + direction: asc + field_path: id + id_DESC: + sort_field: + direction: desc + field_path: id + name_ASC: + sort_field: + direction: asc + field_path: name + name_DESC: + sort_field: + direction: desc + field_path: name + widget_created_at_ASC: + sort_field: + direction: asc + field_path: widget.created_at + widget_created_at_DESC: + sort_field: + direction: desc + field_path: widget.created_at + widget_id_ASC: + sort_field: + direction: asc + field_path: widget.id + widget_id_DESC: + sort_field: + direction: desc + field_path: widget.id +graphql_extension_modules: +- extension_name: ElasticGraph::Apollo::GraphQL::EngineExtension + require_path: elastic_graph/apollo/graphql/engine_extension +index_definitions_by_name: + addresses: + current_sources: + - __self + default_sort_fields: + - direction: desc + field_path: id + fields_by_path: + __counts.shapes: + source: __self + __counts.shapes|coordinates: + source: __self + __counts.shapes|type: + source: __self + full_address: + source: __self + geo_location.lat: + source: __self + geo_location.lon: + source: __self + id: + source: __self + manufacturer_id: + source: __self + shapes.coordinates: + source: __self + shapes.type: + source: __self + timestamps.created_at: + source: __self + route_with: id + components: + current_sources: + - __self + - widget + default_sort_fields: + - direction: desc + field_path: created_at + fields_by_path: + __counts.part_ids: + source: __self + __counts.tags: + source: __self + __counts.widget_tags: + source: widget + created_at: + source: __self + id: + source: __self + name: + source: __self + part_ids: + source: __self + position.x: + source: __self + position.y: + source: __self + tags: + source: __self + widget_cost.amount_cents: + source: widget + widget_cost.currency: + source: widget + widget_name: + source: widget + widget_size: + source: widget + widget_tags: + source: widget + widget_workspace_id3: + source: widget + route_with: id + electrical_parts: + current_sources: + - __self + default_sort_fields: + - direction: desc + field_path: created_at + fields_by_path: + created_at: + source: __self + id: + source: __self + manufacturer_id: + source: __self + name: + source: __self + voltage: + source: __self + route_with: id + manufacturers: + current_sources: + - __self + default_sort_fields: + - direction: desc + field_path: created_at + fields_by_path: + created_at: + source: __self + id: + source: __self + name: + source: __self + route_with: id + mechanical_parts: + current_sources: + - __self + default_sort_fields: + - direction: desc + field_path: created_at + fields_by_path: + created_at: + source: __self + id: + source: __self + manufacturer_id: + source: __self + material: + source: __self + name: + source: __self + route_with: id + sponsors: + current_sources: + - __self + fields_by_path: + id: + source: __self + name: + source: __self + route_with: id + teams: + current_sources: + - __self + fields_by_path: + __counts.current_players_nested: + source: __self + __counts.current_players_object: + source: __self + __counts.current_players_object|affiliations: + source: __self + __counts.current_players_object|affiliations|sponsorships_nested: + source: __self + __counts.current_players_object|affiliations|sponsorships_object: + source: __self + __counts.current_players_object|affiliations|sponsorships_object|annual_total: + source: __self + __counts.current_players_object|affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + __counts.current_players_object|affiliations|sponsorships_object|annual_total|currency: + source: __self + __counts.current_players_object|affiliations|sponsorships_object|sponsor_id: + source: __self + __counts.current_players_object|name: + source: __self + __counts.current_players_object|nicknames: + source: __self + __counts.current_players_object|seasons_nested: + source: __self + __counts.current_players_object|seasons_object: + source: __self + __counts.current_players_object|seasons_object|awards: + source: __self + __counts.current_players_object|seasons_object|games_played: + source: __self + __counts.current_players_object|seasons_object|year: + source: __self + __counts.details|uniform_colors: + source: __self + __counts.forbes_valuation_moneys_nested: + source: __self + __counts.forbes_valuation_moneys_object: + source: __self + __counts.forbes_valuation_moneys_object|amount_cents: + source: __self + __counts.forbes_valuation_moneys_object|currency: + source: __self + __counts.forbes_valuations: + source: __self + __counts.nested_fields2|current_players: + source: __self + __counts.nested_fields2|forbes_valuation_moneys: + source: __self + __counts.nested_fields2|the_seasons: + source: __self + __counts.past_names: + source: __self + __counts.seasons_nested: + source: __self + __counts.seasons_object: + source: __self + __counts.seasons_object|count: + source: __self + __counts.seasons_object|notes: + source: __self + __counts.seasons_object|players_nested: + source: __self + __counts.seasons_object|players_object: + source: __self + __counts.seasons_object|players_object|affiliations: + source: __self + __counts.seasons_object|players_object|affiliations|sponsorships_nested: + source: __self + __counts.seasons_object|players_object|affiliations|sponsorships_object: + source: __self + __counts.seasons_object|players_object|affiliations|sponsorships_object|annual_total: + source: __self + __counts.seasons_object|players_object|affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + __counts.seasons_object|players_object|affiliations|sponsorships_object|annual_total|currency: + source: __self + __counts.seasons_object|players_object|affiliations|sponsorships_object|sponsor_id: + source: __self + __counts.seasons_object|players_object|name: + source: __self + __counts.seasons_object|players_object|nicknames: + source: __self + __counts.seasons_object|players_object|seasons_nested: + source: __self + __counts.seasons_object|players_object|seasons_object: + source: __self + __counts.seasons_object|players_object|seasons_object|awards: + source: __self + __counts.seasons_object|players_object|seasons_object|games_played: + source: __self + __counts.seasons_object|players_object|seasons_object|year: + source: __self + __counts.seasons_object|started_at: + source: __self + __counts.seasons_object|the_record: + source: __self + __counts.seasons_object|the_record|first_win_on: + source: __self + __counts.seasons_object|the_record|last_win_date: + source: __self + __counts.seasons_object|the_record|loss_count: + source: __self + __counts.seasons_object|the_record|win_count: + source: __self + __counts.seasons_object|won_games_at: + source: __self + __counts.seasons_object|year: + source: __self + __counts.the_nested_fields|current_players: + source: __self + __counts.the_nested_fields|forbes_valuation_moneys: + source: __self + __counts.the_nested_fields|the_seasons: + source: __self + __counts.won_championships_at: + source: __self + country_code: + source: __self + current_name: + source: __self + current_players_nested.__counts.affiliations|sponsorships_nested: + source: __self + current_players_nested.__counts.affiliations|sponsorships_object: + source: __self + current_players_nested.__counts.affiliations|sponsorships_object|annual_total: + source: __self + current_players_nested.__counts.affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + current_players_nested.__counts.affiliations|sponsorships_object|annual_total|currency: + source: __self + current_players_nested.__counts.affiliations|sponsorships_object|sponsor_id: + source: __self + current_players_nested.__counts.nicknames: + source: __self + current_players_nested.__counts.seasons_nested: + source: __self + current_players_nested.__counts.seasons_object: + source: __self + current_players_nested.__counts.seasons_object|awards: + source: __self + current_players_nested.__counts.seasons_object|games_played: + source: __self + current_players_nested.__counts.seasons_object|year: + source: __self + current_players_nested.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + current_players_nested.affiliations.sponsorships_nested.annual_total.currency: + source: __self + current_players_nested.affiliations.sponsorships_nested.sponsor_id: + source: __self + current_players_nested.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + current_players_nested.affiliations.sponsorships_object.annual_total.currency: + source: __self + current_players_nested.affiliations.sponsorships_object.sponsor_id: + source: __self + current_players_nested.name: + source: __self + current_players_nested.nicknames: + source: __self + current_players_nested.seasons_nested.__counts.awards: + source: __self + current_players_nested.seasons_nested.awards: + source: __self + current_players_nested.seasons_nested.games_played: + source: __self + current_players_nested.seasons_nested.year: + source: __self + current_players_nested.seasons_object.awards: + source: __self + current_players_nested.seasons_object.games_played: + source: __self + current_players_nested.seasons_object.year: + source: __self + current_players_object.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + current_players_object.affiliations.sponsorships_nested.annual_total.currency: + source: __self + current_players_object.affiliations.sponsorships_nested.sponsor_id: + source: __self + current_players_object.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + current_players_object.affiliations.sponsorships_object.annual_total.currency: + source: __self + current_players_object.affiliations.sponsorships_object.sponsor_id: + source: __self + current_players_object.name: + source: __self + current_players_object.nicknames: + source: __self + current_players_object.seasons_nested.__counts.awards: + source: __self + current_players_object.seasons_nested.awards: + source: __self + current_players_object.seasons_nested.games_played: + source: __self + current_players_object.seasons_nested.year: + source: __self + current_players_object.seasons_object.awards: + source: __self + current_players_object.seasons_object.games_played: + source: __self + current_players_object.seasons_object.year: + source: __self + details.count: + source: __self + details.uniform_colors: + source: __self + forbes_valuation_moneys_nested.amount_cents: + source: __self + forbes_valuation_moneys_nested.currency: + source: __self + forbes_valuation_moneys_object.amount_cents: + source: __self + forbes_valuation_moneys_object.currency: + source: __self + forbes_valuations: + source: __self + formed_on: + source: __self + id: + source: __self + league: + source: __self + nested_fields2.current_players.__counts.affiliations|sponsorships_nested: + source: __self + nested_fields2.current_players.__counts.affiliations|sponsorships_object: + source: __self + nested_fields2.current_players.__counts.affiliations|sponsorships_object|annual_total: + source: __self + nested_fields2.current_players.__counts.affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + nested_fields2.current_players.__counts.affiliations|sponsorships_object|annual_total|currency: + source: __self + nested_fields2.current_players.__counts.affiliations|sponsorships_object|sponsor_id: + source: __self + nested_fields2.current_players.__counts.nicknames: + source: __self + nested_fields2.current_players.__counts.seasons_nested: + source: __self + nested_fields2.current_players.__counts.seasons_object: + source: __self + nested_fields2.current_players.__counts.seasons_object|awards: + source: __self + nested_fields2.current_players.__counts.seasons_object|games_played: + source: __self + nested_fields2.current_players.__counts.seasons_object|year: + source: __self + nested_fields2.current_players.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + nested_fields2.current_players.affiliations.sponsorships_nested.annual_total.currency: + source: __self + nested_fields2.current_players.affiliations.sponsorships_nested.sponsor_id: + source: __self + nested_fields2.current_players.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + nested_fields2.current_players.affiliations.sponsorships_object.annual_total.currency: + source: __self + nested_fields2.current_players.affiliations.sponsorships_object.sponsor_id: + source: __self + nested_fields2.current_players.name: + source: __self + nested_fields2.current_players.nicknames: + source: __self + nested_fields2.current_players.seasons_nested.__counts.awards: + source: __self + nested_fields2.current_players.seasons_nested.awards: + source: __self + nested_fields2.current_players.seasons_nested.games_played: + source: __self + nested_fields2.current_players.seasons_nested.year: + source: __self + nested_fields2.current_players.seasons_object.awards: + source: __self + nested_fields2.current_players.seasons_object.games_played: + source: __self + nested_fields2.current_players.seasons_object.year: + source: __self + nested_fields2.forbes_valuation_moneys.amount_cents: + source: __self + nested_fields2.forbes_valuation_moneys.currency: + source: __self + nested_fields2.the_seasons.__counts.notes: + source: __self + nested_fields2.the_seasons.__counts.players_nested: + source: __self + nested_fields2.the_seasons.__counts.players_object: + source: __self + nested_fields2.the_seasons.__counts.players_object|affiliations: + source: __self + nested_fields2.the_seasons.__counts.players_object|affiliations|sponsorships_nested: + source: __self + nested_fields2.the_seasons.__counts.players_object|affiliations|sponsorships_object: + source: __self + nested_fields2.the_seasons.__counts.players_object|affiliations|sponsorships_object|annual_total: + source: __self + nested_fields2.the_seasons.__counts.players_object|affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + nested_fields2.the_seasons.__counts.players_object|affiliations|sponsorships_object|annual_total|currency: + source: __self + nested_fields2.the_seasons.__counts.players_object|affiliations|sponsorships_object|sponsor_id: + source: __self + nested_fields2.the_seasons.__counts.players_object|name: + source: __self + nested_fields2.the_seasons.__counts.players_object|nicknames: + source: __self + nested_fields2.the_seasons.__counts.players_object|seasons_nested: + source: __self + nested_fields2.the_seasons.__counts.players_object|seasons_object: + source: __self + nested_fields2.the_seasons.__counts.players_object|seasons_object|awards: + source: __self + nested_fields2.the_seasons.__counts.players_object|seasons_object|games_played: + source: __self + nested_fields2.the_seasons.__counts.players_object|seasons_object|year: + source: __self + nested_fields2.the_seasons.__counts.won_games_at: + source: __self + nested_fields2.the_seasons.count: + source: __self + nested_fields2.the_seasons.notes: + source: __self + nested_fields2.the_seasons.players_nested.__counts.affiliations|sponsorships_nested: + source: __self + nested_fields2.the_seasons.players_nested.__counts.affiliations|sponsorships_object: + source: __self + nested_fields2.the_seasons.players_nested.__counts.affiliations|sponsorships_object|annual_total: + source: __self + nested_fields2.the_seasons.players_nested.__counts.affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + nested_fields2.the_seasons.players_nested.__counts.affiliations|sponsorships_object|annual_total|currency: + source: __self + nested_fields2.the_seasons.players_nested.__counts.affiliations|sponsorships_object|sponsor_id: + source: __self + nested_fields2.the_seasons.players_nested.__counts.nicknames: + source: __self + nested_fields2.the_seasons.players_nested.__counts.seasons_nested: + source: __self + nested_fields2.the_seasons.players_nested.__counts.seasons_object: + source: __self + nested_fields2.the_seasons.players_nested.__counts.seasons_object|awards: + source: __self + nested_fields2.the_seasons.players_nested.__counts.seasons_object|games_played: + source: __self + nested_fields2.the_seasons.players_nested.__counts.seasons_object|year: + source: __self + nested_fields2.the_seasons.players_nested.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + nested_fields2.the_seasons.players_nested.affiliations.sponsorships_nested.annual_total.currency: + source: __self + nested_fields2.the_seasons.players_nested.affiliations.sponsorships_nested.sponsor_id: + source: __self + nested_fields2.the_seasons.players_nested.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + nested_fields2.the_seasons.players_nested.affiliations.sponsorships_object.annual_total.currency: + source: __self + nested_fields2.the_seasons.players_nested.affiliations.sponsorships_object.sponsor_id: + source: __self + nested_fields2.the_seasons.players_nested.name: + source: __self + nested_fields2.the_seasons.players_nested.nicknames: + source: __self + nested_fields2.the_seasons.players_nested.seasons_nested.__counts.awards: + source: __self + nested_fields2.the_seasons.players_nested.seasons_nested.awards: + source: __self + nested_fields2.the_seasons.players_nested.seasons_nested.games_played: + source: __self + nested_fields2.the_seasons.players_nested.seasons_nested.year: + source: __self + nested_fields2.the_seasons.players_nested.seasons_object.awards: + source: __self + nested_fields2.the_seasons.players_nested.seasons_object.games_played: + source: __self + nested_fields2.the_seasons.players_nested.seasons_object.year: + source: __self + nested_fields2.the_seasons.players_object.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + nested_fields2.the_seasons.players_object.affiliations.sponsorships_nested.annual_total.currency: + source: __self + nested_fields2.the_seasons.players_object.affiliations.sponsorships_nested.sponsor_id: + source: __self + nested_fields2.the_seasons.players_object.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + nested_fields2.the_seasons.players_object.affiliations.sponsorships_object.annual_total.currency: + source: __self + nested_fields2.the_seasons.players_object.affiliations.sponsorships_object.sponsor_id: + source: __self + nested_fields2.the_seasons.players_object.name: + source: __self + nested_fields2.the_seasons.players_object.nicknames: + source: __self + nested_fields2.the_seasons.players_object.seasons_nested.__counts.awards: + source: __self + nested_fields2.the_seasons.players_object.seasons_nested.awards: + source: __self + nested_fields2.the_seasons.players_object.seasons_nested.games_played: + source: __self + nested_fields2.the_seasons.players_object.seasons_nested.year: + source: __self + nested_fields2.the_seasons.players_object.seasons_object.awards: + source: __self + nested_fields2.the_seasons.players_object.seasons_object.games_played: + source: __self + nested_fields2.the_seasons.players_object.seasons_object.year: + source: __self + nested_fields2.the_seasons.started_at: + source: __self + nested_fields2.the_seasons.the_record.first_win_on: + source: __self + nested_fields2.the_seasons.the_record.last_win_date: + source: __self + nested_fields2.the_seasons.the_record.loss_count: + source: __self + nested_fields2.the_seasons.the_record.win_count: + source: __self + nested_fields2.the_seasons.won_games_at: + source: __self + nested_fields2.the_seasons.year: + source: __self + past_names: + source: __self + seasons_nested.__counts.notes: + source: __self + seasons_nested.__counts.players_nested: + source: __self + seasons_nested.__counts.players_object: + source: __self + seasons_nested.__counts.players_object|affiliations: + source: __self + seasons_nested.__counts.players_object|affiliations|sponsorships_nested: + source: __self + seasons_nested.__counts.players_object|affiliations|sponsorships_object: + source: __self + seasons_nested.__counts.players_object|affiliations|sponsorships_object|annual_total: + source: __self + seasons_nested.__counts.players_object|affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + seasons_nested.__counts.players_object|affiliations|sponsorships_object|annual_total|currency: + source: __self + seasons_nested.__counts.players_object|affiliations|sponsorships_object|sponsor_id: + source: __self + seasons_nested.__counts.players_object|name: + source: __self + seasons_nested.__counts.players_object|nicknames: + source: __self + seasons_nested.__counts.players_object|seasons_nested: + source: __self + seasons_nested.__counts.players_object|seasons_object: + source: __self + seasons_nested.__counts.players_object|seasons_object|awards: + source: __self + seasons_nested.__counts.players_object|seasons_object|games_played: + source: __self + seasons_nested.__counts.players_object|seasons_object|year: + source: __self + seasons_nested.__counts.won_games_at: + source: __self + seasons_nested.count: + source: __self + seasons_nested.notes: + source: __self + seasons_nested.players_nested.__counts.affiliations|sponsorships_nested: + source: __self + seasons_nested.players_nested.__counts.affiliations|sponsorships_object: + source: __self + seasons_nested.players_nested.__counts.affiliations|sponsorships_object|annual_total: + source: __self + seasons_nested.players_nested.__counts.affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + seasons_nested.players_nested.__counts.affiliations|sponsorships_object|annual_total|currency: + source: __self + seasons_nested.players_nested.__counts.affiliations|sponsorships_object|sponsor_id: + source: __self + seasons_nested.players_nested.__counts.nicknames: + source: __self + seasons_nested.players_nested.__counts.seasons_nested: + source: __self + seasons_nested.players_nested.__counts.seasons_object: + source: __self + seasons_nested.players_nested.__counts.seasons_object|awards: + source: __self + seasons_nested.players_nested.__counts.seasons_object|games_played: + source: __self + seasons_nested.players_nested.__counts.seasons_object|year: + source: __self + seasons_nested.players_nested.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + seasons_nested.players_nested.affiliations.sponsorships_nested.annual_total.currency: + source: __self + seasons_nested.players_nested.affiliations.sponsorships_nested.sponsor_id: + source: __self + seasons_nested.players_nested.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + seasons_nested.players_nested.affiliations.sponsorships_object.annual_total.currency: + source: __self + seasons_nested.players_nested.affiliations.sponsorships_object.sponsor_id: + source: __self + seasons_nested.players_nested.name: + source: __self + seasons_nested.players_nested.nicknames: + source: __self + seasons_nested.players_nested.seasons_nested.__counts.awards: + source: __self + seasons_nested.players_nested.seasons_nested.awards: + source: __self + seasons_nested.players_nested.seasons_nested.games_played: + source: __self + seasons_nested.players_nested.seasons_nested.year: + source: __self + seasons_nested.players_nested.seasons_object.awards: + source: __self + seasons_nested.players_nested.seasons_object.games_played: + source: __self + seasons_nested.players_nested.seasons_object.year: + source: __self + seasons_nested.players_object.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + seasons_nested.players_object.affiliations.sponsorships_nested.annual_total.currency: + source: __self + seasons_nested.players_object.affiliations.sponsorships_nested.sponsor_id: + source: __self + seasons_nested.players_object.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + seasons_nested.players_object.affiliations.sponsorships_object.annual_total.currency: + source: __self + seasons_nested.players_object.affiliations.sponsorships_object.sponsor_id: + source: __self + seasons_nested.players_object.name: + source: __self + seasons_nested.players_object.nicknames: + source: __self + seasons_nested.players_object.seasons_nested.__counts.awards: + source: __self + seasons_nested.players_object.seasons_nested.awards: + source: __self + seasons_nested.players_object.seasons_nested.games_played: + source: __self + seasons_nested.players_object.seasons_nested.year: + source: __self + seasons_nested.players_object.seasons_object.awards: + source: __self + seasons_nested.players_object.seasons_object.games_played: + source: __self + seasons_nested.players_object.seasons_object.year: + source: __self + seasons_nested.started_at: + source: __self + seasons_nested.the_record.first_win_on: + source: __self + seasons_nested.the_record.last_win_date: + source: __self + seasons_nested.the_record.loss_count: + source: __self + seasons_nested.the_record.win_count: + source: __self + seasons_nested.won_games_at: + source: __self + seasons_nested.year: + source: __self + seasons_object.count: + source: __self + seasons_object.notes: + source: __self + seasons_object.players_nested.__counts.affiliations|sponsorships_nested: + source: __self + seasons_object.players_nested.__counts.affiliations|sponsorships_object: + source: __self + seasons_object.players_nested.__counts.affiliations|sponsorships_object|annual_total: + source: __self + seasons_object.players_nested.__counts.affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + seasons_object.players_nested.__counts.affiliations|sponsorships_object|annual_total|currency: + source: __self + seasons_object.players_nested.__counts.affiliations|sponsorships_object|sponsor_id: + source: __self + seasons_object.players_nested.__counts.nicknames: + source: __self + seasons_object.players_nested.__counts.seasons_nested: + source: __self + seasons_object.players_nested.__counts.seasons_object: + source: __self + seasons_object.players_nested.__counts.seasons_object|awards: + source: __self + seasons_object.players_nested.__counts.seasons_object|games_played: + source: __self + seasons_object.players_nested.__counts.seasons_object|year: + source: __self + seasons_object.players_nested.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + seasons_object.players_nested.affiliations.sponsorships_nested.annual_total.currency: + source: __self + seasons_object.players_nested.affiliations.sponsorships_nested.sponsor_id: + source: __self + seasons_object.players_nested.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + seasons_object.players_nested.affiliations.sponsorships_object.annual_total.currency: + source: __self + seasons_object.players_nested.affiliations.sponsorships_object.sponsor_id: + source: __self + seasons_object.players_nested.name: + source: __self + seasons_object.players_nested.nicknames: + source: __self + seasons_object.players_nested.seasons_nested.__counts.awards: + source: __self + seasons_object.players_nested.seasons_nested.awards: + source: __self + seasons_object.players_nested.seasons_nested.games_played: + source: __self + seasons_object.players_nested.seasons_nested.year: + source: __self + seasons_object.players_nested.seasons_object.awards: + source: __self + seasons_object.players_nested.seasons_object.games_played: + source: __self + seasons_object.players_nested.seasons_object.year: + source: __self + seasons_object.players_object.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + seasons_object.players_object.affiliations.sponsorships_nested.annual_total.currency: + source: __self + seasons_object.players_object.affiliations.sponsorships_nested.sponsor_id: + source: __self + seasons_object.players_object.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + seasons_object.players_object.affiliations.sponsorships_object.annual_total.currency: + source: __self + seasons_object.players_object.affiliations.sponsorships_object.sponsor_id: + source: __self + seasons_object.players_object.name: + source: __self + seasons_object.players_object.nicknames: + source: __self + seasons_object.players_object.seasons_nested.__counts.awards: + source: __self + seasons_object.players_object.seasons_nested.awards: + source: __self + seasons_object.players_object.seasons_nested.games_played: + source: __self + seasons_object.players_object.seasons_nested.year: + source: __self + seasons_object.players_object.seasons_object.awards: + source: __self + seasons_object.players_object.seasons_object.games_played: + source: __self + seasons_object.players_object.seasons_object.year: + source: __self + seasons_object.started_at: + source: __self + seasons_object.the_record.first_win_on: + source: __self + seasons_object.the_record.last_win_date: + source: __self + seasons_object.the_record.loss_count: + source: __self + seasons_object.the_record.win_count: + source: __self + seasons_object.won_games_at: + source: __self + seasons_object.year: + source: __self + stadium_location.lat: + source: __self + stadium_location.lon: + source: __self + the_nested_fields.current_players.__counts.affiliations|sponsorships_nested: + source: __self + the_nested_fields.current_players.__counts.affiliations|sponsorships_object: + source: __self + the_nested_fields.current_players.__counts.affiliations|sponsorships_object|annual_total: + source: __self + the_nested_fields.current_players.__counts.affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + the_nested_fields.current_players.__counts.affiliations|sponsorships_object|annual_total|currency: + source: __self + the_nested_fields.current_players.__counts.affiliations|sponsorships_object|sponsor_id: + source: __self + the_nested_fields.current_players.__counts.nicknames: + source: __self + the_nested_fields.current_players.__counts.seasons_nested: + source: __self + the_nested_fields.current_players.__counts.seasons_object: + source: __self + the_nested_fields.current_players.__counts.seasons_object|awards: + source: __self + the_nested_fields.current_players.__counts.seasons_object|games_played: + source: __self + the_nested_fields.current_players.__counts.seasons_object|year: + source: __self + the_nested_fields.current_players.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + the_nested_fields.current_players.affiliations.sponsorships_nested.annual_total.currency: + source: __self + the_nested_fields.current_players.affiliations.sponsorships_nested.sponsor_id: + source: __self + the_nested_fields.current_players.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + the_nested_fields.current_players.affiliations.sponsorships_object.annual_total.currency: + source: __self + the_nested_fields.current_players.affiliations.sponsorships_object.sponsor_id: + source: __self + the_nested_fields.current_players.name: + source: __self + the_nested_fields.current_players.nicknames: + source: __self + the_nested_fields.current_players.seasons_nested.__counts.awards: + source: __self + the_nested_fields.current_players.seasons_nested.awards: + source: __self + the_nested_fields.current_players.seasons_nested.games_played: + source: __self + the_nested_fields.current_players.seasons_nested.year: + source: __self + the_nested_fields.current_players.seasons_object.awards: + source: __self + the_nested_fields.current_players.seasons_object.games_played: + source: __self + the_nested_fields.current_players.seasons_object.year: + source: __self + the_nested_fields.forbes_valuation_moneys.amount_cents: + source: __self + the_nested_fields.forbes_valuation_moneys.currency: + source: __self + the_nested_fields.the_seasons.__counts.notes: + source: __self + the_nested_fields.the_seasons.__counts.players_nested: + source: __self + the_nested_fields.the_seasons.__counts.players_object: + source: __self + the_nested_fields.the_seasons.__counts.players_object|affiliations: + source: __self + the_nested_fields.the_seasons.__counts.players_object|affiliations|sponsorships_nested: + source: __self + the_nested_fields.the_seasons.__counts.players_object|affiliations|sponsorships_object: + source: __self + the_nested_fields.the_seasons.__counts.players_object|affiliations|sponsorships_object|annual_total: + source: __self + the_nested_fields.the_seasons.__counts.players_object|affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + the_nested_fields.the_seasons.__counts.players_object|affiliations|sponsorships_object|annual_total|currency: + source: __self + the_nested_fields.the_seasons.__counts.players_object|affiliations|sponsorships_object|sponsor_id: + source: __self + the_nested_fields.the_seasons.__counts.players_object|name: + source: __self + the_nested_fields.the_seasons.__counts.players_object|nicknames: + source: __self + the_nested_fields.the_seasons.__counts.players_object|seasons_nested: + source: __self + the_nested_fields.the_seasons.__counts.players_object|seasons_object: + source: __self + the_nested_fields.the_seasons.__counts.players_object|seasons_object|awards: + source: __self + the_nested_fields.the_seasons.__counts.players_object|seasons_object|games_played: + source: __self + the_nested_fields.the_seasons.__counts.players_object|seasons_object|year: + source: __self + the_nested_fields.the_seasons.__counts.won_games_at: + source: __self + the_nested_fields.the_seasons.count: + source: __self + the_nested_fields.the_seasons.notes: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.affiliations|sponsorships_nested: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.affiliations|sponsorships_object: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.affiliations|sponsorships_object|annual_total: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.affiliations|sponsorships_object|annual_total|amount_cents: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.affiliations|sponsorships_object|annual_total|currency: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.affiliations|sponsorships_object|sponsor_id: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.nicknames: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.seasons_nested: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.seasons_object: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.seasons_object|awards: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.seasons_object|games_played: + source: __self + the_nested_fields.the_seasons.players_nested.__counts.seasons_object|year: + source: __self + the_nested_fields.the_seasons.players_nested.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + the_nested_fields.the_seasons.players_nested.affiliations.sponsorships_nested.annual_total.currency: + source: __self + the_nested_fields.the_seasons.players_nested.affiliations.sponsorships_nested.sponsor_id: + source: __self + the_nested_fields.the_seasons.players_nested.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + the_nested_fields.the_seasons.players_nested.affiliations.sponsorships_object.annual_total.currency: + source: __self + the_nested_fields.the_seasons.players_nested.affiliations.sponsorships_object.sponsor_id: + source: __self + the_nested_fields.the_seasons.players_nested.name: + source: __self + the_nested_fields.the_seasons.players_nested.nicknames: + source: __self + the_nested_fields.the_seasons.players_nested.seasons_nested.__counts.awards: + source: __self + the_nested_fields.the_seasons.players_nested.seasons_nested.awards: + source: __self + the_nested_fields.the_seasons.players_nested.seasons_nested.games_played: + source: __self + the_nested_fields.the_seasons.players_nested.seasons_nested.year: + source: __self + the_nested_fields.the_seasons.players_nested.seasons_object.awards: + source: __self + the_nested_fields.the_seasons.players_nested.seasons_object.games_played: + source: __self + the_nested_fields.the_seasons.players_nested.seasons_object.year: + source: __self + the_nested_fields.the_seasons.players_object.affiliations.sponsorships_nested.annual_total.amount_cents: + source: __self + the_nested_fields.the_seasons.players_object.affiliations.sponsorships_nested.annual_total.currency: + source: __self + the_nested_fields.the_seasons.players_object.affiliations.sponsorships_nested.sponsor_id: + source: __self + the_nested_fields.the_seasons.players_object.affiliations.sponsorships_object.annual_total.amount_cents: + source: __self + the_nested_fields.the_seasons.players_object.affiliations.sponsorships_object.annual_total.currency: + source: __self + the_nested_fields.the_seasons.players_object.affiliations.sponsorships_object.sponsor_id: + source: __self + the_nested_fields.the_seasons.players_object.name: + source: __self + the_nested_fields.the_seasons.players_object.nicknames: + source: __self + the_nested_fields.the_seasons.players_object.seasons_nested.__counts.awards: + source: __self + the_nested_fields.the_seasons.players_object.seasons_nested.awards: + source: __self + the_nested_fields.the_seasons.players_object.seasons_nested.games_played: + source: __self + the_nested_fields.the_seasons.players_object.seasons_nested.year: + source: __self + the_nested_fields.the_seasons.players_object.seasons_object.awards: + source: __self + the_nested_fields.the_seasons.players_object.seasons_object.games_played: + source: __self + the_nested_fields.the_seasons.players_object.seasons_object.year: + source: __self + the_nested_fields.the_seasons.started_at: + source: __self + the_nested_fields.the_seasons.the_record.first_win_on: + source: __self + the_nested_fields.the_seasons.the_record.last_win_date: + source: __self + the_nested_fields.the_seasons.the_record.loss_count: + source: __self + the_nested_fields.the_seasons.the_record.win_count: + source: __self + the_nested_fields.the_seasons.won_games_at: + source: __self + the_nested_fields.the_seasons.year: + source: __self + won_championships_at: + source: __self + rollover: + frequency: yearly + timestamp_field_path: formed_on + route_with: league + widget_currencies: + current_sources: + - __self + fields_by_path: + __counts.widget_fee_currencies: + source: __self + __counts.widget_names2: + source: __self + __counts.widget_options|colors: + source: __self + __counts.widget_options|sizes: + source: __self + __counts.widget_tags: + source: __self + details.symbol: + source: __self + details.unit: + source: __self + id: + source: __self + introduced_on: + source: __self + name: + source: __self + nested_fields.max_widget_cost: + source: __self + oldest_widget_created_at: + source: __self + primary_continent: + source: __self + widget_fee_currencies: + source: __self + widget_names2: + source: __self + widget_options.colors: + source: __self + widget_options.sizes: + source: __self + widget_tags: + source: __self + rollover: + frequency: yearly + timestamp_field_path: introduced_on + route_with: primary_continent + widget_workspaces: + current_sources: + - __self + fields_by_path: + id: + source: __self + name: + source: __self + widget.created_at: + source: __self + widget.id: + source: __self + route_with: id + widgets: + current_sources: + - __self + - workspace + default_sort_fields: + - direction: desc + field_path: created_at + fields_by_path: + __counts.amounts: + source: __self + __counts.component_ids: + source: __self + __counts.fees: + source: __self + __counts.fees|amount_cents: + source: __self + __counts.fees|currency: + source: __self + __counts.release_dates: + source: __self + __counts.release_timestamps: + source: __self + __counts.tags: + source: __self + amount_cents: + source: __self + amounts: + source: __self + component_ids: + source: __self + cost.amount_cents: + source: __self + cost.currency: + source: __self + cost_currency_introduced_on: + source: __self + cost_currency_name: + source: __self + cost_currency_primary_continent: + source: __self + cost_currency_symbol: + source: __self + cost_currency_unit: + source: __self + created_at: + source: __self + created_at_time_of_day: + source: __self + created_on: + source: __self + fees.amount_cents: + source: __self + fees.currency: + source: __self + id: + source: __self + inventor.name: + source: __self + inventor.nationality: + source: __self + inventor.stock_ticker: + source: __self + metadata: + source: __self + name: + source: __self + name_text: + source: __self + named_inventor.name: + source: __self + named_inventor.nationality: + source: __self + named_inventor.stock_ticker: + source: __self + options.color: + source: __self + options.size: + source: __self + options.the_sighs: + source: __self + release_dates: + source: __self + release_timestamps: + source: __self + tags: + source: __self + the_opts.color: + source: __self + the_opts.size: + source: __self + the_opts.the_sighs: + source: __self + weight_in_ng: + source: __self + weight_in_ng_str: + source: __self + workspace_id2: + source: __self + workspace_name: + source: workspace + rollover: + frequency: yearly + timestamp_field_path: created_at + route_with: workspace_id2 +object_types_by_name: + Address: + graphql_fields_by_name: + manufacturer: + relation: + direction: out + foreign_key: manufacturer_id + index_definition_names: + - addresses + update_targets: + - data_params: + full_address: + cardinality: one + geo_location: + cardinality: one + manufacturer_id: + cardinality: one + shapes: + cardinality: one + timestamps: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + routing_value_source: id + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: Address + AddressAggregation: + elasticgraph_category: indexed_aggregation + source_type: Address + AddressAggregationConnection: + elasticgraph_category: relay_connection + AddressAggregationEdge: + elasticgraph_category: relay_edge + AddressConnection: + elasticgraph_category: relay_connection + AddressEdge: + elasticgraph_category: relay_edge + AffiliationsFieldsListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + AggregationCountDetail: + graphql_only_return_type: true + ColorListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + Component: + graphql_fields_by_name: + dollar_widget: + relation: + additional_filter: + cost: + amount_cents: + equal_to_any_of: + - 100 + direction: in + foreign_key: component_ids + part_aggregations: + relation: + direction: out + foreign_key: part_ids + parts: + relation: + direction: out + foreign_key: part_ids + widget: + relation: + direction: in + foreign_key: component_ids + widget_aggregations: + relation: + direction: in + foreign_key: component_ids + widget_workspace_id: + name_in_index: widget_workspace_id3 + widgets: + relation: + direction: in + foreign_key: component_ids + index_definition_names: + - components + update_targets: + - data_params: + created_at: + cardinality: one + name: + cardinality: one + part_ids: + cardinality: one + position: + cardinality: one + tags: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + routing_value_source: id + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: Component + ComponentAggregatedValues: + graphql_fields_by_name: + widget_workspace_id: + name_in_index: widget_workspace_id3 + ComponentAggregation: + elasticgraph_category: indexed_aggregation + source_type: Component + ComponentAggregationConnection: + elasticgraph_category: relay_connection + ComponentAggregationEdge: + elasticgraph_category: relay_edge + ComponentConnection: + elasticgraph_category: relay_connection + ComponentEdge: + elasticgraph_category: relay_edge + ComponentFilterInput: + graphql_fields_by_name: + widget_workspace_id: + name_in_index: widget_workspace_id3 + ComponentGroupedBy: + graphql_fields_by_name: + widget_workspace_id: + name_in_index: widget_workspace_id3 + Country: + graphql_fields_by_name: + team_aggregations: + relation: + direction: in + foreign_key: country_code + teams: + relation: + direction: in + foreign_key: country_code + DateAggregatedValues: + elasticgraph_category: scalar_aggregated_values + graphql_fields_by_name: + approximate_avg: + computation_detail: + function: avg + approximate_distinct_value_count: + computation_detail: + empty_bucket_value: 0 + function: cardinality + exact_max: + computation_detail: + function: max + exact_min: + computation_detail: + function: min + graphql_only_return_type: true + DateGroupedBy: + elasticgraph_category: date_grouped_by_object + graphql_only_return_type: true + DateListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + DateTimeAggregatedValues: + elasticgraph_category: scalar_aggregated_values + graphql_fields_by_name: + approximate_avg: + computation_detail: + function: avg + approximate_distinct_value_count: + computation_detail: + empty_bucket_value: 0 + function: cardinality + exact_max: + computation_detail: + function: max + exact_min: + computation_detail: + function: min + graphql_only_return_type: true + DateTimeGroupedBy: + elasticgraph_category: date_grouped_by_object + graphql_only_return_type: true + DateTimeListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + ElectricalPart: + graphql_fields_by_name: + component_aggregations: + relation: + direction: in + foreign_key: part_ids + components: + relation: + direction: in + foreign_key: part_ids + manufacturer: + relation: + direction: out + foreign_key: manufacturer_id + index_definition_names: + - electrical_parts + update_targets: + - data_params: + created_at: + cardinality: one + manufacturer_id: + cardinality: one + name: + cardinality: one + voltage: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + routing_value_source: id + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: ElectricalPart + ElectricalPartAggregation: + elasticgraph_category: indexed_aggregation + source_type: ElectricalPart + ElectricalPartAggregationConnection: + elasticgraph_category: relay_connection + ElectricalPartAggregationEdge: + elasticgraph_category: relay_edge + ElectricalPartConnection: + elasticgraph_category: relay_connection + ElectricalPartEdge: + elasticgraph_category: relay_edge + FloatAggregatedValues: + elasticgraph_category: scalar_aggregated_values + graphql_fields_by_name: + approximate_avg: + computation_detail: + function: avg + approximate_distinct_value_count: + computation_detail: + empty_bucket_value: 0 + function: cardinality + approximate_sum: + computation_detail: + empty_bucket_value: 0 + function: sum + exact_max: + computation_detail: + function: max + exact_min: + computation_detail: + function: min + graphql_only_return_type: true + GeoLocation: + graphql_fields_by_name: + latitude: + name_in_index: lat + longitude: + name_in_index: lon + IDListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + IntAggregatedValues: + elasticgraph_category: scalar_aggregated_values + graphql_fields_by_name: + approximate_avg: + computation_detail: + function: avg + approximate_distinct_value_count: + computation_detail: + empty_bucket_value: 0 + function: cardinality + approximate_sum: + computation_detail: + empty_bucket_value: 0 + function: sum + exact_max: + computation_detail: + function: max + exact_min: + computation_detail: + function: min + exact_sum: + computation_detail: + empty_bucket_value: 0 + function: sum + graphql_only_return_type: true + IntListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + JsonSafeLongAggregatedValues: + elasticgraph_category: scalar_aggregated_values + graphql_fields_by_name: + approximate_avg: + computation_detail: + function: avg + approximate_distinct_value_count: + computation_detail: + empty_bucket_value: 0 + function: cardinality + approximate_sum: + computation_detail: + empty_bucket_value: 0 + function: sum + exact_max: + computation_detail: + function: max + exact_min: + computation_detail: + function: min + exact_sum: + computation_detail: + empty_bucket_value: 0 + function: sum + graphql_only_return_type: true + JsonSafeLongListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + LocalTimeAggregatedValues: + elasticgraph_category: scalar_aggregated_values + graphql_fields_by_name: + approximate_avg: + computation_detail: + function: avg + approximate_distinct_value_count: + computation_detail: + empty_bucket_value: 0 + function: cardinality + exact_max: + computation_detail: + function: max + exact_min: + computation_detail: + function: min + graphql_only_return_type: true + LongStringAggregatedValues: + elasticgraph_category: scalar_aggregated_values + graphql_fields_by_name: + approximate_avg: + computation_detail: + function: avg + approximate_distinct_value_count: + computation_detail: + empty_bucket_value: 0 + function: cardinality + approximate_max: + computation_detail: + function: max + approximate_min: + computation_detail: + function: min + approximate_sum: + computation_detail: + empty_bucket_value: 0 + function: sum + exact_max: + computation_detail: + function: max + exact_min: + computation_detail: + function: min + exact_sum: + computation_detail: + empty_bucket_value: 0 + function: sum + graphql_only_return_type: true + Manufacturer: + graphql_fields_by_name: + address: + relation: + direction: in + foreign_key: manufacturer_id + manufactured_part_aggregations: + relation: + direction: in + foreign_key: manufacturer_id + manufactured_parts: + relation: + direction: in + foreign_key: manufacturer_id + index_definition_names: + - manufacturers + update_targets: + - data_params: + created_at: + cardinality: one + name: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + routing_value_source: id + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: Manufacturer + ManufacturerAggregation: + elasticgraph_category: indexed_aggregation + source_type: Manufacturer + ManufacturerAggregationConnection: + elasticgraph_category: relay_connection + ManufacturerAggregationEdge: + elasticgraph_category: relay_edge + ManufacturerConnection: + elasticgraph_category: relay_connection + ManufacturerEdge: + elasticgraph_category: relay_edge + MechanicalPart: + graphql_fields_by_name: + component_aggregations: + relation: + direction: in + foreign_key: part_ids + components: + relation: + direction: in + foreign_key: part_ids + manufacturer: + relation: + direction: out + foreign_key: manufacturer_id + index_definition_names: + - mechanical_parts + update_targets: + - data_params: + created_at: + cardinality: one + manufacturer_id: + cardinality: one + material: + cardinality: one + name: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + routing_value_source: id + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: MechanicalPart + MechanicalPartAggregation: + elasticgraph_category: indexed_aggregation + source_type: MechanicalPart + MechanicalPartAggregationConnection: + elasticgraph_category: relay_connection + MechanicalPartAggregationEdge: + elasticgraph_category: relay_edge + MechanicalPartConnection: + elasticgraph_category: relay_connection + MechanicalPartEdge: + elasticgraph_category: relay_edge + MoneyFieldsListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + MoneyListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + NamedEntity: + graphql_fields_by_name: + address: + relation: + direction: in + foreign_key: manufacturer_id + amount_cents2: + name_in_index: amount_cents + component_aggregations: + relation: + direction: out + foreign_key: component_ids + components: + relation: + direction: out + foreign_key: component_ids + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + dollar_widget: + relation: + additional_filter: + cost: + amount_cents: + equal_to_any_of: + - 100 + direction: in + foreign_key: component_ids + manufactured_part_aggregations: + relation: + direction: in + foreign_key: manufacturer_id + manufactured_parts: + relation: + direction: in + foreign_key: manufacturer_id + manufacturer: + relation: + direction: out + foreign_key: manufacturer_id + part_aggregations: + relation: + direction: out + foreign_key: part_ids + parts: + relation: + direction: out + foreign_key: part_ids + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + widget: + relation: + direction: in + foreign_key: component_ids + widget_aggregations: + relation: + direction: in + foreign_key: component_ids + widget_workspace_id: + name_in_index: widget_workspace_id3 + widgets: + relation: + direction: in + foreign_key: component_ids + workspace: + relation: + direction: in + foreign_key: widget.id + workspace_id: + name_in_index: workspace_id2 + NamedEntityAggregatedValues: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + widget_workspace_id: + name_in_index: widget_workspace_id3 + workspace_id: + name_in_index: workspace_id2 + NamedEntityAggregation: + elasticgraph_category: indexed_aggregation + source_type: NamedEntity + NamedEntityAggregationConnection: + elasticgraph_category: relay_connection + NamedEntityAggregationEdge: + elasticgraph_category: relay_edge + NamedEntityConnection: + elasticgraph_category: relay_connection + NamedEntityEdge: + elasticgraph_category: relay_edge + NamedEntityFilterInput: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + widget_workspace_id: + name_in_index: widget_workspace_id3 + workspace_id: + name_in_index: workspace_id2 + NamedEntityGroupedBy: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + release_date: + name_in_index: release_dates + release_timestamp: + name_in_index: release_timestamps + size: + name_in_index: options.size + tag: + name_in_index: tags + the_options: + name_in_index: the_opts + widget_workspace_id: + name_in_index: widget_workspace_id3 + workspace_id: + name_in_index: workspace_id2 + NonNumericAggregatedValues: + elasticgraph_category: scalar_aggregated_values + graphql_fields_by_name: + approximate_distinct_value_count: + computation_detail: + empty_bucket_value: 0 + function: cardinality + graphql_only_return_type: true + PageInfo: + graphql_only_return_type: true + Part: + graphql_fields_by_name: + component_aggregations: + relation: + direction: in + foreign_key: part_ids + components: + relation: + direction: in + foreign_key: part_ids + manufacturer: + relation: + direction: out + foreign_key: manufacturer_id + PartAggregation: + elasticgraph_category: indexed_aggregation + source_type: Part + PartAggregationConnection: + elasticgraph_category: relay_connection + PartAggregationEdge: + elasticgraph_category: relay_edge + PartConnection: + elasticgraph_category: relay_connection + PartEdge: + elasticgraph_category: relay_edge + PlayerFieldsListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + PlayerListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + PlayerSeasonFieldsListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + PlayerSeasonListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + SizeListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + Sponsor: + graphql_fields_by_name: + affiliated_team_from_nested_aggregations: + relation: + direction: in + foreign_key: current_players_nested.affiliations.sponsorships_nested.sponsor_id + foreign_key_nested_paths: + - current_players_nested + - current_players_nested.affiliations.sponsorships_nested + affiliated_team_from_object_aggregations: + relation: + direction: in + foreign_key: current_players_object.affiliations.sponsorships_object.sponsor_id + affiliated_teams_from_nested: + relation: + direction: in + foreign_key: current_players_nested.affiliations.sponsorships_nested.sponsor_id + foreign_key_nested_paths: + - current_players_nested + - current_players_nested.affiliations.sponsorships_nested + affiliated_teams_from_object: + relation: + direction: in + foreign_key: current_players_object.affiliations.sponsorships_object.sponsor_id + index_definition_names: + - sponsors + update_targets: + - data_params: + name: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + routing_value_source: id + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: Sponsor + SponsorAggregation: + elasticgraph_category: indexed_aggregation + source_type: Sponsor + SponsorAggregationConnection: + elasticgraph_category: relay_connection + SponsorAggregationEdge: + elasticgraph_category: relay_edge + SponsorConnection: + elasticgraph_category: relay_connection + SponsorEdge: + elasticgraph_category: relay_edge + SponsorshipFieldsListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + SponsorshipListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + StringConnection: + elasticgraph_category: relay_connection + StringEdge: + elasticgraph_category: relay_edge + StringListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + Team: + graphql_fields_by_name: + nested_fields: + name_in_index: the_nested_fields + index_definition_names: + - teams + update_targets: + - data_params: + country_code: + cardinality: one + current_name: + cardinality: one + current_players_nested: + cardinality: one + current_players_object: + cardinality: one + details: + cardinality: one + forbes_valuation_moneys_nested: + cardinality: one + forbes_valuation_moneys_object: + cardinality: one + forbes_valuations: + cardinality: one + formed_on: + cardinality: one + league: + cardinality: one + nested_fields2: + cardinality: one + past_names: + cardinality: one + seasons_nested: + cardinality: one + seasons_object: + cardinality: one + stadium_location: + cardinality: one + the_nested_fields: + cardinality: one + won_championships_at: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + rollover_timestamp_value_source: formed_on + routing_value_source: league + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: Team + TeamAggregation: + elasticgraph_category: indexed_aggregation + source_type: Team + TeamAggregationConnection: + elasticgraph_category: relay_connection + TeamAggregationEdge: + elasticgraph_category: relay_edge + TeamAggregationNestedFields2SubAggregations: + graphql_fields_by_name: + seasons: + name_in_index: the_seasons + TeamAggregationNestedFieldsSubAggregations: + graphql_fields_by_name: + seasons: + name_in_index: the_seasons + TeamAggregationSubAggregations: + graphql_fields_by_name: + nested_fields: + name_in_index: the_nested_fields + TeamConnection: + elasticgraph_category: relay_connection + TeamEdge: + elasticgraph_category: relay_edge + TeamFilterInput: + graphql_fields_by_name: + nested_fields: + name_in_index: the_nested_fields + TeamMoneySubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamNestedFields: + graphql_fields_by_name: + seasons: + name_in_index: the_seasons + TeamNestedFieldsFilterInput: + graphql_fields_by_name: + seasons: + name_in_index: the_seasons + TeamPlayerPlayerSeasonSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamPlayerSeasonSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamPlayerSponsorshipSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamPlayerSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamRecord: + graphql_fields_by_name: + first_win_on_legacy: + name_in_index: first_win_on + last_win_on: + name_in_index: last_win_date + last_win_on_legacy: + name_in_index: last_win_date + losses: + name_in_index: loss_count + wins: + name_in_index: win_count + TeamRecordAggregatedValues: + graphql_fields_by_name: + first_win_on_legacy: + name_in_index: first_win_on + last_win_on: + name_in_index: last_win_date + last_win_on_legacy: + name_in_index: last_win_date + losses: + name_in_index: loss_count + wins: + name_in_index: win_count + TeamRecordFieldsListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + first_win_on_legacy: + name_in_index: first_win_on + last_win_on: + name_in_index: last_win_date + last_win_on_legacy: + name_in_index: last_win_date + losses: + name_in_index: loss_count + wins: + name_in_index: win_count + TeamRecordFilterInput: + graphql_fields_by_name: + first_win_on_legacy: + name_in_index: first_win_on + last_win_on: + name_in_index: last_win_date + last_win_on_legacy: + name_in_index: last_win_date + losses: + name_in_index: loss_count + wins: + name_in_index: win_count + TeamRecordGroupedBy: + graphql_fields_by_name: + first_win_on_legacy: + name_in_index: first_win_on + last_win_on: + name_in_index: last_win_date + last_win_on_legacy: + name_in_index: last_win_date + losses: + name_in_index: loss_count + wins: + name_in_index: win_count + TeamSeason: + graphql_fields_by_name: + record: + name_in_index: the_record + started_at_legacy: + name_in_index: started_at + won_games_at_legacy: + name_in_index: won_games_at + TeamSeasonAggregatedValues: + graphql_fields_by_name: + record: + name_in_index: the_record + started_at_legacy: + name_in_index: started_at + won_games_at_legacy: + name_in_index: won_games_at + TeamSeasonFieldsListFilterInput: + graphql_fields_by_name: + record: + name_in_index: the_record + started_at_legacy: + name_in_index: started_at + won_games_at_legacy: + name_in_index: won_games_at + TeamSeasonFilterInput: + graphql_fields_by_name: + record: + name_in_index: the_record + started_at_legacy: + name_in_index: started_at + won_games_at_legacy: + name_in_index: won_games_at + TeamSeasonGroupedBy: + graphql_fields_by_name: + note: + name_in_index: notes + record: + name_in_index: the_record + started_at_legacy: + name_in_index: started_at + won_game_at: + name_in_index: won_games_at + won_game_at_legacy: + name_in_index: won_games_at + TeamSeasonListFilterInput: + graphql_fields_by_name: + count: + name_in_index: __counts + TeamSponsorshipSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamTeamSeasonPlayerPlayerSeasonSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamTeamSeasonPlayerSeasonSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamTeamSeasonPlayerSponsorshipSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamTeamSeasonPlayerSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamTeamSeasonSponsorshipSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + TeamTeamSeasonSubAggregationConnection: + elasticgraph_category: nested_sub_aggregation_connection + Widget: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + component_aggregations: + relation: + direction: out + foreign_key: component_ids + components: + relation: + direction: out + foreign_key: component_ids + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + workspace: + relation: + direction: in + foreign_key: widget.id + workspace_id: + name_in_index: workspace_id2 + index_definition_names: + - widgets + update_targets: + - data_params: + cost.amount_cents: + cardinality: many + cost_currency_introduced_on: + cardinality: many + cost_currency_name: + cardinality: many + cost_currency_primary_continent: + cardinality: many + cost_currency_symbol: + cardinality: many + cost_currency_unit: + cardinality: many + created_at: + cardinality: many + fees.currency: + cardinality: many + name: + cardinality: many + options.color: + cardinality: many + options.size: + cardinality: many + tags: + cardinality: many + id_source: cost.currency + rollover_timestamp_value_source: cost_currency_introduced_on + routing_value_source: cost_currency_primary_continent + script_id: update_WidgetCurrency_from_Widget_0f26b3e9ea093af29e5cef02a25e75ca + type: WidgetCurrency + - data_params: + amount_cents: + cardinality: one + amounts: + cardinality: one + component_ids: + cardinality: one + cost: + cardinality: one + cost_currency_introduced_on: + cardinality: one + cost_currency_name: + cardinality: one + cost_currency_primary_continent: + cardinality: one + cost_currency_symbol: + cardinality: one + cost_currency_unit: + cardinality: one + created_at: + cardinality: one + created_at_time_of_day: + cardinality: one + created_on: + cardinality: one + fees: + cardinality: one + inventor: + cardinality: one + metadata: + cardinality: one + name: + cardinality: one + name_text: + cardinality: one + named_inventor: + cardinality: one + options: + cardinality: one + release_dates: + cardinality: one + release_timestamps: + cardinality: one + tags: + cardinality: one + the_opts: + cardinality: one + weight_in_ng: + cardinality: one + weight_in_ng_str: + cardinality: one + workspace_id2: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + rollover_timestamp_value_source: created_at + routing_value_source: workspace_id2 + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: Widget + - data_params: + widget_cost: + cardinality: one + source_path: cost + widget_name: + cardinality: one + source_path: name + widget_size: + cardinality: one + source_path: the_opts.the_sighs + widget_tags: + cardinality: one + source_path: tags + widget_workspace_id3: + cardinality: one + source_path: workspace_id2 + id_source: component_ids + metadata_params: + relationship: + value: widget + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: widget + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: Component + WidgetAggregatedValues: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + workspace_id: + name_in_index: workspace_id2 + WidgetAggregation: + elasticgraph_category: indexed_aggregation + source_type: Widget + WidgetAggregationConnection: + elasticgraph_category: relay_connection + WidgetAggregationEdge: + elasticgraph_category: relay_edge + WidgetConnection: + elasticgraph_category: relay_connection + WidgetCurrency: + graphql_fields_by_name: + widget_names: + name_in_index: widget_names2 + index_definition_names: + - widget_currencies + update_targets: + - data_params: + details: + cardinality: one + introduced_on: + cardinality: one + name: + cardinality: one + nested_fields: + cardinality: one + oldest_widget_created_at: + cardinality: one + primary_continent: + cardinality: one + widget_fee_currencies: + cardinality: one + widget_names2: + cardinality: one + widget_options: + cardinality: one + widget_tags: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + rollover_timestamp_value_source: introduced_on + routing_value_source: primary_continent + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: WidgetCurrency + WidgetCurrencyAggregatedValues: + graphql_fields_by_name: + widget_names: + name_in_index: widget_names2 + WidgetCurrencyAggregation: + elasticgraph_category: indexed_aggregation + source_type: WidgetCurrency + WidgetCurrencyAggregationConnection: + elasticgraph_category: relay_connection + WidgetCurrencyAggregationEdge: + elasticgraph_category: relay_edge + WidgetCurrencyConnection: + elasticgraph_category: relay_connection + WidgetCurrencyEdge: + elasticgraph_category: relay_edge + WidgetCurrencyFilterInput: + graphql_fields_by_name: + widget_names: + name_in_index: widget_names2 + WidgetCurrencyGroupedBy: + graphql_fields_by_name: + widget_name: + name_in_index: widget_names2 + WidgetEdge: + elasticgraph_category: relay_edge + WidgetFilterInput: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + workspace_id: + name_in_index: workspace_id2 + WidgetGroupedBy: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + release_date: + name_in_index: release_dates + release_timestamp: + name_in_index: release_timestamps + size: + name_in_index: options.size + tag: + name_in_index: tags + the_options: + name_in_index: the_opts + workspace_id: + name_in_index: workspace_id2 + WidgetOptions: + graphql_fields_by_name: + the_size: + name_in_index: the_sighs + WidgetOptionsAggregatedValues: + graphql_fields_by_name: + the_size: + name_in_index: the_sighs + WidgetOptionsFilterInput: + graphql_fields_by_name: + the_size: + name_in_index: the_sighs + WidgetOptionsGroupedBy: + graphql_fields_by_name: + the_size: + name_in_index: the_sighs + WidgetOrAddress: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + component_aggregations: + relation: + direction: out + foreign_key: component_ids + components: + relation: + direction: out + foreign_key: component_ids + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + manufacturer: + relation: + direction: out + foreign_key: manufacturer_id + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + workspace: + relation: + direction: in + foreign_key: widget.id + workspace_id: + name_in_index: workspace_id2 + WidgetOrAddressAggregatedValues: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + workspace_id: + name_in_index: workspace_id2 + WidgetOrAddressAggregation: + elasticgraph_category: indexed_aggregation + source_type: WidgetOrAddress + WidgetOrAddressAggregationConnection: + elasticgraph_category: relay_connection + WidgetOrAddressAggregationEdge: + elasticgraph_category: relay_edge + WidgetOrAddressConnection: + elasticgraph_category: relay_connection + WidgetOrAddressEdge: + elasticgraph_category: relay_edge + WidgetOrAddressFilterInput: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + size: + name_in_index: options.size + the_options: + name_in_index: the_opts + workspace_id: + name_in_index: workspace_id2 + WidgetOrAddressGroupedBy: + graphql_fields_by_name: + amount_cents2: + name_in_index: amount_cents + created_at2: + name_in_index: created_at + created_at2_legacy: + name_in_index: created_at + created_at_legacy: + name_in_index: created_at + created_on_legacy: + name_in_index: created_on + release_date: + name_in_index: release_dates + release_timestamp: + name_in_index: release_timestamps + size: + name_in_index: options.size + tag: + name_in_index: tags + the_options: + name_in_index: the_opts + workspace_id: + name_in_index: workspace_id2 + WidgetWorkspace: + index_definition_names: + - widget_workspaces + update_targets: + - data_params: + name: + cardinality: one + widget: + cardinality: one + id_source: id + metadata_params: + relationship: + value: __self + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: __self + routing_value_source: id + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: WidgetWorkspace + - data_params: + workspace_name: + cardinality: one + source_path: name + id_source: widget.id + metadata_params: + relationship: + value: workspace + sourceId: + cardinality: one + source_path: id + sourceType: + cardinality: one + source_path: type + version: + cardinality: one + relationship: workspace + rollover_timestamp_value_source: widget.created_at + routing_value_source: id + script_id: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 + type: Widget + WidgetWorkspaceAggregation: + elasticgraph_category: indexed_aggregation + source_type: WidgetWorkspace + WidgetWorkspaceAggregationConnection: + elasticgraph_category: relay_connection + WidgetWorkspaceAggregationEdge: + elasticgraph_category: relay_edge + WidgetWorkspaceConnection: + elasticgraph_category: relay_connection + WidgetWorkspaceEdge: + elasticgraph_category: relay_edge + _Entity: + graphql_only_return_type: true + _Service: + graphql_only_return_type: true +scalar_types_by_name: + Boolean: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::NoOp + require_path: elastic_graph/graphql/scalar_coercion_adapters/no_op + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + Cursor: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::Cursor + require_path: elastic_graph/graphql/scalar_coercion_adapters/cursor + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + Date: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::Date + require_path: elastic_graph/graphql/scalar_coercion_adapters/date + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + DateTime: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::DateTime + require_path: elastic_graph/graphql/scalar_coercion_adapters/date_time + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + FieldSet: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::NoOp + require_path: elastic_graph/graphql/scalar_coercion_adapters/no_op + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + Float: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::NoOp + require_path: elastic_graph/graphql/scalar_coercion_adapters/no_op + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + ID: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::NoOp + require_path: elastic_graph/graphql/scalar_coercion_adapters/no_op + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + Int: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::NoOp + require_path: elastic_graph/graphql/scalar_coercion_adapters/no_op + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::Integer + require_path: elastic_graph/indexer/indexing_preparers/integer + JsonSafeLong: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::JsonSafeLong + require_path: elastic_graph/graphql/scalar_coercion_adapters/longs + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::Integer + require_path: elastic_graph/indexer/indexing_preparers/integer + LocalTime: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::LocalTime + require_path: elastic_graph/graphql/scalar_coercion_adapters/local_time + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + LongString: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::LongString + require_path: elastic_graph/graphql/scalar_coercion_adapters/longs + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::Integer + require_path: elastic_graph/indexer/indexing_preparers/integer + String: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::NoOp + require_path: elastic_graph/graphql/scalar_coercion_adapters/no_op + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + TimeZone: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::TimeZone + require_path: elastic_graph/graphql/scalar_coercion_adapters/time_zone + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + Untyped: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::Untyped + require_path: elastic_graph/graphql/scalar_coercion_adapters/untyped + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::Untyped + require_path: elastic_graph/indexer/indexing_preparers/untyped + _Any: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::NoOp + require_path: elastic_graph/graphql/scalar_coercion_adapters/no_op + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + federation__Policy: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::NoOp + require_path: elastic_graph/graphql/scalar_coercion_adapters/no_op + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + federation__Scope: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::NoOp + require_path: elastic_graph/graphql/scalar_coercion_adapters/no_op + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op + link__Import: + coercion_adapter: + extension_name: ElasticGraph::GraphQL::ScalarCoercionAdapters::NoOp + require_path: elastic_graph/graphql/scalar_coercion_adapters/no_op + indexing_preparer: + extension_name: ElasticGraph::Indexer::IndexingPreparers::NoOp + require_path: elastic_graph/indexer/indexing_preparers/no_op +schema_element_names: + form: snake_case +static_script_ids_by_scoped_name: + field/as_day_of_week: field_as_day_of_week_f2b5c7d9e8f75bf2457b52412bfb6537 + field/as_time_of_day: field_as_time_of_day_ed82aba44fc66bff5635bec4305c1c66 + filter/by_time_of_day: filter_by_time_of_day_ea12d0561b24961789ab68ed38435612 + update/index_data: update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413 diff --git a/config/schema/artifacts_with_apollo/schema.graphql b/config/schema/artifacts_with_apollo/schema.graphql new file mode 100644 index 00000000..0f9dcc68 --- /dev/null +++ b/config/schema/artifacts_with_apollo/schema.graphql @@ -0,0 +1,15225 @@ +# Generated by `rake schema_artifacts:dump`. +# DO NOT EDIT BY HAND. Any edits will be lost the next time the rake task is run. + +extend schema + @link(import: ["@authenticated", "@composeDirective", "@extends", "@external", "@inaccessible", "@interfaceObject", "@key", "@override", "@policy", "@provides", "@requires", "@requiresScopes", "@shareable", "@tag", "FieldSet"], url: "https://specs.apollo.dev/federation/v2.6") + +directive @authenticated on ENUM | FIELD_DEFINITION | INTERFACE | OBJECT | SCALAR + +directive @composeDirective(name: String!) repeatable on SCHEMA + +""" +Indicates an upper bound on how quickly a query must respond to meet the service-level objective. +ElasticGraph will log a "good event" message if the query latency is less than or equal to this value, +and a "bad event" message if the query latency is greater than this value. These messages can be used +to drive an SLO dashboard. + +Note that the latency compared against this only contains processing time within ElasticGraph itself. +Any time spent on sending the request or response over the network is not included in the comparison. +""" +directive @eg_latency_slo(ms: Int!) on QUERY + +directive @extends on INTERFACE | OBJECT + +directive @external on FIELD_DEFINITION | OBJECT + +directive @inaccessible on ARGUMENT_DEFINITION | ENUM | ENUM_VALUE | FIELD_DEFINITION | INPUT_FIELD_DEFINITION | INPUT_OBJECT | INTERFACE | OBJECT | SCALAR | UNION + +directive @interfaceObject on OBJECT + +directive @key(fields: FieldSet!, resolvable: Boolean = true) repeatable on INTERFACE | OBJECT + +directive @link(as: String, for: link__Purpose, import: [link__Import], url: String!) repeatable on SCHEMA + +directive @override(from: String!) on FIELD_DEFINITION + +directive @policy(policies: [[federation__Policy!]!]!) on ENUM | FIELD_DEFINITION | INTERFACE | OBJECT | SCALAR + +directive @provides(fields: FieldSet!) on FIELD_DEFINITION + +directive @requires(fields: FieldSet!) on FIELD_DEFINITION + +directive @requiresScopes(scopes: [[federation__Scope!]!]!) on ENUM | FIELD_DEFINITION | INTERFACE | OBJECT | SCALAR + +directive @shareable on FIELD_DEFINITION | OBJECT + +directive @tag(name: String!) repeatable on ARGUMENT_DEFINITION | ENUM | ENUM_VALUE | FIELD_DEFINITION | INPUT_FIELD_DEFINITION | INPUT_OBJECT | INTERFACE | OBJECT | SCALAR | UNION + +type Address { + full_address: String! + geo_location: GeoLocation + manufacturer: Manufacturer + shapes: [GeoShape!]! + timestamps: AddressTimestamps +} + +""" +Type used to perform aggregation computations on `Address` fields. +""" +type AddressAggregatedValues { + """ + Computed aggregate values for the `full_address` field. + """ + full_address: NonNumericAggregatedValues + + """ + Computed aggregate values for the `geo_location` field. + """ + geo_location: NonNumericAggregatedValues + + """ + Computed aggregate values for the `shapes` field. + """ + shapes: NonNumericAggregatedValues + + """ + Computed aggregate values for the `timestamps` field. + """ + timestamps: AddressTimestampsAggregatedValues +} + +""" +Return type representing a bucket of `Address` documents for an aggregations query. +""" +type AddressAggregation { + """ + Provides computed aggregated values over all `Address` documents in an aggregation bucket. + """ + aggregated_values: AddressAggregatedValues + + """ + The count of `Address` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `Address` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: AddressGroupedBy +} + +""" +Represents a paginated collection of `AddressAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type AddressAggregationConnection { + """ + Wraps a specific `AddressAggregation` to pair it with its pagination cursor. + """ + edges: [AddressAggregationEdge!]! + + """ + The list of `AddressAggregation` results. + """ + nodes: [AddressAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `AddressAggregation` in the context of a `AddressAggregationConnection`, +providing access to both the `AddressAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type AddressAggregationEdge { + """ + The `Cursor` of this `AddressAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `AddressAggregation`. + """ + cursor: Cursor + + """ + The `AddressAggregation` of this edge. + """ + node: AddressAggregation +} + +""" +Represents a paginated collection of `Address` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type AddressConnection { + """ + Wraps a specific `Address` to pair it with its pagination cursor. + """ + edges: [AddressEdge!]! + + """ + The list of `Address` results. + """ + nodes: [Address!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `Address` in the context of a `AddressConnection`, +providing access to both the `Address` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type AddressEdge { + """ + The `Cursor` of this `Address`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `Address`. + """ + cursor: Cursor + + """ + The `Address` of this edge. + """ + node: Address +} + +""" +Input type used to specify filters on `Address` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input AddressFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [AddressFilterInput!] + + """ + Used to filter on the `full_address` field. + + Will be ignored if `null` or an empty object is passed. + """ + full_address: StringFilterInput + + """ + Used to filter on the `geo_location` field. + + Will be ignored if `null` or an empty object is passed. + """ + geo_location: GeoLocationFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: AddressFilterInput + + """ + Used to filter on the `timestamps` field. + + Will be ignored if `null` or an empty object is passed. + """ + timestamps: AddressTimestampsFilterInput +} + +""" +Type used to specify the `Address` fields to group by for aggregations. +""" +type AddressGroupedBy { + """ + The `full_address` field value for this group. + """ + full_address: String + + """ + The `timestamps` field value for this group. + """ + timestamps: AddressTimestampsGroupedBy +} + +""" +Enumerates the ways `Address`s can be sorted. +""" +enum AddressSortOrderInput { + """ + Sorts ascending by the `full_address` field. + """ + full_address_ASC + + """ + Sorts descending by the `full_address` field. + """ + full_address_DESC + + """ + Sorts ascending by the `timestamps.created_at` field. + """ + timestamps_created_at_ASC + + """ + Sorts descending by the `timestamps.created_at` field. + """ + timestamps_created_at_DESC +} + +type AddressTimestamps { + created_at: DateTime +} + +""" +Type used to perform aggregation computations on `AddressTimestamps` fields. +""" +type AddressTimestampsAggregatedValues { + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues +} + +""" +Input type used to specify filters on `AddressTimestamps` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input AddressTimestampsFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [AddressTimestampsFilterInput!] + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: AddressTimestampsFilterInput +} + +""" +Type used to specify the `AddressTimestamps` fields to group by for aggregations. +""" +type AddressTimestampsGroupedBy { + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy +} + +type Affiliations { + sponsorships_nested: [Sponsorship!]! + sponsorships_object: [Sponsorship!]! +} + +""" +Type used to perform aggregation computations on `Affiliations` fields. +""" +type AffiliationsAggregatedValues { + """ + Computed aggregate values for the `sponsorships_object` field. + """ + sponsorships_object: SponsorshipAggregatedValues +} + +""" +Input type used to specify filters on a `Affiliations` object referenced directly +or transitively from a list field that has been configured to index each leaf field as +its own flattened list of values. + +Will be ignored if passed as an empty object (or as `null`). +""" +input AffiliationsFieldsListFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [AffiliationsFieldsListFilterInput!] + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: AffiliationsFieldsListFilterInput + + """ + Used to filter on the `sponsorships_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + sponsorships_nested: SponsorshipListFilterInput + + """ + Used to filter on the `sponsorships_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + sponsorships_object: SponsorshipFieldsListFilterInput +} + +""" +Input type used to specify filters on `Affiliations` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input AffiliationsFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [AffiliationsFilterInput!] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: AffiliationsFilterInput + + """ + Used to filter on the `sponsorships_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + sponsorships_nested: SponsorshipListFilterInput + + """ + Used to filter on the `sponsorships_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + sponsorships_object: SponsorshipFieldsListFilterInput +} + +""" +Type used to specify the `Affiliations` fields to group by for aggregations. +""" +type AffiliationsGroupedBy { + """ + The `sponsorships_object` field value for this group. + + Note: `sponsorships_object` is a collection field, but selecting this field + will group on individual values of the selected subfields of + `sponsorships_object`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `sponsorships_object` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `sponsorships_object` multiple times for a single document, that document will only be included in the group + once. + """ + sponsorships_object: SponsorshipGroupedBy +} + +""" +Provides detail about an aggregation `count`. +""" +type AggregationCountDetail @shareable { + """ + The (approximate) count of documents in this aggregation bucket. + + When documents in an aggregation bucket are sourced from multiple shards, the count may be only + approximate. The `upper_bound` indicates the maximum value of the true count, but usually + the true count is much closer to this approximate value (which also provides a lower bound on the + true count). + + When this approximation is known to be exact, the same value will be available from `exact_value` + and `upper_bound`. + """ + approximate_value: JsonSafeLong! + + """ + The exact count of documents in this aggregation bucket, if an exact value can be determined. + + When documents in an aggregation bucket are sourced from multiple shards, it may not be possible to + efficiently determine an exact value. When no exact value can be determined, this field will be `null`. + The `approximate_value` field--which will never be `null`--can be used to get an approximation + for the count. + """ + exact_value: JsonSafeLong + + """ + An upper bound on how large the true count of documents in this aggregation bucket could be. + + When documents in an aggregation bucket are sourced from multiple shards, it may not be possible to + efficiently determine an exact value. The `approximate_value` field provides an approximation, + and this field puts an upper bound on the true count. + """ + upper_bound: JsonSafeLong! +} + +enum Color { + BLUE + GREEN + RED +} + +""" +Input type used to specify filters on `Color` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input ColorFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [ColorFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [ColorInput] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: ColorFilterInput +} + +enum ColorInput { + BLUE + GREEN + RED +} + +""" +Input type used to specify filters on elements of a `[Color]` field. + +Will be ignored if passed as an empty object (or as `null`). +""" +input ColorListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [ColorListElementFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [ColorInput!] +} + +""" +Input type used to specify filters on `[Color]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input ColorListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `ColorListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [ColorListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [ColorListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: ColorListElementFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: ColorListFilterInput +} + +type Company implements NamedInventor { + name: String + stock_ticker: String +} + +type Component implements NamedEntity @key(fields: "id") { + created_at: DateTime! + dollar_widget: Widget + id: ID! + name: String + + """ + Aggregations over the `parts` data. + """ + part_aggregations( + """ + Used to forward-paginate through the `part_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `part_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Part` documents that get aggregated over based on the provided criteria. + """ + filter: PartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `part_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `part_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `part_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `part_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): PartAggregationConnection + parts( + """ + Used to forward-paginate through the `parts`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `parts`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `parts` based on the provided criteria. + """ + filter: PartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `parts`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `parts`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `parts`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `parts`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `parts` should be sorted. + """ + order_by: [PartSortOrderInput!] + ): PartConnection + position: Position! + tags: [String!]! + widget: Widget + + """ + Aggregations over the `widgets` data. + """ + widget_aggregations( + """ + Used to forward-paginate through the `widget_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widget_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Widget` documents that get aggregated over based on the provided criteria. + """ + filter: WidgetFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widget_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widget_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widget_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widget_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): WidgetAggregationConnection + widget_cost: Money + widget_name: String + widget_size: Size + widget_tags: [String!] + widget_workspace_id: ID + widgets( + """ + Used to forward-paginate through the `widgets`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widgets`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `widgets` based on the provided criteria. + """ + filter: WidgetFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widgets`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widgets`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widgets`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widgets`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `widgets` should be sorted. + """ + order_by: [WidgetSortOrderInput!] + ): WidgetConnection +} + +""" +Type used to perform aggregation computations on `Component` fields. +""" +type ComponentAggregatedValues { + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `position` field. + """ + position: PositionAggregatedValues + + """ + Computed aggregate values for the `tags` field. + """ + tags: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_cost` field. + """ + widget_cost: MoneyAggregatedValues + + """ + Computed aggregate values for the `widget_name` field. + """ + widget_name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_size` field. + """ + widget_size: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_tags` field. + """ + widget_tags: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_workspace_id` field. + """ + widget_workspace_id: NonNumericAggregatedValues +} + +""" +Return type representing a bucket of `Component` documents for an aggregations query. +""" +type ComponentAggregation { + """ + Provides computed aggregated values over all `Component` documents in an aggregation bucket. + """ + aggregated_values: ComponentAggregatedValues + + """ + The count of `Component` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `Component` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: ComponentGroupedBy +} + +""" +Represents a paginated collection of `ComponentAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type ComponentAggregationConnection { + """ + Wraps a specific `ComponentAggregation` to pair it with its pagination cursor. + """ + edges: [ComponentAggregationEdge!]! + + """ + The list of `ComponentAggregation` results. + """ + nodes: [ComponentAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `ComponentAggregation` in the context of a `ComponentAggregationConnection`, +providing access to both the `ComponentAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type ComponentAggregationEdge { + """ + The `Cursor` of this `ComponentAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `ComponentAggregation`. + """ + cursor: Cursor + + """ + The `ComponentAggregation` of this edge. + """ + node: ComponentAggregation +} + +""" +Represents a paginated collection of `Component` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type ComponentConnection { + """ + Wraps a specific `Component` to pair it with its pagination cursor. + """ + edges: [ComponentEdge!]! + + """ + The list of `Component` results. + """ + nodes: [Component!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `Component` in the context of a `ComponentConnection`, +providing access to both the `Component` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type ComponentEdge { + """ + The `Cursor` of this `Component`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `Component`. + """ + cursor: Cursor + + """ + The `Component` of this edge. + """ + node: Component +} + +""" +Input type used to specify filters on `Component` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input ComponentFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [ComponentFilterInput!] + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: ComponentFilterInput + + """ + Used to filter on the `position` field. + + Will be ignored if `null` or an empty object is passed. + """ + position: PositionFilterInput + + """ + Used to filter on the `tags` field. + + Will be ignored if `null` or an empty object is passed. + """ + tags: StringListFilterInput + + """ + Used to filter on the `widget_cost` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_cost: MoneyFilterInput + + """ + Used to filter on the `widget_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_name: StringFilterInput + + """ + Used to filter on the `widget_size` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_size: SizeFilterInput + + """ + Used to filter on the `widget_tags` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_tags: StringListFilterInput + + """ + Used to filter on the `widget_workspace_id` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_workspace_id: IDFilterInput +} + +""" +Type used to specify the `Component` fields to group by for aggregations. +""" +type ComponentGroupedBy { + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + The `name` field value for this group. + """ + name: String + + """ + The `position` field value for this group. + """ + position: PositionGroupedBy + + """ + The `widget_cost` field value for this group. + """ + widget_cost: MoneyGroupedBy + + """ + The `widget_name` field value for this group. + """ + widget_name: String + + """ + The `widget_size` field value for this group. + """ + widget_size: Size + + """ + The `widget_workspace_id` field value for this group. + """ + widget_workspace_id: ID +} + +""" +Enumerates the ways `Component`s can be sorted. +""" +enum ComponentSortOrderInput { + """ + Sorts ascending by the `created_at` field. + """ + created_at_ASC + + """ + Sorts descending by the `created_at` field. + """ + created_at_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC + + """ + Sorts ascending by the `position.x` field. + """ + position_x_ASC + + """ + Sorts descending by the `position.x` field. + """ + position_x_DESC + + """ + Sorts ascending by the `position.y` field. + """ + position_y_ASC + + """ + Sorts descending by the `position.y` field. + """ + position_y_DESC + + """ + Sorts ascending by the `widget_cost.amount_cents` field. + """ + widget_cost_amount_cents_ASC + + """ + Sorts descending by the `widget_cost.amount_cents` field. + """ + widget_cost_amount_cents_DESC + + """ + Sorts ascending by the `widget_cost.currency` field. + """ + widget_cost_currency_ASC + + """ + Sorts descending by the `widget_cost.currency` field. + """ + widget_cost_currency_DESC + + """ + Sorts ascending by the `widget_name` field. + """ + widget_name_ASC + + """ + Sorts descending by the `widget_name` field. + """ + widget_name_DESC + + """ + Sorts ascending by the `widget_size` field. + """ + widget_size_ASC + + """ + Sorts descending by the `widget_size` field. + """ + widget_size_DESC + + """ + Sorts ascending by the `widget_workspace_id` field. + """ + widget_workspace_id_ASC + + """ + Sorts descending by the `widget_workspace_id` field. + """ + widget_workspace_id_DESC +} + +type Country @key(fields: "id") { + currency: String @external + id: ID! + names( + """ + Used to forward-paginate through the `names`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `names`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used in conjunction with the `after` argument to forward-paginate through the `names`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `names`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `names`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `names`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): StringConnection @external + + """ + Aggregations over the `teams` data. + """ + team_aggregations( + """ + Used to forward-paginate through the `team_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `team_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Team` documents that get aggregated over based on the provided criteria. + """ + filter: TeamFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `team_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `team_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `team_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `team_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): TeamAggregationConnection + teams( + """ + Used to forward-paginate through the `teams`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `teams`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `teams` based on the provided criteria. + """ + filter: TeamFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `teams`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `teams`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `teams`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `teams`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `teams` should be sorted. + """ + order_by: [TeamSortOrderInput!] + ): TeamConnection +} + +type CurrencyDetails { + symbol: String + unit: String +} + +""" +Type used to perform aggregation computations on `CurrencyDetails` fields. +""" +type CurrencyDetailsAggregatedValues { + """ + Computed aggregate values for the `symbol` field. + """ + symbol: NonNumericAggregatedValues + + """ + Computed aggregate values for the `unit` field. + """ + unit: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on `CurrencyDetails` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input CurrencyDetailsFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [CurrencyDetailsFilterInput!] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: CurrencyDetailsFilterInput + + """ + Used to filter on the `symbol` field. + + Will be ignored if `null` or an empty object is passed. + """ + symbol: StringFilterInput + + """ + Used to filter on the `unit` field. + + Will be ignored if `null` or an empty object is passed. + """ + unit: StringFilterInput +} + +""" +Type used to specify the `CurrencyDetails` fields to group by for aggregations. +""" +type CurrencyDetailsGroupedBy { + """ + The `symbol` field value for this group. + """ + symbol: String + + """ + The `unit` field value for this group. + """ + unit: String +} + +""" +An opaque string value representing a specific location in a paginated connection type. +Returned cursors can be passed back in the next query via the `before` or `after` +arguments to continue paginating from that point. +""" +scalar Cursor + +""" +A date, represented as an [ISO 8601 date string](https://en.wikipedia.org/wiki/ISO_8601). +""" +scalar Date + +""" +A return type used from aggregations to provided aggregated values over `Date` fields. +""" +type DateAggregatedValues @shareable { + """ + The average (mean) of the field values within this grouping. + The returned value will be rounded to the nearest `Date` value. + """ + approximate_avg: Date + + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in + Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + approximate_distinct_value_count: JsonSafeLong + + """ + The maximum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_max: Date + + """ + The minimum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_min: Date +} + +""" +Input type used to specify filters on `Date` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input DateFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [DateFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [Date] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: Date + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: Date + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: Date + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: Date + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: DateFilterInput +} + +""" +Allows for grouping `Date` values based on the desired return type. +""" +type DateGroupedBy @shareable { + """ + Used when grouping on the full `Date` value. + """ + as_date( + """ + Amount of offset (positive or negative) to shift the `Date` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 1 day to change what + day-of-week weeks are considered to start on. + """ + offset: DateGroupingOffsetInput + + """ + Determines the grouping truncation unit for this field. + """ + truncation_unit: DateGroupingTruncationUnitInput! + ): Date + + """ + An alternative to `as_date` for when grouping on the day-of-week is desired. + """ + as_day_of_week( + """ + Amount of offset (positive or negative) to shift the `DayOfWeek` boundaries of each grouping bucket. + + For example, you can apply an offset of -2 hours to shift `DateTime` values to the prior `DayOfWeek` + when they fall between midnight and 2 AM. + """ + offset: DayOfWeekGroupingOffsetInput + ): DayOfWeek +} + +""" +Enumerates the supported granularities of a `Date`. +""" +enum DateGroupingGranularityInput { + """ + The exact day of a `Date`. + """ + DAY + + """ + The month a `Date` falls in. + """ + MONTH + + """ + The quarter a `Date` falls in. + """ + QUARTER + + """ + The week, beginning on Monday, a `Date` falls in. + """ + WEEK + + """ + The year a `Date` falls in. + """ + YEAR +} + +""" +Input type offered when grouping on `Date` fields, representing the amount of offset +(positive or negative) to shift the `Date` boundaries of each grouping bucket. + +For example, when grouping by `WEEK`, you can shift by 1 day to change +what day-of-week weeks are considered to start on. +""" +input DateGroupingOffsetInput { + """ + Number (positive or negative) of the given `unit` to offset the boundaries of the `Date` groupings. + """ + amount: Int! + + """ + Unit of offsetting to apply to the boundaries of the `Date` groupings. + """ + unit: DateUnitInput! +} + +""" +Enumerates the supported truncation units of a `Date`. +""" +enum DateGroupingTruncationUnitInput { + """ + The exact day of a `Date`. + """ + DAY + + """ + The month a `Date` falls in. + """ + MONTH + + """ + The quarter a `Date` falls in. + """ + QUARTER + + """ + The week, beginning on Monday, a `Date` falls in. + """ + WEEK + + """ + The year a `Date` falls in. + """ + YEAR +} + +""" +Input type used to specify filters on elements of a `[Date]` field. + +Will be ignored if passed as an empty object (or as `null`). +""" +input DateListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [DateListElementFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [Date!] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: Date + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: Date + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: Date + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: Date +} + +""" +Input type used to specify filters on `[Date]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input DateListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `DateListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [DateListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [DateListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: DateListElementFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: DateListFilterInput +} + +""" +A timestamp, represented as an [ISO 8601 time string](https://en.wikipedia.org/wiki/ISO_8601). +""" +scalar DateTime + +""" +A return type used from aggregations to provided aggregated values over `DateTime` fields. +""" +type DateTimeAggregatedValues @shareable { + """ + The average (mean) of the field values within this grouping. + The returned value will be rounded to the nearest `DateTime` value. + """ + approximate_avg: DateTime + + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in + Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + approximate_distinct_value_count: JsonSafeLong + + """ + The maximum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_max: DateTime + + """ + The minimum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_min: DateTime +} + +""" +Input type used to specify filters on `DateTime` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input DateTimeFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [DateTimeFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [DateTime] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: DateTime + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: DateTime + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: DateTime + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: DateTime + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: DateTimeFilterInput + + """ + Matches records based on the time-of-day of the `DateTime` values. + + Will be ignored when `null` or an empty list is passed. + """ + time_of_day: DateTimeTimeOfDayFilterInput +} + +""" +Allows for grouping `DateTime` values based on the desired return type. +""" +type DateTimeGroupedBy @shareable { + """ + An alternative to `as_date_time` for when grouping on just the date is desired. + """ + as_date( + """ + Amount of offset (positive or negative) to shift the `Date` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 1 day to change what + day-of-week weeks are considered to start on. + """ + offset: DateGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `Date` value falls in. + """ + time_zone: TimeZone! = "UTC" + + """ + Determines the grouping truncation unit for this field. + """ + truncation_unit: DateGroupingTruncationUnitInput! + ): Date + + """ + Used when grouping on the full `DateTime` value. + """ + as_date_time( + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 1 day to change what + day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone! = "UTC" + + """ + Determines the grouping truncation unit for this field. + """ + truncation_unit: DateTimeGroupingTruncationUnitInput! + ): DateTime + + """ + An alternative to `as_date_time` for when grouping on the day-of-week is desired. + """ + as_day_of_week( + """ + Amount of offset (positive or negative) to shift the `DayOfWeek` boundaries of each grouping bucket. + + For example, you can apply an offset of -2 hours to shift `DateTime` values to the prior `DayOfWeek` + when they fall between midnight and 2 AM. + """ + offset: DayOfWeekGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DayOfWeek` value falls in. + """ + time_zone: TimeZone! = "UTC" + ): DayOfWeek + + """ + An alternative to `as_date_time` for when grouping on just the time-of-day is desired. + """ + as_time_of_day( + """ + Amount of offset (positive or negative) to shift the `LocalTime` boundaries of each grouping bucket. + + For example, when grouping by `HOUR`, you can apply an offset of -5 minutes to shift `LocalTime` + values to the prior hour when they fall between the the top of an hour and 5 after. + """ + offset: LocalTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `LocalTime` value falls in. + """ + time_zone: TimeZone! = "UTC" + + """ + Determines the grouping truncation unit for this field. + """ + truncation_unit: LocalTimeGroupingTruncationUnitInput! + ): LocalTime +} + +""" +Enumerates the supported granularities of a `DateTime`. +""" +enum DateTimeGroupingGranularityInput { + """ + The day a `DateTime` falls in. + """ + DAY + + """ + The hour a `DateTime` falls in. + """ + HOUR + + """ + The minute a `DateTime` falls in. + """ + MINUTE + + """ + The month a `DateTime` falls in. + """ + MONTH + + """ + The quarter a `DateTime` falls in. + """ + QUARTER + + """ + The second a `DateTime` falls in. + """ + SECOND + + """ + The week, beginning on Monday, a `DateTime` falls in. + """ + WEEK + + """ + The year a `DateTime` falls in. + """ + YEAR +} + +""" +Input type offered when grouping on `DateTime` fields, representing the amount of offset +(positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + +For example, when grouping by `WEEK`, you can shift by 1 day to change +what day-of-week weeks are considered to start on. +""" +input DateTimeGroupingOffsetInput { + """ + Number (positive or negative) of the given `unit` to offset the boundaries of the `DateTime` groupings. + """ + amount: Int! + + """ + Unit of offsetting to apply to the boundaries of the `DateTime` groupings. + """ + unit: DateTimeUnitInput! +} + +""" +Enumerates the supported truncation units of a `DateTime`. +""" +enum DateTimeGroupingTruncationUnitInput { + """ + The day a `DateTime` falls in. + """ + DAY + + """ + The hour a `DateTime` falls in. + """ + HOUR + + """ + The minute a `DateTime` falls in. + """ + MINUTE + + """ + The month a `DateTime` falls in. + """ + MONTH + + """ + The quarter a `DateTime` falls in. + """ + QUARTER + + """ + The second a `DateTime` falls in. + """ + SECOND + + """ + The week, beginning on Monday, a `DateTime` falls in. + """ + WEEK + + """ + The year a `DateTime` falls in. + """ + YEAR +} + +""" +Input type used to specify filters on elements of a `[DateTime]` field. + +Will be ignored if passed as an empty object (or as `null`). +""" +input DateTimeListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [DateTimeListElementFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [DateTime!] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: DateTime + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: DateTime + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: DateTime + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: DateTime + + """ + Matches records based on the time-of-day of the `DateTime` values. + + Will be ignored when `null` or an empty list is passed. + """ + time_of_day: DateTimeTimeOfDayFilterInput +} + +""" +Input type used to specify filters on `[DateTime]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input DateTimeListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `DateTimeListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [DateTimeListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [DateTimeListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: DateTimeListElementFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: DateTimeListFilterInput +} + +""" +Input type used to specify filters on the time-of-day of `DateTime` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input DateTimeTimeOfDayFilterInput { + """ + Matches records where the time of day of the `DateTime` field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [LocalTime!] + + """ + Matches records where the time of day of the `DateTime` field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: LocalTime + + """ + Matches records where the time of day of the `DateTime` field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: LocalTime + + """ + Matches records where the time of day of the `DateTime` field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: LocalTime + + """ + Matches records where the time of day of the `DateTime` field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: LocalTime + + """ + TimeZone to use when comparing the `DateTime` values against the provided `LocalTime` values. + """ + time_zone: TimeZone! = "UTC" +} + +""" +Enumeration of `DateTime` units. +""" +enum DateTimeUnitInput { + """ + The time period of a full rotation of the Earth with respect to the Sun. + """ + DAY + + """ + 1/24th of a day. + """ + HOUR + + """ + 1/1000th of a second. + """ + MILLISECOND + + """ + 1/60th of an hour. + """ + MINUTE + + """ + 1/60th of a minute. + """ + SECOND +} + +""" +Enumeration of `Date` units. +""" +enum DateUnitInput { + """ + The time period of a full rotation of the Earth with respect to the Sun. + """ + DAY +} + +""" +Indicates the specific day of the week. +""" +enum DayOfWeek { + """ + Friday. + """ + FRIDAY + + """ + Monday. + """ + MONDAY + + """ + Saturday. + """ + SATURDAY + + """ + Sunday. + """ + SUNDAY + + """ + Thursday. + """ + THURSDAY + + """ + Tuesday. + """ + TUESDAY + + """ + Wednesday. + """ + WEDNESDAY +} + +""" +Input type offered when grouping on `DayOfWeek` fields, representing the amount of offset +(positive or negative) to shift the `DayOfWeek` boundaries of each grouping bucket. + +For example, you can apply an offset of -2 hours to shift `DateTime` values to the prior `DayOfWeek` +when they fall between midnight and 2 AM. +""" +input DayOfWeekGroupingOffsetInput { + """ + Number (positive or negative) of the given `unit` to offset the boundaries of the `DayOfWeek` groupings. + """ + amount: Int! + + """ + Unit of offsetting to apply to the boundaries of the `DayOfWeek` groupings. + """ + unit: DateTimeUnitInput! +} + +""" +Enumerates the supported distance units. +""" +enum DistanceUnitInput { + """ + A metric system unit equal to 1/100th of a meter. + """ + CENTIMETER + + """ + A United States customary unit of 12 inches. + """ + FOOT + + """ + A United States customary unit equal to 1/12th of a foot. + """ + INCH + + """ + A metric system unit equal to 1,000 meters. + """ + KILOMETER + + """ + The base unit of length in the metric system. + """ + METER + + """ + A United States customary unit of 5,280 feet. + """ + MILE + + """ + A metric system unit equal to 1/1,000th of a meter. + """ + MILLIMETER + + """ + An international unit of length used for air, marine, and space navigation. Equivalent to 1,852 meters. + """ + NAUTICAL_MILE + + """ + A United States customary unit of 3 feet. + """ + YARD +} + +type ElectricalPart implements NamedEntity @key(fields: "id") { + """ + Aggregations over the `components` data. + """ + component_aggregations( + """ + Used to forward-paginate through the `component_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `component_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Component` documents that get aggregated over based on the provided criteria. + """ + filter: ComponentFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `component_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `component_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `component_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `component_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): ComponentAggregationConnection + components( + """ + Used to forward-paginate through the `components`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `components`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `components` based on the provided criteria. + """ + filter: ComponentFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `components`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `components`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `components`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `components`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `components` should be sorted. + """ + order_by: [ComponentSortOrderInput!] + ): ComponentConnection + created_at: DateTime! + id: ID! + manufacturer: Manufacturer + name: String + voltage: Int! +} + +""" +Type used to perform aggregation computations on `ElectricalPart` fields. +""" +type ElectricalPartAggregatedValues { + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `voltage` field. + """ + voltage: IntAggregatedValues +} + +""" +Return type representing a bucket of `ElectricalPart` documents for an aggregations query. +""" +type ElectricalPartAggregation { + """ + Provides computed aggregated values over all `ElectricalPart` documents in an aggregation bucket. + """ + aggregated_values: ElectricalPartAggregatedValues + + """ + The count of `ElectricalPart` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `ElectricalPart` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: ElectricalPartGroupedBy +} + +""" +Represents a paginated collection of `ElectricalPartAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type ElectricalPartAggregationConnection { + """ + Wraps a specific `ElectricalPartAggregation` to pair it with its pagination cursor. + """ + edges: [ElectricalPartAggregationEdge!]! + + """ + The list of `ElectricalPartAggregation` results. + """ + nodes: [ElectricalPartAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `ElectricalPartAggregation` in the context of a `ElectricalPartAggregationConnection`, +providing access to both the `ElectricalPartAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type ElectricalPartAggregationEdge { + """ + The `Cursor` of this `ElectricalPartAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `ElectricalPartAggregation`. + """ + cursor: Cursor + + """ + The `ElectricalPartAggregation` of this edge. + """ + node: ElectricalPartAggregation +} + +""" +Represents a paginated collection of `ElectricalPart` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type ElectricalPartConnection { + """ + Wraps a specific `ElectricalPart` to pair it with its pagination cursor. + """ + edges: [ElectricalPartEdge!]! + + """ + The list of `ElectricalPart` results. + """ + nodes: [ElectricalPart!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `ElectricalPart` in the context of a `ElectricalPartConnection`, +providing access to both the `ElectricalPart` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type ElectricalPartEdge { + """ + The `Cursor` of this `ElectricalPart`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `ElectricalPart`. + """ + cursor: Cursor + + """ + The `ElectricalPart` of this edge. + """ + node: ElectricalPart +} + +""" +Input type used to specify filters on `ElectricalPart` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input ElectricalPartFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [ElectricalPartFilterInput!] + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: ElectricalPartFilterInput + + """ + Used to filter on the `voltage` field. + + Will be ignored if `null` or an empty object is passed. + """ + voltage: IntFilterInput +} + +""" +Type used to specify the `ElectricalPart` fields to group by for aggregations. +""" +type ElectricalPartGroupedBy { + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + The `name` field value for this group. + """ + name: String + + """ + The `voltage` field value for this group. + """ + voltage: Int +} + +""" +Enumerates the ways `ElectricalPart`s can be sorted. +""" +enum ElectricalPartSortOrderInput { + """ + Sorts ascending by the `created_at` field. + """ + created_at_ASC + + """ + Sorts descending by the `created_at` field. + """ + created_at_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC + + """ + Sorts ascending by the `voltage` field. + """ + voltage_ASC + + """ + Sorts descending by the `voltage` field. + """ + voltage_DESC +} + +""" +A custom scalar type required by the [Apollo Federation subgraph +spec](https://www.apollographql.com/docs/federation/subgraph-spec/#scalar-fieldset): + +> This string-serialized scalar represents a set of fields that's passed to a federated directive, +> such as `@key`, `@requires`, or `@provides`. +> +> Grammatically, a `FieldSet` is a [selection set](http://spec.graphql.org/draft/#sec-Selection-Sets) +> minus the outermost curly braces. It can represent a single field (`"upc"`), multiple fields +> (`"id countryCode"`), and even nested selection sets (`"id organization { id }"`). + +Not intended for use by clients other than Apollo. +""" +scalar FieldSet + +""" +A return type used from aggregations to provided aggregated values over `Float` fields. +""" +type FloatAggregatedValues @shareable { + """ + The average (mean) of the field values within this grouping. + + The computation of this value may introduce additional imprecision (on top of the + natural imprecision of floats) when it deals with intermediary values that are + outside the `JsonSafeLong` range (-9,007,199,254,740,991 to 9,007,199,254,740,991). + """ + approximate_avg: Float + + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in + Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + approximate_distinct_value_count: JsonSafeLong + + """ + The sum of the field values within this grouping. + + As with all double-precision `Float` values, operations are subject to floating-point loss + of precision, so the value may be approximate. + """ + approximate_sum: Float! + + """ + The maximum of the field values within this grouping. + + The value will be "exact" in that the aggregation computation will return + the exact value of the largest float that has been indexed, without + introducing any new imprecision. However, floats by their nature are + naturally imprecise since they cannot precisely represent all real numbers. + """ + exact_max: Float + + """ + The minimum of the field values within this grouping. + + The value will be "exact" in that the aggregation computation will return + the exact value of the smallest float that has been indexed, without + introducing any new imprecision. However, floats by their nature are + naturally imprecise since they cannot precisely represent all real numbers. + """ + exact_min: Float +} + +""" +Input type used to specify filters on `Float` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input FloatFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [FloatFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [Float] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: Float + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: Float + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: Float + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: Float + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: FloatFilterInput +} + +""" +Geographic coordinates representing a location on the Earth's surface. +""" +type GeoLocation @shareable { + """ + Angular distance north or south of the Earth's equator, measured in degrees from -90 to +90. + """ + latitude: Float + + """ + Angular distance east or west of the Prime Meridian at Greenwich, UK, measured in degrees from -180 to +180. + """ + longitude: Float +} + +""" +Input type used to specify distance filtering parameters on `GeoLocation` fields. +""" +input GeoLocationDistanceFilterInput { + """ + Angular distance north or south of the Earth's equator, measured in degrees from -90 to +90. + """ + latitude: Float! + + """ + Angular distance east or west of the Prime Meridian at Greenwich, UK, measured in degrees from -180 to +180. + """ + longitude: Float! + + """ + Maximum distance (of the provided `unit`) to consider "near" the location identified + by `latitude` and `longitude`. + """ + max_distance: Float! + + """ + Determines the unit of the specified `max_distance`. + """ + unit: DistanceUnitInput! +} + +""" +Input type used to specify filters on `GeoLocation` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input GeoLocationFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [GeoLocationFilterInput!] + + """ + Matches records where the field's geographic location is within a specified distance from the + location identified by `latitude` and `longitude`. + + Will be ignored when `null` or an empty object is passed. + """ + near: GeoLocationDistanceFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: GeoLocationFilterInput +} + +type GeoShape { + coordinates: [Float!]! + type: String +} + +""" +Input type used to specify filters on `ID` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input IDFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [IDFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [ID] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: IDFilterInput +} + +""" +Input type used to specify filters on elements of a `[ID]` field. + +Will be ignored if passed as an empty object (or as `null`). +""" +input IDListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [IDListElementFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [ID!] +} + +""" +Input type used to specify filters on `[ID]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input IDListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `IDListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [IDListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [IDListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: IDListElementFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: IDListFilterInput +} + +""" +A return type used from aggregations to provided aggregated values over `Int` fields. +""" +type IntAggregatedValues @shareable { + """ + The average (mean) of the field values within this grouping. + + Note that the returned value is approximate. Imprecision can be introduced by the computation if + any intermediary values fall outside the `JsonSafeLong` range (-9,007,199,254,740,991 + to 9,007,199,254,740,991). + """ + approximate_avg: Float + + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in + Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + approximate_distinct_value_count: JsonSafeLong + + """ + The (approximate) sum of the field values within this grouping. + + Sums of large `Int` values can result in overflow, where the exact sum cannot + fit in a `JsonSafeLong` return value. This field, as a double-precision `Float`, can + represent larger sums, but the value may only be approximate. + """ + approximate_sum: Float! + + """ + The maximum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_max: Int + + """ + The minimum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_min: Int + + """ + The exact sum of the field values within this grouping, if it fits in a `JsonSafeLong`. + + Sums of large `Int` values can result in overflow, where the exact sum cannot + fit in a `JsonSafeLong`. In that case, `null` will be returned, and `approximate_sum` + can be used to get an approximate value. + """ + exact_sum: JsonSafeLong +} + +""" +Input type used to specify filters on `Int` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input IntFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [IntFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [Int] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: Int + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: Int + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: Int + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: Int + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: IntFilterInput +} + +""" +Input type used to specify filters on elements of a `[Int]` field. + +Will be ignored if passed as an empty object (or as `null`). +""" +input IntListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [IntListElementFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [Int!] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: Int + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: Int + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: Int + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: Int +} + +""" +Input type used to specify filters on `[Int]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input IntListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `IntListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [IntListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [IntListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: IntListElementFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: IntListFilterInput +} + +union Inventor = Company | Person + +""" +Type used to perform aggregation computations on `Inventor` fields. +""" +type InventorAggregatedValues { + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `nationality` field. + """ + nationality: NonNumericAggregatedValues + + """ + Computed aggregate values for the `stock_ticker` field. + """ + stock_ticker: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on `Inventor` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input InventorFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [InventorFilterInput!] + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Used to filter on the `nationality` field. + + Will be ignored if `null` or an empty object is passed. + """ + nationality: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: InventorFilterInput + + """ + Used to filter on the `stock_ticker` field. + + Will be ignored if `null` or an empty object is passed. + """ + stock_ticker: StringFilterInput +} + +""" +Type used to specify the `Inventor` fields to group by for aggregations. +""" +type InventorGroupedBy { + """ + The `name` field value for this group. + """ + name: String + + """ + The `nationality` field value for this group. + """ + nationality: String + + """ + The `stock_ticker` field value for this group. + """ + stock_ticker: String +} + +""" +A numeric type for large integer values that can serialize safely as JSON. + +While JSON itself has no hard limit on the size of integers, the RFC-7159 spec +mentions that values outside of the range -9,007,199,254,740,991 (-(2^53) + 1) +to 9,007,199,254,740,991 (2^53 - 1) may not be interopable with all JSON +implementations. As it turns out, the number implementation used by JavaScript +has this issue. When you parse a JSON string that contains a numeric value like +`4693522397653681111`, the parsed result will contain a rounded value like +`4693522397653681000`. + +While this is entirely a client-side problem, we want to preserve maximum compatibility +with common client languages. Given the ubiquity of GraphiQL as a GraphQL client, +we want to avoid this problem. + +Our solution is to support two separate types: + +- This type (`JsonSafeLong`) is serialized as a number, but limits values to the safely + serializable range. +- The `LongString` type supports long values that use all 64 bits, but serializes as a + string rather than a number, avoiding the JavaScript compatibility problems. + +For more background, see the [JavaScript `Number.MAX_SAFE_INTEGER` +docs](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER). +""" +scalar JsonSafeLong + +""" +A return type used from aggregations to provided aggregated values over `JsonSafeLong` fields. +""" +type JsonSafeLongAggregatedValues @shareable { + """ + The average (mean) of the field values within this grouping. + + Note that the returned value is approximate. Imprecision can be introduced by the computation if + any intermediary values fall outside the `JsonSafeLong` range (-9,007,199,254,740,991 + to 9,007,199,254,740,991). + """ + approximate_avg: Float + + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in + Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + approximate_distinct_value_count: JsonSafeLong + + """ + The (approximate) sum of the field values within this grouping. + + Sums of large `JsonSafeLong` values can result in overflow, where the exact sum cannot + fit in a `JsonSafeLong` return value. This field, as a double-precision `Float`, can + represent larger sums, but the value may only be approximate. + """ + approximate_sum: Float! + + """ + The maximum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_max: JsonSafeLong + + """ + The minimum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_min: JsonSafeLong + + """ + The exact sum of the field values within this grouping, if it fits in a `JsonSafeLong`. + + Sums of large `JsonSafeLong` values can result in overflow, where the exact sum cannot + fit in a `JsonSafeLong`. In that case, `null` will be returned, and `approximate_sum` + can be used to get an approximate value. + """ + exact_sum: JsonSafeLong +} + +""" +Input type used to specify filters on `JsonSafeLong` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input JsonSafeLongFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [JsonSafeLongFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [JsonSafeLong] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: JsonSafeLong + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: JsonSafeLong + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: JsonSafeLong + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: JsonSafeLong + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: JsonSafeLongFilterInput +} + +""" +Input type used to specify filters on elements of a `[JsonSafeLong]` field. + +Will be ignored if passed as an empty object (or as `null`). +""" +input JsonSafeLongListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [JsonSafeLongListElementFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [JsonSafeLong!] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: JsonSafeLong + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: JsonSafeLong + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: JsonSafeLong + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: JsonSafeLong +} + +""" +Input type used to specify filters on `[JsonSafeLong]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input JsonSafeLongListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `JsonSafeLongListFilterInput` input because of + collisions between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [JsonSafeLongListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [JsonSafeLongListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: JsonSafeLongListElementFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: JsonSafeLongListFilterInput +} + +""" +A local time such as `"23:59:33"` or `"07:20:47.454"` without a time zone or offset, formatted based on the +[partial-time portion of RFC3339](https://datatracker.ietf.org/doc/html/rfc3339#section-5.6). +""" +scalar LocalTime + +""" +A return type used from aggregations to provided aggregated values over `LocalTime` fields. +""" +type LocalTimeAggregatedValues @shareable { + """ + The average (mean) of the field values within this grouping. + The returned value will be rounded to the nearest `LocalTime` value. + """ + approximate_avg: LocalTime + + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in + Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + approximate_distinct_value_count: JsonSafeLong + + """ + The maximum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_max: LocalTime + + """ + The minimum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + exact_min: LocalTime +} + +""" +Input type used to specify filters on `LocalTime` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input LocalTimeFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [LocalTimeFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [LocalTime] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: LocalTime + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: LocalTime + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: LocalTime + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: LocalTime + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: LocalTimeFilterInput +} + +""" +Input type offered when grouping on `LocalTime` fields, representing the amount of offset +(positive or negative) to shift the `LocalTime` boundaries of each grouping bucket. + +For example, when grouping by `HOUR`, you can shift by 30 minutes to change +what minute-of-hour hours are considered to start on. +""" +input LocalTimeGroupingOffsetInput { + """ + Number (positive or negative) of the given `unit` to offset the boundaries of the `LocalTime` groupings. + """ + amount: Int! + + """ + Unit of offsetting to apply to the boundaries of the `LocalTime` groupings. + """ + unit: LocalTimeUnitInput! +} + +""" +Enumerates the supported truncation units of a `LocalTime`. +""" +enum LocalTimeGroupingTruncationUnitInput { + """ + The hour a `LocalTime` falls in. + """ + HOUR + + """ + The minute a `LocalTime` falls in. + """ + MINUTE + + """ + The second a `LocalTime` falls in. + """ + SECOND +} + +""" +Enumeration of `LocalTime` units. +""" +enum LocalTimeUnitInput { + """ + 1/24th of a day. + """ + HOUR + + """ + 1/1000th of a second. + """ + MILLISECOND + + """ + 1/60th of an hour. + """ + MINUTE + + """ + 1/60th of a minute. + """ + SECOND +} + +""" +A numeric type for large integer values in the inclusive range -2^63 +(-9,223,372,036,854,775,808) to (2^63 - 1) (9,223,372,036,854,775,807). + +Note that `LongString` values are serialized as strings within JSON, to avoid +interopability problems with JavaScript. If you want a large integer type that +serializes within JSON as a number, use `JsonSafeLong`. +""" +scalar LongString + +""" +A return type used from aggregations to provided aggregated values over `LongString` fields. +""" +type LongStringAggregatedValues @shareable { + """ + The average (mean) of the field values within this grouping. + + Note that the returned value is approximate. Imprecision can be introduced by the computation if + any intermediary values fall outside the `JsonSafeLong` range (-9,007,199,254,740,991 + to 9,007,199,254,740,991). + """ + approximate_avg: Float + + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in + Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + approximate_distinct_value_count: JsonSafeLong + + """ + The maximum of the field values within this grouping. + + The aggregation computation performed to identify the largest value is not able + to maintain exact precision when dealing with values that are outside the `JsonSafeLong` + range (-9,007,199,254,740,991 to 9,007,199,254,740,991). + In that case, the `exact_max` field will return `null`, but this field will provide + a value which may be approximate. + """ + approximate_max: LongString + + """ + The minimum of the field values within this grouping. + + The aggregation computation performed to identify the smallest value is not able + to maintain exact precision when dealing with values that are outside the `JsonSafeLong` + range (-9,007,199,254,740,991 to 9,007,199,254,740,991). + In that case, the `exact_min` field will return `null`, but this field will provide + a value which may be approximate. + """ + approximate_min: LongString + + """ + The (approximate) sum of the field values within this grouping. + + Sums of large `LongString` values can result in overflow, where the exact sum cannot + fit in a `LongString` return value. This field, as a double-precision `Float`, can + represent larger sums, but the value may only be approximate. + """ + approximate_sum: Float! + + """ + The maximum of the field values within this grouping. + + So long as the grouping contains at least one non-null value, and no values exceed the + `JsonSafeLong` range in the underlying indexed field, this will return an exact non-null value. + + If no non-null values are available, or if the maximum value is outside the `JsonSafeLong` + range, `null` will be returned. `approximate_max` can be used to differentiate between these + cases and to get an approximate value. + """ + exact_max: JsonSafeLong + + """ + The minimum of the field values within this grouping. + + So long as the grouping contains at least one non-null value, and no values exceed the + `JsonSafeLong` range in the underlying indexed field, this will return an exact non-null value. + + If no non-null values are available, or if the minimum value is outside the `JsonSafeLong` + range, `null` will be returned. `approximate_min` can be used to differentiate between these + cases and to get an approximate value. + """ + exact_min: JsonSafeLong + + """ + The exact sum of the field values within this grouping, if it fits in a `JsonSafeLong`. + + Sums of large `LongString` values can result in overflow, where the exact sum cannot + fit in a `JsonSafeLong`. In that case, `null` will be returned, and `approximate_sum` + can be used to get an approximate value. + """ + exact_sum: JsonSafeLong +} + +""" +Input type used to specify filters on `LongString` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input LongStringFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [LongStringFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [LongString] + + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + gt: LongString + + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + gte: LongString + + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + lt: LongString + + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: LongString + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: LongStringFilterInput +} + +type Manufacturer implements NamedEntity @key(fields: "id") { + address: Address + created_at: DateTime! + id: ID! + + """ + Aggregations over the `manufactured_parts` data. + """ + manufactured_part_aggregations( + """ + Used to forward-paginate through the `manufactured_part_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `manufactured_part_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Part` documents that get aggregated over based on the provided criteria. + """ + filter: PartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `manufactured_part_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `manufactured_part_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `manufactured_part_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `manufactured_part_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): PartAggregationConnection + manufactured_parts( + """ + Used to forward-paginate through the `manufactured_parts`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `manufactured_parts`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `manufactured_parts` based on the provided criteria. + """ + filter: PartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `manufactured_parts`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `manufactured_parts`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `manufactured_parts`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `manufactured_parts`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `manufactured_parts` should be sorted. + """ + order_by: [PartSortOrderInput!] + ): PartConnection + name: String +} + +""" +Type used to perform aggregation computations on `Manufacturer` fields. +""" +type ManufacturerAggregatedValues { + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues +} + +""" +Return type representing a bucket of `Manufacturer` documents for an aggregations query. +""" +type ManufacturerAggregation { + """ + Provides computed aggregated values over all `Manufacturer` documents in an aggregation bucket. + """ + aggregated_values: ManufacturerAggregatedValues + + """ + The count of `Manufacturer` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `Manufacturer` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: ManufacturerGroupedBy +} + +""" +Represents a paginated collection of `ManufacturerAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type ManufacturerAggregationConnection { + """ + Wraps a specific `ManufacturerAggregation` to pair it with its pagination cursor. + """ + edges: [ManufacturerAggregationEdge!]! + + """ + The list of `ManufacturerAggregation` results. + """ + nodes: [ManufacturerAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `ManufacturerAggregation` in the context of a `ManufacturerAggregationConnection`, +providing access to both the `ManufacturerAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type ManufacturerAggregationEdge { + """ + The `Cursor` of this `ManufacturerAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `ManufacturerAggregation`. + """ + cursor: Cursor + + """ + The `ManufacturerAggregation` of this edge. + """ + node: ManufacturerAggregation +} + +""" +Represents a paginated collection of `Manufacturer` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type ManufacturerConnection { + """ + Wraps a specific `Manufacturer` to pair it with its pagination cursor. + """ + edges: [ManufacturerEdge!]! + + """ + The list of `Manufacturer` results. + """ + nodes: [Manufacturer!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `Manufacturer` in the context of a `ManufacturerConnection`, +providing access to both the `Manufacturer` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type ManufacturerEdge { + """ + The `Cursor` of this `Manufacturer`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `Manufacturer`. + """ + cursor: Cursor + + """ + The `Manufacturer` of this edge. + """ + node: Manufacturer +} + +""" +Input type used to specify filters on `Manufacturer` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input ManufacturerFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [ManufacturerFilterInput!] + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: ManufacturerFilterInput +} + +""" +Type used to specify the `Manufacturer` fields to group by for aggregations. +""" +type ManufacturerGroupedBy { + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + The `name` field value for this group. + """ + name: String +} + +""" +Enumerates the ways `Manufacturer`s can be sorted. +""" +enum ManufacturerSortOrderInput { + """ + Sorts ascending by the `created_at` field. + """ + created_at_ASC + + """ + Sorts descending by the `created_at` field. + """ + created_at_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC +} + +""" +Input type used to specify parameters for the `matches_phrase` filtering operator. + +Will be ignored if passed as `null`. +""" +input MatchesPhraseFilterInput { + """ + The input phrase to search for. + """ + phrase: String! +} + +""" +Enumeration of allowed values for the `matches_query: {allowed_edits_per_term: ...}` filter option. +""" +enum MatchesQueryAllowedEditsPerTermInput { + """ + Allowed edits per term is dynamically chosen based on the length of the term. + """ + DYNAMIC + + """ + No allowed edits per term. + """ + NONE + + """ + One allowed edit per term. + """ + ONE + + """ + Two allowed edits per term. + """ + TWO +} + +""" +Input type used to specify parameters for the `matches_query` filtering operator. + +Will be ignored if passed as `null`. +""" +input MatchesQueryFilterInput { + """ + Number of allowed modifications per term to arrive at a match. For example, if set to 'ONE', the input + term 'glue' would match 'blue' but not 'clued', since the latter requires two modifications. + """ + allowed_edits_per_term: MatchesQueryAllowedEditsPerTermInput! = DYNAMIC + + """ + The input query to search for. + """ + query: String! + + """ + Set to `true` to match only if all terms in `query` are found, or + `false` to only require one term to be found. + """ + require_all_terms: Boolean! = false +} + +enum Material { + ALLOY + CARBON_FIBER +} + +""" +Input type used to specify filters on `Material` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input MaterialFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [MaterialFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [MaterialInput] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: MaterialFilterInput +} + +enum MaterialInput { + ALLOY + CARBON_FIBER +} + +type MechanicalPart implements NamedEntity @key(fields: "id") { + """ + Aggregations over the `components` data. + """ + component_aggregations( + """ + Used to forward-paginate through the `component_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `component_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Component` documents that get aggregated over based on the provided criteria. + """ + filter: ComponentFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `component_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `component_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `component_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `component_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): ComponentAggregationConnection + components( + """ + Used to forward-paginate through the `components`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `components`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `components` based on the provided criteria. + """ + filter: ComponentFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `components`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `components`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `components`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `components`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `components` should be sorted. + """ + order_by: [ComponentSortOrderInput!] + ): ComponentConnection + created_at: DateTime! + id: ID! + manufacturer: Manufacturer + material: Material + name: String +} + +""" +Type used to perform aggregation computations on `MechanicalPart` fields. +""" +type MechanicalPartAggregatedValues { + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `material` field. + """ + material: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues +} + +""" +Return type representing a bucket of `MechanicalPart` documents for an aggregations query. +""" +type MechanicalPartAggregation { + """ + Provides computed aggregated values over all `MechanicalPart` documents in an aggregation bucket. + """ + aggregated_values: MechanicalPartAggregatedValues + + """ + The count of `MechanicalPart` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `MechanicalPart` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: MechanicalPartGroupedBy +} + +""" +Represents a paginated collection of `MechanicalPartAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type MechanicalPartAggregationConnection { + """ + Wraps a specific `MechanicalPartAggregation` to pair it with its pagination cursor. + """ + edges: [MechanicalPartAggregationEdge!]! + + """ + The list of `MechanicalPartAggregation` results. + """ + nodes: [MechanicalPartAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `MechanicalPartAggregation` in the context of a `MechanicalPartAggregationConnection`, +providing access to both the `MechanicalPartAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type MechanicalPartAggregationEdge { + """ + The `Cursor` of this `MechanicalPartAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `MechanicalPartAggregation`. + """ + cursor: Cursor + + """ + The `MechanicalPartAggregation` of this edge. + """ + node: MechanicalPartAggregation +} + +""" +Represents a paginated collection of `MechanicalPart` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type MechanicalPartConnection { + """ + Wraps a specific `MechanicalPart` to pair it with its pagination cursor. + """ + edges: [MechanicalPartEdge!]! + + """ + The list of `MechanicalPart` results. + """ + nodes: [MechanicalPart!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `MechanicalPart` in the context of a `MechanicalPartConnection`, +providing access to both the `MechanicalPart` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type MechanicalPartEdge { + """ + The `Cursor` of this `MechanicalPart`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `MechanicalPart`. + """ + cursor: Cursor + + """ + The `MechanicalPart` of this edge. + """ + node: MechanicalPart +} + +""" +Input type used to specify filters on `MechanicalPart` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input MechanicalPartFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [MechanicalPartFilterInput!] + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `material` field. + + Will be ignored if `null` or an empty object is passed. + """ + material: MaterialFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: MechanicalPartFilterInput +} + +""" +Type used to specify the `MechanicalPart` fields to group by for aggregations. +""" +type MechanicalPartGroupedBy { + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + The `material` field value for this group. + """ + material: Material + + """ + The `name` field value for this group. + """ + name: String +} + +""" +Enumerates the ways `MechanicalPart`s can be sorted. +""" +enum MechanicalPartSortOrderInput { + """ + Sorts ascending by the `created_at` field. + """ + created_at_ASC + + """ + Sorts descending by the `created_at` field. + """ + created_at_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `material` field. + """ + material_ASC + + """ + Sorts descending by the `material` field. + """ + material_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC +} + +type Money { + amount_cents: Int + currency: String! +} + +""" +Type used to perform aggregation computations on `Money` fields. +""" +type MoneyAggregatedValues { + """ + Computed aggregate values for the `amount_cents` field. + """ + amount_cents: IntAggregatedValues + + """ + Computed aggregate values for the `currency` field. + """ + currency: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on a `Money` object referenced directly +or transitively from a list field that has been configured to index each leaf field as +its own flattened list of values. + +Will be ignored if passed as an empty object (or as `null`). +""" +input MoneyFieldsListFilterInput { + """ + Used to filter on the `amount_cents` field. + + Will be ignored if `null` or an empty object is passed. + """ + amount_cents: IntListFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [MoneyFieldsListFilterInput!] + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Used to filter on the `currency` field. + + Will be ignored if `null` or an empty object is passed. + """ + currency: StringListFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: MoneyFieldsListFilterInput +} + +""" +Input type used to specify filters on `Money` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input MoneyFilterInput { + """ + Used to filter on the `amount_cents` field. + + Will be ignored if `null` or an empty object is passed. + """ + amount_cents: IntFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [MoneyFilterInput!] + + """ + Used to filter on the `currency` field. + + Will be ignored if `null` or an empty object is passed. + """ + currency: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: MoneyFilterInput +} + +""" +Type used to specify the `Money` fields to group by for aggregations. +""" +type MoneyGroupedBy { + """ + The `amount_cents` field value for this group. + """ + amount_cents: Int + + """ + The `currency` field value for this group. + """ + currency: String +} + +""" +Input type used to specify filters on `[Money]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input MoneyListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `MoneyListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [MoneyListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [MoneyListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: MoneyFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: MoneyListFilterInput +} + +interface NamedEntity { + id: ID! + name: String +} + +""" +Type used to perform aggregation computations on `NamedEntity` fields. +""" +type NamedEntityAggregatedValues { + """ + Computed aggregate values for the `amount_cents` field. + """ + amount_cents: IntAggregatedValues + + """ + Computed aggregate values for the `amount_cents2` field. + """ + amount_cents2: IntAggregatedValues + + """ + Computed aggregate values for the `amounts` field. + """ + amounts: IntAggregatedValues + + """ + Computed aggregate values for the `cost` field. + """ + cost: MoneyAggregatedValues + + """ + Computed aggregate values for the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on: DateAggregatedValues + + """ + Computed aggregate values for the `cost_currency_name` field. + """ + cost_currency_name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_symbol` field. + """ + cost_currency_symbol: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_unit` field. + """ + cost_currency_unit: NonNumericAggregatedValues + + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at2` field. + """ + created_at2: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at2_legacy` field. + """ + created_at2_legacy: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at_legacy` field. + """ + created_at_legacy: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at_time_of_day` field. + """ + created_at_time_of_day: LocalTimeAggregatedValues + + """ + Computed aggregate values for the `created_on` field. + """ + created_on: DateAggregatedValues + + """ + Computed aggregate values for the `created_on_legacy` field. + """ + created_on_legacy: DateAggregatedValues + + """ + Computed aggregate values for the `fees` field. + """ + fees: MoneyAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `inventor` field. + """ + inventor: InventorAggregatedValues + + """ + Computed aggregate values for the `material` field. + """ + material: NonNumericAggregatedValues + + """ + Computed aggregate values for the `metadata` field. + """ + metadata: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `named_inventor` field. + """ + named_inventor: NamedInventorAggregatedValues + + """ + Computed aggregate values for the `options` field. + """ + options: WidgetOptionsAggregatedValues + + """ + Computed aggregate values for the `position` field. + """ + position: PositionAggregatedValues + + """ + Computed aggregate values for the `release_dates` field. + """ + release_dates: DateAggregatedValues + + """ + Computed aggregate values for the `release_timestamps` field. + """ + release_timestamps: DateTimeAggregatedValues + + """ + Computed aggregate values for the `size` field. + """ + size: NonNumericAggregatedValues + + """ + Computed aggregate values for the `tags` field. + """ + tags: NonNumericAggregatedValues + + """ + Computed aggregate values for the `the_options` field. + """ + the_options: WidgetOptionsAggregatedValues + + """ + Computed aggregate values for the `voltage` field. + """ + voltage: IntAggregatedValues + + """ + Computed aggregate values for the `weight_in_ng` field. + """ + weight_in_ng: JsonSafeLongAggregatedValues + + """ + Computed aggregate values for the `weight_in_ng_str` field. + """ + weight_in_ng_str: LongStringAggregatedValues + + """ + Computed aggregate values for the `widget_cost` field. + """ + widget_cost: MoneyAggregatedValues + + """ + Computed aggregate values for the `widget_name` field. + """ + widget_name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_size` field. + """ + widget_size: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_tags` field. + """ + widget_tags: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_workspace_id` field. + """ + widget_workspace_id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `workspace_id` field. + """ + workspace_id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `workspace_name` field. + """ + workspace_name: NonNumericAggregatedValues +} + +""" +Return type representing a bucket of `NamedEntity` documents for an aggregations query. +""" +type NamedEntityAggregation { + """ + Provides computed aggregated values over all `NamedEntity` documents in an aggregation bucket. + """ + aggregated_values: NamedEntityAggregatedValues + + """ + The count of `NamedEntity` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `NamedEntity` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: NamedEntityGroupedBy +} + +""" +Represents a paginated collection of `NamedEntityAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type NamedEntityAggregationConnection { + """ + Wraps a specific `NamedEntityAggregation` to pair it with its pagination cursor. + """ + edges: [NamedEntityAggregationEdge!]! + + """ + The list of `NamedEntityAggregation` results. + """ + nodes: [NamedEntityAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `NamedEntityAggregation` in the context of a `NamedEntityAggregationConnection`, +providing access to both the `NamedEntityAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type NamedEntityAggregationEdge { + """ + The `Cursor` of this `NamedEntityAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `NamedEntityAggregation`. + """ + cursor: Cursor + + """ + The `NamedEntityAggregation` of this edge. + """ + node: NamedEntityAggregation +} + +""" +Represents a paginated collection of `NamedEntity` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type NamedEntityConnection { + """ + Wraps a specific `NamedEntity` to pair it with its pagination cursor. + """ + edges: [NamedEntityEdge!]! + + """ + The list of `NamedEntity` results. + """ + nodes: [NamedEntity!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `NamedEntity` in the context of a `NamedEntityConnection`, +providing access to both the `NamedEntity` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type NamedEntityEdge { + """ + The `Cursor` of this `NamedEntity`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `NamedEntity`. + """ + cursor: Cursor + + """ + The `NamedEntity` of this edge. + """ + node: NamedEntity +} + +""" +Input type used to specify filters on `NamedEntity` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input NamedEntityFilterInput { + """ + Used to filter on the `amount_cents` field. + + Will be ignored if `null` or an empty object is passed. + """ + amount_cents: IntFilterInput + + """ + Used to filter on the `amount_cents2` field. + + Will be ignored if `null` or an empty object is passed. + """ + amount_cents2: IntFilterInput + + """ + Used to filter on the `amounts` field. + + Will be ignored if `null` or an empty object is passed. + """ + amounts: IntListFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [NamedEntityFilterInput!] + + """ + Used to filter on the `cost` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost: MoneyFilterInput + + """ + Used to filter on the `cost_currency_introduced_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_introduced_on: DateFilterInput + + """ + Used to filter on the `cost_currency_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_name: StringFilterInput + + """ + Used to filter on the `cost_currency_primary_continent` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_primary_continent: StringFilterInput + + """ + Used to filter on the `cost_currency_symbol` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_symbol: StringFilterInput + + """ + Used to filter on the `cost_currency_unit` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_unit: StringFilterInput + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `created_at2` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at2: DateTimeFilterInput + + """ + Used to filter on the `created_at2_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at2_legacy: DateTimeFilterInput + + """ + Used to filter on the `created_at_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at_legacy: DateTimeFilterInput + + """ + Used to filter on the `created_at_time_of_day` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at_time_of_day: LocalTimeFilterInput + + """ + Used to filter on the `created_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_on: DateFilterInput + + """ + Used to filter on the `created_on_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_on_legacy: DateFilterInput + + """ + Used to filter on the `fees` field. + + Will be ignored if `null` or an empty object is passed. + """ + fees: MoneyFieldsListFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `inventor` field. + + Will be ignored if `null` or an empty object is passed. + """ + inventor: InventorFilterInput + + """ + Used to filter on the `material` field. + + Will be ignored if `null` or an empty object is passed. + """ + material: MaterialFilterInput + + """ + Used to filter on the `metadata` field. + + Will be ignored if `null` or an empty object is passed. + """ + metadata: UntypedFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Used to filter on the `name_text` field. + + Will be ignored if `null` or an empty object is passed. + """ + name_text: TextFilterInput + + """ + Used to filter on the `named_inventor` field. + + Will be ignored if `null` or an empty object is passed. + """ + named_inventor: NamedInventorFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: NamedEntityFilterInput + + """ + Used to filter on the `options` field. + + Will be ignored if `null` or an empty object is passed. + """ + options: WidgetOptionsFilterInput + + """ + Used to filter on the `position` field. + + Will be ignored if `null` or an empty object is passed. + """ + position: PositionFilterInput + + """ + Used to filter on the `release_dates` field. + + Will be ignored if `null` or an empty object is passed. + """ + release_dates: DateListFilterInput + + """ + Used to filter on the `release_timestamps` field. + + Will be ignored if `null` or an empty object is passed. + """ + release_timestamps: DateTimeListFilterInput + + """ + Used to filter on the `size` field. + + Will be ignored if `null` or an empty object is passed. + """ + size: SizeFilterInput + + """ + Used to filter on the `tags` field. + + Will be ignored if `null` or an empty object is passed. + """ + tags: StringListFilterInput + + """ + Used to filter on the `the_options` field. + + Will be ignored if `null` or an empty object is passed. + """ + the_options: WidgetOptionsFilterInput + + """ + Used to filter on the `voltage` field. + + Will be ignored if `null` or an empty object is passed. + """ + voltage: IntFilterInput + + """ + Used to filter on the `weight_in_ng` field. + + Will be ignored if `null` or an empty object is passed. + """ + weight_in_ng: JsonSafeLongFilterInput + + """ + Used to filter on the `weight_in_ng_str` field. + + Will be ignored if `null` or an empty object is passed. + """ + weight_in_ng_str: LongStringFilterInput + + """ + Used to filter on the `widget_cost` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_cost: MoneyFilterInput + + """ + Used to filter on the `widget_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_name: StringFilterInput + + """ + Used to filter on the `widget_size` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_size: SizeFilterInput + + """ + Used to filter on the `widget_tags` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_tags: StringListFilterInput + + """ + Used to filter on the `widget_workspace_id` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_workspace_id: IDFilterInput + + """ + Used to filter on the `workspace_id` field. + + Will be ignored if `null` or an empty object is passed. + """ + workspace_id: IDFilterInput + + """ + Used to filter on the `workspace_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + workspace_name: StringFilterInput +} + +""" +Type used to specify the `NamedEntity` fields to group by for aggregations. +""" +type NamedEntityGroupedBy { + """ + The `amount_cents` field value for this group. + """ + amount_cents: Int + + """ + The `amount_cents2` field value for this group. + """ + amount_cents2: Int + + """ + The `cost` field value for this group. + """ + cost: MoneyGroupedBy + + """ + Offers the different grouping options for the `cost_currency_introduced_on` value within this group. + """ + cost_currency_introduced_on: DateGroupedBy + + """ + The `cost_currency_name` field value for this group. + """ + cost_currency_name: String + + """ + The `cost_currency_primary_continent` field value for this group. + """ + cost_currency_primary_continent: String + + """ + The `cost_currency_symbol` field value for this group. + """ + cost_currency_symbol: String + + """ + The `cost_currency_unit` field value for this group. + """ + cost_currency_unit: String + + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + Offers the different grouping options for the `created_at2` value within this group. + """ + created_at2: DateTimeGroupedBy + + """ + The `created_at2_legacy` field value for this group. + """ + created_at2_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateTimeGroupingGranularityInput! + + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change + what day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone = "UTC" + ): DateTime + + """ + The `created_at_legacy` field value for this group. + """ + created_at_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateTimeGroupingGranularityInput! + + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change + what day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone = "UTC" + ): DateTime + + """ + The `created_at_time_of_day` field value for this group. + """ + created_at_time_of_day: LocalTime + + """ + Offers the different grouping options for the `created_on` value within this group. + """ + created_on: DateGroupedBy + + """ + The `created_on_legacy` field value for this group. + """ + created_on_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateGroupingGranularityInput! + + """ + Number of days (positive or negative) to shift the `Date` boundaries of each date grouping bucket. + + For example, when grouping by `YEAR`, this can be used to align the buckets + with fiscal or school years instead of calendar years. + """ + offset_days: Int + ): Date + + """ + The `fees` field value for this group. + + Note: `fees` is a collection field, but selecting this field will group on + individual values of the selected subfields of `fees`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `fees` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `fees` multiple times for a single document, that document will only be included in the group + once. + """ + fees: MoneyGroupedBy + + """ + The `inventor` field value for this group. + """ + inventor: InventorGroupedBy + + """ + The `material` field value for this group. + """ + material: Material + + """ + The `metadata` field value for this group. + """ + metadata: Untyped + + """ + The `name` field value for this group. + """ + name: String + + """ + The `named_inventor` field value for this group. + """ + named_inventor: NamedInventorGroupedBy + + """ + The `options` field value for this group. + """ + options: WidgetOptionsGroupedBy + + """ + The `position` field value for this group. + """ + position: PositionGroupedBy + + """ + The individual value from `release_dates` for this group. + + Note: `release_dates` is a collection field, but selecting this field will group on individual values of `release_dates`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `release_dates` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `release_dates` multiple times for a single document, that document will only be included in the group + once. + """ + release_date: DateGroupedBy + + """ + The individual value from `release_timestamps` for this group. + + Note: `release_timestamps` is a collection field, but selecting this field + will group on individual values of `release_timestamps`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `release_timestamps` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `release_timestamps` multiple times for a single document, that document will only be included in the group + once. + """ + release_timestamp: DateTimeGroupedBy + + """ + The `size` field value for this group. + """ + size: Size + + """ + The individual value from `tags` for this group. + + Note: `tags` is a collection field, but selecting this field will group on individual values of `tags`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `tags` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `tags` multiple times for a single document, that document will only be included in the group + once. + """ + tag: String + + """ + The `the_options` field value for this group. + """ + the_options: WidgetOptionsGroupedBy + + """ + The `voltage` field value for this group. + """ + voltage: Int + + """ + The `weight_in_ng` field value for this group. + """ + weight_in_ng: JsonSafeLong + + """ + The `weight_in_ng_str` field value for this group. + """ + weight_in_ng_str: LongString + + """ + The `widget_cost` field value for this group. + """ + widget_cost: MoneyGroupedBy + + """ + The `widget_name` field value for this group. + """ + widget_name: String + + """ + The `widget_size` field value for this group. + """ + widget_size: Size + + """ + The `widget_workspace_id` field value for this group. + """ + widget_workspace_id: ID + + """ + The `workspace_id` field value for this group. + """ + workspace_id: ID + + """ + The `workspace_name` field value for this group. + """ + workspace_name: String +} + +""" +Enumerates the ways `NamedEntity`s can be sorted. +""" +enum NamedEntitySortOrderInput { + """ + Sorts ascending by the `amount_cents2` field. + """ + amount_cents2_ASC + + """ + Sorts descending by the `amount_cents2` field. + """ + amount_cents2_DESC + + """ + Sorts ascending by the `amount_cents` field. + """ + amount_cents_ASC + + """ + Sorts descending by the `amount_cents` field. + """ + amount_cents_DESC + + """ + Sorts ascending by the `cost.amount_cents` field. + """ + cost_amount_cents_ASC + + """ + Sorts descending by the `cost.amount_cents` field. + """ + cost_amount_cents_DESC + + """ + Sorts ascending by the `cost.currency` field. + """ + cost_currency_ASC + + """ + Sorts descending by the `cost.currency` field. + """ + cost_currency_DESC + + """ + Sorts ascending by the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on_ASC + + """ + Sorts descending by the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on_DESC + + """ + Sorts ascending by the `cost_currency_name` field. + """ + cost_currency_name_ASC + + """ + Sorts descending by the `cost_currency_name` field. + """ + cost_currency_name_DESC + + """ + Sorts ascending by the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent_ASC + + """ + Sorts descending by the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent_DESC + + """ + Sorts ascending by the `cost_currency_symbol` field. + """ + cost_currency_symbol_ASC + + """ + Sorts descending by the `cost_currency_symbol` field. + """ + cost_currency_symbol_DESC + + """ + Sorts ascending by the `cost_currency_unit` field. + """ + cost_currency_unit_ASC + + """ + Sorts descending by the `cost_currency_unit` field. + """ + cost_currency_unit_DESC + + """ + Sorts ascending by the `created_at2` field. + """ + created_at2_ASC + + """ + Sorts descending by the `created_at2` field. + """ + created_at2_DESC + + """ + Sorts ascending by the `created_at2_legacy` field. + """ + created_at2_legacy_ASC + + """ + Sorts descending by the `created_at2_legacy` field. + """ + created_at2_legacy_DESC + + """ + Sorts ascending by the `created_at` field. + """ + created_at_ASC + + """ + Sorts descending by the `created_at` field. + """ + created_at_DESC + + """ + Sorts ascending by the `created_at_legacy` field. + """ + created_at_legacy_ASC + + """ + Sorts descending by the `created_at_legacy` field. + """ + created_at_legacy_DESC + + """ + Sorts ascending by the `created_at_time_of_day` field. + """ + created_at_time_of_day_ASC + + """ + Sorts descending by the `created_at_time_of_day` field. + """ + created_at_time_of_day_DESC + + """ + Sorts ascending by the `created_on` field. + """ + created_on_ASC + + """ + Sorts descending by the `created_on` field. + """ + created_on_DESC + + """ + Sorts ascending by the `created_on_legacy` field. + """ + created_on_legacy_ASC + + """ + Sorts descending by the `created_on_legacy` field. + """ + created_on_legacy_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `inventor.name` field. + """ + inventor_name_ASC + + """ + Sorts descending by the `inventor.name` field. + """ + inventor_name_DESC + + """ + Sorts ascending by the `inventor.nationality` field. + """ + inventor_nationality_ASC + + """ + Sorts descending by the `inventor.nationality` field. + """ + inventor_nationality_DESC + + """ + Sorts ascending by the `inventor.stock_ticker` field. + """ + inventor_stock_ticker_ASC + + """ + Sorts descending by the `inventor.stock_ticker` field. + """ + inventor_stock_ticker_DESC + + """ + Sorts ascending by the `material` field. + """ + material_ASC + + """ + Sorts descending by the `material` field. + """ + material_DESC + + """ + Sorts ascending by the `metadata` field. + """ + metadata_ASC + + """ + Sorts descending by the `metadata` field. + """ + metadata_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC + + """ + Sorts ascending by the `named_inventor.name` field. + """ + named_inventor_name_ASC + + """ + Sorts descending by the `named_inventor.name` field. + """ + named_inventor_name_DESC + + """ + Sorts ascending by the `named_inventor.nationality` field. + """ + named_inventor_nationality_ASC + + """ + Sorts descending by the `named_inventor.nationality` field. + """ + named_inventor_nationality_DESC + + """ + Sorts ascending by the `named_inventor.stock_ticker` field. + """ + named_inventor_stock_ticker_ASC + + """ + Sorts descending by the `named_inventor.stock_ticker` field. + """ + named_inventor_stock_ticker_DESC + + """ + Sorts ascending by the `options.color` field. + """ + options_color_ASC + + """ + Sorts descending by the `options.color` field. + """ + options_color_DESC + + """ + Sorts ascending by the `options.size` field. + """ + options_size_ASC + + """ + Sorts descending by the `options.size` field. + """ + options_size_DESC + + """ + Sorts ascending by the `options.the_size` field. + """ + options_the_size_ASC + + """ + Sorts descending by the `options.the_size` field. + """ + options_the_size_DESC + + """ + Sorts ascending by the `position.x` field. + """ + position_x_ASC + + """ + Sorts descending by the `position.x` field. + """ + position_x_DESC + + """ + Sorts ascending by the `position.y` field. + """ + position_y_ASC + + """ + Sorts descending by the `position.y` field. + """ + position_y_DESC + + """ + Sorts ascending by the `size` field. + """ + size_ASC + + """ + Sorts descending by the `size` field. + """ + size_DESC + + """ + Sorts ascending by the `the_options.color` field. + """ + the_options_color_ASC + + """ + Sorts descending by the `the_options.color` field. + """ + the_options_color_DESC + + """ + Sorts ascending by the `the_options.size` field. + """ + the_options_size_ASC + + """ + Sorts descending by the `the_options.size` field. + """ + the_options_size_DESC + + """ + Sorts ascending by the `the_options.the_size` field. + """ + the_options_the_size_ASC + + """ + Sorts descending by the `the_options.the_size` field. + """ + the_options_the_size_DESC + + """ + Sorts ascending by the `voltage` field. + """ + voltage_ASC + + """ + Sorts descending by the `voltage` field. + """ + voltage_DESC + + """ + Sorts ascending by the `weight_in_ng` field. + """ + weight_in_ng_ASC + + """ + Sorts descending by the `weight_in_ng` field. + """ + weight_in_ng_DESC + + """ + Sorts ascending by the `weight_in_ng_str` field. + """ + weight_in_ng_str_ASC + + """ + Sorts descending by the `weight_in_ng_str` field. + """ + weight_in_ng_str_DESC + + """ + Sorts ascending by the `widget_cost.amount_cents` field. + """ + widget_cost_amount_cents_ASC + + """ + Sorts descending by the `widget_cost.amount_cents` field. + """ + widget_cost_amount_cents_DESC + + """ + Sorts ascending by the `widget_cost.currency` field. + """ + widget_cost_currency_ASC + + """ + Sorts descending by the `widget_cost.currency` field. + """ + widget_cost_currency_DESC + + """ + Sorts ascending by the `widget_name` field. + """ + widget_name_ASC + + """ + Sorts descending by the `widget_name` field. + """ + widget_name_DESC + + """ + Sorts ascending by the `widget_size` field. + """ + widget_size_ASC + + """ + Sorts descending by the `widget_size` field. + """ + widget_size_DESC + + """ + Sorts ascending by the `widget_workspace_id` field. + """ + widget_workspace_id_ASC + + """ + Sorts descending by the `widget_workspace_id` field. + """ + widget_workspace_id_DESC + + """ + Sorts ascending by the `workspace_id` field. + """ + workspace_id_ASC + + """ + Sorts descending by the `workspace_id` field. + """ + workspace_id_DESC + + """ + Sorts ascending by the `workspace_name` field. + """ + workspace_name_ASC + + """ + Sorts descending by the `workspace_name` field. + """ + workspace_name_DESC +} + +interface NamedInventor { + name: String +} + +""" +Type used to perform aggregation computations on `NamedInventor` fields. +""" +type NamedInventorAggregatedValues { + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `nationality` field. + """ + nationality: NonNumericAggregatedValues + + """ + Computed aggregate values for the `stock_ticker` field. + """ + stock_ticker: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on `NamedInventor` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input NamedInventorFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [NamedInventorFilterInput!] + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Used to filter on the `nationality` field. + + Will be ignored if `null` or an empty object is passed. + """ + nationality: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: NamedInventorFilterInput + + """ + Used to filter on the `stock_ticker` field. + + Will be ignored if `null` or an empty object is passed. + """ + stock_ticker: StringFilterInput +} + +""" +Type used to specify the `NamedInventor` fields to group by for aggregations. +""" +type NamedInventorGroupedBy { + """ + The `name` field value for this group. + """ + name: String + + """ + The `nationality` field value for this group. + """ + nationality: String + + """ + The `stock_ticker` field value for this group. + """ + stock_ticker: String +} + +""" +A return type used from aggregations to provided aggregated values over non-numeric fields. +""" +type NonNumericAggregatedValues @shareable { + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in + Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + approximate_distinct_value_count: JsonSafeLong +} + +""" +Provides information about the specific fetched page. This implements the `PageInfo` +specification from the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-undefined.PageInfo). +""" +type PageInfo @shareable { + """ + The `Cursor` of the last edge of the current page. This can be passed in the next query as + a `after` argument to paginate forwards. + """ + end_cursor: Cursor + + """ + Indicates if there is another page of results available after the current one. + """ + has_next_page: Boolean! + + """ + Indicates if there is another page of results available before the current one. + """ + has_previous_page: Boolean! + + """ + The `Cursor` of the first edge of the current page. This can be passed in the next query as + a `before` argument to paginate backwards. + """ + start_cursor: Cursor +} + +union Part = ElectricalPart | MechanicalPart + +""" +Type used to perform aggregation computations on `Part` fields. +""" +type PartAggregatedValues { + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `material` field. + """ + material: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `voltage` field. + """ + voltage: IntAggregatedValues +} + +""" +Return type representing a bucket of `Part` documents for an aggregations query. +""" +type PartAggregation { + """ + Provides computed aggregated values over all `Part` documents in an aggregation bucket. + """ + aggregated_values: PartAggregatedValues + + """ + The count of `Part` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `Part` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: PartGroupedBy +} + +""" +Represents a paginated collection of `PartAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type PartAggregationConnection { + """ + Wraps a specific `PartAggregation` to pair it with its pagination cursor. + """ + edges: [PartAggregationEdge!]! + + """ + The list of `PartAggregation` results. + """ + nodes: [PartAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `PartAggregation` in the context of a `PartAggregationConnection`, +providing access to both the `PartAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type PartAggregationEdge { + """ + The `Cursor` of this `PartAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `PartAggregation`. + """ + cursor: Cursor + + """ + The `PartAggregation` of this edge. + """ + node: PartAggregation +} + +""" +Represents a paginated collection of `Part` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type PartConnection { + """ + Wraps a specific `Part` to pair it with its pagination cursor. + """ + edges: [PartEdge!]! + + """ + The list of `Part` results. + """ + nodes: [Part!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `Part` in the context of a `PartConnection`, +providing access to both the `Part` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type PartEdge { + """ + The `Cursor` of this `Part`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `Part`. + """ + cursor: Cursor + + """ + The `Part` of this edge. + """ + node: Part +} + +""" +Input type used to specify filters on `Part` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input PartFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [PartFilterInput!] + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `material` field. + + Will be ignored if `null` or an empty object is passed. + """ + material: MaterialFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: PartFilterInput + + """ + Used to filter on the `voltage` field. + + Will be ignored if `null` or an empty object is passed. + """ + voltage: IntFilterInput +} + +""" +Type used to specify the `Part` fields to group by for aggregations. +""" +type PartGroupedBy { + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + The `material` field value for this group. + """ + material: Material + + """ + The `name` field value for this group. + """ + name: String + + """ + The `voltage` field value for this group. + """ + voltage: Int +} + +""" +Enumerates the ways `Part`s can be sorted. +""" +enum PartSortOrderInput { + """ + Sorts ascending by the `created_at` field. + """ + created_at_ASC + + """ + Sorts descending by the `created_at` field. + """ + created_at_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `material` field. + """ + material_ASC + + """ + Sorts descending by the `material` field. + """ + material_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC + + """ + Sorts ascending by the `voltage` field. + """ + voltage_ASC + + """ + Sorts descending by the `voltage` field. + """ + voltage_DESC +} + +type Person implements NamedInventor { + name: String + nationality: String +} + +type Player { + affiliations: Affiliations! + name: String + nicknames: [String!]! + seasons_nested: [PlayerSeason!]! + seasons_object: [PlayerSeason!]! +} + +""" +Type used to perform aggregation computations on `Player` fields. +""" +type PlayerAggregatedValues { + """ + Computed aggregate values for the `affiliations` field. + """ + affiliations: AffiliationsAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `nicknames` field. + """ + nicknames: NonNumericAggregatedValues + + """ + Computed aggregate values for the `seasons_object` field. + """ + seasons_object: PlayerSeasonAggregatedValues +} + +""" +Input type used to specify filters on a `Player` object referenced directly +or transitively from a list field that has been configured to index each leaf field as +its own flattened list of values. + +Will be ignored if passed as an empty object (or as `null`). +""" +input PlayerFieldsListFilterInput { + """ + Used to filter on the `affiliations` field. + + Will be ignored if `null` or an empty object is passed. + """ + affiliations: AffiliationsFieldsListFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [PlayerFieldsListFilterInput!] + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringListFilterInput + + """ + Used to filter on the `nicknames` field. + + Will be ignored if `null` or an empty object is passed. + """ + nicknames: StringListFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: PlayerFieldsListFilterInput + + """ + Used to filter on the `seasons_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + seasons_nested: PlayerSeasonListFilterInput + + """ + Used to filter on the `seasons_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + seasons_object: PlayerSeasonFieldsListFilterInput +} + +""" +Input type used to specify filters on `Player` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input PlayerFilterInput { + """ + Used to filter on the `affiliations` field. + + Will be ignored if `null` or an empty object is passed. + """ + affiliations: AffiliationsFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [PlayerFilterInput!] + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Used to filter on the `nicknames` field. + + Will be ignored if `null` or an empty object is passed. + """ + nicknames: StringListFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: PlayerFilterInput + + """ + Used to filter on the `seasons_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + seasons_nested: PlayerSeasonListFilterInput + + """ + Used to filter on the `seasons_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + seasons_object: PlayerSeasonFieldsListFilterInput +} + +""" +Type used to specify the `Player` fields to group by for aggregations. +""" +type PlayerGroupedBy { + """ + The `affiliations` field value for this group. + """ + affiliations: AffiliationsGroupedBy + + """ + The `name` field value for this group. + """ + name: String + + """ + The `seasons_object` field value for this group. + + Note: `seasons_object` is a collection field, but selecting this field will + group on individual values of the selected subfields of `seasons_object`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `seasons_object` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `seasons_object` multiple times for a single document, that document will only be included in the group + once. + """ + seasons_object: PlayerSeasonGroupedBy +} + +""" +Input type used to specify filters on `[Player]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input PlayerListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `PlayerListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [PlayerListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [PlayerListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: PlayerFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: PlayerListFilterInput +} + +type PlayerSeason { + awards( + """ + Used to forward-paginate through the `awards`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `awards`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used in conjunction with the `after` argument to forward-paginate through the `awards`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `awards`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `awards`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `awards`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): StringConnection + games_played: Int + year: Int +} + +""" +Type used to perform aggregation computations on `PlayerSeason` fields. +""" +type PlayerSeasonAggregatedValues { + """ + Computed aggregate values for the `awards` field. + """ + awards: NonNumericAggregatedValues + + """ + Computed aggregate values for the `games_played` field. + """ + games_played: IntAggregatedValues + + """ + Computed aggregate values for the `year` field. + """ + year: IntAggregatedValues +} + +""" +Input type used to specify filters on a `PlayerSeason` object referenced directly +or transitively from a list field that has been configured to index each leaf field as +its own flattened list of values. + +Will be ignored if passed as an empty object (or as `null`). +""" +input PlayerSeasonFieldsListFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [PlayerSeasonFieldsListFilterInput!] + + """ + Used to filter on the `awards` field. + + Will be ignored if `null` or an empty object is passed. + """ + awards: StringListFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Used to filter on the `games_played` field. + + Will be ignored if `null` or an empty object is passed. + """ + games_played: IntListFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: PlayerSeasonFieldsListFilterInput + + """ + Used to filter on the `year` field. + + Will be ignored if `null` or an empty object is passed. + """ + year: IntListFilterInput +} + +""" +Input type used to specify filters on `PlayerSeason` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input PlayerSeasonFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [PlayerSeasonFilterInput!] + + """ + Used to filter on the `awards` field. + + Will be ignored if `null` or an empty object is passed. + """ + awards: StringListFilterInput + + """ + Used to filter on the `games_played` field. + + Will be ignored if `null` or an empty object is passed. + """ + games_played: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: PlayerSeasonFilterInput + + """ + Used to filter on the `year` field. + + Will be ignored if `null` or an empty object is passed. + """ + year: IntFilterInput +} + +""" +Type used to specify the `PlayerSeason` fields to group by for aggregations. +""" +type PlayerSeasonGroupedBy { + """ + The `games_played` field value for this group. + """ + games_played: Int + + """ + The `year` field value for this group. + """ + year: Int +} + +""" +Input type used to specify filters on `[PlayerSeason]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input PlayerSeasonListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `PlayerSeasonListFilterInput` input because of + collisions between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [PlayerSeasonListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [PlayerSeasonListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: PlayerSeasonFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: PlayerSeasonListFilterInput +} + +type Position { + x: Float! + y: Float! +} + +""" +Type used to perform aggregation computations on `Position` fields. +""" +type PositionAggregatedValues { + """ + Computed aggregate values for the `x` field. + """ + x: FloatAggregatedValues + + """ + Computed aggregate values for the `y` field. + """ + y: FloatAggregatedValues +} + +""" +Input type used to specify filters on `Position` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input PositionFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [PositionFilterInput!] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: PositionFilterInput + + """ + Used to filter on the `x` field. + + Will be ignored if `null` or an empty object is passed. + """ + x: FloatFilterInput + + """ + Used to filter on the `y` field. + + Will be ignored if `null` or an empty object is passed. + """ + y: FloatFilterInput +} + +""" +Type used to specify the `Position` fields to group by for aggregations. +""" +type PositionGroupedBy { + """ + The `x` field value for this group. + """ + x: Float + + """ + The `y` field value for this group. + """ + y: Float +} + +""" +The query entry point for the entire schema. +""" +type Query { + """ + A field required by the [Apollo Federation subgraph + spec](https://www.apollographql.com/docs/federation/subgraph-spec/#query_entities): + + > The graph router uses this root-level `Query` field to directly fetch fields of entities defined by a subgraph. + > + > This field must take a `representations` argument of type `[_Any!]!` (a non-nullable list of non-nullable + > [`_Any` scalars](https://www.apollographql.com/docs/federation/subgraph-spec/#scalar-_any)). Its return type must be `[_Entity]!` (a non-nullable list of _nullable_ + > objects that belong to the [`_Entity` union](https://www.apollographql.com/docs/federation/subgraph-spec/#union-_entity)). + > + > Each entry in the `representations` list must be validated with the following rules: + > + > - A representation must include a `__typename` string field. + > - A representation must contain all fields included in the fieldset of a + `@key` directive applied to the corresponding entity definition. + > + > For details, see [Resolving entity fields with `Query._entities`](https://www.apollographql.com/docs/federation/subgraph-spec/#resolving-entity-fields-with-query_entities). + + Not intended for use by clients other than Apollo. + """ + _entities( + """ + A list of entity data blobs from other apollo subgraphs. For more information (and + to see an example of what form this argument takes), see the [Apollo Federation subgraph + spec](https://www.apollographql.com/docs/federation/subgraph-spec/#resolve-requests-for-entities). + """ + representations: [_Any!]! + ): [_Entity]! + + """ + A field required by the [Apollo Federation subgraph + spec](https://www.apollographql.com/docs/federation/subgraph-spec/#query_service): + + > This field of the root `Query` type must return a non-nullable [`_Service` type](https://www.apollographql.com/docs/federation/subgraph-spec/#type-_service). + + > For details, see [Enhanced introspection with `Query._service`](https://www.apollographql.com/docs/federation/subgraph-spec/#enhanced-introspection-with-query_service). + + Not intended for use by clients other than Apollo. + """ + _service: _Service! + + """ + Aggregations over the `addresses` data: + + > Fetches `Address`s based on the provided arguments. + """ + address_aggregations( + """ + Used to forward-paginate through the `address_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `address_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Address` documents that get aggregated over based on the provided criteria. + """ + filter: AddressFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `address_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `address_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `address_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `address_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): AddressAggregationConnection + + """ + Fetches `Address`s based on the provided arguments. + """ + addresses( + """ + Used to forward-paginate through the `addresses`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `addresses`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `addresses` based on the provided criteria. + """ + filter: AddressFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `addresses`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `addresses`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `addresses`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `addresses`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `addresses` should be sorted. + """ + order_by: [AddressSortOrderInput!] + ): AddressConnection + + """ + Aggregations over the `components` data: + + > Fetches `Component`s based on the provided arguments. + """ + component_aggregations( + """ + Used to forward-paginate through the `component_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `component_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Component` documents that get aggregated over based on the provided criteria. + """ + filter: ComponentFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `component_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `component_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `component_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `component_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): ComponentAggregationConnection + + """ + Fetches `Component`s based on the provided arguments. + """ + components( + """ + Used to forward-paginate through the `components`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `components`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `components` based on the provided criteria. + """ + filter: ComponentFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `components`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `components`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `components`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `components`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `components` should be sorted. + """ + order_by: [ComponentSortOrderInput!] + ): ComponentConnection + + """ + Aggregations over the `electrical_parts` data: + + > Fetches `ElectricalPart`s based on the provided arguments. + """ + electrical_part_aggregations( + """ + Used to forward-paginate through the `electrical_part_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `electrical_part_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `ElectricalPart` documents that get aggregated over based on the provided criteria. + """ + filter: ElectricalPartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `electrical_part_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `electrical_part_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `electrical_part_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `electrical_part_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): ElectricalPartAggregationConnection + + """ + Fetches `ElectricalPart`s based on the provided arguments. + """ + electrical_parts( + """ + Used to forward-paginate through the `electrical_parts`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `electrical_parts`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `electrical_parts` based on the provided criteria. + """ + filter: ElectricalPartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `electrical_parts`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `electrical_parts`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `electrical_parts`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `electrical_parts`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `electrical_parts` should be sorted. + """ + order_by: [ElectricalPartSortOrderInput!] + ): ElectricalPartConnection + + """ + Aggregations over the `manufacturers` data: + + > Fetches `Manufacturer`s based on the provided arguments. + """ + manufacturer_aggregations( + """ + Used to forward-paginate through the `manufacturer_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `manufacturer_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Manufacturer` documents that get aggregated over based on the provided criteria. + """ + filter: ManufacturerFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `manufacturer_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `manufacturer_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `manufacturer_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `manufacturer_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): ManufacturerAggregationConnection + + """ + Fetches `Manufacturer`s based on the provided arguments. + """ + manufacturers( + """ + Used to forward-paginate through the `manufacturers`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `manufacturers`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `manufacturers` based on the provided criteria. + """ + filter: ManufacturerFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `manufacturers`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `manufacturers`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `manufacturers`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `manufacturers`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `manufacturers` should be sorted. + """ + order_by: [ManufacturerSortOrderInput!] + ): ManufacturerConnection + + """ + Aggregations over the `mechanical_parts` data: + + > Fetches `MechanicalPart`s based on the provided arguments. + """ + mechanical_part_aggregations( + """ + Used to forward-paginate through the `mechanical_part_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `mechanical_part_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `MechanicalPart` documents that get aggregated over based on the provided criteria. + """ + filter: MechanicalPartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `mechanical_part_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `mechanical_part_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `mechanical_part_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `mechanical_part_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): MechanicalPartAggregationConnection + + """ + Fetches `MechanicalPart`s based on the provided arguments. + """ + mechanical_parts( + """ + Used to forward-paginate through the `mechanical_parts`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `mechanical_parts`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `mechanical_parts` based on the provided criteria. + """ + filter: MechanicalPartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `mechanical_parts`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `mechanical_parts`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `mechanical_parts`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `mechanical_parts`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `mechanical_parts` should be sorted. + """ + order_by: [MechanicalPartSortOrderInput!] + ): MechanicalPartConnection + + """ + Fetches `NamedEntity`s based on the provided arguments. + """ + named_entities( + """ + Used to forward-paginate through the `named_entities`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `named_entities`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `named_entities` based on the provided criteria. + """ + filter: NamedEntityFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `named_entities`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `named_entities`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `named_entities`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `named_entities`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `named_entities` should be sorted. + """ + order_by: [NamedEntitySortOrderInput!] + ): NamedEntityConnection + + """ + Aggregations over the `named_entities` data: + + > Fetches `NamedEntity`s based on the provided arguments. + """ + named_entity_aggregations( + """ + Used to forward-paginate through the `named_entity_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `named_entity_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `NamedEntity` documents that get aggregated over based on the provided criteria. + """ + filter: NamedEntityFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `named_entity_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `named_entity_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `named_entity_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `named_entity_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): NamedEntityAggregationConnection + + """ + Aggregations over the `parts` data: + + > Fetches `Part`s based on the provided arguments. + """ + part_aggregations( + """ + Used to forward-paginate through the `part_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `part_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Part` documents that get aggregated over based on the provided criteria. + """ + filter: PartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `part_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `part_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `part_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `part_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): PartAggregationConnection + + """ + Fetches `Part`s based on the provided arguments. + """ + parts( + """ + Used to forward-paginate through the `parts`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `parts`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `parts` based on the provided criteria. + """ + filter: PartFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `parts`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `parts`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `parts`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `parts`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `parts` should be sorted. + """ + order_by: [PartSortOrderInput!] + ): PartConnection + + """ + Aggregations over the `sponsors` data: + + > Fetches `Sponsor`s based on the provided arguments. + """ + sponsor_aggregations( + """ + Used to forward-paginate through the `sponsor_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `sponsor_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Sponsor` documents that get aggregated over based on the provided criteria. + """ + filter: SponsorFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `sponsor_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `sponsor_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `sponsor_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `sponsor_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): SponsorAggregationConnection + + """ + Fetches `Sponsor`s based on the provided arguments. + """ + sponsors( + """ + Used to forward-paginate through the `sponsors`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `sponsors`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `sponsors` based on the provided criteria. + """ + filter: SponsorFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `sponsors`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `sponsors`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `sponsors`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `sponsors`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `sponsors` should be sorted. + """ + order_by: [SponsorSortOrderInput!] + ): SponsorConnection + + """ + Aggregations over the `teams` data: + + > Fetches `Team`s based on the provided arguments. + """ + team_aggregations( + """ + Used to forward-paginate through the `team_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `team_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Team` documents that get aggregated over based on the provided criteria. + """ + filter: TeamFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `team_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `team_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `team_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `team_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): TeamAggregationConnection + + """ + Fetches `Team`s based on the provided arguments. + """ + teams( + """ + Used to forward-paginate through the `teams`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `teams`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `teams` based on the provided criteria. + """ + filter: TeamFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `teams`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `teams`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `teams`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `teams`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `teams` should be sorted. + """ + order_by: [TeamSortOrderInput!] + ): TeamConnection + + """ + Aggregations over the `widgets` data: + + > Fetches `Widget`s based on the provided arguments. + + Note: aggregation queries are relatively expensive, and some fields have been pre-aggregated to allow + more efficient queries for some common aggregation cases: + + - The root `widget_currencies` field groups by `cost.currency` + """ + widget_aggregations( + """ + Used to forward-paginate through the `widget_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widget_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Widget` documents that get aggregated over based on the provided criteria. + """ + filter: WidgetFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widget_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widget_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widget_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widget_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): WidgetAggregationConnection + + """ + Fetches `WidgetCurrency`s based on the provided arguments. + """ + widget_currencies( + """ + Used to forward-paginate through the `widget_currencies`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widget_currencies`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `widget_currencies` based on the provided criteria. + """ + filter: WidgetCurrencyFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widget_currencies`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widget_currencies`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widget_currencies`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widget_currencies`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `widget_currencies` should be sorted. + """ + order_by: [WidgetCurrencySortOrderInput!] + ): WidgetCurrencyConnection + + """ + Aggregations over the `widget_currencies` data: + + > Fetches `WidgetCurrency`s based on the provided arguments. + """ + widget_currency_aggregations( + """ + Used to forward-paginate through the `widget_currency_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widget_currency_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `WidgetCurrency` documents that get aggregated over based on the provided criteria. + """ + filter: WidgetCurrencyFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widget_currency_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widget_currency_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widget_currency_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widget_currency_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): WidgetCurrencyAggregationConnection + + """ + Aggregations over the `widgets_or_addresses` data: + + > Fetches `WidgetOrAddress`s based on the provided arguments. + """ + widget_or_address_aggregations( + """ + Used to forward-paginate through the `widget_or_address_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widget_or_address_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `WidgetOrAddress` documents that get aggregated over based on the provided criteria. + """ + filter: WidgetOrAddressFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widget_or_address_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widget_or_address_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widget_or_address_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widget_or_address_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): WidgetOrAddressAggregationConnection + + """ + Aggregations over the `widget_workspaces` data: + + > Fetches `WidgetWorkspace`s based on the provided arguments. + """ + widget_workspace_aggregations( + """ + Used to forward-paginate through the `widget_workspace_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widget_workspace_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `WidgetWorkspace` documents that get aggregated over based on the provided criteria. + """ + filter: WidgetWorkspaceFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widget_workspace_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widget_workspace_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widget_workspace_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widget_workspace_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): WidgetWorkspaceAggregationConnection + + """ + Fetches `WidgetWorkspace`s based on the provided arguments. + """ + widget_workspaces( + """ + Used to forward-paginate through the `widget_workspaces`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widget_workspaces`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `widget_workspaces` based on the provided criteria. + """ + filter: WidgetWorkspaceFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widget_workspaces`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widget_workspaces`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widget_workspaces`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widget_workspaces`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `widget_workspaces` should be sorted. + """ + order_by: [WidgetWorkspaceSortOrderInput!] + ): WidgetWorkspaceConnection + + """ + Fetches `Widget`s based on the provided arguments. + """ + widgets( + """ + Used to forward-paginate through the `widgets`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widgets`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `widgets` based on the provided criteria. + """ + filter: WidgetFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widgets`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widgets`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widgets`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widgets`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `widgets` should be sorted. + """ + order_by: [WidgetSortOrderInput!] + ): WidgetConnection + + """ + Fetches `WidgetOrAddress`s based on the provided arguments. + """ + widgets_or_addresses( + """ + Used to forward-paginate through the `widgets_or_addresses`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widgets_or_addresses`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `widgets_or_addresses` based on the provided criteria. + """ + filter: WidgetOrAddressFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widgets_or_addresses`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widgets_or_addresses`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widgets_or_addresses`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widgets_or_addresses`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `widgets_or_addresses` should be sorted. + """ + order_by: [WidgetOrAddressSortOrderInput!] + ): WidgetOrAddressConnection +} + +enum Size { + LARGE + MEDIUM + SMALL +} + +""" +Input type used to specify filters on `Size` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input SizeFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [SizeFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [SizeInput] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: SizeFilterInput +} + +enum SizeInput { + LARGE + MEDIUM + SMALL +} + +""" +Input type used to specify filters on elements of a `[Size]` field. + +Will be ignored if passed as an empty object (or as `null`). +""" +input SizeListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [SizeListElementFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [SizeInput!] +} + +""" +Input type used to specify filters on `[Size]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input SizeListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `SizeListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [SizeListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [SizeListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: SizeListElementFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: SizeListFilterInput +} + +type Sponsor @key(fields: "id") { + """ + Aggregations over the `affiliated_teams_from_nested` data. + """ + affiliated_team_from_nested_aggregations( + """ + Used to forward-paginate through the `affiliated_team_from_nested_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the + `affiliated_team_from_nested_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Team` documents that get aggregated over based on the provided criteria. + """ + filter: TeamFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through + the `affiliated_team_from_nested_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `affiliated_team_from_nested_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through + the `affiliated_team_from_nested_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `affiliated_team_from_nested_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): TeamAggregationConnection + + """ + Aggregations over the `affiliated_teams_from_object` data. + """ + affiliated_team_from_object_aggregations( + """ + Used to forward-paginate through the `affiliated_team_from_object_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the + `affiliated_team_from_object_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Team` documents that get aggregated over based on the provided criteria. + """ + filter: TeamFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through + the `affiliated_team_from_object_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `affiliated_team_from_object_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through + the `affiliated_team_from_object_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `affiliated_team_from_object_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): TeamAggregationConnection + affiliated_teams_from_nested( + """ + Used to forward-paginate through the `affiliated_teams_from_nested`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `affiliated_teams_from_nested`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `affiliated_teams_from_nested` based on the provided criteria. + """ + filter: TeamFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `affiliated_teams_from_nested`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `affiliated_teams_from_nested`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `affiliated_teams_from_nested`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `affiliated_teams_from_nested`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `affiliated_teams_from_nested` should be sorted. + """ + order_by: [TeamSortOrderInput!] + ): TeamConnection + affiliated_teams_from_object( + """ + Used to forward-paginate through the `affiliated_teams_from_object`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `affiliated_teams_from_object`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `affiliated_teams_from_object` based on the provided criteria. + """ + filter: TeamFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `affiliated_teams_from_object`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `affiliated_teams_from_object`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `affiliated_teams_from_object`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `affiliated_teams_from_object`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `affiliated_teams_from_object` should be sorted. + """ + order_by: [TeamSortOrderInput!] + ): TeamConnection + id: ID! + name: String +} + +""" +Type used to perform aggregation computations on `Sponsor` fields. +""" +type SponsorAggregatedValues { + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues +} + +""" +Return type representing a bucket of `Sponsor` documents for an aggregations query. +""" +type SponsorAggregation { + """ + Provides computed aggregated values over all `Sponsor` documents in an aggregation bucket. + """ + aggregated_values: SponsorAggregatedValues + + """ + The count of `Sponsor` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `Sponsor` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: SponsorGroupedBy +} + +""" +Represents a paginated collection of `SponsorAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type SponsorAggregationConnection { + """ + Wraps a specific `SponsorAggregation` to pair it with its pagination cursor. + """ + edges: [SponsorAggregationEdge!]! + + """ + The list of `SponsorAggregation` results. + """ + nodes: [SponsorAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `SponsorAggregation` in the context of a `SponsorAggregationConnection`, +providing access to both the `SponsorAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type SponsorAggregationEdge { + """ + The `Cursor` of this `SponsorAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `SponsorAggregation`. + """ + cursor: Cursor + + """ + The `SponsorAggregation` of this edge. + """ + node: SponsorAggregation +} + +""" +Represents a paginated collection of `Sponsor` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type SponsorConnection { + """ + Wraps a specific `Sponsor` to pair it with its pagination cursor. + """ + edges: [SponsorEdge!]! + + """ + The list of `Sponsor` results. + """ + nodes: [Sponsor!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `Sponsor` in the context of a `SponsorConnection`, +providing access to both the `Sponsor` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type SponsorEdge { + """ + The `Cursor` of this `Sponsor`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `Sponsor`. + """ + cursor: Cursor + + """ + The `Sponsor` of this edge. + """ + node: Sponsor +} + +""" +Input type used to specify filters on `Sponsor` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input SponsorFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [SponsorFilterInput!] + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: SponsorFilterInput +} + +""" +Type used to specify the `Sponsor` fields to group by for aggregations. +""" +type SponsorGroupedBy { + """ + The `name` field value for this group. + """ + name: String +} + +""" +Enumerates the ways `Sponsor`s can be sorted. +""" +enum SponsorSortOrderInput { + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC +} + +type Sponsorship { + annual_total: Money! + sponsor_id: ID! +} + +""" +Type used to perform aggregation computations on `Sponsorship` fields. +""" +type SponsorshipAggregatedValues { + """ + Computed aggregate values for the `annual_total` field. + """ + annual_total: MoneyAggregatedValues + + """ + Computed aggregate values for the `sponsor_id` field. + """ + sponsor_id: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on a `Sponsorship` object referenced directly +or transitively from a list field that has been configured to index each leaf field as +its own flattened list of values. + +Will be ignored if passed as an empty object (or as `null`). +""" +input SponsorshipFieldsListFilterInput { + """ + Used to filter on the `annual_total` field. + + Will be ignored if `null` or an empty object is passed. + """ + annual_total: MoneyFieldsListFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [SponsorshipFieldsListFilterInput!] + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: SponsorshipFieldsListFilterInput + + """ + Used to filter on the `sponsor_id` field. + + Will be ignored if `null` or an empty object is passed. + """ + sponsor_id: IDListFilterInput +} + +""" +Input type used to specify filters on `Sponsorship` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input SponsorshipFilterInput { + """ + Used to filter on the `annual_total` field. + + Will be ignored if `null` or an empty object is passed. + """ + annual_total: MoneyFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [SponsorshipFilterInput!] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: SponsorshipFilterInput + + """ + Used to filter on the `sponsor_id` field. + + Will be ignored if `null` or an empty object is passed. + """ + sponsor_id: IDFilterInput +} + +""" +Type used to specify the `Sponsorship` fields to group by for aggregations. +""" +type SponsorshipGroupedBy { + """ + The `annual_total` field value for this group. + """ + annual_total: MoneyGroupedBy + + """ + The `sponsor_id` field value for this group. + """ + sponsor_id: ID +} + +""" +Input type used to specify filters on `[Sponsorship]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input SponsorshipListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `SponsorshipListFilterInput` input because of + collisions between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [SponsorshipListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [SponsorshipListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: SponsorshipFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: SponsorshipListFilterInput +} + +""" +Represents a paginated collection of `String` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type StringConnection @shareable { + """ + Wraps a specific `String` to pair it with its pagination cursor. + """ + edges: [StringEdge!]! + + """ + The list of `String` results. + """ + nodes: [String!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `String` in the context of a `StringConnection`, +providing access to both the `String` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type StringEdge @shareable { + """ + The `Cursor` of this `String`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `String`. + """ + cursor: Cursor + + """ + The `String` of this edge. + """ + node: String +} + +""" +Input type used to specify filters on `String` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input StringFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [StringFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [String] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: StringFilterInput +} + +""" +Input type used to specify filters on elements of a `[String]` field. + +Will be ignored if passed as an empty object (or as `null`). +""" +input StringListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [StringListElementFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [String!] +} + +""" +Input type used to specify filters on `[String]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input StringListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `StringListFilterInput` input because of collisions + between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [StringListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [StringListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: StringListElementFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: StringListFilterInput +} + +""" +For more performant queries on this type, please filter on `league` if possible. +""" +type Team @key(fields: "id") { + country_code: ID! + current_name: String + current_players_nested: [Player!]! + current_players_object: [Player!]! + details: TeamDetails + forbes_valuation_moneys_nested: [Money!]! + forbes_valuation_moneys_object: [Money!]! + forbes_valuations: [JsonSafeLong!]! + formed_on: Date + id: ID! + league: String + nested_fields: TeamNestedFields + nested_fields2: TeamNestedFields + past_names: [String!]! + seasons_nested: [TeamSeason!]! + seasons_object: [TeamSeason!]! + stadium_location: GeoLocation + won_championships_at: [DateTime!]! +} + +""" +Type used to perform aggregation computations on `Team` fields. +""" +type TeamAggregatedValues { + """ + Computed aggregate values for the `country_code` field. + """ + country_code: NonNumericAggregatedValues + + """ + Computed aggregate values for the `current_name` field. + """ + current_name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `current_players_object` field. + """ + current_players_object: PlayerAggregatedValues + + """ + Computed aggregate values for the `details` field. + """ + details: TeamDetailsAggregatedValues + + """ + Computed aggregate values for the `forbes_valuation_moneys_object` field. + """ + forbes_valuation_moneys_object: MoneyAggregatedValues + + """ + Computed aggregate values for the `forbes_valuations` field. + """ + forbes_valuations: JsonSafeLongAggregatedValues + + """ + Computed aggregate values for the `formed_on` field. + """ + formed_on: DateAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `league` field. + """ + league: NonNumericAggregatedValues + + """ + Computed aggregate values for the `past_names` field. + """ + past_names: NonNumericAggregatedValues + + """ + Computed aggregate values for the `seasons_object` field. + """ + seasons_object: TeamSeasonAggregatedValues + + """ + Computed aggregate values for the `stadium_location` field. + """ + stadium_location: NonNumericAggregatedValues + + """ + Computed aggregate values for the `won_championships_at` field. + """ + won_championships_at: DateTimeAggregatedValues +} + +""" +Return type representing a bucket of `Team` documents for an aggregations query. +""" +type TeamAggregation { + """ + Provides computed aggregated values over all `Team` documents in an aggregation bucket. + """ + aggregated_values: TeamAggregatedValues + + """ + The count of `Team` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `Team` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: TeamGroupedBy + + """ + Used to perform sub-aggregations of `TeamAggregation` data. + """ + sub_aggregations: TeamAggregationSubAggregations +} + +""" +Represents a paginated collection of `TeamAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type TeamAggregationConnection { + """ + Wraps a specific `TeamAggregation` to pair it with its pagination cursor. + """ + edges: [TeamAggregationEdge!]! + + """ + The list of `TeamAggregation` results. + """ + nodes: [TeamAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Provides access to the `sub_aggregations` under `current_players_object.affiliations` within each `TeamAggregation`. +""" +type TeamAggregationCurrentPlayersObjectAffiliationsSubAggregations { + """ + Used to perform a sub-aggregation of `sponsorships_nested`. + """ + sponsorships_nested( + """ + Used to filter the `Sponsorship` documents included in this sub-aggregation based on the provided criteria. + """ + filter: SponsorshipFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamSponsorshipSubAggregationConnection +} + +""" +Provides access to the `sub_aggregations` under `current_players_object` within each `TeamAggregation`. +""" +type TeamAggregationCurrentPlayersObjectSubAggregations { + """ + Used to perform a sub-aggregation of `affiliations`. + """ + affiliations: TeamAggregationCurrentPlayersObjectAffiliationsSubAggregations + + """ + Used to perform a sub-aggregation of `seasons_nested`. + """ + seasons_nested( + """ + Used to filter the `PlayerSeason` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerSeasonFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamPlayerSeasonSubAggregationConnection +} + +""" +Represents a specific `TeamAggregation` in the context of a `TeamAggregationConnection`, +providing access to both the `TeamAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type TeamAggregationEdge { + """ + The `Cursor` of this `TeamAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `TeamAggregation`. + """ + cursor: Cursor + + """ + The `TeamAggregation` of this edge. + """ + node: TeamAggregation +} + +""" +Provides access to the `sub_aggregations` under `nested_fields2` within each `TeamAggregation`. +""" +type TeamAggregationNestedFields2SubAggregations { + """ + Used to perform a sub-aggregation of `current_players`. + """ + current_players( + """ + Used to filter the `Player` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamPlayerSubAggregationConnection + + """ + Used to perform a sub-aggregation of `forbes_valuation_moneys`. + """ + forbes_valuation_moneys( + """ + Used to filter the `Money` documents included in this sub-aggregation based on the provided criteria. + """ + filter: MoneyFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamMoneySubAggregationConnection + + """ + Used to perform a sub-aggregation of `seasons`. + """ + seasons( + """ + Used to filter the `TeamSeason` documents included in this sub-aggregation based on the provided criteria. + """ + filter: TeamSeasonFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamTeamSeasonSubAggregationConnection +} + +""" +Provides access to the `sub_aggregations` under `nested_fields` within each `TeamAggregation`. +""" +type TeamAggregationNestedFieldsSubAggregations { + """ + Used to perform a sub-aggregation of `current_players`. + """ + current_players( + """ + Used to filter the `Player` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamPlayerSubAggregationConnection + + """ + Used to perform a sub-aggregation of `forbes_valuation_moneys`. + """ + forbes_valuation_moneys( + """ + Used to filter the `Money` documents included in this sub-aggregation based on the provided criteria. + """ + filter: MoneyFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamMoneySubAggregationConnection + + """ + Used to perform a sub-aggregation of `seasons`. + """ + seasons( + """ + Used to filter the `TeamSeason` documents included in this sub-aggregation based on the provided criteria. + """ + filter: TeamSeasonFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamTeamSeasonSubAggregationConnection +} + +""" +Provides access to the `sub_aggregations` under `seasons_object.players_object.affiliations` within each `TeamAggregation`. +""" +type TeamAggregationSeasonsObjectPlayersObjectAffiliationsSubAggregations { + """ + Used to perform a sub-aggregation of `sponsorships_nested`. + """ + sponsorships_nested( + """ + Used to filter the `Sponsorship` documents included in this sub-aggregation based on the provided criteria. + """ + filter: SponsorshipFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamSponsorshipSubAggregationConnection +} + +""" +Provides access to the `sub_aggregations` under `seasons_object.players_object` within each `TeamAggregation`. +""" +type TeamAggregationSeasonsObjectPlayersObjectSubAggregations { + """ + Used to perform a sub-aggregation of `affiliations`. + """ + affiliations: TeamAggregationSeasonsObjectPlayersObjectAffiliationsSubAggregations + + """ + Used to perform a sub-aggregation of `seasons_nested`. + """ + seasons_nested( + """ + Used to filter the `PlayerSeason` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerSeasonFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamPlayerSeasonSubAggregationConnection +} + +""" +Provides access to the `sub_aggregations` under `seasons_object` within each `TeamAggregation`. +""" +type TeamAggregationSeasonsObjectSubAggregations { + """ + Used to perform a sub-aggregation of `players_nested`. + """ + players_nested( + """ + Used to filter the `Player` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamPlayerSubAggregationConnection + + """ + Used to perform a sub-aggregation of `players_object`. + """ + players_object: TeamAggregationSeasonsObjectPlayersObjectSubAggregations +} + +""" +Provides access to the `sub_aggregations` within each `TeamAggregation`. +""" +type TeamAggregationSubAggregations { + """ + Used to perform a sub-aggregation of `current_players_nested`. + """ + current_players_nested( + """ + Used to filter the `Player` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamPlayerSubAggregationConnection + + """ + Used to perform a sub-aggregation of `current_players_object`. + """ + current_players_object: TeamAggregationCurrentPlayersObjectSubAggregations + + """ + Used to perform a sub-aggregation of `forbes_valuation_moneys_nested`. + """ + forbes_valuation_moneys_nested( + """ + Used to filter the `Money` documents included in this sub-aggregation based on the provided criteria. + """ + filter: MoneyFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamMoneySubAggregationConnection + + """ + Used to perform a sub-aggregation of `nested_fields`. + """ + nested_fields: TeamAggregationNestedFieldsSubAggregations + + """ + Used to perform a sub-aggregation of `nested_fields2`. + """ + nested_fields2: TeamAggregationNestedFields2SubAggregations + + """ + Used to perform a sub-aggregation of `seasons_nested`. + """ + seasons_nested( + """ + Used to filter the `TeamSeason` documents included in this sub-aggregation based on the provided criteria. + """ + filter: TeamSeasonFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamTeamSeasonSubAggregationConnection + + """ + Used to perform a sub-aggregation of `seasons_object`. + """ + seasons_object: TeamAggregationSeasonsObjectSubAggregations +} + +""" +Represents a paginated collection of `Team` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type TeamConnection { + """ + Wraps a specific `Team` to pair it with its pagination cursor. + """ + edges: [TeamEdge!]! + + """ + The list of `Team` results. + """ + nodes: [Team!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +type TeamDetails { + count: Int + uniform_colors: [String!]! +} + +""" +Type used to perform aggregation computations on `TeamDetails` fields. +""" +type TeamDetailsAggregatedValues { + """ + Computed aggregate values for the `count` field. + """ + count: IntAggregatedValues + + """ + Computed aggregate values for the `uniform_colors` field. + """ + uniform_colors: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on `TeamDetails` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TeamDetailsFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TeamDetailsFilterInput!] + + """ + Used to filter on the `count` field. + + Will be ignored if `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TeamDetailsFilterInput + + """ + Used to filter on the `uniform_colors` field. + + Will be ignored if `null` or an empty object is passed. + """ + uniform_colors: StringListFilterInput +} + +""" +Type used to specify the `TeamDetails` fields to group by for aggregations. +""" +type TeamDetailsGroupedBy { + """ + The `count` field value for this group. + """ + count: Int +} + +""" +Represents a specific `Team` in the context of a `TeamConnection`, +providing access to both the `Team` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type TeamEdge { + """ + The `Cursor` of this `Team`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `Team`. + """ + cursor: Cursor + + """ + The `Team` of this edge. + """ + node: Team +} + +""" +Input type used to specify filters on `Team` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TeamFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TeamFilterInput!] + + """ + Used to filter on the `country_code` field. + + Will be ignored if `null` or an empty object is passed. + """ + country_code: IDFilterInput + + """ + Used to filter on the `current_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + current_name: StringFilterInput + + """ + Used to filter on the `current_players_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + current_players_nested: PlayerListFilterInput + + """ + Used to filter on the `current_players_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + current_players_object: PlayerFieldsListFilterInput + + """ + Used to filter on the `details` field. + + Will be ignored if `null` or an empty object is passed. + """ + details: TeamDetailsFilterInput + + """ + Used to filter on the `forbes_valuation_moneys_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + forbes_valuation_moneys_nested: MoneyListFilterInput + + """ + Used to filter on the `forbes_valuation_moneys_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + forbes_valuation_moneys_object: MoneyFieldsListFilterInput + + """ + Used to filter on the `forbes_valuations` field. + + Will be ignored if `null` or an empty object is passed. + """ + forbes_valuations: JsonSafeLongListFilterInput + + """ + Used to filter on the `formed_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + formed_on: DateFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `league` field. + + Will be ignored if `null` or an empty object is passed. + """ + league: StringFilterInput + + """ + Used to filter on the `nested_fields` field. + + Will be ignored if `null` or an empty object is passed. + """ + nested_fields: TeamNestedFieldsFilterInput + + """ + Used to filter on the `nested_fields2` field. + + Will be ignored if `null` or an empty object is passed. + """ + nested_fields2: TeamNestedFieldsFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TeamFilterInput + + """ + Used to filter on the `past_names` field. + + Will be ignored if `null` or an empty object is passed. + """ + past_names: StringListFilterInput + + """ + Used to filter on the `seasons_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + seasons_nested: TeamSeasonListFilterInput + + """ + Used to filter on the `seasons_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + seasons_object: TeamSeasonFieldsListFilterInput + + """ + Used to filter on the `stadium_location` field. + + Will be ignored if `null` or an empty object is passed. + """ + stadium_location: GeoLocationFilterInput + + """ + Used to filter on the `won_championships_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + won_championships_at: DateTimeListFilterInput +} + +""" +Type used to specify the `Team` fields to group by for aggregations. +""" +type TeamGroupedBy { + """ + The `country_code` field value for this group. + """ + country_code: ID + + """ + The `current_name` field value for this group. + """ + current_name: String + + """ + The `current_players_object` field value for this group. + + Note: `current_players_object` is a collection field, but selecting this field + will group on individual values of the selected subfields of + `current_players_object`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `current_players_object` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `current_players_object` multiple times for a single document, that document will only be included in the group + once. + """ + current_players_object: PlayerGroupedBy + + """ + The `details` field value for this group. + """ + details: TeamDetailsGroupedBy + + """ + The `forbes_valuation_moneys_object` field value for this group. + + Note: `forbes_valuation_moneys_object` is a collection field, but selecting + this field will group on individual values of the selected subfields of + `forbes_valuation_moneys_object`. + That means that a document may be grouped into multiple aggregation groupings + (i.e. when its `forbes_valuation_moneys_object` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `forbes_valuation_moneys_object` multiple times for a single document, + that document will only be included in the group + once. + """ + forbes_valuation_moneys_object: MoneyGroupedBy + + """ + Offers the different grouping options for the `formed_on` value within this group. + """ + formed_on: DateGroupedBy + + """ + The `league` field value for this group. + """ + league: String + + """ + The `seasons_object` field value for this group. + + Note: `seasons_object` is a collection field, but selecting this field will + group on individual values of the selected subfields of `seasons_object`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `seasons_object` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `seasons_object` multiple times for a single document, that document will only be included in the group + once. + """ + seasons_object: TeamSeasonGroupedBy +} + +""" +Return type representing a bucket of `Money` objects for a sub-aggregation within each `TeamAggregation`. +""" +type TeamMoneySubAggregation { + """ + Provides computed aggregated values over all `Money` documents in a sub-aggregation bucket. + """ + aggregated_values: MoneyAggregatedValues + + """ + Details of the count of `Money` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `Money` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: MoneyGroupedBy +} + +""" +Represents a collection of `TeamMoneySubAggregation` results. +""" +type TeamMoneySubAggregationConnection { + """ + The list of `TeamMoneySubAggregation` results. + """ + nodes: [TeamMoneySubAggregation!]! +} + +type TeamNestedFields { + current_players: [Player!]! + forbes_valuation_moneys: [Money!]! + seasons: [TeamSeason!]! +} + +""" +Input type used to specify filters on `TeamNestedFields` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TeamNestedFieldsFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TeamNestedFieldsFilterInput!] + + """ + Used to filter on the `current_players` field. + + Will be ignored if `null` or an empty object is passed. + """ + current_players: PlayerListFilterInput + + """ + Used to filter on the `forbes_valuation_moneys` field. + + Will be ignored if `null` or an empty object is passed. + """ + forbes_valuation_moneys: MoneyListFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TeamNestedFieldsFilterInput + + """ + Used to filter on the `seasons` field. + + Will be ignored if `null` or an empty object is passed. + """ + seasons: TeamSeasonListFilterInput +} + +""" +Return type representing a bucket of `PlayerSeason` objects for a sub-aggregation within each `TeamPlayerSubAggregation`. +""" +type TeamPlayerPlayerSeasonSubAggregation { + """ + Provides computed aggregated values over all `PlayerSeason` documents in a sub-aggregation bucket. + """ + aggregated_values: PlayerSeasonAggregatedValues + + """ + Details of the count of `PlayerSeason` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `PlayerSeason` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: PlayerSeasonGroupedBy +} + +""" +Represents a collection of `TeamPlayerPlayerSeasonSubAggregation` results. +""" +type TeamPlayerPlayerSeasonSubAggregationConnection { + """ + The list of `TeamPlayerPlayerSeasonSubAggregation` results. + """ + nodes: [TeamPlayerPlayerSeasonSubAggregation!]! +} + +""" +Return type representing a bucket of `PlayerSeason` objects for a sub-aggregation within each `TeamAggregation`. +""" +type TeamPlayerSeasonSubAggregation { + """ + Provides computed aggregated values over all `PlayerSeason` documents in a sub-aggregation bucket. + """ + aggregated_values: PlayerSeasonAggregatedValues + + """ + Details of the count of `PlayerSeason` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `PlayerSeason` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: PlayerSeasonGroupedBy +} + +""" +Represents a collection of `TeamPlayerSeasonSubAggregation` results. +""" +type TeamPlayerSeasonSubAggregationConnection { + """ + The list of `TeamPlayerSeasonSubAggregation` results. + """ + nodes: [TeamPlayerSeasonSubAggregation!]! +} + +""" +Return type representing a bucket of `Sponsorship` objects for a sub-aggregation within each `TeamPlayerSubAggregation`. +""" +type TeamPlayerSponsorshipSubAggregation { + """ + Provides computed aggregated values over all `Sponsorship` documents in a sub-aggregation bucket. + """ + aggregated_values: SponsorshipAggregatedValues + + """ + Details of the count of `Sponsorship` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `Sponsorship` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: SponsorshipGroupedBy +} + +""" +Represents a collection of `TeamPlayerSponsorshipSubAggregation` results. +""" +type TeamPlayerSponsorshipSubAggregationConnection { + """ + The list of `TeamPlayerSponsorshipSubAggregation` results. + """ + nodes: [TeamPlayerSponsorshipSubAggregation!]! +} + +""" +Return type representing a bucket of `Player` objects for a sub-aggregation within each `TeamAggregation`. +""" +type TeamPlayerSubAggregation { + """ + Provides computed aggregated values over all `Player` documents in a sub-aggregation bucket. + """ + aggregated_values: PlayerAggregatedValues + + """ + Details of the count of `Player` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `Player` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: PlayerGroupedBy + + """ + Used to perform sub-aggregations of `TeamPlayerSubAggregation` data. + """ + sub_aggregations: TeamPlayerSubAggregationSubAggregations +} + +""" +Provides access to the `sub_aggregations` under `affiliations` within each `TeamPlayerSubAggregation`. +""" +type TeamPlayerSubAggregationAffiliationsSubAggregations { + """ + Used to perform a sub-aggregation of `sponsorships_nested`. + """ + sponsorships_nested( + """ + Used to filter the `Sponsorship` documents included in this sub-aggregation based on the provided criteria. + """ + filter: SponsorshipFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamPlayerSponsorshipSubAggregationConnection +} + +""" +Represents a collection of `TeamPlayerSubAggregation` results. +""" +type TeamPlayerSubAggregationConnection { + """ + The list of `TeamPlayerSubAggregation` results. + """ + nodes: [TeamPlayerSubAggregation!]! +} + +""" +Provides access to the `sub_aggregations` within each `TeamPlayerSubAggregation`. +""" +type TeamPlayerSubAggregationSubAggregations { + """ + Used to perform a sub-aggregation of `affiliations`. + """ + affiliations: TeamPlayerSubAggregationAffiliationsSubAggregations + + """ + Used to perform a sub-aggregation of `seasons_nested`. + """ + seasons_nested( + """ + Used to filter the `PlayerSeason` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerSeasonFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamPlayerPlayerSeasonSubAggregationConnection +} + +type TeamRecord { + first_win_on: Date + first_win_on_legacy: Date + last_win_on: Date + last_win_on_legacy: Date + losses: Int + wins: Int +} + +""" +Type used to perform aggregation computations on `TeamRecord` fields. +""" +type TeamRecordAggregatedValues { + """ + Computed aggregate values for the `first_win_on` field. + """ + first_win_on: DateAggregatedValues + + """ + Computed aggregate values for the `first_win_on_legacy` field. + """ + first_win_on_legacy: DateAggregatedValues + + """ + Computed aggregate values for the `last_win_on` field. + """ + last_win_on: DateAggregatedValues + + """ + Computed aggregate values for the `last_win_on_legacy` field. + """ + last_win_on_legacy: DateAggregatedValues + + """ + Computed aggregate values for the `losses` field. + """ + losses: IntAggregatedValues + + """ + Computed aggregate values for the `wins` field. + """ + wins: IntAggregatedValues +} + +""" +Input type used to specify filters on a `TeamRecord` object referenced directly +or transitively from a list field that has been configured to index each leaf field as +its own flattened list of values. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TeamRecordFieldsListFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TeamRecordFieldsListFilterInput!] + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Used to filter on the `first_win_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + first_win_on: DateListFilterInput + + """ + Used to filter on the `first_win_on_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + first_win_on_legacy: DateListFilterInput + + """ + Used to filter on the `last_win_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + last_win_on: DateListFilterInput + + """ + Used to filter on the `last_win_on_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + last_win_on_legacy: DateListFilterInput + + """ + Used to filter on the `losses` field. + + Will be ignored if `null` or an empty object is passed. + """ + losses: IntListFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TeamRecordFieldsListFilterInput + + """ + Used to filter on the `wins` field. + + Will be ignored if `null` or an empty object is passed. + """ + wins: IntListFilterInput +} + +""" +Input type used to specify filters on `TeamRecord` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TeamRecordFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TeamRecordFilterInput!] + + """ + Used to filter on the `first_win_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + first_win_on: DateFilterInput + + """ + Used to filter on the `first_win_on_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + first_win_on_legacy: DateFilterInput + + """ + Used to filter on the `last_win_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + last_win_on: DateFilterInput + + """ + Used to filter on the `last_win_on_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + last_win_on_legacy: DateFilterInput + + """ + Used to filter on the `losses` field. + + Will be ignored if `null` or an empty object is passed. + """ + losses: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TeamRecordFilterInput + + """ + Used to filter on the `wins` field. + + Will be ignored if `null` or an empty object is passed. + """ + wins: IntFilterInput +} + +""" +Type used to specify the `TeamRecord` fields to group by for aggregations. +""" +type TeamRecordGroupedBy { + """ + Offers the different grouping options for the `first_win_on` value within this group. + """ + first_win_on: DateGroupedBy + + """ + The `first_win_on_legacy` field value for this group. + """ + first_win_on_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateGroupingGranularityInput! + + """ + Number of days (positive or negative) to shift the `Date` boundaries of each date grouping bucket. + + For example, when grouping by `YEAR`, this can be used to align the buckets + with fiscal or school years instead of calendar years. + """ + offset_days: Int + ): Date + + """ + Offers the different grouping options for the `last_win_on` value within this group. + """ + last_win_on: DateGroupedBy + + """ + The `last_win_on_legacy` field value for this group. + """ + last_win_on_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateGroupingGranularityInput! + + """ + Number of days (positive or negative) to shift the `Date` boundaries of each date grouping bucket. + + For example, when grouping by `YEAR`, this can be used to align the buckets + with fiscal or school years instead of calendar years. + """ + offset_days: Int + ): Date + + """ + The `losses` field value for this group. + """ + losses: Int + + """ + The `wins` field value for this group. + """ + wins: Int +} + +type TeamSeason { + count: Int + notes: [String!]! + players_nested: [Player!]! + players_object: [Player!]! + record: TeamRecord + started_at: DateTime + started_at_legacy: DateTime + won_games_at: [DateTime!]! + won_games_at_legacy: [DateTime!]! + year: Int +} + +""" +Type used to perform aggregation computations on `TeamSeason` fields. +""" +type TeamSeasonAggregatedValues { + """ + Computed aggregate values for the `count` field. + """ + count: IntAggregatedValues + + """ + Computed aggregate values for the `notes` field. + """ + notes: NonNumericAggregatedValues + + """ + Computed aggregate values for the `players_object` field. + """ + players_object: PlayerAggregatedValues + + """ + Computed aggregate values for the `record` field. + """ + record: TeamRecordAggregatedValues + + """ + Computed aggregate values for the `started_at` field. + """ + started_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `started_at_legacy` field. + """ + started_at_legacy: DateTimeAggregatedValues + + """ + Computed aggregate values for the `won_games_at` field. + """ + won_games_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `won_games_at_legacy` field. + """ + won_games_at_legacy: DateTimeAggregatedValues + + """ + Computed aggregate values for the `year` field. + """ + year: IntAggregatedValues +} + +""" +Input type used to specify filters on a `TeamSeason` object referenced directly +or transitively from a list field that has been configured to index each leaf field as +its own flattened list of values. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TeamSeasonFieldsListFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TeamSeasonFieldsListFilterInput!] + + """ + Used to filter on the `count` field. + + Will be ignored if `null` or an empty object is passed. + """ + count: IntListFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TeamSeasonFieldsListFilterInput + + """ + Used to filter on the `notes` field. + + Will be ignored if `null` or an empty object is passed. + """ + notes: StringListFilterInput + + """ + Used to filter on the `players_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + players_nested: PlayerListFilterInput + + """ + Used to filter on the `players_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + players_object: PlayerFieldsListFilterInput + + """ + Used to filter on the `record` field. + + Will be ignored if `null` or an empty object is passed. + """ + record: TeamRecordFieldsListFilterInput + + """ + Used to filter on the `started_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + started_at: DateTimeListFilterInput + + """ + Used to filter on the `started_at_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + started_at_legacy: DateTimeListFilterInput + + """ + Used to filter on the `won_games_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + won_games_at: DateTimeListFilterInput + + """ + Used to filter on the `won_games_at_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + won_games_at_legacy: DateTimeListFilterInput + + """ + Used to filter on the `year` field. + + Will be ignored if `null` or an empty object is passed. + """ + year: IntListFilterInput +} + +""" +Input type used to specify filters on `TeamSeason` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TeamSeasonFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TeamSeasonFilterInput!] + + """ + Used to filter on the `count` field. + + Will be ignored if `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TeamSeasonFilterInput + + """ + Used to filter on the `notes` field. + + Will be ignored if `null` or an empty object is passed. + """ + notes: StringListFilterInput + + """ + Used to filter on the `players_nested` field. + + Will be ignored if `null` or an empty object is passed. + """ + players_nested: PlayerListFilterInput + + """ + Used to filter on the `players_object` field. + + Will be ignored if `null` or an empty object is passed. + """ + players_object: PlayerFieldsListFilterInput + + """ + Used to filter on the `record` field. + + Will be ignored if `null` or an empty object is passed. + """ + record: TeamRecordFilterInput + + """ + Used to filter on the `started_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + started_at: DateTimeFilterInput + + """ + Used to filter on the `started_at_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + started_at_legacy: DateTimeFilterInput + + """ + Used to filter on the `won_games_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + won_games_at: DateTimeListFilterInput + + """ + Used to filter on the `won_games_at_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + won_games_at_legacy: DateTimeListFilterInput + + """ + Used to filter on the `year` field. + + Will be ignored if `null` or an empty object is passed. + """ + year: IntFilterInput +} + +""" +Type used to specify the `TeamSeason` fields to group by for aggregations. +""" +type TeamSeasonGroupedBy { + """ + The `count` field value for this group. + """ + count: Int + + """ + The individual value from `notes` for this group. + + Note: `notes` is a collection field, but selecting this field will group on individual values of `notes`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `notes` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `notes` multiple times for a single document, that document will only be included in the group + once. + """ + note: String + + """ + The `players_object` field value for this group. + + Note: `players_object` is a collection field, but selecting this field will + group on individual values of the selected subfields of `players_object`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `players_object` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `players_object` multiple times for a single document, that document will only be included in the group + once. + """ + players_object: PlayerGroupedBy + + """ + The `record` field value for this group. + """ + record: TeamRecordGroupedBy + + """ + Offers the different grouping options for the `started_at` value within this group. + """ + started_at: DateTimeGroupedBy + + """ + The `started_at_legacy` field value for this group. + """ + started_at_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateTimeGroupingGranularityInput! + + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change + what day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone = "UTC" + ): DateTime + + """ + The individual value from `won_games_at` for this group. + + Note: `won_games_at` is a collection field, but selecting this field will group on individual values of `won_games_at`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `won_games_at` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `won_games_at` multiple times for a single document, that document will only be included in the group + once. + """ + won_game_at: DateTimeGroupedBy + + """ + The individual value from `won_games_at_legacy` for this group. + + Note: `won_games_at_legacy` is a collection field, but selecting this field + will group on individual values of `won_games_at_legacy`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `won_games_at_legacy` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `won_games_at_legacy` multiple times for a single document, that document will only be included in the group + once. + """ + won_game_at_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateTimeGroupingGranularityInput! + + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change + what day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone = "UTC" + ): DateTime + + """ + The `year` field value for this group. + """ + year: Int +} + +""" +Input type used to specify filters on `[TeamSeason]` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TeamSeasonListFilterInput { + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `TeamSeasonListFilterInput` input because of + collisions between key names. For example, if you want to provide + multiple `any_satisfy: ...` filters, you could do `all_of: [{any_satisfy: ...}, {any_satisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + all_of: [TeamSeasonListFilterInput!] + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TeamSeasonListFilterInput!] + + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + any_satisfy: TeamSeasonFilterInput + + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TeamSeasonListFilterInput +} + +""" +Enumerates the ways `Team`s can be sorted. +""" +enum TeamSortOrderInput { + """ + Sorts ascending by the `country_code` field. + """ + country_code_ASC + + """ + Sorts descending by the `country_code` field. + """ + country_code_DESC + + """ + Sorts ascending by the `current_name` field. + """ + current_name_ASC + + """ + Sorts descending by the `current_name` field. + """ + current_name_DESC + + """ + Sorts ascending by the `details.count` field. + """ + details_count_ASC + + """ + Sorts descending by the `details.count` field. + """ + details_count_DESC + + """ + Sorts ascending by the `formed_on` field. + """ + formed_on_ASC + + """ + Sorts descending by the `formed_on` field. + """ + formed_on_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `league` field. + """ + league_ASC + + """ + Sorts descending by the `league` field. + """ + league_DESC +} + +""" +Return type representing a bucket of `Sponsorship` objects for a sub-aggregation within each `TeamAggregation`. +""" +type TeamSponsorshipSubAggregation { + """ + Provides computed aggregated values over all `Sponsorship` documents in a sub-aggregation bucket. + """ + aggregated_values: SponsorshipAggregatedValues + + """ + Details of the count of `Sponsorship` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `Sponsorship` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: SponsorshipGroupedBy +} + +""" +Represents a collection of `TeamSponsorshipSubAggregation` results. +""" +type TeamSponsorshipSubAggregationConnection { + """ + The list of `TeamSponsorshipSubAggregation` results. + """ + nodes: [TeamSponsorshipSubAggregation!]! +} + +""" +Return type representing a bucket of `PlayerSeason` objects for a +sub-aggregation within each `TeamTeamSeasonPlayerSubAggregation`. +""" +type TeamTeamSeasonPlayerPlayerSeasonSubAggregation { + """ + Provides computed aggregated values over all `PlayerSeason` documents in a sub-aggregation bucket. + """ + aggregated_values: PlayerSeasonAggregatedValues + + """ + Details of the count of `PlayerSeason` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `PlayerSeason` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: PlayerSeasonGroupedBy +} + +""" +Represents a collection of `TeamTeamSeasonPlayerPlayerSeasonSubAggregation` results. +""" +type TeamTeamSeasonPlayerPlayerSeasonSubAggregationConnection { + """ + The list of `TeamTeamSeasonPlayerPlayerSeasonSubAggregation` results. + """ + nodes: [TeamTeamSeasonPlayerPlayerSeasonSubAggregation!]! +} + +""" +Return type representing a bucket of `PlayerSeason` objects for a +sub-aggregation within each `TeamTeamSeasonSubAggregation`. +""" +type TeamTeamSeasonPlayerSeasonSubAggregation { + """ + Provides computed aggregated values over all `PlayerSeason` documents in a sub-aggregation bucket. + """ + aggregated_values: PlayerSeasonAggregatedValues + + """ + Details of the count of `PlayerSeason` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `PlayerSeason` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: PlayerSeasonGroupedBy +} + +""" +Represents a collection of `TeamTeamSeasonPlayerSeasonSubAggregation` results. +""" +type TeamTeamSeasonPlayerSeasonSubAggregationConnection { + """ + The list of `TeamTeamSeasonPlayerSeasonSubAggregation` results. + """ + nodes: [TeamTeamSeasonPlayerSeasonSubAggregation!]! +} + +""" +Return type representing a bucket of `Sponsorship` objects for a sub-aggregation +within each `TeamTeamSeasonPlayerSubAggregation`. +""" +type TeamTeamSeasonPlayerSponsorshipSubAggregation { + """ + Provides computed aggregated values over all `Sponsorship` documents in a sub-aggregation bucket. + """ + aggregated_values: SponsorshipAggregatedValues + + """ + Details of the count of `Sponsorship` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `Sponsorship` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: SponsorshipGroupedBy +} + +""" +Represents a collection of `TeamTeamSeasonPlayerSponsorshipSubAggregation` results. +""" +type TeamTeamSeasonPlayerSponsorshipSubAggregationConnection { + """ + The list of `TeamTeamSeasonPlayerSponsorshipSubAggregation` results. + """ + nodes: [TeamTeamSeasonPlayerSponsorshipSubAggregation!]! +} + +""" +Return type representing a bucket of `Player` objects for a sub-aggregation within each `TeamTeamSeasonSubAggregation`. +""" +type TeamTeamSeasonPlayerSubAggregation { + """ + Provides computed aggregated values over all `Player` documents in a sub-aggregation bucket. + """ + aggregated_values: PlayerAggregatedValues + + """ + Details of the count of `Player` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `Player` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: PlayerGroupedBy + + """ + Used to perform sub-aggregations of `TeamTeamSeasonPlayerSubAggregation` data. + """ + sub_aggregations: TeamTeamSeasonPlayerSubAggregationSubAggregations +} + +""" +Provides access to the `sub_aggregations` under `affiliations` within each `TeamTeamSeasonPlayerSubAggregation`. +""" +type TeamTeamSeasonPlayerSubAggregationAffiliationsSubAggregations { + """ + Used to perform a sub-aggregation of `sponsorships_nested`. + """ + sponsorships_nested( + """ + Used to filter the `Sponsorship` documents included in this sub-aggregation based on the provided criteria. + """ + filter: SponsorshipFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamTeamSeasonPlayerSponsorshipSubAggregationConnection +} + +""" +Represents a collection of `TeamTeamSeasonPlayerSubAggregation` results. +""" +type TeamTeamSeasonPlayerSubAggregationConnection { + """ + The list of `TeamTeamSeasonPlayerSubAggregation` results. + """ + nodes: [TeamTeamSeasonPlayerSubAggregation!]! +} + +""" +Provides access to the `sub_aggregations` within each `TeamTeamSeasonPlayerSubAggregation`. +""" +type TeamTeamSeasonPlayerSubAggregationSubAggregations { + """ + Used to perform a sub-aggregation of `affiliations`. + """ + affiliations: TeamTeamSeasonPlayerSubAggregationAffiliationsSubAggregations + + """ + Used to perform a sub-aggregation of `seasons_nested`. + """ + seasons_nested( + """ + Used to filter the `PlayerSeason` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerSeasonFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamTeamSeasonPlayerPlayerSeasonSubAggregationConnection +} + +""" +Return type representing a bucket of `Sponsorship` objects for a sub-aggregation within each `TeamTeamSeasonSubAggregation`. +""" +type TeamTeamSeasonSponsorshipSubAggregation { + """ + Provides computed aggregated values over all `Sponsorship` documents in a sub-aggregation bucket. + """ + aggregated_values: SponsorshipAggregatedValues + + """ + Details of the count of `Sponsorship` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `Sponsorship` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: SponsorshipGroupedBy +} + +""" +Represents a collection of `TeamTeamSeasonSponsorshipSubAggregation` results. +""" +type TeamTeamSeasonSponsorshipSubAggregationConnection { + """ + The list of `TeamTeamSeasonSponsorshipSubAggregation` results. + """ + nodes: [TeamTeamSeasonSponsorshipSubAggregation!]! +} + +""" +Return type representing a bucket of `TeamSeason` objects for a sub-aggregation within each `TeamAggregation`. +""" +type TeamTeamSeasonSubAggregation { + """ + Provides computed aggregated values over all `TeamSeason` documents in a sub-aggregation bucket. + """ + aggregated_values: TeamSeasonAggregatedValues + + """ + Details of the count of `TeamSeason` documents in a sub-aggregation bucket. + """ + count_detail: AggregationCountDetail + + """ + Used to specify the `TeamSeason` fields to group by. The returned values identify each sub-aggregation bucket. + """ + grouped_by: TeamSeasonGroupedBy + + """ + Used to perform sub-aggregations of `TeamTeamSeasonSubAggregation` data. + """ + sub_aggregations: TeamTeamSeasonSubAggregationSubAggregations +} + +""" +Represents a collection of `TeamTeamSeasonSubAggregation` results. +""" +type TeamTeamSeasonSubAggregationConnection { + """ + The list of `TeamTeamSeasonSubAggregation` results. + """ + nodes: [TeamTeamSeasonSubAggregation!]! +} + +""" +Provides access to the `sub_aggregations` under `players_object.affiliations` within each `TeamTeamSeasonSubAggregation`. +""" +type TeamTeamSeasonSubAggregationPlayersObjectAffiliationsSubAggregations { + """ + Used to perform a sub-aggregation of `sponsorships_nested`. + """ + sponsorships_nested( + """ + Used to filter the `Sponsorship` documents included in this sub-aggregation based on the provided criteria. + """ + filter: SponsorshipFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamTeamSeasonSponsorshipSubAggregationConnection +} + +""" +Provides access to the `sub_aggregations` under `players_object` within each `TeamTeamSeasonSubAggregation`. +""" +type TeamTeamSeasonSubAggregationPlayersObjectSubAggregations { + """ + Used to perform a sub-aggregation of `affiliations`. + """ + affiliations: TeamTeamSeasonSubAggregationPlayersObjectAffiliationsSubAggregations + + """ + Used to perform a sub-aggregation of `seasons_nested`. + """ + seasons_nested( + """ + Used to filter the `PlayerSeason` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerSeasonFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamTeamSeasonPlayerSeasonSubAggregationConnection +} + +""" +Provides access to the `sub_aggregations` within each `TeamTeamSeasonSubAggregation`. +""" +type TeamTeamSeasonSubAggregationSubAggregations { + """ + Used to perform a sub-aggregation of `players_nested`. + """ + players_nested( + """ + Used to filter the `Player` documents included in this sub-aggregation based on the provided criteria. + """ + filter: PlayerFilterInput + + """ + Determines how many sub-aggregation buckets should be returned. + """ + first: Int + ): TeamTeamSeasonPlayerSubAggregationConnection + + """ + Used to perform a sub-aggregation of `players_object`. + """ + players_object: TeamTeamSeasonSubAggregationPlayersObjectSubAggregations +} + +""" +Input type used to specify filters on `String` fields that have been indexed for full text search. + +Will be ignored if passed as an empty object (or as `null`). +""" +input TextFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [TextFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [String] + + """ + Matches records where the field value matches the provided value using full text search. + + Will be ignored when `null` is passed. + """ + matches: String @deprecated(reason: "Use `matches_query` instead.") + + """ + Matches records where the field value has a phrase matching the provided phrase using + full text search. This is stricter than `matches_query`: all terms must match + and be in the same order as the provided phrase. + + Will be ignored when `null` is passed. + """ + matches_phrase: MatchesPhraseFilterInput + + """ + Matches records where the field value matches the provided query using full text search. + This is more lenient than `matches_phrase`: the order of terms is ignored, and, + by default, only one search term is required to be in the field value. + + Will be ignored when `null` is passed. + """ + matches_query: MatchesQueryFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: TextFilterInput +} + +""" +An [IANA time zone identifier](https://www.iana.org/time-zones), such as `America/Los_Angeles` or `UTC`. + +For a full list of valid identifiers, see the [wikipedia +article](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List). +""" +scalar TimeZone + +""" +A custom scalar type that allows any type of data, including: + +- strings +- numbers +- objects and arrays (nested as deeply as you like) +- booleans + +Note: fields of this type are effectively untyped. We recommend it only be used for +parts of your schema that can't be statically typed. +""" +scalar Untyped + +""" +Input type used to specify filters on `Untyped` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input UntypedFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [UntypedFilterInput!] + + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + equal_to_any_of: [Untyped] + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: UntypedFilterInput +} + +""" +For more performant queries on this type, please filter on `workspace_id` if possible. +""" +type Widget implements NamedEntity @key(fields: "id") { + amount_cents: Int! + amount_cents2: Int! + amounts: [Int!]! + + """ + Aggregations over the `components` data. + """ + component_aggregations( + """ + Used to forward-paginate through the `component_aggregations`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `component_aggregations`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the `Component` documents that get aggregated over based on the provided criteria. + """ + filter: ComponentFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `component_aggregations`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `component_aggregations`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `component_aggregations`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `component_aggregations`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): ComponentAggregationConnection + components( + """ + Used to forward-paginate through the `components`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `components`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used to filter the returned `components` based on the provided criteria. + """ + filter: ComponentFilterInput + + """ + Used in conjunction with the `after` argument to forward-paginate through the `components`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `components`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `components`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `components`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + + """ + Used to specify how the returned `components` should be sorted. + """ + order_by: [ComponentSortOrderInput!] + ): ComponentConnection + cost: Money + cost_currency_introduced_on: Date + cost_currency_name: String + cost_currency_primary_continent: String + cost_currency_symbol: String + cost_currency_unit: String + created_at: DateTime! + created_at2: DateTime! + created_at2_legacy: DateTime! + created_at_legacy: DateTime! + created_at_time_of_day: LocalTime + created_on: Date + created_on_legacy: Date + fees: [Money!]! + id: ID! + inventor: Inventor + metadata: Untyped + name: String + name_text: String + named_inventor: NamedInventor + options: WidgetOptions + release_dates: [Date!]! + release_timestamps: [DateTime!]! + size: Size + tags: [String!]! + the_options: WidgetOptions + weight_in_ng: JsonSafeLong! + weight_in_ng_str: LongString! + workspace: WidgetWorkspace + workspace_id: ID + workspace_name: String +} + +""" +Type used to perform aggregation computations on `Widget` fields. +""" +type WidgetAggregatedValues { + """ + Computed aggregate values for the `amount_cents` field. + """ + amount_cents: IntAggregatedValues + + """ + Computed aggregate values for the `amount_cents2` field. + """ + amount_cents2: IntAggregatedValues + + """ + Computed aggregate values for the `amounts` field. + """ + amounts: IntAggregatedValues + + """ + Computed aggregate values for the `cost` field. + """ + cost: MoneyAggregatedValues + + """ + Computed aggregate values for the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on: DateAggregatedValues + + """ + Computed aggregate values for the `cost_currency_name` field. + """ + cost_currency_name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_symbol` field. + """ + cost_currency_symbol: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_unit` field. + """ + cost_currency_unit: NonNumericAggregatedValues + + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at2` field. + """ + created_at2: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at2_legacy` field. + """ + created_at2_legacy: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at_legacy` field. + """ + created_at_legacy: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at_time_of_day` field. + """ + created_at_time_of_day: LocalTimeAggregatedValues + + """ + Computed aggregate values for the `created_on` field. + """ + created_on: DateAggregatedValues + + """ + Computed aggregate values for the `created_on_legacy` field. + """ + created_on_legacy: DateAggregatedValues + + """ + Computed aggregate values for the `fees` field. + """ + fees: MoneyAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `inventor` field. + """ + inventor: InventorAggregatedValues + + """ + Computed aggregate values for the `metadata` field. + """ + metadata: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `named_inventor` field. + """ + named_inventor: NamedInventorAggregatedValues + + """ + Computed aggregate values for the `options` field. + """ + options: WidgetOptionsAggregatedValues + + """ + Computed aggregate values for the `release_dates` field. + """ + release_dates: DateAggregatedValues + + """ + Computed aggregate values for the `release_timestamps` field. + """ + release_timestamps: DateTimeAggregatedValues + + """ + Computed aggregate values for the `size` field. + """ + size: NonNumericAggregatedValues + + """ + Computed aggregate values for the `tags` field. + """ + tags: NonNumericAggregatedValues + + """ + Computed aggregate values for the `the_options` field. + """ + the_options: WidgetOptionsAggregatedValues + + """ + Computed aggregate values for the `weight_in_ng` field. + """ + weight_in_ng: JsonSafeLongAggregatedValues + + """ + Computed aggregate values for the `weight_in_ng_str` field. + """ + weight_in_ng_str: LongStringAggregatedValues + + """ + Computed aggregate values for the `workspace_id` field. + """ + workspace_id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `workspace_name` field. + """ + workspace_name: NonNumericAggregatedValues +} + +""" +Return type representing a bucket of `Widget` documents for an aggregations query. +""" +type WidgetAggregation { + """ + Provides computed aggregated values over all `Widget` documents in an aggregation bucket. + """ + aggregated_values: WidgetAggregatedValues + + """ + The count of `Widget` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `Widget` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: WidgetGroupedBy +} + +""" +Represents a paginated collection of `WidgetAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type WidgetAggregationConnection { + """ + Wraps a specific `WidgetAggregation` to pair it with its pagination cursor. + """ + edges: [WidgetAggregationEdge!]! + + """ + The list of `WidgetAggregation` results. + """ + nodes: [WidgetAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `WidgetAggregation` in the context of a `WidgetAggregationConnection`, +providing access to both the `WidgetAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type WidgetAggregationEdge { + """ + The `Cursor` of this `WidgetAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `WidgetAggregation`. + """ + cursor: Cursor + + """ + The `WidgetAggregation` of this edge. + """ + node: WidgetAggregation +} + +""" +Represents a paginated collection of `Widget` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type WidgetConnection { + """ + Wraps a specific `Widget` to pair it with its pagination cursor. + """ + edges: [WidgetEdge!]! + + """ + The list of `Widget` results. + """ + nodes: [Widget!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +For more performant queries on this type, please filter on `primary_continent` if possible. +""" +type WidgetCurrency @key(fields: "id") { + details: CurrencyDetails + id: ID! + introduced_on: Date + name: String + nested_fields: WidgetCurrencyNestedFields + oldest_widget_created_at: DateTime + primary_continent: String + widget_fee_currencies: [String!]! + widget_names( + """ + Used to forward-paginate through the `widget_names`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + + """ + Used to backward-paginate through the `widget_names`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor + + """ + Used in conjunction with the `after` argument to forward-paginate through the `widget_names`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `widget_names`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + + """ + Used in conjunction with the `before` argument to backward-paginate through the `widget_names`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `widget_names`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + ): StringConnection + widget_options: WidgetOptionSets + widget_tags: [String!]! +} + +""" +Type used to perform aggregation computations on `WidgetCurrency` fields. +""" +type WidgetCurrencyAggregatedValues { + """ + Computed aggregate values for the `details` field. + """ + details: CurrencyDetailsAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `introduced_on` field. + """ + introduced_on: DateAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `nested_fields` field. + """ + nested_fields: WidgetCurrencyNestedFieldsAggregatedValues + + """ + Computed aggregate values for the `oldest_widget_created_at` field. + """ + oldest_widget_created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `primary_continent` field. + """ + primary_continent: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_fee_currencies` field. + """ + widget_fee_currencies: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_names` field. + """ + widget_names: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget_options` field. + """ + widget_options: WidgetOptionSetsAggregatedValues + + """ + Computed aggregate values for the `widget_tags` field. + """ + widget_tags: NonNumericAggregatedValues +} + +""" +Return type representing a bucket of `WidgetCurrency` documents for an aggregations query. +""" +type WidgetCurrencyAggregation { + """ + Provides computed aggregated values over all `WidgetCurrency` documents in an aggregation bucket. + """ + aggregated_values: WidgetCurrencyAggregatedValues + + """ + The count of `WidgetCurrency` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `WidgetCurrency` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: WidgetCurrencyGroupedBy +} + +""" +Represents a paginated collection of `WidgetCurrencyAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type WidgetCurrencyAggregationConnection { + """ + Wraps a specific `WidgetCurrencyAggregation` to pair it with its pagination cursor. + """ + edges: [WidgetCurrencyAggregationEdge!]! + + """ + The list of `WidgetCurrencyAggregation` results. + """ + nodes: [WidgetCurrencyAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `WidgetCurrencyAggregation` in the context of a `WidgetCurrencyAggregationConnection`, +providing access to both the `WidgetCurrencyAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type WidgetCurrencyAggregationEdge { + """ + The `Cursor` of this `WidgetCurrencyAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `WidgetCurrencyAggregation`. + """ + cursor: Cursor + + """ + The `WidgetCurrencyAggregation` of this edge. + """ + node: WidgetCurrencyAggregation +} + +""" +Represents a paginated collection of `WidgetCurrency` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type WidgetCurrencyConnection { + """ + Wraps a specific `WidgetCurrency` to pair it with its pagination cursor. + """ + edges: [WidgetCurrencyEdge!]! + + """ + The list of `WidgetCurrency` results. + """ + nodes: [WidgetCurrency!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `WidgetCurrency` in the context of a `WidgetCurrencyConnection`, +providing access to both the `WidgetCurrency` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type WidgetCurrencyEdge { + """ + The `Cursor` of this `WidgetCurrency`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `WidgetCurrency`. + """ + cursor: Cursor + + """ + The `WidgetCurrency` of this edge. + """ + node: WidgetCurrency +} + +""" +Input type used to specify filters on `WidgetCurrency` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input WidgetCurrencyFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [WidgetCurrencyFilterInput!] + + """ + Used to filter on the `details` field. + + Will be ignored if `null` or an empty object is passed. + """ + details: CurrencyDetailsFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `introduced_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + introduced_on: DateFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Used to filter on the `nested_fields` field. + + Will be ignored if `null` or an empty object is passed. + """ + nested_fields: WidgetCurrencyNestedFieldsFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: WidgetCurrencyFilterInput + + """ + Used to filter on the `oldest_widget_created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + oldest_widget_created_at: DateTimeFilterInput + + """ + Used to filter on the `primary_continent` field. + + Will be ignored if `null` or an empty object is passed. + """ + primary_continent: StringFilterInput + + """ + Used to filter on the `widget_fee_currencies` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_fee_currencies: StringListFilterInput + + """ + Used to filter on the `widget_names` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_names: StringListFilterInput + + """ + Used to filter on the `widget_options` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_options: WidgetOptionSetsFilterInput + + """ + Used to filter on the `widget_tags` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget_tags: StringListFilterInput +} + +""" +Type used to specify the `WidgetCurrency` fields to group by for aggregations. +""" +type WidgetCurrencyGroupedBy { + """ + The `details` field value for this group. + """ + details: CurrencyDetailsGroupedBy + + """ + Offers the different grouping options for the `introduced_on` value within this group. + """ + introduced_on: DateGroupedBy + + """ + The `name` field value for this group. + """ + name: String + + """ + The `nested_fields` field value for this group. + """ + nested_fields: WidgetCurrencyNestedFieldsGroupedBy + + """ + Offers the different grouping options for the `oldest_widget_created_at` value within this group. + """ + oldest_widget_created_at: DateTimeGroupedBy + + """ + The `primary_continent` field value for this group. + """ + primary_continent: String + + """ + The individual value from `widget_names` for this group. + + Note: `widget_names` is a collection field, but selecting this field will group on individual values of `widget_names`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `widget_names` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `widget_names` multiple times for a single document, that document will only be included in the group + once. + """ + widget_name: String +} + +type WidgetCurrencyNestedFields { + max_widget_cost: Int +} + +""" +Type used to perform aggregation computations on `WidgetCurrencyNestedFields` fields. +""" +type WidgetCurrencyNestedFieldsAggregatedValues { + """ + Computed aggregate values for the `max_widget_cost` field. + """ + max_widget_cost: IntAggregatedValues +} + +""" +Input type used to specify filters on `WidgetCurrencyNestedFields` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input WidgetCurrencyNestedFieldsFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [WidgetCurrencyNestedFieldsFilterInput!] + + """ + Used to filter on the `max_widget_cost` field. + + Will be ignored if `null` or an empty object is passed. + """ + max_widget_cost: IntFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: WidgetCurrencyNestedFieldsFilterInput +} + +""" +Type used to specify the `WidgetCurrencyNestedFields` fields to group by for aggregations. +""" +type WidgetCurrencyNestedFieldsGroupedBy { + """ + The `max_widget_cost` field value for this group. + """ + max_widget_cost: Int +} + +""" +Enumerates the ways `WidgetCurrency`s can be sorted. +""" +enum WidgetCurrencySortOrderInput { + """ + Sorts ascending by the `details.symbol` field. + """ + details_symbol_ASC + + """ + Sorts descending by the `details.symbol` field. + """ + details_symbol_DESC + + """ + Sorts ascending by the `details.unit` field. + """ + details_unit_ASC + + """ + Sorts descending by the `details.unit` field. + """ + details_unit_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `introduced_on` field. + """ + introduced_on_ASC + + """ + Sorts descending by the `introduced_on` field. + """ + introduced_on_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC + + """ + Sorts ascending by the `nested_fields.max_widget_cost` field. + """ + nested_fields_max_widget_cost_ASC + + """ + Sorts descending by the `nested_fields.max_widget_cost` field. + """ + nested_fields_max_widget_cost_DESC + + """ + Sorts ascending by the `oldest_widget_created_at` field. + """ + oldest_widget_created_at_ASC + + """ + Sorts descending by the `oldest_widget_created_at` field. + """ + oldest_widget_created_at_DESC + + """ + Sorts ascending by the `primary_continent` field. + """ + primary_continent_ASC + + """ + Sorts descending by the `primary_continent` field. + """ + primary_continent_DESC +} + +""" +Represents a specific `Widget` in the context of a `WidgetConnection`, +providing access to both the `Widget` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type WidgetEdge { + """ + The `Cursor` of this `Widget`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `Widget`. + """ + cursor: Cursor + + """ + The `Widget` of this edge. + """ + node: Widget +} + +""" +Input type used to specify filters on `Widget` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input WidgetFilterInput { + """ + Used to filter on the `amount_cents` field. + + Will be ignored if `null` or an empty object is passed. + """ + amount_cents: IntFilterInput + + """ + Used to filter on the `amount_cents2` field. + + Will be ignored if `null` or an empty object is passed. + """ + amount_cents2: IntFilterInput + + """ + Used to filter on the `amounts` field. + + Will be ignored if `null` or an empty object is passed. + """ + amounts: IntListFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [WidgetFilterInput!] + + """ + Used to filter on the `cost` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost: MoneyFilterInput + + """ + Used to filter on the `cost_currency_introduced_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_introduced_on: DateFilterInput + + """ + Used to filter on the `cost_currency_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_name: StringFilterInput + + """ + Used to filter on the `cost_currency_primary_continent` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_primary_continent: StringFilterInput + + """ + Used to filter on the `cost_currency_symbol` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_symbol: StringFilterInput + + """ + Used to filter on the `cost_currency_unit` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_unit: StringFilterInput + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `created_at2` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at2: DateTimeFilterInput + + """ + Used to filter on the `created_at2_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at2_legacy: DateTimeFilterInput + + """ + Used to filter on the `created_at_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at_legacy: DateTimeFilterInput + + """ + Used to filter on the `created_at_time_of_day` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at_time_of_day: LocalTimeFilterInput + + """ + Used to filter on the `created_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_on: DateFilterInput + + """ + Used to filter on the `created_on_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_on_legacy: DateFilterInput + + """ + Used to filter on the `fees` field. + + Will be ignored if `null` or an empty object is passed. + """ + fees: MoneyFieldsListFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `inventor` field. + + Will be ignored if `null` or an empty object is passed. + """ + inventor: InventorFilterInput + + """ + Used to filter on the `metadata` field. + + Will be ignored if `null` or an empty object is passed. + """ + metadata: UntypedFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Used to filter on the `name_text` field. + + Will be ignored if `null` or an empty object is passed. + """ + name_text: TextFilterInput + + """ + Used to filter on the `named_inventor` field. + + Will be ignored if `null` or an empty object is passed. + """ + named_inventor: NamedInventorFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: WidgetFilterInput + + """ + Used to filter on the `options` field. + + Will be ignored if `null` or an empty object is passed. + """ + options: WidgetOptionsFilterInput + + """ + Used to filter on the `release_dates` field. + + Will be ignored if `null` or an empty object is passed. + """ + release_dates: DateListFilterInput + + """ + Used to filter on the `release_timestamps` field. + + Will be ignored if `null` or an empty object is passed. + """ + release_timestamps: DateTimeListFilterInput + + """ + Used to filter on the `size` field. + + Will be ignored if `null` or an empty object is passed. + """ + size: SizeFilterInput + + """ + Used to filter on the `tags` field. + + Will be ignored if `null` or an empty object is passed. + """ + tags: StringListFilterInput + + """ + Used to filter on the `the_options` field. + + Will be ignored if `null` or an empty object is passed. + """ + the_options: WidgetOptionsFilterInput + + """ + Used to filter on the `weight_in_ng` field. + + Will be ignored if `null` or an empty object is passed. + """ + weight_in_ng: JsonSafeLongFilterInput + + """ + Used to filter on the `weight_in_ng_str` field. + + Will be ignored if `null` or an empty object is passed. + """ + weight_in_ng_str: LongStringFilterInput + + """ + Used to filter on the `workspace_id` field. + + Will be ignored if `null` or an empty object is passed. + """ + workspace_id: IDFilterInput + + """ + Used to filter on the `workspace_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + workspace_name: StringFilterInput +} + +""" +Type used to specify the `Widget` fields to group by for aggregations. +""" +type WidgetGroupedBy { + """ + The `amount_cents` field value for this group. + """ + amount_cents: Int + + """ + The `amount_cents2` field value for this group. + """ + amount_cents2: Int + + """ + The `cost` field value for this group. + """ + cost: MoneyGroupedBy + + """ + Offers the different grouping options for the `cost_currency_introduced_on` value within this group. + """ + cost_currency_introduced_on: DateGroupedBy + + """ + The `cost_currency_name` field value for this group. + """ + cost_currency_name: String + + """ + The `cost_currency_primary_continent` field value for this group. + """ + cost_currency_primary_continent: String + + """ + The `cost_currency_symbol` field value for this group. + """ + cost_currency_symbol: String + + """ + The `cost_currency_unit` field value for this group. + """ + cost_currency_unit: String + + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + Offers the different grouping options for the `created_at2` value within this group. + """ + created_at2: DateTimeGroupedBy + + """ + The `created_at2_legacy` field value for this group. + """ + created_at2_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateTimeGroupingGranularityInput! + + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change + what day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone = "UTC" + ): DateTime + + """ + The `created_at_legacy` field value for this group. + """ + created_at_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateTimeGroupingGranularityInput! + + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change + what day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone = "UTC" + ): DateTime + + """ + The `created_at_time_of_day` field value for this group. + """ + created_at_time_of_day: LocalTime + + """ + Offers the different grouping options for the `created_on` value within this group. + """ + created_on: DateGroupedBy + + """ + The `created_on_legacy` field value for this group. + """ + created_on_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateGroupingGranularityInput! + + """ + Number of days (positive or negative) to shift the `Date` boundaries of each date grouping bucket. + + For example, when grouping by `YEAR`, this can be used to align the buckets + with fiscal or school years instead of calendar years. + """ + offset_days: Int + ): Date + + """ + The `fees` field value for this group. + + Note: `fees` is a collection field, but selecting this field will group on + individual values of the selected subfields of `fees`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `fees` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `fees` multiple times for a single document, that document will only be included in the group + once. + """ + fees: MoneyGroupedBy + + """ + The `inventor` field value for this group. + """ + inventor: InventorGroupedBy + + """ + The `metadata` field value for this group. + """ + metadata: Untyped + + """ + The `name` field value for this group. + """ + name: String + + """ + The `named_inventor` field value for this group. + """ + named_inventor: NamedInventorGroupedBy + + """ + The `options` field value for this group. + """ + options: WidgetOptionsGroupedBy + + """ + The individual value from `release_dates` for this group. + + Note: `release_dates` is a collection field, but selecting this field will group on individual values of `release_dates`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `release_dates` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `release_dates` multiple times for a single document, that document will only be included in the group + once. + """ + release_date: DateGroupedBy + + """ + The individual value from `release_timestamps` for this group. + + Note: `release_timestamps` is a collection field, but selecting this field + will group on individual values of `release_timestamps`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `release_timestamps` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `release_timestamps` multiple times for a single document, that document will only be included in the group + once. + """ + release_timestamp: DateTimeGroupedBy + + """ + The `size` field value for this group. + """ + size: Size + + """ + The individual value from `tags` for this group. + + Note: `tags` is a collection field, but selecting this field will group on individual values of `tags`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `tags` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `tags` multiple times for a single document, that document will only be included in the group + once. + """ + tag: String + + """ + The `the_options` field value for this group. + """ + the_options: WidgetOptionsGroupedBy + + """ + The `weight_in_ng` field value for this group. + """ + weight_in_ng: JsonSafeLong + + """ + The `weight_in_ng_str` field value for this group. + """ + weight_in_ng_str: LongString + + """ + The `workspace_id` field value for this group. + """ + workspace_id: ID + + """ + The `workspace_name` field value for this group. + """ + workspace_name: String +} + +type WidgetOptionSets { + colors: [Color!]! + sizes: [Size!]! +} + +""" +Type used to perform aggregation computations on `WidgetOptionSets` fields. +""" +type WidgetOptionSetsAggregatedValues { + """ + Computed aggregate values for the `colors` field. + """ + colors: NonNumericAggregatedValues + + """ + Computed aggregate values for the `sizes` field. + """ + sizes: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on `WidgetOptionSets` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input WidgetOptionSetsFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [WidgetOptionSetsFilterInput!] + + """ + Used to filter on the `colors` field. + + Will be ignored if `null` or an empty object is passed. + """ + colors: ColorListFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: WidgetOptionSetsFilterInput + + """ + Used to filter on the `sizes` field. + + Will be ignored if `null` or an empty object is passed. + """ + sizes: SizeListFilterInput +} + +type WidgetOptions { + color: Color + size: Size + the_size: Size +} + +""" +Type used to perform aggregation computations on `WidgetOptions` fields. +""" +type WidgetOptionsAggregatedValues { + """ + Computed aggregate values for the `color` field. + """ + color: NonNumericAggregatedValues + + """ + Computed aggregate values for the `size` field. + """ + size: NonNumericAggregatedValues + + """ + Computed aggregate values for the `the_size` field. + """ + the_size: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on `WidgetOptions` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input WidgetOptionsFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [WidgetOptionsFilterInput!] + + """ + Used to filter on the `color` field. + + Will be ignored if `null` or an empty object is passed. + """ + color: ColorFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: WidgetOptionsFilterInput + + """ + Used to filter on the `size` field. + + Will be ignored if `null` or an empty object is passed. + """ + size: SizeFilterInput + + """ + Used to filter on the `the_size` field. + + Will be ignored if `null` or an empty object is passed. + """ + the_size: SizeFilterInput +} + +""" +Type used to specify the `WidgetOptions` fields to group by for aggregations. +""" +type WidgetOptionsGroupedBy { + """ + The `color` field value for this group. + """ + color: Color + + """ + The `size` field value for this group. + """ + size: Size + + """ + The `the_size` field value for this group. + """ + the_size: Size +} + +union WidgetOrAddress = Address | Widget + +""" +Type used to perform aggregation computations on `WidgetOrAddress` fields. +""" +type WidgetOrAddressAggregatedValues { + """ + Computed aggregate values for the `amount_cents` field. + """ + amount_cents: IntAggregatedValues + + """ + Computed aggregate values for the `amount_cents2` field. + """ + amount_cents2: IntAggregatedValues + + """ + Computed aggregate values for the `amounts` field. + """ + amounts: IntAggregatedValues + + """ + Computed aggregate values for the `cost` field. + """ + cost: MoneyAggregatedValues + + """ + Computed aggregate values for the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on: DateAggregatedValues + + """ + Computed aggregate values for the `cost_currency_name` field. + """ + cost_currency_name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_symbol` field. + """ + cost_currency_symbol: NonNumericAggregatedValues + + """ + Computed aggregate values for the `cost_currency_unit` field. + """ + cost_currency_unit: NonNumericAggregatedValues + + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at2` field. + """ + created_at2: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at2_legacy` field. + """ + created_at2_legacy: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at_legacy` field. + """ + created_at_legacy: DateTimeAggregatedValues + + """ + Computed aggregate values for the `created_at_time_of_day` field. + """ + created_at_time_of_day: LocalTimeAggregatedValues + + """ + Computed aggregate values for the `created_on` field. + """ + created_on: DateAggregatedValues + + """ + Computed aggregate values for the `created_on_legacy` field. + """ + created_on_legacy: DateAggregatedValues + + """ + Computed aggregate values for the `fees` field. + """ + fees: MoneyAggregatedValues + + """ + Computed aggregate values for the `full_address` field. + """ + full_address: NonNumericAggregatedValues + + """ + Computed aggregate values for the `geo_location` field. + """ + geo_location: NonNumericAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `inventor` field. + """ + inventor: InventorAggregatedValues + + """ + Computed aggregate values for the `metadata` field. + """ + metadata: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `named_inventor` field. + """ + named_inventor: NamedInventorAggregatedValues + + """ + Computed aggregate values for the `options` field. + """ + options: WidgetOptionsAggregatedValues + + """ + Computed aggregate values for the `release_dates` field. + """ + release_dates: DateAggregatedValues + + """ + Computed aggregate values for the `release_timestamps` field. + """ + release_timestamps: DateTimeAggregatedValues + + """ + Computed aggregate values for the `shapes` field. + """ + shapes: NonNumericAggregatedValues + + """ + Computed aggregate values for the `size` field. + """ + size: NonNumericAggregatedValues + + """ + Computed aggregate values for the `tags` field. + """ + tags: NonNumericAggregatedValues + + """ + Computed aggregate values for the `the_options` field. + """ + the_options: WidgetOptionsAggregatedValues + + """ + Computed aggregate values for the `timestamps` field. + """ + timestamps: AddressTimestampsAggregatedValues + + """ + Computed aggregate values for the `weight_in_ng` field. + """ + weight_in_ng: JsonSafeLongAggregatedValues + + """ + Computed aggregate values for the `weight_in_ng_str` field. + """ + weight_in_ng_str: LongStringAggregatedValues + + """ + Computed aggregate values for the `workspace_id` field. + """ + workspace_id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `workspace_name` field. + """ + workspace_name: NonNumericAggregatedValues +} + +""" +Return type representing a bucket of `WidgetOrAddress` documents for an aggregations query. +""" +type WidgetOrAddressAggregation { + """ + Provides computed aggregated values over all `WidgetOrAddress` documents in an aggregation bucket. + """ + aggregated_values: WidgetOrAddressAggregatedValues + + """ + The count of `WidgetOrAddress` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `WidgetOrAddress` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: WidgetOrAddressGroupedBy +} + +""" +Represents a paginated collection of `WidgetOrAddressAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type WidgetOrAddressAggregationConnection { + """ + Wraps a specific `WidgetOrAddressAggregation` to pair it with its pagination cursor. + """ + edges: [WidgetOrAddressAggregationEdge!]! + + """ + The list of `WidgetOrAddressAggregation` results. + """ + nodes: [WidgetOrAddressAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `WidgetOrAddressAggregation` in the context of a `WidgetOrAddressAggregationConnection`, +providing access to both the `WidgetOrAddressAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type WidgetOrAddressAggregationEdge { + """ + The `Cursor` of this `WidgetOrAddressAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `WidgetOrAddressAggregation`. + """ + cursor: Cursor + + """ + The `WidgetOrAddressAggregation` of this edge. + """ + node: WidgetOrAddressAggregation +} + +""" +Represents a paginated collection of `WidgetOrAddress` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type WidgetOrAddressConnection { + """ + Wraps a specific `WidgetOrAddress` to pair it with its pagination cursor. + """ + edges: [WidgetOrAddressEdge!]! + + """ + The list of `WidgetOrAddress` results. + """ + nodes: [WidgetOrAddress!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `WidgetOrAddress` in the context of a `WidgetOrAddressConnection`, +providing access to both the `WidgetOrAddress` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type WidgetOrAddressEdge { + """ + The `Cursor` of this `WidgetOrAddress`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `WidgetOrAddress`. + """ + cursor: Cursor + + """ + The `WidgetOrAddress` of this edge. + """ + node: WidgetOrAddress +} + +""" +Input type used to specify filters on `WidgetOrAddress` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input WidgetOrAddressFilterInput { + """ + Used to filter on the `amount_cents` field. + + Will be ignored if `null` or an empty object is passed. + """ + amount_cents: IntFilterInput + + """ + Used to filter on the `amount_cents2` field. + + Will be ignored if `null` or an empty object is passed. + """ + amount_cents2: IntFilterInput + + """ + Used to filter on the `amounts` field. + + Will be ignored if `null` or an empty object is passed. + """ + amounts: IntListFilterInput + + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [WidgetOrAddressFilterInput!] + + """ + Used to filter on the `cost` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost: MoneyFilterInput + + """ + Used to filter on the `cost_currency_introduced_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_introduced_on: DateFilterInput + + """ + Used to filter on the `cost_currency_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_name: StringFilterInput + + """ + Used to filter on the `cost_currency_primary_continent` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_primary_continent: StringFilterInput + + """ + Used to filter on the `cost_currency_symbol` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_symbol: StringFilterInput + + """ + Used to filter on the `cost_currency_unit` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost_currency_unit: StringFilterInput + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `created_at2` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at2: DateTimeFilterInput + + """ + Used to filter on the `created_at2_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at2_legacy: DateTimeFilterInput + + """ + Used to filter on the `created_at_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at_legacy: DateTimeFilterInput + + """ + Used to filter on the `created_at_time_of_day` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at_time_of_day: LocalTimeFilterInput + + """ + Used to filter on the `created_on` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_on: DateFilterInput + + """ + Used to filter on the `created_on_legacy` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_on_legacy: DateFilterInput + + """ + Used to filter on the `fees` field. + + Will be ignored if `null` or an empty object is passed. + """ + fees: MoneyFieldsListFilterInput + + """ + Used to filter on the `full_address` field. + + Will be ignored if `null` or an empty object is passed. + """ + full_address: StringFilterInput + + """ + Used to filter on the `geo_location` field. + + Will be ignored if `null` or an empty object is passed. + """ + geo_location: GeoLocationFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `inventor` field. + + Will be ignored if `null` or an empty object is passed. + """ + inventor: InventorFilterInput + + """ + Used to filter on the `metadata` field. + + Will be ignored if `null` or an empty object is passed. + """ + metadata: UntypedFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Used to filter on the `name_text` field. + + Will be ignored if `null` or an empty object is passed. + """ + name_text: TextFilterInput + + """ + Used to filter on the `named_inventor` field. + + Will be ignored if `null` or an empty object is passed. + """ + named_inventor: NamedInventorFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: WidgetOrAddressFilterInput + + """ + Used to filter on the `options` field. + + Will be ignored if `null` or an empty object is passed. + """ + options: WidgetOptionsFilterInput + + """ + Used to filter on the `release_dates` field. + + Will be ignored if `null` or an empty object is passed. + """ + release_dates: DateListFilterInput + + """ + Used to filter on the `release_timestamps` field. + + Will be ignored if `null` or an empty object is passed. + """ + release_timestamps: DateTimeListFilterInput + + """ + Used to filter on the `size` field. + + Will be ignored if `null` or an empty object is passed. + """ + size: SizeFilterInput + + """ + Used to filter on the `tags` field. + + Will be ignored if `null` or an empty object is passed. + """ + tags: StringListFilterInput + + """ + Used to filter on the `the_options` field. + + Will be ignored if `null` or an empty object is passed. + """ + the_options: WidgetOptionsFilterInput + + """ + Used to filter on the `timestamps` field. + + Will be ignored if `null` or an empty object is passed. + """ + timestamps: AddressTimestampsFilterInput + + """ + Used to filter on the `weight_in_ng` field. + + Will be ignored if `null` or an empty object is passed. + """ + weight_in_ng: JsonSafeLongFilterInput + + """ + Used to filter on the `weight_in_ng_str` field. + + Will be ignored if `null` or an empty object is passed. + """ + weight_in_ng_str: LongStringFilterInput + + """ + Used to filter on the `workspace_id` field. + + Will be ignored if `null` or an empty object is passed. + """ + workspace_id: IDFilterInput + + """ + Used to filter on the `workspace_name` field. + + Will be ignored if `null` or an empty object is passed. + """ + workspace_name: StringFilterInput +} + +""" +Type used to specify the `WidgetOrAddress` fields to group by for aggregations. +""" +type WidgetOrAddressGroupedBy { + """ + The `amount_cents` field value for this group. + """ + amount_cents: Int + + """ + The `amount_cents2` field value for this group. + """ + amount_cents2: Int + + """ + The `cost` field value for this group. + """ + cost: MoneyGroupedBy + + """ + Offers the different grouping options for the `cost_currency_introduced_on` value within this group. + """ + cost_currency_introduced_on: DateGroupedBy + + """ + The `cost_currency_name` field value for this group. + """ + cost_currency_name: String + + """ + The `cost_currency_primary_continent` field value for this group. + """ + cost_currency_primary_continent: String + + """ + The `cost_currency_symbol` field value for this group. + """ + cost_currency_symbol: String + + """ + The `cost_currency_unit` field value for this group. + """ + cost_currency_unit: String + + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + Offers the different grouping options for the `created_at2` value within this group. + """ + created_at2: DateTimeGroupedBy + + """ + The `created_at2_legacy` field value for this group. + """ + created_at2_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateTimeGroupingGranularityInput! + + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change + what day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone = "UTC" + ): DateTime + + """ + The `created_at_legacy` field value for this group. + """ + created_at_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateTimeGroupingGranularityInput! + + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change + what day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput + + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + time_zone: TimeZone = "UTC" + ): DateTime + + """ + The `created_at_time_of_day` field value for this group. + """ + created_at_time_of_day: LocalTime + + """ + Offers the different grouping options for the `created_on` value within this group. + """ + created_on: DateGroupedBy + + """ + The `created_on_legacy` field value for this group. + """ + created_on_legacy( + """ + Determines the grouping granularity for this field. + """ + granularity: DateGroupingGranularityInput! + + """ + Number of days (positive or negative) to shift the `Date` boundaries of each date grouping bucket. + + For example, when grouping by `YEAR`, this can be used to align the buckets + with fiscal or school years instead of calendar years. + """ + offset_days: Int + ): Date + + """ + The `fees` field value for this group. + + Note: `fees` is a collection field, but selecting this field will group on + individual values of the selected subfields of `fees`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `fees` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `fees` multiple times for a single document, that document will only be included in the group + once. + """ + fees: MoneyGroupedBy + + """ + The `full_address` field value for this group. + """ + full_address: String + + """ + The `inventor` field value for this group. + """ + inventor: InventorGroupedBy + + """ + The `metadata` field value for this group. + """ + metadata: Untyped + + """ + The `name` field value for this group. + """ + name: String + + """ + The `named_inventor` field value for this group. + """ + named_inventor: NamedInventorGroupedBy + + """ + The `options` field value for this group. + """ + options: WidgetOptionsGroupedBy + + """ + The individual value from `release_dates` for this group. + + Note: `release_dates` is a collection field, but selecting this field will group on individual values of `release_dates`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `release_dates` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `release_dates` multiple times for a single document, that document will only be included in the group + once. + """ + release_date: DateGroupedBy + + """ + The individual value from `release_timestamps` for this group. + + Note: `release_timestamps` is a collection field, but selecting this field + will group on individual values of `release_timestamps`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `release_timestamps` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `release_timestamps` multiple times for a single document, that document will only be included in the group + once. + """ + release_timestamp: DateTimeGroupedBy + + """ + The `size` field value for this group. + """ + size: Size + + """ + The individual value from `tags` for this group. + + Note: `tags` is a collection field, but selecting this field will group on individual values of `tags`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `tags` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `tags` multiple times for a single document, that document will only be included in the group + once. + """ + tag: String + + """ + The `the_options` field value for this group. + """ + the_options: WidgetOptionsGroupedBy + + """ + The `timestamps` field value for this group. + """ + timestamps: AddressTimestampsGroupedBy + + """ + The `weight_in_ng` field value for this group. + """ + weight_in_ng: JsonSafeLong + + """ + The `weight_in_ng_str` field value for this group. + """ + weight_in_ng_str: LongString + + """ + The `workspace_id` field value for this group. + """ + workspace_id: ID + + """ + The `workspace_name` field value for this group. + """ + workspace_name: String +} + +""" +Enumerates the ways `WidgetOrAddress`s can be sorted. +""" +enum WidgetOrAddressSortOrderInput { + """ + Sorts ascending by the `amount_cents2` field. + """ + amount_cents2_ASC + + """ + Sorts descending by the `amount_cents2` field. + """ + amount_cents2_DESC + + """ + Sorts ascending by the `amount_cents` field. + """ + amount_cents_ASC + + """ + Sorts descending by the `amount_cents` field. + """ + amount_cents_DESC + + """ + Sorts ascending by the `cost.amount_cents` field. + """ + cost_amount_cents_ASC + + """ + Sorts descending by the `cost.amount_cents` field. + """ + cost_amount_cents_DESC + + """ + Sorts ascending by the `cost.currency` field. + """ + cost_currency_ASC + + """ + Sorts descending by the `cost.currency` field. + """ + cost_currency_DESC + + """ + Sorts ascending by the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on_ASC + + """ + Sorts descending by the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on_DESC + + """ + Sorts ascending by the `cost_currency_name` field. + """ + cost_currency_name_ASC + + """ + Sorts descending by the `cost_currency_name` field. + """ + cost_currency_name_DESC + + """ + Sorts ascending by the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent_ASC + + """ + Sorts descending by the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent_DESC + + """ + Sorts ascending by the `cost_currency_symbol` field. + """ + cost_currency_symbol_ASC + + """ + Sorts descending by the `cost_currency_symbol` field. + """ + cost_currency_symbol_DESC + + """ + Sorts ascending by the `cost_currency_unit` field. + """ + cost_currency_unit_ASC + + """ + Sorts descending by the `cost_currency_unit` field. + """ + cost_currency_unit_DESC + + """ + Sorts ascending by the `created_at2` field. + """ + created_at2_ASC + + """ + Sorts descending by the `created_at2` field. + """ + created_at2_DESC + + """ + Sorts ascending by the `created_at2_legacy` field. + """ + created_at2_legacy_ASC + + """ + Sorts descending by the `created_at2_legacy` field. + """ + created_at2_legacy_DESC + + """ + Sorts ascending by the `created_at` field. + """ + created_at_ASC + + """ + Sorts descending by the `created_at` field. + """ + created_at_DESC + + """ + Sorts ascending by the `created_at_legacy` field. + """ + created_at_legacy_ASC + + """ + Sorts descending by the `created_at_legacy` field. + """ + created_at_legacy_DESC + + """ + Sorts ascending by the `created_at_time_of_day` field. + """ + created_at_time_of_day_ASC + + """ + Sorts descending by the `created_at_time_of_day` field. + """ + created_at_time_of_day_DESC + + """ + Sorts ascending by the `created_on` field. + """ + created_on_ASC + + """ + Sorts descending by the `created_on` field. + """ + created_on_DESC + + """ + Sorts ascending by the `created_on_legacy` field. + """ + created_on_legacy_ASC + + """ + Sorts descending by the `created_on_legacy` field. + """ + created_on_legacy_DESC + + """ + Sorts ascending by the `full_address` field. + """ + full_address_ASC + + """ + Sorts descending by the `full_address` field. + """ + full_address_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `inventor.name` field. + """ + inventor_name_ASC + + """ + Sorts descending by the `inventor.name` field. + """ + inventor_name_DESC + + """ + Sorts ascending by the `inventor.nationality` field. + """ + inventor_nationality_ASC + + """ + Sorts descending by the `inventor.nationality` field. + """ + inventor_nationality_DESC + + """ + Sorts ascending by the `inventor.stock_ticker` field. + """ + inventor_stock_ticker_ASC + + """ + Sorts descending by the `inventor.stock_ticker` field. + """ + inventor_stock_ticker_DESC + + """ + Sorts ascending by the `metadata` field. + """ + metadata_ASC + + """ + Sorts descending by the `metadata` field. + """ + metadata_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC + + """ + Sorts ascending by the `named_inventor.name` field. + """ + named_inventor_name_ASC + + """ + Sorts descending by the `named_inventor.name` field. + """ + named_inventor_name_DESC + + """ + Sorts ascending by the `named_inventor.nationality` field. + """ + named_inventor_nationality_ASC + + """ + Sorts descending by the `named_inventor.nationality` field. + """ + named_inventor_nationality_DESC + + """ + Sorts ascending by the `named_inventor.stock_ticker` field. + """ + named_inventor_stock_ticker_ASC + + """ + Sorts descending by the `named_inventor.stock_ticker` field. + """ + named_inventor_stock_ticker_DESC + + """ + Sorts ascending by the `options.color` field. + """ + options_color_ASC + + """ + Sorts descending by the `options.color` field. + """ + options_color_DESC + + """ + Sorts ascending by the `options.size` field. + """ + options_size_ASC + + """ + Sorts descending by the `options.size` field. + """ + options_size_DESC + + """ + Sorts ascending by the `options.the_size` field. + """ + options_the_size_ASC + + """ + Sorts descending by the `options.the_size` field. + """ + options_the_size_DESC + + """ + Sorts ascending by the `size` field. + """ + size_ASC + + """ + Sorts descending by the `size` field. + """ + size_DESC + + """ + Sorts ascending by the `the_options.color` field. + """ + the_options_color_ASC + + """ + Sorts descending by the `the_options.color` field. + """ + the_options_color_DESC + + """ + Sorts ascending by the `the_options.size` field. + """ + the_options_size_ASC + + """ + Sorts descending by the `the_options.size` field. + """ + the_options_size_DESC + + """ + Sorts ascending by the `the_options.the_size` field. + """ + the_options_the_size_ASC + + """ + Sorts descending by the `the_options.the_size` field. + """ + the_options_the_size_DESC + + """ + Sorts ascending by the `timestamps.created_at` field. + """ + timestamps_created_at_ASC + + """ + Sorts descending by the `timestamps.created_at` field. + """ + timestamps_created_at_DESC + + """ + Sorts ascending by the `weight_in_ng` field. + """ + weight_in_ng_ASC + + """ + Sorts descending by the `weight_in_ng` field. + """ + weight_in_ng_DESC + + """ + Sorts ascending by the `weight_in_ng_str` field. + """ + weight_in_ng_str_ASC + + """ + Sorts descending by the `weight_in_ng_str` field. + """ + weight_in_ng_str_DESC + + """ + Sorts ascending by the `workspace_id` field. + """ + workspace_id_ASC + + """ + Sorts descending by the `workspace_id` field. + """ + workspace_id_DESC + + """ + Sorts ascending by the `workspace_name` field. + """ + workspace_name_ASC + + """ + Sorts descending by the `workspace_name` field. + """ + workspace_name_DESC +} + +""" +Enumerates the ways `Widget`s can be sorted. +""" +enum WidgetSortOrderInput { + """ + Sorts ascending by the `amount_cents2` field. + """ + amount_cents2_ASC + + """ + Sorts descending by the `amount_cents2` field. + """ + amount_cents2_DESC + + """ + Sorts ascending by the `amount_cents` field. + """ + amount_cents_ASC + + """ + Sorts descending by the `amount_cents` field. + """ + amount_cents_DESC + + """ + Sorts ascending by the `cost.amount_cents` field. + """ + cost_amount_cents_ASC + + """ + Sorts descending by the `cost.amount_cents` field. + """ + cost_amount_cents_DESC + + """ + Sorts ascending by the `cost.currency` field. + """ + cost_currency_ASC + + """ + Sorts descending by the `cost.currency` field. + """ + cost_currency_DESC + + """ + Sorts ascending by the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on_ASC + + """ + Sorts descending by the `cost_currency_introduced_on` field. + """ + cost_currency_introduced_on_DESC + + """ + Sorts ascending by the `cost_currency_name` field. + """ + cost_currency_name_ASC + + """ + Sorts descending by the `cost_currency_name` field. + """ + cost_currency_name_DESC + + """ + Sorts ascending by the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent_ASC + + """ + Sorts descending by the `cost_currency_primary_continent` field. + """ + cost_currency_primary_continent_DESC + + """ + Sorts ascending by the `cost_currency_symbol` field. + """ + cost_currency_symbol_ASC + + """ + Sorts descending by the `cost_currency_symbol` field. + """ + cost_currency_symbol_DESC + + """ + Sorts ascending by the `cost_currency_unit` field. + """ + cost_currency_unit_ASC + + """ + Sorts descending by the `cost_currency_unit` field. + """ + cost_currency_unit_DESC + + """ + Sorts ascending by the `created_at2` field. + """ + created_at2_ASC + + """ + Sorts descending by the `created_at2` field. + """ + created_at2_DESC + + """ + Sorts ascending by the `created_at2_legacy` field. + """ + created_at2_legacy_ASC + + """ + Sorts descending by the `created_at2_legacy` field. + """ + created_at2_legacy_DESC + + """ + Sorts ascending by the `created_at` field. + """ + created_at_ASC + + """ + Sorts descending by the `created_at` field. + """ + created_at_DESC + + """ + Sorts ascending by the `created_at_legacy` field. + """ + created_at_legacy_ASC + + """ + Sorts descending by the `created_at_legacy` field. + """ + created_at_legacy_DESC + + """ + Sorts ascending by the `created_at_time_of_day` field. + """ + created_at_time_of_day_ASC + + """ + Sorts descending by the `created_at_time_of_day` field. + """ + created_at_time_of_day_DESC + + """ + Sorts ascending by the `created_on` field. + """ + created_on_ASC + + """ + Sorts descending by the `created_on` field. + """ + created_on_DESC + + """ + Sorts ascending by the `created_on_legacy` field. + """ + created_on_legacy_ASC + + """ + Sorts descending by the `created_on_legacy` field. + """ + created_on_legacy_DESC + + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `inventor.name` field. + """ + inventor_name_ASC + + """ + Sorts descending by the `inventor.name` field. + """ + inventor_name_DESC + + """ + Sorts ascending by the `inventor.nationality` field. + """ + inventor_nationality_ASC + + """ + Sorts descending by the `inventor.nationality` field. + """ + inventor_nationality_DESC + + """ + Sorts ascending by the `inventor.stock_ticker` field. + """ + inventor_stock_ticker_ASC + + """ + Sorts descending by the `inventor.stock_ticker` field. + """ + inventor_stock_ticker_DESC + + """ + Sorts ascending by the `metadata` field. + """ + metadata_ASC + + """ + Sorts descending by the `metadata` field. + """ + metadata_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC + + """ + Sorts ascending by the `named_inventor.name` field. + """ + named_inventor_name_ASC + + """ + Sorts descending by the `named_inventor.name` field. + """ + named_inventor_name_DESC + + """ + Sorts ascending by the `named_inventor.nationality` field. + """ + named_inventor_nationality_ASC + + """ + Sorts descending by the `named_inventor.nationality` field. + """ + named_inventor_nationality_DESC + + """ + Sorts ascending by the `named_inventor.stock_ticker` field. + """ + named_inventor_stock_ticker_ASC + + """ + Sorts descending by the `named_inventor.stock_ticker` field. + """ + named_inventor_stock_ticker_DESC + + """ + Sorts ascending by the `options.color` field. + """ + options_color_ASC + + """ + Sorts descending by the `options.color` field. + """ + options_color_DESC + + """ + Sorts ascending by the `options.size` field. + """ + options_size_ASC + + """ + Sorts descending by the `options.size` field. + """ + options_size_DESC + + """ + Sorts ascending by the `options.the_size` field. + """ + options_the_size_ASC + + """ + Sorts descending by the `options.the_size` field. + """ + options_the_size_DESC + + """ + Sorts ascending by the `size` field. + """ + size_ASC + + """ + Sorts descending by the `size` field. + """ + size_DESC + + """ + Sorts ascending by the `the_options.color` field. + """ + the_options_color_ASC + + """ + Sorts descending by the `the_options.color` field. + """ + the_options_color_DESC + + """ + Sorts ascending by the `the_options.size` field. + """ + the_options_size_ASC + + """ + Sorts descending by the `the_options.size` field. + """ + the_options_size_DESC + + """ + Sorts ascending by the `the_options.the_size` field. + """ + the_options_the_size_ASC + + """ + Sorts descending by the `the_options.the_size` field. + """ + the_options_the_size_DESC + + """ + Sorts ascending by the `weight_in_ng` field. + """ + weight_in_ng_ASC + + """ + Sorts descending by the `weight_in_ng` field. + """ + weight_in_ng_DESC + + """ + Sorts ascending by the `weight_in_ng_str` field. + """ + weight_in_ng_str_ASC + + """ + Sorts descending by the `weight_in_ng_str` field. + """ + weight_in_ng_str_DESC + + """ + Sorts ascending by the `workspace_id` field. + """ + workspace_id_ASC + + """ + Sorts descending by the `workspace_id` field. + """ + workspace_id_DESC + + """ + Sorts ascending by the `workspace_name` field. + """ + workspace_name_ASC + + """ + Sorts descending by the `workspace_name` field. + """ + workspace_name_DESC +} + +type WidgetWorkspace @key(fields: "id") { + id: ID! + name: String + widget: WorkspaceWidget +} + +""" +Type used to perform aggregation computations on `WidgetWorkspace` fields. +""" +type WidgetWorkspaceAggregatedValues { + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues + + """ + Computed aggregate values for the `name` field. + """ + name: NonNumericAggregatedValues + + """ + Computed aggregate values for the `widget` field. + """ + widget: WorkspaceWidgetAggregatedValues +} + +""" +Return type representing a bucket of `WidgetWorkspace` documents for an aggregations query. +""" +type WidgetWorkspaceAggregation { + """ + Provides computed aggregated values over all `WidgetWorkspace` documents in an aggregation bucket. + """ + aggregated_values: WidgetWorkspaceAggregatedValues + + """ + The count of `WidgetWorkspace` documents in an aggregation bucket. + """ + count: JsonSafeLong! + + """ + Used to specify the `WidgetWorkspace` fields to group by. The returned values identify each aggregation bucket. + """ + grouped_by: WidgetWorkspaceGroupedBy +} + +""" +Represents a paginated collection of `WidgetWorkspaceAggregation` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type WidgetWorkspaceAggregationConnection { + """ + Wraps a specific `WidgetWorkspaceAggregation` to pair it with its pagination cursor. + """ + edges: [WidgetWorkspaceAggregationEdge!]! + + """ + The list of `WidgetWorkspaceAggregation` results. + """ + nodes: [WidgetWorkspaceAggregation!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! +} + +""" +Represents a specific `WidgetWorkspaceAggregation` in the context of a `WidgetWorkspaceAggregationConnection`, +providing access to both the `WidgetWorkspaceAggregation` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type WidgetWorkspaceAggregationEdge { + """ + The `Cursor` of this `WidgetWorkspaceAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `WidgetWorkspaceAggregation`. + """ + cursor: Cursor + + """ + The `WidgetWorkspaceAggregation` of this edge. + """ + node: WidgetWorkspaceAggregation +} + +""" +Represents a paginated collection of `WidgetWorkspace` results. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. +""" +type WidgetWorkspaceConnection { + """ + Wraps a specific `WidgetWorkspace` to pair it with its pagination cursor. + """ + edges: [WidgetWorkspaceEdge!]! + + """ + The list of `WidgetWorkspace` results. + """ + nodes: [WidgetWorkspace!]! + + """ + Provides pagination-related information. + """ + page_info: PageInfo! + + """ + The total number of edges available in this connection to paginate over. + """ + total_edge_count: JsonSafeLong! +} + +""" +Represents a specific `WidgetWorkspace` in the context of a `WidgetWorkspaceConnection`, +providing access to both the `WidgetWorkspace` and a pagination `Cursor`. + +See the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. +""" +type WidgetWorkspaceEdge { + """ + The `Cursor` of this `WidgetWorkspace`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `WidgetWorkspace`. + """ + cursor: Cursor + + """ + The `WidgetWorkspace` of this edge. + """ + node: WidgetWorkspace +} + +""" +Input type used to specify filters on `WidgetWorkspace` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input WidgetWorkspaceFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [WidgetWorkspaceFilterInput!] + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Used to filter on the `name` field. + + Will be ignored if `null` or an empty object is passed. + """ + name: StringFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: WidgetWorkspaceFilterInput + + """ + Used to filter on the `widget` field. + + Will be ignored if `null` or an empty object is passed. + """ + widget: WorkspaceWidgetFilterInput +} + +""" +Type used to specify the `WidgetWorkspace` fields to group by for aggregations. +""" +type WidgetWorkspaceGroupedBy { + """ + The `name` field value for this group. + """ + name: String + + """ + The `widget` field value for this group. + """ + widget: WorkspaceWidgetGroupedBy +} + +""" +Enumerates the ways `WidgetWorkspace`s can be sorted. +""" +enum WidgetWorkspaceSortOrderInput { + """ + Sorts ascending by the `id` field. + """ + id_ASC + + """ + Sorts descending by the `id` field. + """ + id_DESC + + """ + Sorts ascending by the `name` field. + """ + name_ASC + + """ + Sorts descending by the `name` field. + """ + name_DESC + + """ + Sorts ascending by the `widget.created_at` field. + """ + widget_created_at_ASC + + """ + Sorts descending by the `widget.created_at` field. + """ + widget_created_at_DESC + + """ + Sorts ascending by the `widget.id` field. + """ + widget_id_ASC + + """ + Sorts descending by the `widget.id` field. + """ + widget_id_DESC +} + +type WorkspaceWidget { + created_at: DateTime + id: ID! +} + +""" +Type used to perform aggregation computations on `WorkspaceWidget` fields. +""" +type WorkspaceWidgetAggregatedValues { + """ + Computed aggregate values for the `created_at` field. + """ + created_at: DateTimeAggregatedValues + + """ + Computed aggregate values for the `id` field. + """ + id: NonNumericAggregatedValues +} + +""" +Input type used to specify filters on `WorkspaceWidget` fields. + +Will be ignored if passed as an empty object (or as `null`). +""" +input WorkspaceWidgetFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + any_of: [WorkspaceWidgetFilterInput!] + + """ + Used to filter on the `created_at` field. + + Will be ignored if `null` or an empty object is passed. + """ + created_at: DateTimeFilterInput + + """ + Used to filter on the `id` field. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: WorkspaceWidgetFilterInput +} + +""" +Type used to specify the `WorkspaceWidget` fields to group by for aggregations. +""" +type WorkspaceWidgetGroupedBy { + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + + """ + The `id` field value for this group. + """ + id: ID +} + +""" +A custom scalar type required by the [Apollo Federation subgraph +spec](https://www.apollographql.com/docs/federation/subgraph-spec/#scalar-_any): + +> This scalar is the type used for entity **representations** that the graph router +> passes to the `Query._entities` field. An `_Any` scalar is validated by matching +> its `__typename` and `@key` fields against entities defined in the subgraph schema. +> +> An `_Any` is serialized as a JSON object, like so: +> +> ``` +> { +> "__typename": "Product", +> "upc": "abc123" +> } +> ``` + +Not intended for use by clients other than Apollo. +""" +scalar _Any + +""" +A union type required by the [Apollo Federation subgraph +spec](https://www.apollographql.com/docs/federation/subgraph-spec/#union-_entity): + +> **⚠️ This union type is generated dynamically based on the input subgraph schema!** +> +> This union's possible types must include all entities that the subgraph defines. +> It's the return type of the `Query._entities` field, which the graph router uses +> to directly access a subgraph's entity fields. +> +> For details, see [Defining the `_Entity` union](https://www.apollographql.com/docs/federation/subgraph-spec/#defining-the-_entity-union). + +In an ElasticGraph schema, this is a union of all indexed types. + +Not intended for use by clients other than Apollo. +""" +union _Entity = Component | Country | ElectricalPart | Manufacturer | MechanicalPart | Sponsor | Team | Widget | WidgetCurrency | WidgetWorkspace + +""" +An object type required by the [Apollo Federation subgraph +spec](https://www.apollographql.com/docs/federation/subgraph-spec/#type-_service): + +> This object type must have an `sdl: String!` field, which returns the SDL of the subgraph schema as a string. +> +> - The returned schema string _must_ include all uses of federation-specific directives (`@key`, `@requires`, etc.). +> - **If supporting Federation 1,** the schema _must not_ include any +definitions from [Subgraph schema additions](https://www.apollographql.com/docs/federation/subgraph-spec/#subgraph-schema-additions). +> +> For details, see [Enhanced introspection with `Query._service`](https://www.apollographql.com/docs/federation/subgraph-spec/#enhanced-introspection-with-query_service). + +Not intended for use by clients other than Apollo. +""" +type _Service { + """ + A field required by the [Apollo Federation subgraph + spec](https://www.apollographql.com/docs/federation/subgraph-spec/#required-resolvers-for-introspection): + + > The returned `sdl` string has the following requirements: + > + > - It must **include** all uses of all federation-specific directives, such as `@key`. + > - All of these directives are shown in [Subgraph schema additions](https://www.apollographql.com/docs/federation/subgraph-spec/#subgraph-schema-additions). + > - **If supporting Federation 1,** `sdl` must **omit** all automatically added definitions from + > [Subgraph schema additions](https://www.apollographql.com/docs/federation/subgraph-spec/#subgraph-schema-additions), + > such as `Query._service` and `_Service.sdl`! + > - If your library is _only_ supporting Federation 2, `sdl` can include these definitions. + + Not intended for use by clients other than Apollo. + """ + sdl: String +} + +""" +Scalar type used by the `@policy` directive required for Apollo Federation V2.6+. +""" +scalar federation__Policy + +""" +Scalar type used by the `@requiresScopes` directive required for Apollo Federation V2.5+. +""" +scalar federation__Scope + +""" +Scalar type used by the `@link` directive required for Apollo Federation V2. +""" +scalar link__Import + +""" +Enum type used by the `@link` directive required for Apollo Federation V2. +""" +enum link__Purpose { + """ + `EXECUTION` features provide metadata necessary for operation execution. + """ + EXECUTION + + """ + `SECURITY` features provide metadata necessary to securely resolve fields. + """ + SECURITY +} \ No newline at end of file diff --git a/config/schema/teams.rb b/config/schema/teams.rb new file mode 100644 index 00000000..15c6ff43 --- /dev/null +++ b/config/schema/teams.rb @@ -0,0 +1,187 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# These types have been designed to focus on list fields: +# - scalar lists +# - nested object lists +# - embedded object lists +# - multiple levels of lists +# - lists under a singleton object +# - any of the above with an alternate `name_in_index`. +ElasticGraph.define_schema do |schema| + schema.object_type "TeamDetails" do |t| + t.field "uniform_colors", "[String!]!" + + # `details.count` isn't really meaningful on our team model here, but we need this field + # to test that ElasticGraph handles a domain field named `count` even while it offers a + # `count` operator on list fields. + t.field schema.state.schema_elements.count, "Int" + end + + schema.object_type "TeamNestedFields" do |t| + t.field "forbes_valuation_moneys", "[Money!]!" do |f| + f.mapping type: "nested" + end + + t.field "current_players", "[Player!]!" do |f| + f.mapping type: "nested" + end + + t.field "seasons", "[TeamSeason!]!", name_in_index: "the_seasons" do |f| + f.mapping type: "nested" + end + end + + schema.object_type "Team" do |t| + t.root_query_fields plural: "teams" + t.field "id", "ID!" + t.field "league", "String" + t.field "country_code", "ID!" + t.field "formed_on", "Date" + t.field "current_name", "String" + t.field "past_names", "[String!]!" + t.field "won_championships_at", "[DateTime!]!" + t.field "details", "TeamDetails" + t.field "stadium_location", "GeoLocation" + t.field "forbes_valuations", "[JsonSafeLong!]!" + + t.field "forbes_valuation_moneys_nested", "[Money!]!" do |f| + f.mapping type: "nested" + end + + t.field "forbes_valuation_moneys_object", "[Money!]!" do |f| + f.mapping type: "object" + end + + t.field "current_players_nested", "[Player!]!" do |f| + f.mapping type: "nested" + end + + t.field "current_players_object", "[Player!]!" do |f| + f.mapping type: "object" + end + + t.field "seasons_nested", "[TeamSeason!]!" do |f| + f.mapping type: "nested" + end + + t.field "seasons_object", "[TeamSeason!]!" do |f| + f.mapping type: "object" + end + + t.field "nested_fields", "TeamNestedFields", name_in_index: "the_nested_fields" + + # To exercise an edge case, we need: Two different fields of an object type which both have a `nested` field of the same name. + # Here we duplicate `nested_fields` as `nested_fields2` to achieve that. + t.field "nested_fields2", "TeamNestedFields" + + t.index "teams" do |i| + i.route_with "league" + i.rollover :yearly, "formed_on" + end + end + + schema.object_type "Player" do |t| + t.field "name", "String" + t.field "nicknames", "[String!]!" + t.field "affiliations", "Affiliations!" + + t.field "seasons_nested", "[PlayerSeason!]!" do |f| + f.mapping type: "nested" + end + + t.field "seasons_object", "[PlayerSeason!]!" do |f| + f.mapping type: "object" + end + end + + schema.object_type "TeamRecord" do |t| + t.field "wins", "Int", name_in_index: "win_count" + t.field "losses", "Int", name_in_index: "loss_count" + t.field "last_win_on", "Date", name_in_index: "last_win_date" + t.field "last_win_on_legacy", "Date", name_in_index: "last_win_date", graphql_only: true, legacy_grouping_schema: true + t.field "first_win_on", "Date" + t.field "first_win_on_legacy", "Date", name_in_index: "first_win_on", graphql_only: true, legacy_grouping_schema: true + end + + schema.object_type "TeamSeason" do |t| + t.field "record", "TeamRecord", name_in_index: "the_record" + t.field "year", "Int" + t.field "notes", "[String!]!", singular: "note" + # `details.count` isn't really meaningful on our team model here, but we need this field + # to test that ElasticGraph handles a domain field named `count` on a list-of-object field + # even while it also offers a `count` operator on all list fields. + t.field schema.state.schema_elements.count, "Int" + t.field "started_at", "DateTime" + t.field "started_at_legacy", "DateTime", name_in_index: "started_at", graphql_only: true, legacy_grouping_schema: true + t.field "won_games_at", "[DateTime!]!", singular: "won_game_at" + t.field "won_games_at_legacy", "[DateTime!]!", singular: "won_game_at_legacy", name_in_index: "won_games_at", graphql_only: true, legacy_grouping_schema: true + + t.field "players_nested", "[Player!]!" do |f| + f.mapping type: "nested" + end + + t.field "players_object", "[Player!]!" do |f| + f.mapping type: "object" + end + end + + schema.object_type "PlayerSeason" do |t| + t.field "year", "Int" + t.field "games_played", "Int" + t.paginated_collection_field "awards", "String" + end + + schema.object_type "Sponsorship" do |t| + t.field "sponsor_id", "ID!" + t.field "annual_total", "Money!" + end + + schema.object_type "Affiliations" do |t| + t.field "sponsorships_nested", "[Sponsorship!]!" do |f| + f.mapping type: "nested" + end + + t.field "sponsorships_object", "[Sponsorship!]!" do |f| + f.mapping type: "object" + end + end + + schema.object_type "Sponsor" do |t| + t.root_query_fields plural: "sponsors" + t.field "id", "ID!" + t.field "name", "String" + t.relates_to_many "affiliated_teams_from_nested", "Team", via: "current_players_nested.affiliations.sponsorships_nested.sponsor_id", dir: :in, singular: "affiliated_team_from_nested" + t.relates_to_many "affiliated_teams_from_object", "Team", via: "current_players_object.affiliations.sponsorships_object.sponsor_id", dir: :in, singular: "affiliated_team_from_object" + + t.index "sponsors" + end + + schema.object_type "Country" do |t| + t.field "id", "ID!" + t.relates_to_many "teams", "Team", via: "country_code", dir: :in, singular: "team" + + # :nocov: -- only one side of these conditionals is executed in our test suite (but both both are covered by rake tasks) + t.apollo_key fields: "id" if t.respond_to?(:apollo_key) + # :nocov: + + # Note: we use `paginated_collection_field` here in order to exercise a case with the Apollo entity resolver and + # a paginated collection field, which initially yielded an exception. + t.paginated_collection_field "names", "String" do |f| + # :nocov: -- only one side of these conditionals is executed in our test suite (but both both are covered by rake tasks) + f.apollo_external if f.respond_to?(:apollo_external) + # :nocov: + end + + t.field "currency", "String" do |f| + # :nocov: -- only one side of these conditionals is executed in our test suite (but both both are covered by rake tasks) + f.apollo_external if f.respond_to?(:apollo_external) + # :nocov: + end + end +end diff --git a/config/schema/widgets.rb b/config/schema/widgets.rb new file mode 100644 index 00000000..1466de3b --- /dev/null +++ b/config/schema/widgets.rb @@ -0,0 +1,360 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# The types in this schema file are used by most tests. +ElasticGraph.define_schema do |schema| + schema.enum_type "Color" do |e| + e.values %w[RED BLUE GREEN] + end + + schema.enum_type "Size" do |e| + e.values %w[SMALL MEDIUM LARGE] + end + + schema.enum_type "Material" do |e| + e.values %w[ALLOY CARBON_FIBER] + end + + schema.object_type "WidgetOptions" do |t| + t.field "size", "Size" + # `the_size` is defined with a different `name_in_index` so we can demonstrate grouping works + # when a selected `group_by` option has a different name in the index vs GraphQL. + t.field "the_size", "Size", name_in_index: "the_sighs" + t.field "color", "Color" + end + + schema.object_type "Person" do |t| + t.implements "NamedInventor" + t.field "name", "String" + t.field "nationality", "String" + end + + schema.object_type "Company" do |t| + t.implements "NamedInventor" + t.field "name", "String" + t.field "stock_ticker", "String" + end + + schema.union_type "Inventor" do |t| + t.subtypes "Person", "Company" + end + + # Embedded interface type. + schema.interface_type "NamedInventor" do |t| + t.field "name", "String" + end + + # Indexed interfae type. + schema.interface_type "NamedEntity" do |t| + t.root_query_fields plural: "named_entities" + t.field "id", "ID!" + t.field "name", "String" + end + + # Note: it is important that no list field is added to this `Money` type. We are using it as an example + # of a `nested` list field with no list fields of its own in the `teams.rb` schema. + schema.object_type "Money" do |t| + t.field "currency", "String!" + t.field "amount_cents", "Int" + end + + schema.object_type "Position" do |t| + t.field "x", "Float!" + t.field "y", "Float!" + end + + schema.object_type "Widget" do |t| + t.root_query_fields plural: "widgets" + t.implements "NamedEntity" + t.field "id", "ID!" + + # Here we use an alternate name for this field since it's the routing field and want to verify + # that `name_in_index` works correctly on routing fields. + t.field "workspace_id", "ID", name_in_index: "workspace_id2" + + # It's a bit funny we have both `amount_cents` and `cost` but it's nice to be able to test + # aggregations on both a root numeric field and on a nested one, so we are keeping both here. + t.field "amount_cents", "Int!" + # Note: we are naming this field differently so we can demonstrate that filtering/aggregating + # on a field that is named differently in the index works. + t.field "amount_cents2", "Int!", name_in_index: "amount_cents", graphql_only: true + t.field "cost", "Money" + t.field "cost_currency_unit", "String" + t.field "cost_currency_name", "String" + t.field "cost_currency_symbol", "String" + t.field "cost_currency_primary_continent", "String" + t.field "cost_currency_introduced_on", "Date" + t.field "name", "String" + t.field "name_text", "String" do |f| + f.mapping type: "text" + end + t.field "created_at", "DateTime!" + t.field "created_at_legacy", "DateTime!", name_in_index: "created_at", graphql_only: true, legacy_grouping_schema: true + # `created_at2` is defined with a different `name_in_index` so we can demonstrate grouping works + # when a selected grouping field has a different name in the index vs GraphQL. + t.field "created_at2", "DateTime!", name_in_index: "created_at", graphql_only: true + t.field "created_at2_legacy", "DateTime!", name_in_index: "created_at", graphql_only: true, legacy_grouping_schema: true + t.field "created_at_time_of_day", "LocalTime" + t.field "created_on", "Date" + t.field "created_on_legacy", "Date", name_in_index: "created_on", graphql_only: true, legacy_grouping_schema: true + t.field "release_timestamps", "[DateTime!]!", singular: "release_timestamp" + t.field "release_dates", "[Date!]!", singular: "release_date" + t.relates_to_many "components", "Component", via: "component_ids", dir: :out, singular: "component" + t.field "options", "WidgetOptions" + + # Demonstrate using `name_in_index` with a graphql-only embedded field. + t.field "size", "Size", name_in_index: "options.size", graphql_only: true + + # `the_options` is defined with a different `name_in_index` so we can demonstrate grouping works when a parent field + # of a selected `group_by` option has a different name in the index vs GraphQL. + t.field "the_options", "WidgetOptions", name_in_index: "the_opts" + t.field "inventor", "Inventor" + t.field "named_inventor", "NamedInventor" + t.field "weight_in_ng_str", "LongString!" # Weight in nanograms, to exercise Long support. + t.field "weight_in_ng", "JsonSafeLong!" # Weight in nanograms, to exercise Long support. + t.field "tags", "[String!]!", sortable: false, singular: "tag" + t.field "amounts", "[Int!]!", sortable: false do |f| + f.mapping index: false + end + t.field "fees", "[Money!]!", sortable: false do |f| + f.mapping type: "object" + end + t.field "metadata", "Untyped" + + # TODO: change `widget.` in these field paths to `widgets.` when we can support `sourced_from` with that. + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget.id", dir: :in do |rel| + rel.equivalent_field "id", locally_named: "workspace_id" + rel.equivalent_field "widget.created_at", locally_named: "created_at" + end + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + # Customize the index so we can demonstrate that index customization works. + # Also, to demonstrate that custom shard routing works correctly, we need multiple shards. + # That way, our documents wind up on multiple shards and we can demonstrate that our + # queries are directly routed to the correct shards. + t.index "widgets", number_of_shards: 3 do |i| + i.rollover :yearly, "created_at" + i.route_with "workspace_id" + i.default_sort "created_at", :desc + end + + t.derive_indexed_type_fields "WidgetCurrency", from_id: "cost.currency", route_with: "cost_currency_primary_continent", rollover_with: "cost_currency_introduced_on" do |derive| + derive.immutable_value "name", from: "cost_currency_name" + derive.immutable_value "introduced_on", from: "cost_currency_introduced_on" + derive.immutable_value "primary_continent", from: "cost_currency_primary_continent" + derive.immutable_value "details.unit", from: "cost_currency_unit", nullable: false + derive.immutable_value "details.symbol", from: "cost_currency_symbol", can_change_from_null: true + + # named `widget_names2` to match `name_in_index` of `WidgetCurrency.widget_names` + # Note: `sourced_from` handles `name_in_index` better and should avoid the need to use the + # `name_in_index` here. + derive.append_only_set "widget_names2", from: "name" + + derive.append_only_set "widget_options.colors", from: "options.color" + derive.append_only_set "widget_options.sizes", from: "options.size" + derive.append_only_set "widget_tags", from: "tags" + derive.append_only_set "widget_fee_currencies", from: "fees.currency" + derive.max_value "nested_fields.max_widget_cost", from: "cost.amount_cents" + derive.min_value "oldest_widget_created_at", from: "created_at" + end + end + + schema.object_type "WidgetOptionSets" do |t| + t.field "sizes", "[Size!]!" + t.field "colors", "[Color!]!" + end + + schema.object_type "WidgetCurrencyNestedFields" do |t| + t.field "max_widget_cost", "Int" + end + + schema.object_type "CurrencyDetails" do |t| + t.field "unit", "String" + t.field "symbol", "String" + end + + schema.object_type "WidgetCurrency" do |t| + t.root_query_fields plural: "widget_currencies" + t.field "id", "ID!" + t.field "name", "String" + t.field "introduced_on", "Date" + t.field "primary_continent", "String" + t.field "details", "CurrencyDetails" + t.paginated_collection_field "widget_names", "String", name_in_index: "widget_names2", singular: "widget_name" + t.field "widget_tags", "[String!]!" + t.field "widget_fee_currencies", "[String!]!" + t.field "widget_options", "WidgetOptionSets" + t.field "nested_fields", "WidgetCurrencyNestedFields" + t.field "oldest_widget_created_at", "DateTime" + t.index "widget_currencies" do |i| + i.rollover :yearly, "introduced_on" + i.route_with "primary_continent" + end + end + + schema.object_type "WidgetWorkspace" do |t| + t.root_query_fields plural: "widget_workspaces" + t.field "id", "ID!" + t.field "name", "String" + + # TODO: replace `widget` with `widgets` when we can support `sourced_from` with that. + t.field "widget", "WorkspaceWidget" + # t.field "widgets", "[WorkspaceWidget!]!" + + t.index "widget_workspaces" + end + + schema.object_type "WorkspaceWidget" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime" + end + + schema.object_type "Component" do |t| + t.root_query_fields plural: "components" + t.implements "NamedEntity" + t.field "id", "ID!" + t.field "name", "String" + t.field "created_at", "DateTime!" + t.field "position", "Position!" + t.field "tags", "[String!]!" + + t.field "widget_name", "String" do |f| + f.sourced_from "widget", "name" + end + + t.field "widget_tags", "[String!]" do |f| + f.sourced_from "widget", "tags" + end + + # We use `name_in_index` here to demonstrate that a `sourced_from` field can flow into an alternately named field. + t.field "widget_workspace_id", "ID", name_in_index: "widget_workspace_id3" do |f| + f.sourced_from "widget", "workspace_id" + end + + t.field "widget_size", "Size" do |f| + # Here we're demonstrating usage on a nested field, and on fields which use an alternative `name_in_index`. + f.sourced_from "widget", "the_options.the_size" + end + + # `Money` is an object type, so this is defined to demonstrate that denormalizing object values works. + t.field "widget_cost", "Money" do |f| + f.sourced_from "widget", "cost" + end + + t.relates_to_one "widget", "Widget", via: "component_ids", dir: :in + t.relates_to_one "dollar_widget", "Widget", via: "component_ids", dir: :in do |rel| + rel.additional_filter "cost" => {"amount_cents" => {"equal_to_any_of" => [100]}} + end + # In practice, there is one widget to many components. But to exercise an edge case it is useful to have a + # many-to-many as well, so here we expose a list even though it will only ever be a list of 1. + t.relates_to_many "widgets", "Widget", via: "component_ids", dir: :in, singular: "widget" + t.relates_to_many "parts", "Part", via: "part_ids", dir: :out, singular: "part" + + t.index "components" do |i| + i.default_sort "created_at", :desc + end + end + + schema.object_type "MechanicalPart" do |t| + t.root_query_fields plural: "mechanical_parts" + t.implements "NamedEntity" + t.field "id", "ID!" + t.field "name", "String" + t.field "created_at", "DateTime!" + t.field "material", "Material" + t.relates_to_many "components", "Component", via: "part_ids", dir: :in, singular: "component" + t.relates_to_one "manufacturer", "Manufacturer", via: "manufacturer_id", dir: :out + + t.index "mechanical_parts" do |i| + i.default_sort "created_at", :desc + end + end + + schema.object_type "ElectricalPart" do |t| + t.root_query_fields plural: "electrical_parts" + t.implements "NamedEntity" + t.field "id", "ID!" + t.field "name", "String" + t.field "created_at", "DateTime!" + t.field "voltage", "Int!" + t.relates_to_many "components", "Component", via: "part_ids", dir: :in, singular: "component" + t.relates_to_one "manufacturer", "Manufacturer", via: "manufacturer_id", dir: :out + + t.index "electrical_parts" do |i| + i.default_sort "created_at", :desc + end + end + + schema.union_type "Part" do |t| + t.subtypes "MechanicalPart", "ElectricalPart" + end + + # Note: `Manufacturer` is used in our tests as an example of an indexed type that has no list fields, so we should + # not add any list fields to this type in the future. + schema.object_type "Manufacturer" do |t| + t.root_query_fields plural: "manufacturers" + t.implements "NamedEntity" + t.field "id", "ID!" + t.field "name", "String" + t.field "created_at", "DateTime!" + t.relates_to_many "manufactured_parts", "Part", via: "manufacturer_id", dir: :in, singular: "manufactured_part" + t.relates_to_one "address", "Address", via: "manufacturer_id", dir: :in + + t.index "manufacturers" do |i| + i.default_sort "created_at", :desc + end + end + + schema.object_type "AddressTimestamps" do |t| + t.field "created_at", "DateTime" + end + + schema.object_type "GeoShape" do |t| + t.field "type", "String" + t.field "coordinates", "[Float!]!" + + # Here we are using a custom mapping type on an object type so we can verify that the schema + # artifact generation works as expected in this case. + # + # Note: `geo_shape` is one of the few custom mapping types available on both OpenSearch and Elasticsearch, + # which is why we've chosen it here. + # https://www.elastic.co/guide/en/elasticsearch/reference/8.9/mapping-types.html + # https://opensearch.org/docs/latest/field-types/supported-field-types/index/ + t.mapping type: "geo_shape" + end + + schema.object_type "Address" do |t| + t.root_query_fields plural: "addresses" + # We use `indexing_only: true` here to verify that `id` can be an indexing-only field. + t.field "id", "ID!", indexing_only: true + + t.field "full_address", "String!" + t.field "timestamps", "AddressTimestamps" + t.field "geo_location", "GeoLocation" + + # Not used by anything, but defined so we can test how a list-of-objects-with-custom-mapping + # works in our schema generation. + t.field "shapes", "[GeoShape!]!" + + t.relates_to_one "manufacturer", "Manufacturer", via: "manufacturer_id", dir: :out + + t.index "addresses" do |i| + # We don't yet support a default sort of a nested field so we use id here instead of `timestamps.created_at`. + i.default_sort "id", :desc + end + end + + # Defined so we can exercise having a union type with a subtype that's uses a rollover index. + schema.union_type "WidgetOrAddress" do |t| + t.subtypes "Widget", "Address" + t.root_query_fields plural: "widgets_or_addresses" + end +end diff --git a/config/settings/development.yaml b/config/settings/development.yaml new file mode 100644 index 00000000..16a9fe3b --- /dev/null +++ b/config/settings/development.yaml @@ -0,0 +1,37 @@ +datastore: + client_faraday_adapter: + name: httpx + require: httpx/adapters/faraday + clusters: + main: + url: http://localhost:9334 + backend: elasticsearch + settings: {} + index_definitions: + addresses: &main_index_settings + query_cluster: "main" + index_into_clusters: ["main"] + ignore_routing_values: [] + custom_timestamp_ranges: [] + setting_overrides: + number_of_shards: 1 + setting_overrides_by_timestamp: {} + components: *main_index_settings + electrical_parts: *main_index_settings + manufacturers: *main_index_settings + mechanical_parts: *main_index_settings + teams: *main_index_settings + widget_currencies: *main_index_settings + widgets: *main_index_settings + widget_workspaces: *main_index_settings + sponsors: *main_index_settings + max_client_retries: 3 +graphql: + default_page_size: 50 + max_page_size: 500 +logger: + device: log/development.log +indexer: + latency_slo_thresholds_by_timestamp_in_ms: {} +schema_artifacts: + directory: config/schema/artifacts diff --git a/config/settings/development_with_apollo.yaml b/config/settings/development_with_apollo.yaml new file mode 100644 index 00000000..5be15711 --- /dev/null +++ b/config/settings/development_with_apollo.yaml @@ -0,0 +1,38 @@ +datastore: + client_faraday_adapter: + name: httpx + require: httpx/adapters/faraday + clusters: + main: + url: http://localhost:9334 + backend: elasticsearch + settings: {} + index_definitions: + addresses: &main_index_settings + query_cluster: "main" + index_into_clusters: ["main"] + ignore_routing_values: [] + custom_timestamp_ranges: [] + setting_overrides: + number_of_shards: 1 + setting_overrides_by_timestamp: {} + components: *main_index_settings + electrical_parts: *main_index_settings + manufacturers: *main_index_settings + mechanical_parts: *main_index_settings + teams: *main_index_settings + widget_currencies: *main_index_settings + widgets: *main_index_settings + widget_workspaces: *main_index_settings + sponsors: *main_index_settings + max_client_retries: 3 +graphql: + default_page_size: 50 + max_page_size: 500 +logger: + device: log/development.log +indexer: + latency_slo_thresholds_by_timestamp_in_ms: {} +schema_artifacts: + # Note: this is the only difference between development.yaml and development_with_apollo.yaml + directory: config/schema/artifacts_with_apollo diff --git a/config/settings/test.yaml.template b/config/settings/test.yaml.template new file mode 100644 index 00000000..2f2aecca --- /dev/null +++ b/config/settings/test.yaml.template @@ -0,0 +1,86 @@ +datastore: + client_faraday_adapter: + name: httpx + require: httpx/adapters/faraday + clusters: + main: + url: http://localhost:9234 + backend: "%{datastore_backend}" + settings: + # We set this to be quite high, in order to support a parallel test suite runner. When we run specs in parallel we + # add a per-worker-process prefix to the index names used by that worker in order to isolate the tests that get run + # on each worker. This leads to a large number of indices being created. To support up to 16 workers on 16-core + # machines, we've increased this from 1000 (the default) to 16000. + cluster.max_shards_per_node: 16000 + other1: + url: http://localhost:9234 + backend: "opensearch" + settings: + cluster.max_shards_per_node: 16001 # to be different from the above; a spec asserts on this. + other2: + url: http://localhost:9234 + backend: "%{datastore_backend}" + settings: {} + other3: + url: http://localhost:9234 + backend: "%{datastore_backend}" + settings: {} + index_definitions: + addresses: &main_index_settings + query_cluster: "main" + index_into_clusters: ["main"] + ignore_routing_values: [] + custom_timestamp_ranges: [] + setting_overrides: + number_of_shards: 1 + setting_overrides_by_timestamp: {} + components: *main_index_settings + electrical_parts: *main_index_settings + manufacturers: *main_index_settings + mechanical_parts: *main_index_settings + teams: *main_index_settings + widget_currencies: *main_index_settings + widgets: + query_cluster: "main" + index_into_clusters: ["main"] + ignore_routing_values: ["ignored_workspace_1", "ignored_workspace_2"] + setting_overrides: + number_of_shards: 1 + # The settings here are designed to remove some non-determinism from the tests, related to + # index expressions used in queries. The logic that selects which indices to query based on + # a timestamp filter excludes known indices that could not match the query (based on the + # timestamp filter being used), which means that the body of the request is based partially + # on what indices exist for our rollover indices (of which `widgets` is the only one). But + # without configuring things here, the indices created from the rollover template would be + # created lazily when a document is indexed having a timestamp that falls in a new index's + # time range. If we relied on that, our tests would be slightly non-deterministic, where the + # indices available to exclude would depend on what tests had run before. + # + # Here we avoid that non-determism by explicitly defining the indices: one per year between + # 2019 and 2021, one for before 2019, and one for after 2021. All possible timestamps are + # covered by these defined indices so no new indices will be created lazily when documents + # are indexed. + custom_timestamp_ranges: + - index_name_suffix: "before_2019" + lt: "2019-01-01T00:00:00Z" + setting_overrides: {} + - index_name_suffix: "after_2021" + gte: "2022-01-01T00:00:00Z" + setting_overrides: {} + setting_overrides_by_timestamp: + "2019-01-01T00:00:00Z": {} + "2020-01-01T00:00:00Z": {} + "2021-01-01T00:00:00Z": {} + widget_workspaces: *main_index_settings + sponsors: *main_index_settings + max_client_retries: 3 +graphql: + # default_page_size of 50 is duplicated in `elasticgraph/spec/support/aggregations_helpers.rb`. + default_page_size: 50 + max_page_size: 500 +logger: + device: log/test.log +indexer: + latency_slo_thresholds_by_timestamp_in_ms: {} +schema_artifacts: + directory: config/schema/artifacts diff --git a/config/site/README.md b/config/site/README.md new file mode 100644 index 00000000..f7e2098b --- /dev/null +++ b/config/site/README.md @@ -0,0 +1,27 @@ +# Site Config README + +The documentation site is built on the `gh-pages` branch using the GitHub action `publish-site.yml`. + +To build the site and documentation locally, run `bundle exec rake -T | grep site` to view a list of commands. These commands are sourced from the Rake tasks in `config/site/Rakefile`. + +## CSS Styling + +CSS Styling is powerd by [Tailwind CSS](https://tailwindcss.com/) via the `package.json` script: `npm run build:css`. + +Tip: most LLM's do a good job of generating Tailwind CSS classes. + +Extract common classes to `src/_config.yaml` + +We're using the `@tailwindcss/typography` plugin to style the markdown content automatically. See https://github.com/tailwindlabs/tailwindcss-typography for more information. + +If a new class is used in an HTML file, you'll need to restart the site serve rake task to ensure the new class is included in `main.css` by Tailwind. + +## Markdown + +Write any standalone content (non-documentation) as regular Markdown files with front matter. (see `src/about.md` for example). + +## Icons + +Icons are SVG's copied from [heroicons](https://heroicons.com/) (MIT licensed). + +Include them via `{% include svg/document-duplicate.svg %}` diff --git a/config/site/Rakefile b/config/site/Rakefile new file mode 100644 index 00000000..a6579273 --- /dev/null +++ b/config/site/Rakefile @@ -0,0 +1,290 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/local/rake_tasks" +require "elastic_graph/query_registry/rake_tasks" +require "pathname" + +module ElasticGraph + class SiteRakeTasks < ::Rake::TaskLib + SITE_CONFIG_DIR = ::Pathname.new(__dir__) + REPO_ROOT = SITE_CONFIG_DIR.parent.parent + SITE_SOURCE_DIR = SITE_CONFIG_DIR / "src" + YARD_OUTPUT_DIR = SITE_SOURCE_DIR / "docs" / "main" + JEKYLL_SITE_DIR = SITE_CONFIG_DIR / "_site" + JEKYLL_DATA_DIR = SITE_SOURCE_DIR / "_data" + EXAMPLE_SCHEMA_FILES_BY_NAME = SITE_CONFIG_DIR.glob("examples/*/schema.rb").to_h do |file| + [file.parent.basename.to_s, file] + end + + # The list of currently undocumented gems. Over time we're going to shrink this until it's empty. + undocumented_gems = %w[ + elasticgraph-admin + elasticgraph-datastore_core + elasticgraph-graphql + elasticgraph-health_check + elasticgraph-indexer + elasticgraph-lambda_support + elasticgraph-query_interceptor + elasticgraph-query_registry + elasticgraph-schema_artifacts + ] + + require_relative "../../script/list_eg_gems" + DOCUMENTED_GEMS = ::ElasticGraphGems.list - undocumented_gems + + def initialize + namespace :site do + task build_docs: :render_mermaid do + # Clean the docs output directory + FileUtils.rm_rf(YARD_OUTPUT_DIR) + run_yard_doc_ignoring_expected_warnings + end + + # Note: we tried to get this setup as a single task that serves load for all ElasticGraph gems, + # but we haven't succeeded in getting that to work. We've only gotten it to work by changing + # to a specific gem directory to run the command. + namespace :preview_docs do + ::ElasticGraphGems.list.each do |gem| + desc "Boots a reloading doc server for `#{gem}`." + task gem do + ::Dir.chdir(gem) do + sh "bundle exec yard server --reload" + end + end + end + end + + desc "Check documentation coverage" + task :docs_coverage do + doc_output = run_yard_doc_ignoring_expected_warnings + + coverage = doc_output[/([\d\.]+)% documented/, 1] + warning_count = doc_output.scan("[warn]:").count + error_count = doc_output.scan("[error]:").count + + if coverage.to_f < 100 + # Since we do not have 100% coverage, we want to list what is undocumented. + # + # Note: we don't use this as the main command above because we've observed that + # `stats` does not produce as many warnings as `doc`--so we'd rather run `doc` + # when detecting warnings, and use `stats --list-undoc` for supplemental output. + undoc_output = IO.popen(yard_cmd("stats --list-undoc")).read + + # Just print the output starting with `Undocumented Objects" + puts "\n#{undoc_output[/^Undocumented .*/m]}" + end + + issues = [] + issues << "Missing documentation coverage (currently at #{coverage}%)." if coverage.to_f < 100 + issues << "YARD emitted #{warning_count} documentation warning(s)." if warning_count > 0 + issues << "YARD emitted #{error_count} documentation error(s)." if error_count > 0 + + unless issues.empty? + abort <<~EOS + + Documentation has #{issues.size} issues: + + #{issues.map { |i| " - #{i}" }.join("\n")} + EOS + end + end + + desc "Tests all documentation examples." + task :doctest do + require "yard-doctest" + + # Change the log level to ERROR in order to silence yard warnings. + # (We deal with yard warnings in `docs_coverage` and don't want to also print them here.) + ::YARD::Logger.instance.enter_level(::Logger::ERROR) do + # We change into this directory because `yard-doctest` loads `doctest_helper.rb` from specific paths + # such as `support/doctest_helper.rb`. + ::Dir.chdir(__dir__) do + paths_with_yard_examples = ::ElasticGraphGems.list.map do |gem| + "#{REPO_ROOT}/#{gem}/lib" + end + + ::YARD::CLI::Doctest.run(*paths_with_yard_examples) + end + end + end + + task :npm_install do + ::Dir.chdir(SITE_CONFIG_DIR) do + sh "npm install" + end + end + + task render_mermaid: :npm_install do + ::Dir.chdir(SITE_CONFIG_DIR) do + sh "npx -p @mermaid-js/mermaid-cli mmdc -i ../../README.md -o ../../tmp/README.md" + end + end + + desc "Build Jekyll site with freshly generated YARD documentation" + task build: [:build_docs, :build_css, "examples:compile_queries"] do + sh "bundle exec jekyll build --source #{SITE_SOURCE_DIR} --destination #{JEKYLL_SITE_DIR}" + end + + desc "Serve Jekyll site locally" + task serve: [:build_docs, :build_css, "examples:compile_queries"] do + require "filewatcher" + + # Regenerate the YARD docs anytime we change anything. + ::Thread.new do + ::Filewatcher.new(DOCUMENTED_GEMS.map { |g| "#{g}/" }).watch do |changes| + changed_files = changes.keys.map { |f| ::Pathname.new(f).relative_path_from(REPO_ROOT) }.sort + puts "#{changed_files.size} files changed (#{changed_files.join(", ")}). Regenerating YARD docs..." + run_yard_doc_ignoring_expected_warnings + end + end + + # Re-compile queries when any of the examples change. + ::Thread.new do + ::Filewatcher.new(EXAMPLE_SCHEMA_FILES_BY_NAME.values.map { |f| "#{f.parent}/" }, exclude: "**/*.variables.yaml").watch do |changes| + changed_files = changes.keys.map { |f| ::Pathname.new(f).relative_path_from(REPO_ROOT) }.sort + + puts "#{changed_files.size} files changed (#{changed_files.join(", ")}). Recompiling GraphQL queries into data files..." + changed_files.each do |file| + t1 = ::Time.now + example_schema = file.to_s.split("/")[3] + task = ::Rake::Task["site:examples:#{example_schema}:compile_queries"] + task.all_prerequisite_tasks.each(&:reenable) + task.tap(&:reenable).invoke + t2 = ::Time.now + + puts "Done in #{(t2 - t1).round(3)} seconds." + rescue => e + # Print validation errors and allow the filewatcher to continue. + puts <<~EOS + #{e.class}: #{e.message} + + #{e.backtrace.join("\n")} + EOS + end + end + end + + sh "bundle exec jekyll serve --source #{SITE_SOURCE_DIR} --destination #{JEKYLL_SITE_DIR} --trace" + end + + desc "Perform validations of the website, including doc tests and doc coverage" + task validate: [:build, :docs_coverage, :doctest] + + task build_css: :npm_install do + require "rouge" + + ::Dir.chdir(SITE_SOURCE_DIR) do + sh "npm run build:css" + # tulip appears to provide the best looking syntax highlighting theme of all the built-in rouge themes. + ::File.write("assets/css/highlight.css", ::Rouge::Theme.find("tulip").render(scope: ".highlight")) + end + end + + namespace :examples do + task compile_queries: EXAMPLE_SCHEMA_FILES_BY_NAME.keys.map { |schema| "#{schema}:compile_queries" } + + EXAMPLE_SCHEMA_FILES_BY_NAME.each do |schema_name, schema_file| + example_dir = schema_file.parent + settings_file = example_dir / "local_settings.yaml" + queries_dir = example_dir / "queries" + + namespace schema_name do + ::ElasticGraph::Local::RakeTasks.new(local_config_yaml: settings_file, path_to_schema: schema_file) do |tasks| + tasks.opensearch_versions = [] + tasks.enforce_json_schema_version = false + end + ::ElasticGraph::QueryRegistry::RakeTasks.from_yaml_file(settings_file, queries_dir) + + task "query_registry:validate_queries" => ["schema_artifacts:dump", "query_registry:dump_variables:all"] + + task compile_queries: "query_registry:validate_queries" do + queries_by_name_by_category = queries_dir.children.to_h do |category_path| + queries_by_name = category_path.glob("*.graphql").to_h do |query_path| + [query_path.basename.sub_ext("").to_s, query_path.read.strip] + end + + [category_path.basename.to_s, queries_by_name] + end + + ::FileUtils.mkdir_p JEKYLL_DATA_DIR + ::File.write(::File.join(JEKYLL_DATA_DIR, "#{schema_name}_queries.yaml"), ::YAML.dump(queries_by_name_by_category)) + end + end + end + end + end + end + + # Yard doesn't allow us to suppress warnings. Warnings are for code constructs its not able to understand, and sometimes we're ok + # with that (e.g. when the alternative is making the code worse or more verbose). Here we suppress warnings using custom logic: + # we filter out specific warnings that we don't want to be notified about. + YARD_WARNINGS_TO_IGNORE = { + # `Data.define` metaprograms a superclass and there's not a way to avoid the warning: + # https://github.com/lsegal/yard/issues/1533 + # https://github.com/lsegal/yard/issues/1477#issuecomment-1399339983 + "`Data.define` superclass" => + # https://rubular.com/r/lecYmbp981T4LY + /^\[warn\]: in YARD::Handlers::Ruby::ClassHandler: Undocumentable superclass \(class was added without superclass\)\n\s+in file[^\n]+\n\n\s+\d+:[^\n]*?(Data\.define|Struct.new)[^\n]*\n\n/m, + + # We sometimes include/extend/prepend a mixin that is a dynamic module + # (e.g. `include Mixins::HasReadableToSAndInspect.new`) and YARD isn't able to understand this. + # That's fine, and we don't want a warning for this. + "Undocumentable mixin" => + # https://rubular.com/r/o0Daj0rKgNLes0 + /^\[warn\]: in YARD::Handlers::Ruby::(Extend|Mixin)Handler: Undocumentable mixin: YARD::Parser::UndocumentableError[^\n]*\n\s+in file[^\n]+\n\n\s+\d+: (include|extend|prepend) [A-Za-z:]+\.new[^\n]*\n\n/m + } + + def run_yard_doc_ignoring_expected_warnings + command = yard_cmd("doc") + puts "#{command}\n\n" + + t1 = ::Time.now + doc_output = IO.popen(command).read + t2 = ::Time.now + + ignored_warnings_output = YARD_WARNINGS_TO_IGNORE.filter_map do |warning, regex| + if (count = doc_output.scan(regex).count) > 0 + "Ignored #{count} #{warning} warning(s)." + end + end.join("\n") + + filtered_output = YARD_WARNINGS_TO_IGNORE.values.reduce(doc_output) { |accum, regex| accum.gsub(regex, "") } + + <<~EOS.strip.tap { |output| puts output } + YARD doc generation took #{(t2 - t1).round(3)} seconds. + #{ignored_warnings_output} + + #{filtered_output} + EOS + end + + yardopts_from_file = ::File.read(::File.expand_path("yardopts", __dir__)) + .split("\n") + .map(&:strip) + .reject(&:empty?) + + YARD_DOC_OPTS = [ + "--output-dir #{YARD_OUTPUT_DIR}", + "--db #{SITE_CONFIG_DIR}/.yardoc", + "--main tmp/README.md", + *yardopts_from_file, + *DOCUMENTED_GEMS + ].join(" ") + + def yard_cmd(subcmd) + assets = Dir.glob("tmp/README-*.svg").map do |svg_file| + "--asset #{svg_file}:#{::File.basename(svg_file)}" + end + + "bundle exec yard #{subcmd} #{YARD_DOC_OPTS} #{assets.join(" ")}" + end + end +end + +ElasticGraph::SiteRakeTasks.new diff --git a/config/site/examples/music/local_settings.yaml b/config/site/examples/music/local_settings.yaml new file mode 100644 index 00000000..b2d29eac --- /dev/null +++ b/config/site/examples/music/local_settings.yaml @@ -0,0 +1,34 @@ +datastore: + client_faraday_adapter: + name: httpx + require: httpx/adapters/faraday + clusters: + main: + url: http://localhost:9444 + backend: elasticsearch + settings: {} + index_definitions: + artists: &main_index_settings + query_cluster: "main" + index_into_clusters: ["main"] + ignore_routing_values: [] + custom_timestamp_ranges: [] + setting_overrides: + number_of_shards: 1 + setting_overrides_by_timestamp: {} + venues: *main_index_settings + max_client_retries: 3 +graphql: + default_page_size: 50 + max_page_size: 500 +logger: + device: stderr +indexer: + latency_slo_thresholds_by_timestamp_in_ms: {} +schema_artifacts: + directory: config/site/examples/music/schema_artifacts +query_registry: + # Allow any query by any client since this is for local use. + allow_unregistered_clients: true + allow_any_query_for_clients: [] + path_to_registry: config/site/examples/music/queries diff --git a/config/site/examples/music/queries/aggregations/AlbumCount.graphql b/config/site/examples/music/queries/aggregations/AlbumCount.graphql new file mode 100644 index 00000000..d1b461aa --- /dev/null +++ b/config/site/examples/music/queries/aggregations/AlbumCount.graphql @@ -0,0 +1,17 @@ +query AlbumCount { + artistAggregations { + nodes { + subAggregations { + albums { + nodes { + countDetail { + approximateValue + exactValue + upperBound + } + } + } + } + } + } +} diff --git a/config/site/examples/music/queries/aggregations/AlbumSalesByReleaseDayOfWeek.graphql b/config/site/examples/music/queries/aggregations/AlbumSalesByReleaseDayOfWeek.graphql new file mode 100644 index 00000000..f13628d8 --- /dev/null +++ b/config/site/examples/music/queries/aggregations/AlbumSalesByReleaseDayOfWeek.graphql @@ -0,0 +1,23 @@ +query AlbumSalesByReleaseDayOfWeek { + artistAggregations { + nodes { + subAggregations { + albums { + nodes { + groupedBy { + releasedOn { + asDayOfWeek + } + } + + aggregatedValues { + soldUnits { + exactSum + } + } + } + } + } + } + } +} diff --git a/config/site/examples/music/queries/aggregations/AlbumSalesByReleaseMonth.graphql b/config/site/examples/music/queries/aggregations/AlbumSalesByReleaseMonth.graphql new file mode 100644 index 00000000..dc8018a3 --- /dev/null +++ b/config/site/examples/music/queries/aggregations/AlbumSalesByReleaseMonth.graphql @@ -0,0 +1,23 @@ +query AlbumSalesByReleaseMonth { + artistAggregations { + nodes { + subAggregations { + albums(first: 100) { + nodes { + groupedBy { + releasedOn { + asDate(truncationUnit: MONTH) + } + } + + aggregatedValues { + soldUnits { + exactSum + } + } + } + } + } + } + } +} diff --git a/config/site/examples/music/queries/aggregations/AlbumSalesByReleaseWeek.graphql b/config/site/examples/music/queries/aggregations/AlbumSalesByReleaseWeek.graphql new file mode 100644 index 00000000..04e0409c --- /dev/null +++ b/config/site/examples/music/queries/aggregations/AlbumSalesByReleaseWeek.graphql @@ -0,0 +1,23 @@ +query AlbumSalesByReleaseWeek { + artistAggregations { + nodes { + subAggregations { + albums { + nodes { + groupedBy { + releasedOn { + asDate(truncationUnit: WEEK, offset: {amount: -1, unit: DAY}) + } + } + + aggregatedValues { + soldUnits { + exactSum + } + } + } + } + } + } + } +} diff --git a/config/site/examples/music/queries/aggregations/AlbumSalesByReleaseYear.graphql b/config/site/examples/music/queries/aggregations/AlbumSalesByReleaseYear.graphql new file mode 100644 index 00000000..de1f271c --- /dev/null +++ b/config/site/examples/music/queries/aggregations/AlbumSalesByReleaseYear.graphql @@ -0,0 +1,23 @@ +query AlbumSalesByReleaseYear { + artistAggregations { + nodes { + subAggregations { + albums { + nodes { + groupedBy { + releasedOn { + asDate(truncationUnit: YEAR) + } + } + + aggregatedValues { + soldUnits { + exactSum + } + } + } + } + } + } + } +} diff --git a/config/site/examples/music/queries/aggregations/ArtistCountsByCountry.graphql b/config/site/examples/music/queries/aggregations/ArtistCountsByCountry.graphql new file mode 100644 index 00000000..df53740e --- /dev/null +++ b/config/site/examples/music/queries/aggregations/ArtistCountsByCountry.graphql @@ -0,0 +1,11 @@ +query ArtistCountsByCountry { + artistAggregations { + nodes { + groupedBy { + bio { homeCountry } + } + + count + } + } +} diff --git a/config/site/examples/music/queries/aggregations/ArtistCountsByYearFormedAndHomeCountry.graphql b/config/site/examples/music/queries/aggregations/ArtistCountsByYearFormedAndHomeCountry.graphql new file mode 100644 index 00000000..23cf6bc6 --- /dev/null +++ b/config/site/examples/music/queries/aggregations/ArtistCountsByYearFormedAndHomeCountry.graphql @@ -0,0 +1,14 @@ +query ArtistCountsByYearFormedAndHomeCountry { + artistAggregations { + nodes { + groupedBy { + bio { + yearFormed + homeCountry + } + } + + count + } + } +} diff --git a/config/site/examples/music/queries/aggregations/BluegrassArtistAggregations.graphql b/config/site/examples/music/queries/aggregations/BluegrassArtistAggregations.graphql new file mode 100644 index 00000000..f2ad0a15 --- /dev/null +++ b/config/site/examples/music/queries/aggregations/BluegrassArtistAggregations.graphql @@ -0,0 +1,40 @@ +query BluegrassArtistAggregations($cursor: Cursor) { + artistAggregations( + first: 10 + after: $cursor + filter: {bio: {description: {matchesQuery: {query: "bluegrass"}}}} + ) { + pageInfo { + hasNextPage + endCursor + } + + nodes { + groupedBy { + bio { yearFormed } + } + + aggregatedValues { + lifetimeSales { + approximateAvg + exactMin + exactMax + } + } + + count + + subAggregations { + albums( + first: 3 + filter: {tracks: {count: {gt: 10}}} + ) { + nodes { + countDetail { approximateValue } + } + } + } + } + } +} + diff --git a/config/site/examples/music/queries/aggregations/BluegrassArtistLifetimeSales.graphql b/config/site/examples/music/queries/aggregations/BluegrassArtistLifetimeSales.graphql new file mode 100644 index 00000000..b06dfd66 --- /dev/null +++ b/config/site/examples/music/queries/aggregations/BluegrassArtistLifetimeSales.graphql @@ -0,0 +1,23 @@ +query BluegrassArtistLifetimeSales { + artistAggregations( + filter: {bio: {description: {matchesQuery: {query: "bluegrass"}}}} + ) { + nodes { + groupedBy { + bio { yearFormed } + } + + aggregatedValues { + lifetimeSales { + exactMin + exactMax + + exactSum + approximateSum + + approximateAvg + } + } + } + } +} diff --git a/config/site/examples/music/queries/aggregations/SkaArtistHomeCountries.graphql b/config/site/examples/music/queries/aggregations/SkaArtistHomeCountries.graphql new file mode 100644 index 00000000..d52b8e49 --- /dev/null +++ b/config/site/examples/music/queries/aggregations/SkaArtistHomeCountries.graphql @@ -0,0 +1,20 @@ +query SkaArtistHomeCountries { + artistAggregations( + filter: {bio: {description: {matchesQuery: {query: "ska"}}}} + ) { + nodes { + groupedBy { + bio { yearFormed } + } + + aggregatedValues { + bio { + homeCountry { + approximateDistinctValueCount + } + } + } + } + } +} + diff --git a/config/site/examples/music/queries/aggregations/TotalAlbumSales.graphql b/config/site/examples/music/queries/aggregations/TotalAlbumSales.graphql new file mode 100644 index 00000000..5bf7abef --- /dev/null +++ b/config/site/examples/music/queries/aggregations/TotalAlbumSales.graphql @@ -0,0 +1,17 @@ +query TotalAlbumSales { + artistAggregations { + nodes { + subAggregations { + albums { + nodes { + aggregatedValues { + soldUnits { + exactSum + } + } + } + } + } + } + } +} diff --git a/config/site/examples/music/queries/aggregations/TotalAlbumSalesByArtistHomeCountry.graphql b/config/site/examples/music/queries/aggregations/TotalAlbumSalesByArtistHomeCountry.graphql new file mode 100644 index 00000000..220fac20 --- /dev/null +++ b/config/site/examples/music/queries/aggregations/TotalAlbumSalesByArtistHomeCountry.graphql @@ -0,0 +1,23 @@ +query TotalAlbumSalesByArtistHomeCountry { + artistAggregations { + nodes { + groupedBy { + bio { + homeCountry + } + } + + subAggregations { + albums { + nodes { + aggregatedValues { + soldUnits { + exactSum + } + } + } + } + } + } + } +} diff --git a/config/site/examples/music/queries/aggregations/TourAttendanceByHour.graphql b/config/site/examples/music/queries/aggregations/TourAttendanceByHour.graphql new file mode 100644 index 00000000..a2b730ad --- /dev/null +++ b/config/site/examples/music/queries/aggregations/TourAttendanceByHour.graphql @@ -0,0 +1,29 @@ +query TourAttendanceByHour { + artistAggregations { + nodes { + subAggregations { + tours { + nodes { + subAggregations { + shows { + nodes { + groupedBy { + startedAt { + asDateTime(truncationUnit: HOUR) + } + } + + aggregatedValues { + attendance { + exactSum + } + } + } + } + } + } + } + } + } + } +} diff --git a/config/site/examples/music/queries/aggregations/TourAttendanceByHourOfDay.graphql b/config/site/examples/music/queries/aggregations/TourAttendanceByHourOfDay.graphql new file mode 100644 index 00000000..f630e628 --- /dev/null +++ b/config/site/examples/music/queries/aggregations/TourAttendanceByHourOfDay.graphql @@ -0,0 +1,29 @@ +query TourAttendanceByHourOfDay { + artistAggregations { + nodes { + subAggregations { + tours { + nodes { + subAggregations { + shows { + nodes { + groupedBy { + startedAt { + asTimeOfDay(truncationUnit: HOUR) + } + } + + aggregatedValues { + attendance { + exactSum + } + } + } + } + } + } + } + } + } + } +} diff --git a/config/site/examples/music/queries/aggregations/TourAttendanceByYear.graphql b/config/site/examples/music/queries/aggregations/TourAttendanceByYear.graphql new file mode 100644 index 00000000..f4fe3488 --- /dev/null +++ b/config/site/examples/music/queries/aggregations/TourAttendanceByYear.graphql @@ -0,0 +1,32 @@ +query TourAttendanceByYear { + artistAggregations { + nodes { + subAggregations { + tours { + nodes { + subAggregations { + shows { + nodes { + groupedBy { + startedAt { + asDate( + truncationUnit: YEAR + timeZone: "America/Los_Angeles" + ) + } + } + + aggregatedValues { + attendance { + exactSum + } + } + } + } + } + } + } + } + } + } +} diff --git a/config/site/examples/music/queries/aggregations/TwentyFirstCenturyAlbumSales.graphql b/config/site/examples/music/queries/aggregations/TwentyFirstCenturyAlbumSales.graphql new file mode 100644 index 00000000..688cf165 --- /dev/null +++ b/config/site/examples/music/queries/aggregations/TwentyFirstCenturyAlbumSales.graphql @@ -0,0 +1,19 @@ +query TwentyFirstCenturyAlbumSales { + artistAggregations { + nodes { + subAggregations { + albums(filter: { + releasedOn: {gte: "2000-01-01"} + }) { + nodes { + aggregatedValues { + soldUnits { + exactSum + } + } + } + } + } + } + } +} diff --git a/config/site/examples/music/queries/basic/ListArtistAlbums.graphql b/config/site/examples/music/queries/basic/ListArtistAlbums.graphql new file mode 100644 index 00000000..5c7ddafc --- /dev/null +++ b/config/site/examples/music/queries/basic/ListArtistAlbums.graphql @@ -0,0 +1,15 @@ +query ListArtistAlbums { + artists { + nodes { + name + albums { + name + releasedOn + + tracks { + name + } + } + } + } +} diff --git a/config/site/examples/music/queries/filtering/AccordionAndViolinStrictSearch.graphql b/config/site/examples/music/queries/filtering/AccordionAndViolinStrictSearch.graphql new file mode 100644 index 00000000..60e2d534 --- /dev/null +++ b/config/site/examples/music/queries/filtering/AccordionAndViolinStrictSearch.graphql @@ -0,0 +1,20 @@ +query AccordionAndViolinStrictSearch { + artists(filter: { + bio: { + description:{ + matchesQuery: { + query: "accordion violin" + requireAllTerms: true + allowedEditsPerTerm: NONE + } + } + } + }) { + nodes { + name + bio { + description + } + } + } +} diff --git a/config/site/examples/music/queries/filtering/AccordionOrViolinSearch.graphql b/config/site/examples/music/queries/filtering/AccordionOrViolinSearch.graphql new file mode 100644 index 00000000..f0895185 --- /dev/null +++ b/config/site/examples/music/queries/filtering/AccordionOrViolinSearch.graphql @@ -0,0 +1,18 @@ +query AccordionOrViolinSearch { + artists(filter: { + bio: { + description: { + matchesQuery: { + query: "accordion violin" + } + } + } + }) { + nodes { + name + bio { + description + } + } + } +} diff --git a/config/site/examples/music/queries/filtering/AnyOfGotcha.graphql b/config/site/examples/music/queries/filtering/AnyOfGotcha.graphql new file mode 100644 index 00000000..32fe1a20 --- /dev/null +++ b/config/site/examples/music/queries/filtering/AnyOfGotcha.graphql @@ -0,0 +1,18 @@ +query AnyOfGotcha { + artists(filter: { + bio: { + anyOf: { + yearFormed: {gt: 2000} + description: {matchesQuery: {query: "accordion"}} + } + } + }) { + nodes { + name + bio { + yearFormed + description + } + } + } +} diff --git a/config/site/examples/music/queries/filtering/ArtistsWith90sAlbumAndPlatinumAlbum.graphql b/config/site/examples/music/queries/filtering/ArtistsWith90sAlbumAndPlatinumAlbum.graphql new file mode 100644 index 00000000..93ec3f91 --- /dev/null +++ b/config/site/examples/music/queries/filtering/ArtistsWith90sAlbumAndPlatinumAlbum.graphql @@ -0,0 +1,19 @@ +query ArtistsWith90sAlbumAndPlatinumAlbum { + artists(filter: { + albums: { + allOf: [ + {anySatisfy: {soldUnits: {gte: 1000000}}} + {anySatisfy: {releasedOn: {gte: "1990-01-01", lt: "2000-01-01"}}} + ] + } + }) { + nodes { + name + albums { + name + releasedOn + soldUnits + } + } + } +} diff --git a/config/site/examples/music/queries/filtering/ArtistsWithPlatinum90sAlbum.graphql b/config/site/examples/music/queries/filtering/ArtistsWithPlatinum90sAlbum.graphql new file mode 100644 index 00000000..cbcb2fa4 --- /dev/null +++ b/config/site/examples/music/queries/filtering/ArtistsWithPlatinum90sAlbum.graphql @@ -0,0 +1,19 @@ +query ArtistsWithPlatinum90sAlbum { + artists(filter: { + albums: { + anySatisfy: { + soldUnits: {gte: 1000000} + releasedOn: {gte: "1990-01-01", lt: "2000-01-01"} + } + } + }) { + nodes { + name + albums { + name + releasedOn + soldUnits + } + } + } +} diff --git a/config/site/examples/music/queries/filtering/EqualityFilter.graphql b/config/site/examples/music/queries/filtering/EqualityFilter.graphql new file mode 100644 index 00000000..494a25bb --- /dev/null +++ b/config/site/examples/music/queries/filtering/EqualityFilter.graphql @@ -0,0 +1,12 @@ +query EqualityFilter { + artists(filter: { + name: {equalToAnyOf: ["U2", "Radiohead"]} + }) { + nodes { + name + bio { + yearFormed + } + } + } +} diff --git a/config/site/examples/music/queries/filtering/EqualityFilterNull.graphql b/config/site/examples/music/queries/filtering/EqualityFilterNull.graphql new file mode 100644 index 00000000..53158793 --- /dev/null +++ b/config/site/examples/music/queries/filtering/EqualityFilterNull.graphql @@ -0,0 +1,12 @@ +query EqualityFilterNull { + artists(filter: { + name: {equalToAnyOf: [null]} + }) { + nodes { + name + bio { + yearFormed + } + } + } +} diff --git a/config/site/examples/music/queries/filtering/FindArtist.graphql b/config/site/examples/music/queries/filtering/FindArtist.graphql new file mode 100644 index 00000000..7ece1820 --- /dev/null +++ b/config/site/examples/music/queries/filtering/FindArtist.graphql @@ -0,0 +1,23 @@ +query FindArtist { + byName: artists(filter: { + name: {equalToAnyOf: ["U2"]} + }) { + nodes { + name + bio { + yearFormed + } + } + } + + byBioYearFounded: artists(filter: { + bio: {yearFormed: {gt: 2000}} + }) { + nodes { + name + bio { + yearFormed + } + } + } +} diff --git a/config/site/examples/music/queries/filtering/FindArtists.graphql b/config/site/examples/music/queries/filtering/FindArtists.graphql new file mode 100644 index 00000000..f7bf871c --- /dev/null +++ b/config/site/examples/music/queries/filtering/FindArtists.graphql @@ -0,0 +1,21 @@ +query FindArtists( + $names: [String!] = null + $yearFormed_gt: Int = null + $albumNames: [String!] = null +) { + artists(filter: { + name: {equalToAnyOf: $names} + bio: {yearFormed: {gt: $yearFormed_gt}} + albums: {anySatisfy: {name: {equalToAnyOf: $albumNames}}} + }) { + nodes { + name + bio { + yearFormed + } + albums { + name + } + } + } +} diff --git a/config/site/examples/music/queries/filtering/FindArtistsFormedIn90s.graphql b/config/site/examples/music/queries/filtering/FindArtistsFormedIn90s.graphql new file mode 100644 index 00000000..30824a80 --- /dev/null +++ b/config/site/examples/music/queries/filtering/FindArtistsFormedIn90s.graphql @@ -0,0 +1,15 @@ +query FindArtistsFormedIn90s { + artists(filter: { + bio: {yearFormed: {gte: 1990, lt: 2000}} + }) { + nodes { + name + bio { + yearFormed + } + albums { + name + } + } + } +} diff --git a/config/site/examples/music/queries/filtering/FindArtistsWithBios.graphql b/config/site/examples/music/queries/filtering/FindArtistsWithBios.graphql new file mode 100644 index 00000000..ff877f30 --- /dev/null +++ b/config/site/examples/music/queries/filtering/FindArtistsWithBios.graphql @@ -0,0 +1,12 @@ +query FindArtistsWithBios { + artists(filter: { + bio: {description: {not: {equalToAnyOf: [null]}}} + }) { + nodes { + name + bio { + description + } + } + } +} diff --git a/config/site/examples/music/queries/filtering/FindEarlyAfternoonShows.graphql b/config/site/examples/music/queries/filtering/FindEarlyAfternoonShows.graphql new file mode 100644 index 00000000..78a18bf0 --- /dev/null +++ b/config/site/examples/music/queries/filtering/FindEarlyAfternoonShows.graphql @@ -0,0 +1,26 @@ +query FindEarlyAfternoonShows { + artists(filter: { + tours: {anySatisfy: {shows: {anySatisfy: { + startedAt: { + timeOfDay: { + timeZone: "America/Los_Angeles" + gte: "12:00:00" + lt: "15:00:00" + } + } + }}}} + }) { + nodes { + name + + tours { + shows { + venue { + id + } + startedAt + } + } + } + } +} diff --git a/config/site/examples/music/queries/filtering/FindProlificArtists.graphql b/config/site/examples/music/queries/filtering/FindProlificArtists.graphql new file mode 100644 index 00000000..2f016774 --- /dev/null +++ b/config/site/examples/music/queries/filtering/FindProlificArtists.graphql @@ -0,0 +1,12 @@ +query FindProlificArtists { + artists(filter: { + albums: {count: {gte: 20}} + }) { + nodes { + name + albums { + name + } + } + } +} diff --git a/config/site/examples/music/queries/filtering/FindRecentAccordionArtists.graphql b/config/site/examples/music/queries/filtering/FindRecentAccordionArtists.graphql new file mode 100644 index 00000000..af6dd043 --- /dev/null +++ b/config/site/examples/music/queries/filtering/FindRecentAccordionArtists.graphql @@ -0,0 +1,16 @@ +query FindRecentAccordionArtists { + artists(filter: { + bio: { + yearFormed: {gt: 2000} + description: {matchesQuery: {query: "accordion"}} + } + }) { + nodes { + name + bio { + yearFormed + description + } + } + } +} diff --git a/config/site/examples/music/queries/filtering/FindRecentOrAccordionArtists.graphql b/config/site/examples/music/queries/filtering/FindRecentOrAccordionArtists.graphql new file mode 100644 index 00000000..ea952e2e --- /dev/null +++ b/config/site/examples/music/queries/filtering/FindRecentOrAccordionArtists.graphql @@ -0,0 +1,18 @@ +query FindRecentOrAccordionArtists { + artists(filter: { + bio: { + anyOf: [ + {yearFormed: {gt: 2000}} + {description: {matchesQuery: {query: "accordion"}}} + ] + } + }) { + nodes { + name + bio { + yearFormed + description + } + } + } +} diff --git a/config/site/examples/music/queries/filtering/FindSeattleVenues.graphql b/config/site/examples/music/queries/filtering/FindSeattleVenues.graphql new file mode 100644 index 00000000..09868b73 --- /dev/null +++ b/config/site/examples/music/queries/filtering/FindSeattleVenues.graphql @@ -0,0 +1,15 @@ +query FindSeattleVenues { + venues(filter: { + location: {near: { + latitude: 47.621 + longitude: -122.349 + maxDistance: 10 + unit: MILE + }} + }) { + nodes { + name + capacity + } + } +} diff --git a/config/site/examples/music/queries/filtering/IgnoredFilters.graphql b/config/site/examples/music/queries/filtering/IgnoredFilters.graphql new file mode 100644 index 00000000..4e94097c --- /dev/null +++ b/config/site/examples/music/queries/filtering/IgnoredFilters.graphql @@ -0,0 +1,13 @@ +query IgnoredFilters { + artists(filter: { + name: {equalToAnyOf: null} + bio: {yearFormed: {}} + }) { + nodes { + name + bio { + yearFormed + } + } + } +} diff --git a/config/site/examples/music/queries/filtering/PhraseSearch.graphql b/config/site/examples/music/queries/filtering/PhraseSearch.graphql new file mode 100644 index 00000000..29a8bd61 --- /dev/null +++ b/config/site/examples/music/queries/filtering/PhraseSearch.graphql @@ -0,0 +1,18 @@ +query PhraseSearch { + artists(filter: { + bio: { + description:{ + matchesPhrase: { + phrase: "Ed Sullivan Show" + } + } + } + }) { + nodes { + name + bio { + description + } + } + } +} diff --git a/config/site/examples/music/queries/pagination/Count21stCenturyArtists.graphql b/config/site/examples/music/queries/pagination/Count21stCenturyArtists.graphql new file mode 100644 index 00000000..b376f1e0 --- /dev/null +++ b/config/site/examples/music/queries/pagination/Count21stCenturyArtists.graphql @@ -0,0 +1,7 @@ +query Count21stCenturyArtists { + artists(filter: { + bio: {yearFormed: {gte: 2000}} + }) { + totalEdgeCount + } +} diff --git a/config/site/examples/music/queries/pagination/PaginationExample.graphql b/config/site/examples/music/queries/pagination/PaginationExample.graphql new file mode 100644 index 00000000..c84cce1b --- /dev/null +++ b/config/site/examples/music/queries/pagination/PaginationExample.graphql @@ -0,0 +1,16 @@ +query PaginationExample($cursor: Cursor) { + artists(first: 10, after: $cursor) { + pageInfo { + hasNextPage + endCursor + } + edges { + node { + name + albums { + name + } + } + } + } +} diff --git a/config/site/examples/music/queries/pagination/PaginationNodes.graphql b/config/site/examples/music/queries/pagination/PaginationNodes.graphql new file mode 100644 index 00000000..0079451d --- /dev/null +++ b/config/site/examples/music/queries/pagination/PaginationNodes.graphql @@ -0,0 +1,14 @@ +query PaginationNodes($cursor: Cursor) { + artists(first: 10, after: $cursor) { + pageInfo { + hasNextPage + endCursor + } + nodes { + name + albums { + name + } + } + } +} diff --git a/config/site/examples/music/queries/sorting/ListArtists.graphql b/config/site/examples/music/queries/sorting/ListArtists.graphql new file mode 100644 index 00000000..fc7257ac --- /dev/null +++ b/config/site/examples/music/queries/sorting/ListArtists.graphql @@ -0,0 +1,10 @@ +query ListArtists { + artists(orderBy: [name_ASC, bio_yearFormed_DESC]) { + nodes { + name + albums { + name + } + } + } +} diff --git a/config/site/examples/music/schema.rb b/config/site/examples/music/schema.rb new file mode 100644 index 00000000..0ac5da1c --- /dev/null +++ b/config/site/examples/music/schema.rb @@ -0,0 +1,74 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +ElasticGraph.define_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Artist" do |t| + t.field "id", "ID" + t.field "name", "String" + t.field "lifetimeSales", "Int" + t.field "bio", "ArtistBio" + + t.field "albums", "[Album!]!" do |f| + f.mapping type: "nested" + end + + t.field "tours", "[Tour!]!" do |f| + f.mapping type: "nested" + end + + t.index "artists" + end + + schema.object_type "ArtistBio" do |t| + t.field "yearFormed", "Int" + t.field "homeCountry", "String" + t.field "description", "String" do |f| + f.mapping type: "text" + end + end + + schema.object_type "Album" do |t| + t.field "name", "String" + t.field "releasedOn", "Date" + t.field "soldUnits", "Int" + t.field "tracks", "[AlbumTrack!]!" do |f| + f.mapping type: "nested" + end + end + + schema.object_type "AlbumTrack" do |t| + t.field "name", "String" + t.field "trackNumber", "Int" + t.field "lengthInSeconds", "Int" + end + + schema.object_type "Tour" do |t| + t.field "name", "String" + t.field "shows", "[Show!]!" do |f| + f.mapping type: "nested" + end + end + + schema.object_type "Show" do |t| + t.relates_to_one "venue", "Venue", via: "venueId", dir: :out + t.field "attendance", "Int" + t.field "startedAt", "DateTime" + end + + schema.object_type "Venue" do |t| + t.field "id", "ID" + t.field "name", "String" + t.field "location", "GeoLocation" + t.field "capacity", "Int" + t.relates_to_many "featuredArtists", "Artist", via: "tours.shows.venueId", dir: :in, singular: "featuredArtist" + + t.index "venues" + end +end diff --git a/config/site/package.json b/config/site/package.json new file mode 100644 index 00000000..d810a15f --- /dev/null +++ b/config/site/package.json @@ -0,0 +1,11 @@ +{ + "devDependencies": { + "@tailwindcss/typography": "0.5.14", + "tailwindcss": "3.4.10", + "@mermaid-js/mermaid-cli": "11.2.1" + }, + "scripts": { + "build:css": "npx tailwindcss -i ./src/assets/css/tailwind.css -o ./src/assets/css/main.css --minify", + "format:md": "npx prettier --write \"src/**/*.md\"" + } +} diff --git a/config/site/src/_config.yaml b/config/site/src/_config.yaml new file mode 100644 index 00000000..7b557527 --- /dev/null +++ b/config/site/src/_config.yaml @@ -0,0 +1,14 @@ +markdown: kramdown +kramdown: + input: GFM # Enables GitHub-Flavored Markdown + +# Site-wide variables available to all pages via the `site` variable, ex: `{{ site.github_url }}` +github_url: https://github.com/block/elasticgraph +x_url: https://x.com/elasticgraph +x_username: "@ElasticGraph" +support_email: elasticgraph@squareup.com + +# Reusable CSS styling for the site, available via the `site.style` variable, ex: `{{ site.style.body }}` +style: + body: bg-gray-50 dark:bg-gray-900 text-gray-900 dark:text-gray-100 flex flex-col h-screen + link: text-blue-600 dark:text-blue-400 hover:underline diff --git a/config/site/src/_includes/cta.html b/config/site/src/_includes/cta.html new file mode 100644 index 00000000..8335ec0c --- /dev/null +++ b/config/site/src/_includes/cta.html @@ -0,0 +1,22 @@ + +
+
+

+ Try ElasticGraph Now +

+

+ Get started by installing ElasticGraph with the following command: +

+
+ + gem install elasticgraph + + +
+
+
diff --git a/config/site/src/_includes/filtering_predicate_definitions/comparison.md b/config/site/src/_includes/filtering_predicate_definitions/comparison.md new file mode 100644 index 00000000..c45b28d7 --- /dev/null +++ b/config/site/src/_includes/filtering_predicate_definitions/comparison.md @@ -0,0 +1,19 @@ +[`gt`]({% link query-api/filtering/comparison.md %}) +: Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + +[`gte`]({% link query-api/filtering/comparison.md %}) +: Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + +[`lt`]({% link query-api/filtering/comparison.md %}) +: Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + +[`lte`]({% link query-api/filtering/comparison.md %}) +: Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. diff --git a/config/site/src/_includes/filtering_predicate_definitions/conjunctions.md b/config/site/src/_includes/filtering_predicate_definitions/conjunctions.md new file mode 100644 index 00000000..cbf8a051 --- /dev/null +++ b/config/site/src/_includes/filtering_predicate_definitions/conjunctions.md @@ -0,0 +1,16 @@ +[`allOf`]({% link query-api/filtering/conjunctions.md %}) +: Matches records where all of the provided sub-filters evaluate to true. + This works just like an `AND` operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple + filters that can't be provided on a single filter input because of collisions between key names. + For example, if you want to provide multiple `anySatisfy: ...` filters, you could do `allOf: [{anySatisfy: ...}, {anySatisfy: ...}]`. + + Will be ignored when `null` or an empty list is passed. + +[`anyOf`]({% link query-api/filtering/conjunctions.md %}) +: Matches records where any of the provided sub-filters evaluate to true. + This works just like an `OR` operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will + cause this part of the filter to match no documents. diff --git a/config/site/src/_includes/filtering_predicate_definitions/equality.md b/config/site/src/_includes/filtering_predicate_definitions/equality.md new file mode 100644 index 00000000..b515f1b5 --- /dev/null +++ b/config/site/src/_includes/filtering_predicate_definitions/equality.md @@ -0,0 +1,7 @@ +[`equalToAnyOf`]({% link query-api/filtering/equality.md %}) +: Matches records where the field value is equal to any of the provided values. + This works just like an `IN` operator in SQL. + + Will be ignored when `null` is passed. + When an empty list is passed, will cause this part of the filter to match no documents. + When `null` is passed in the list, will match records where the field value is `null`. diff --git a/config/site/src/_includes/filtering_predicate_definitions/fulltext.md b/config/site/src/_includes/filtering_predicate_definitions/fulltext.md new file mode 100644 index 00000000..13acce90 --- /dev/null +++ b/config/site/src/_includes/filtering_predicate_definitions/fulltext.md @@ -0,0 +1,13 @@ +[`matchesPhrase`]({% link query-api/filtering/full-text-search.md %}) +: Matches records where the field value has a phrase matching the provided phrase using + full text search. This is stricter than `matchesQuery`: all terms must match + and be in the same order as the provided phrase. + + Will be ignored when `null` is passed. + +[`matchesQuery`]({% link query-api/filtering/full-text-search.md %}) +: Matches records where the field value matches the provided query using full text search. + This is more lenient than `matchesPhrase`: the order of terms is ignored, and, by default, + only one search term is required to be in the field value. + + Will be ignored when `null` is passed. diff --git a/config/site/src/_includes/filtering_predicate_definitions/list.md b/config/site/src/_includes/filtering_predicate_definitions/list.md new file mode 100644 index 00000000..65a8d1fa --- /dev/null +++ b/config/site/src/_includes/filtering_predicate_definitions/list.md @@ -0,0 +1,9 @@ +[`anySatisfy`]({% link query-api/filtering/list.md %}) +: Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + +[`count`]({% link query-api/filtering/list.md %}) +: Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. diff --git a/config/site/src/_includes/filtering_predicate_definitions/near.md b/config/site/src/_includes/filtering_predicate_definitions/near.md new file mode 100644 index 00000000..3e3922a6 --- /dev/null +++ b/config/site/src/_includes/filtering_predicate_definitions/near.md @@ -0,0 +1,5 @@ +[`near`]({% link query-api/filtering/geographic.md %}) +: Matches records where the field's geographic location is within a specified distance + from the location identified by `latitude` and `longitude`. + + Will be ignored when `null` or an empty object is passed. diff --git a/config/site/src/_includes/filtering_predicate_definitions/not.md b/config/site/src/_includes/filtering_predicate_definitions/not.md new file mode 100644 index 00000000..9d674527 --- /dev/null +++ b/config/site/src/_includes/filtering_predicate_definitions/not.md @@ -0,0 +1,5 @@ +[`not`]({% link query-api/filtering/negation.md %}) +: Matches records where the provided sub-filter evaluates to false. + This works just like a `NOT` operator in SQL. + + Will be ignored when `null` or an empty object is passed. diff --git a/config/site/src/_includes/filtering_predicate_definitions/time_of_day.md b/config/site/src/_includes/filtering_predicate_definitions/time_of_day.md new file mode 100644 index 00000000..33ab4cf2 --- /dev/null +++ b/config/site/src/_includes/filtering_predicate_definitions/time_of_day.md @@ -0,0 +1,4 @@ +[`timeOfDay`]({% link query-api/filtering/date-time.md %}) +: Matches records based on the time-of-day of the DateTime values. + + Will be ignored when `null` of an empty object is passed. diff --git a/config/site/src/_includes/footer.html b/config/site/src/_includes/footer.html new file mode 100644 index 00000000..565b1a41 --- /dev/null +++ b/config/site/src/_includes/footer.html @@ -0,0 +1,15 @@ + + diff --git a/config/site/src/_includes/navbar.html b/config/site/src/_includes/navbar.html new file mode 100644 index 00000000..5005c9e1 --- /dev/null +++ b/config/site/src/_includes/navbar.html @@ -0,0 +1,51 @@ + + diff --git a/config/site/src/_includes/subpages.html b/config/site/src/_includes/subpages.html new file mode 100644 index 00000000..73a04fca --- /dev/null +++ b/config/site/src/_includes/subpages.html @@ -0,0 +1,17 @@ +{% assign needed_subpage_part_count = page.url | split: '/' | size | plus: 1 %} +
+

Subpages

+ +
diff --git a/config/site/src/_includes/svg/document-duplicate.svg b/config/site/src/_includes/svg/document-duplicate.svg new file mode 100644 index 00000000..cab6814e --- /dev/null +++ b/config/site/src/_includes/svg/document-duplicate.svg @@ -0,0 +1,3 @@ + + + diff --git a/config/site/src/_includes/testimonials.html b/config/site/src/_includes/testimonials.html new file mode 100644 index 00000000..266f25c1 --- /dev/null +++ b/config/site/src/_includes/testimonials.html @@ -0,0 +1,35 @@ + +
+
+

+ What People Are Saying +

+
+
+

+ "ElasticGraph has revolutionized the way we manage and analyze our + data." +

+
+ - John Doe, CTO at TechCorp +
+
+
+

+ "A must-have tool for any data-driven organization." +

+
+ - Jane Smith, Data Scientist +
+
+
+

+ "ElasticGraph makes complex data operations simple and efficient." +

+
+ - Gandalf Skywalker, Software Engineer +
+
+
+
+
diff --git a/config/site/src/_layouts/default.html b/config/site/src/_layouts/default.html new file mode 100644 index 00000000..33d107c4 --- /dev/null +++ b/config/site/src/_layouts/default.html @@ -0,0 +1,20 @@ + + + + + + + {{ page.title }} + + + + + + {% include navbar.html %} + + {{ content }} + + {% include footer.html %} + + + diff --git a/config/site/src/_layouts/markdown.html b/config/site/src/_layouts/markdown.html new file mode 100644 index 00000000..93f77b68 --- /dev/null +++ b/config/site/src/_layouts/markdown.html @@ -0,0 +1,28 @@ + + + + + + + {{ page.title }} + + + + + + {% include navbar.html %} + +
+
+
+

{{ page.title }}

+ + {{ content }} +
+
+
+ + {% include footer.html %} + + + diff --git a/config/site/src/about.md b/config/site/src/about.md new file mode 100644 index 00000000..28438094 --- /dev/null +++ b/config/site/src/about.md @@ -0,0 +1,31 @@ +--- +layout: markdown +title: What is ElasticGraph? +permalink: /about/ +--- + +ElasticGraph is a general purpose, near real-time data query and search platform that is scalable and performant, serves rich interactive queries, and dramatically simplifies the creation of complex reports. The platform combines the power of indexing and search of Elasticsearch or OpenSearch with the query flexibility of GraphQL language. Optimized for AWS cloud, it also offers scale and reliability. + +ElasticGraph is a naturally flexible framework with many different possible applications. However, the main motivation we have for building it is to power various data APIs, UIs and reports. These modern reports require filtering and aggregations across a body of ever growing data sets. Modern APIs allow us to: + +- Minimize network trips to retrieve your data +- Get exactly what you want in a single query. No over- or under-serving the data. +- Push filtering complex calculations to the backend. + +## What can I do with it? + +The ElasticGraph platform will allow you to query your data in many different configurations. To do so requires defining a schema which ElasticGraph will use to both index data and also query it. Besides all basic GraphQL query features, ElasticGraph also supports: + +- Real-time indexing and data querying +- Filtering, sorting, pagination, grouping, aggregations, and sub-aggregations +- Navigating across data sets in a single query +- Robust, safe schema evolution support via Query Registry mechanism +- Derived indexes to power commonly accessed queries and aggregations +- Client and Publisher libraries + +## Get started + + +```shell +gem install elasticgraph +``` diff --git a/config/site/src/assets/css/tailwind.css b/config/site/src/assets/css/tailwind.css new file mode 100644 index 00000000..b5c61c95 --- /dev/null +++ b/config/site/src/assets/css/tailwind.css @@ -0,0 +1,3 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; diff --git a/config/site/src/index.html b/config/site/src/index.html new file mode 100644 index 00000000..c7bb66c4 --- /dev/null +++ b/config/site/src/index.html @@ -0,0 +1,25 @@ +--- +layout: default +title: ElasticGraph +--- + + +
+
+

+ ElasticGraph +

+

+ A general-purpose framework and platform for indexing, searching, + grouping, and aggregating data. +

+ + Documentation + +
+
+ +{% include cta.html %} +{% include testimonials.html %} + diff --git a/config/site/src/query-api.md b/config/site/src/query-api.md new file mode 100644 index 00000000..c7f1f667 --- /dev/null +++ b/config/site/src/query-api.md @@ -0,0 +1,19 @@ +--- +layout: markdown +title: ElasticGraph Query API +permalink: /query-api/ +--- + +ElasticGraph provides an extremely flexible GraphQL query API. As with every GraphQL API, you request the fields you want: + +{% highlight graphql %} +{{ site.data.music_queries.basic.ListArtistAlbums }} +{% endhighlight %} + +If you're just getting started with GraphQL, we recommend you review the [graphql.org learning materials](https://graphql.org/learn/queries/). + +ElasticGraph offers a number of query features that go far beyond a traditional GraphQL +API. Each of these features is implemented directly by the ElasticGraph framework, ensuring +consistent, predictable behavior across your entire schema. + +{% include subpages.html %} diff --git a/config/site/src/query-api/aggregations.md b/config/site/src/query-api/aggregations.md new file mode 100644 index 00000000..73bf2009 --- /dev/null +++ b/config/site/src/query-api/aggregations.md @@ -0,0 +1,26 @@ +--- +layout: markdown +title: "ElasticGraph Query API: Aggregations" +permalink: /query-api/aggregations/ +subpage_title: "Aggregations" +--- + +ElasticGraph offers a powerful aggregations API. Each indexed type gets a corresponding `*Aggregations` field. +Here's a complete example: + +{% highlight graphql %} +{{ site.data.music_queries.aggregations.BluegrassArtistAggregations }} +{% endhighlight %} + +Aggregation fields support [filtering]({% link query-api/filtering.md %}) and [pagination]({% link query-api/pagination.md %}) +but do _not_ support client-specified [sorting]({% link query-api/sorting.md %})[^1]. Under an aggregations field, each node +represents a grouping of documents. When [`groupedBy` fields]({% link query-api/aggregations/grouping.md %}) have been requested, +each node represents the grouping of documents that have the `groupedBy` values. When no `groupedBy` fields have been requested, +a single node will be returned containing a grouping for all documents matched by the filter. + +Aggregation nodes in turn offer 4 different aggregation features. + +{% include subpages.html %} + +[^1]: Aggregation sorting is implicitly controlled by the [groupings]({% link query-api/aggregations/grouping.md %})--the + nodes will sort ascending by each of the grouping fields. diff --git a/config/site/src/query-api/aggregations/aggregated-values.md b/config/site/src/query-api/aggregations/aggregated-values.md new file mode 100644 index 00000000..a4cffe46 --- /dev/null +++ b/config/site/src/query-api/aggregations/aggregated-values.md @@ -0,0 +1,41 @@ +--- +layout: markdown +title: "ElasticGraph Query API: Aggregated Values" +permalink: /query-api/aggregations/aggregated-values/ +subpage_title: "Aggregated Values" +--- + +Aggregated values can be computed from all values of a particular field from all documents backing an aggregation node. +Here's an example: + +{% highlight graphql %} +{{ site.data.music_queries.aggregations.BluegrassArtistLifetimeSales }} +{% endhighlight %} + +This example query aggregates the values of the `Artist.lifetimeSales` field using all 4 of the standard numeric +aggregated values: `min`, `max`, `avg`, and `sum`. These are qualified with `approximate` or `exact` to indicate +the level of precision they offer. The documentation for `approximateSum` and `exactSum` provide more detail: + +`approximateSum` +: The (approximate) sum of the field values within this grouping. + + Sums of large `Int` values can result in overflow, where the exact sum cannot + fit in a `JsonSafeLong` return value. This field, as a double-precision `Float`, can + represent larger sums, but the value may only be approximate. + +`exactSum` +: The exact sum of the field values within this grouping, if it fits in a `JsonSafeLong`. + + Sums of large `Int` values can result in overflow, where the exact sum cannot + fit in a `JsonSafeLong`. In that case, `null` will be returned, and `approximateSum` + can be used to get an approximate value. + +Besides these standard numeric aggregated values, ElasticGraph offers one more: + +{% highlight graphql %} +{{ site.data.music_queries.aggregations.SkaArtistHomeCountries }} +{% endhighlight %} + +The `approximateDistinctValueCount` field uses the [HyperLogLog++ algorithm](https://research.google.com/pubs/archive/40671.pdf) +to provide an approximate count of distinct values for the field. In this case, it can give us an idea of how many countries ska +bands were formed in, in each year. diff --git a/config/site/src/query-api/aggregations/counts.md b/config/site/src/query-api/aggregations/counts.md new file mode 100644 index 00000000..0798064d --- /dev/null +++ b/config/site/src/query-api/aggregations/counts.md @@ -0,0 +1,15 @@ +--- +layout: markdown +title: "ElasticGraph Query API: Aggregation Counts" +permalink: /query-api/aggregations/counts/ +subpage_title: "Counts" +--- + +The aggregations API allows you to count documents within a grouping: + +{% highlight graphql %} +{{ site.data.music_queries.aggregations.ArtistCountsByCountry }} +{% endhighlight %} + +This query, for example, returns a grouping for each country, and provides a count of how many artists +call each country home. diff --git a/config/site/src/query-api/aggregations/grouping.md b/config/site/src/query-api/aggregations/grouping.md new file mode 100644 index 00000000..8fc4eaae --- /dev/null +++ b/config/site/src/query-api/aggregations/grouping.md @@ -0,0 +1,64 @@ +--- +layout: markdown +title: "ElasticGraph Query API: Aggregation Grouping" +permalink: /query-api/aggregations/grouping/ +subpage_title: "Grouping" +--- + +When aggregating documents, the groupings are defined by `groupedBy`. Here's an example: + +{% highlight graphql %} +{{ site.data.music_queries.aggregations.ArtistCountsByYearFormedAndHomeCountry }} +{% endhighlight %} + +In this case, we're grouping by multiple fields; a grouping will be returned for each +combination of `Artist.bio.yearFormed` and `Artist.bio.homeCountry` found in the data. + +### Date Grouping + +In the example above, the grouping was performed on the raw values of the `groupedBy` fields. +However, for `Date` fields it's generally more useful to group by _truncated_ values. +Here's an example: + +{% highlight graphql %} +{{ site.data.music_queries.aggregations.AlbumSalesByReleaseYear }} +{% endhighlight %} + +In this case, we're truncating the `Album.releaseOn` dates to the year to give us one grouping per +year rather than one grouping per distinct date. The `truncationUnit` argument supports `DAY`, `MONTH`, +`QUARTER`, `WEEK` and `YEAR` values. In addition, an `offset` argument is supported, which can be used +to shift what grouping a `Date` falls into. This is particularly useful when using `WEEK`: + +{% highlight graphql %} +{{ site.data.music_queries.aggregations.AlbumSalesByReleaseWeek }} +{% endhighlight %} + +With no offset, grouped weeks run Monday to Sunday, but we can shift it using `offset`. In this case, the weeks have been +shifted to run Sunday to Saturday. + +Finally, we can also group `Date` fields by what day of week they fall into using `asDayOfWeek` instead of `asDate`: + +{% highlight graphql %} +{{ site.data.music_queries.aggregations.AlbumSalesByReleaseDayOfWeek }} +{% endhighlight %} + +### DateTime Grouping + +`DateTime` fields offer a similar grouping API. `asDate` and `asDayOfWeek` work the same, but they accept an optional `timeZone` +argument (default is "UTC"): + +{% highlight graphql %} +{{ site.data.music_queries.aggregations.TourAttendanceByYear }} +{% endhighlight %} + +Sub-day granualarities (`HOUR`, `MINUTE`, `SECOND`) are supported when you use `asDateTime` instead of `asDate`: + +{% highlight graphql %} +{{ site.data.music_queries.aggregations.TourAttendanceByHour }} +{% endhighlight %} + +Finally, you can group by the time of day (while ignoring the date) by using `asTimeOfDay`: + +{% highlight graphql %} +{{ site.data.music_queries.aggregations.TourAttendanceByHourOfDay }} +{% endhighlight %} diff --git a/config/site/src/query-api/aggregations/sub-aggregations.md b/config/site/src/query-api/aggregations/sub-aggregations.md new file mode 100644 index 00000000..aaf3fb40 --- /dev/null +++ b/config/site/src/query-api/aggregations/sub-aggregations.md @@ -0,0 +1,87 @@ +--- +layout: markdown +title: "ElasticGraph Query API: Sub-Aggregations" +permalink: /query-api/aggregations/sub-aggregations/ +subpage_title: "Sub-Aggregations" +--- + +The example schema used throughout this guide has a number of lists-of-object fields nested +within the overall `Artist` type: + +* `Artist.albums` + * `Artist.albums[].tracks` +* `Artist.tours` + * `Artist.tours[].shows` + +ElasticGraph supports aggregations on these nested fields via `subAggregations`. This can be used +to aggregation directly on the data of one of these fields. For example, this query returns the +total sales for all albums of all artists: + +{% highlight graphql %} +{{ site.data.music_queries.aggregations.TotalAlbumSales }} +{% endhighlight %} + +Sub-aggregations can also be performed under the groupings of an outer aggregations. For example, +this query returns the total album sales grouped by the home country of the artist: + +{% highlight graphql %} +{{ site.data.music_queries.aggregations.TotalAlbumSalesByArtistHomeCountry }} +{% endhighlight %} + +Sub-aggregation nodes offer the standard set of aggregation operations: + +* [Aggregated Values]({% link query-api/aggregations/aggregated-values.md %}) +* [Counts]({% link query-api/aggregations/counts.md %}) +* [Grouping]({% link query-api/aggregations/grouping.md %}) +* Sub-aggregations + +### Filtering Sub-Aggregations + +The data included in a sub-aggregation can be filtered. For example, this query gets the total +sales of all albums released in the 21st century: + +{% highlight graphql %} +{{ site.data.music_queries.aggregations.TwentyFirstCenturyAlbumSales }} +{% endhighlight %} + +### Sub-Aggregation Limitations + +Sub-aggregation pagination support is limited. You can use `first` to request how many +nodes are returned, but there is no `pageInfo` and you cannot request the next page of data: + +{% highlight graphql %} +{{ site.data.music_queries.aggregations.AlbumSalesByReleaseMonth }} +{% endhighlight %} + +Sub-aggregation counts are approximate. Instead of `count`, ElasticGraph offers `countDetail` +with multiple subfields: + +{% highlight graphql %} +{{ site.data.music_queries.aggregations.AlbumCount }} +{% endhighlight %} + +`approximateValue` +: The (approximate) count of documents in this aggregation bucket. + + When documents in an aggregation bucket are sourced from multiple shards, the count may be only + approximate. The `upperBound` indicates the maximum value of the true count, but usually + the true count is much closer to this approximate value (which also provides a lower bound on the + true count). + + When this approximation is known to be exact, the same value will be available from `exactValue` + and `upperBound`. + +`exactValue` +: The exact count of documents in this aggregation bucket, if an exact value can be determined. + + When documents in an aggregation bucket are sourced from multiple shards, it may not be possible to + efficiently determine an exact value. When no exact value can be determined, this field will be `null`. + The `approximateValue` field--which will never be `null`--can be used to get an approximation + for the count. + +`upperBound` +: An upper bound on how large the true count of documents in this aggregation bucket could be. + + When documents in an aggregation bucket are sourced from multiple shards, it may not be possible to + efficiently determine an exact value. The `approximateValue` field provides an approximation, + and this field puts an upper bound on the true count. diff --git a/config/site/src/query-api/filtering.md b/config/site/src/query-api/filtering.md new file mode 100644 index 00000000..1fcdf640 --- /dev/null +++ b/config/site/src/query-api/filtering.md @@ -0,0 +1,36 @@ +--- +layout: markdown +title: "ElasticGraph Query API: Filtering" +permalink: /query-api/filtering/ +subpage_title: "Filtering" +--- + +Use `filter:` on a root query field to narrow down the returned results: + +{% highlight graphql %} +{{ site.data.music_queries.filtering.FindArtist }} +{% endhighlight %} + +As shown here, filters have two basic parts: + +* A _field path_: this specifies which field you want to filter on. When dealing with a nested field (e.g. `bio.yearFormed`), + you'll need to provide a nested object matching the field structure. +* A _filtering predicate_: this specifies a filtering operator to apply at the field path. + +### Ignored Filters + +Filters with a value of `null` or empty object (`{}`) are ignored. The filters in this query are all ignored: + +{% highlight graphql %} +{{ site.data.music_queries.filtering.IgnoredFilters }} +{% endhighlight %} + +This particularly comes in handy when using [query variables](https://graphql.org/learn/queries/#variables). +It allows a query to flexibly support a wide array of filters without requiring them to all be used for an +individual request. + +{% highlight graphql %} +{{ site.data.music_queries.filtering.FindArtists }} +{% endhighlight %} + +{% include subpages.html %} diff --git a/config/site/src/query-api/filtering/available-predicates.md b/config/site/src/query-api/filtering/available-predicates.md new file mode 100644 index 00000000..d5aaa383 --- /dev/null +++ b/config/site/src/query-api/filtering/available-predicates.md @@ -0,0 +1,23 @@ +--- +layout: markdown +title: "ElasticGraph Query API: Available Filter Predicates" +permalink: /query-api/filtering/available-predicates/ +subpage_title: "Available Filter Predicates" +--- + +ElasticGraph offers a variety of filtering predicates: + +{% comment %} + Note: these are ordered so that the predicates are sorted alphabetically. The file + names are not alphabetical because they are named after "categories" rather than + the predicates themselves. +{% endcomment %} + +{% include filtering_predicate_definitions/conjunctions.md %} +{% include filtering_predicate_definitions/list.md %} +{% include filtering_predicate_definitions/equality.md %} +{% include filtering_predicate_definitions/comparison.md %} +{% include filtering_predicate_definitions/fulltext.md %} +{% include filtering_predicate_definitions/near.md %} +{% include filtering_predicate_definitions/not.md %} +{% include filtering_predicate_definitions/time_of_day.md %} diff --git a/config/site/src/query-api/filtering/comparison.md b/config/site/src/query-api/filtering/comparison.md new file mode 100644 index 00000000..ee5bbcd0 --- /dev/null +++ b/config/site/src/query-api/filtering/comparison.md @@ -0,0 +1,14 @@ +--- +layout: markdown +title: "ElasticGraph Query API: Comparison Filtering" +permalink: /query-api/filtering/comparison/ +subpage_title: "Comparison Filtering" +--- + +ElasticGraph offers a standard set of comparison filter predicates: + +{% include filtering_predicate_definitions/comparison.md %} + +{% highlight graphql %} +{{ site.data.music_queries.filtering.FindArtistsFormedIn90s }} +{% endhighlight %} diff --git a/config/site/src/query-api/filtering/conjunctions.md b/config/site/src/query-api/filtering/conjunctions.md new file mode 100644 index 00000000..21d6ca4c --- /dev/null +++ b/config/site/src/query-api/filtering/conjunctions.md @@ -0,0 +1,98 @@ +--- +layout: markdown +title: "ElasticGraph Query API: Filter Conjunctions" +permalink: /query-api/filtering/conjunctions/ +subpage_title: "Conjunctions" +--- + +ElasticGraph supports two conjunction predicates: + +{% include filtering_predicate_definitions/conjunctions.md %} + +By default, multiple filters are ANDed together. For example, this query finds artists +formed after the year 2000 with "accordion" in their bio: + +{% highlight graphql %} +{{ site.data.music_queries.filtering.FindRecentAccordionArtists }} +{% endhighlight %} + +### ORing subfilters with `anyOf` + +To instead find artists formed after the year 2000 OR with "accordion" in their bio, you +can wrap the sub-filters in an `anyOf`: + +{% highlight graphql %} +{{ site.data.music_queries.filtering.FindRecentOrAccordionArtists }} +{% endhighlight %} + +`anyOf` is available at all levels of the filtering structure so that you can OR +sub-filters anywhere you like. + +### ANDing subfilters with `allOf` + +`allOf` is rarely needed since multiple filters are ANDed together by default. But it can +come in handy when you'd otherwise have a duplicate key collision on a filter input. One +case where this comes in handy is when using `anySatisfy` to [filter on a +list]({% link query-api/filtering/list.md %}). Consider this query: + +{% highlight graphql %} +{{ site.data.music_queries.filtering.ArtistsWithPlatinum90sAlbum }} +{% endhighlight %} + +This query finds artists who released an album in the 90's that sold more than million copies. +If you wanted to broaden the query to find artists with at least one 90's album and at least one +platinum-selling album--without requiring it to be the same album--you could do this: + +{% highlight graphql %} +{{ site.data.music_queries.filtering.ArtistsWith90sAlbumAndPlatinumAlbum }} +{% endhighlight %} + +GraphQL input objects don't allow duplicate keys, so +`albums: {anySatisfy: {...}, anySatisfy: {...}}` isn't supported, but `allOf` +enables this use case. + +{% comment %}TODO: figure out a way to highlight this section as a warning.{% endcomment %} +### Warning: Always Pass a List + +When using `allOf` or `anyOf`, be sure to pass the sub-filters as a list. If you instead +pass them as an object, it won't work as expected. Consider this query: + +{% highlight graphql %} +{{ site.data.music_queries.filtering.AnyOfGotcha }} +{% endhighlight %} + +While this query will return results, it doesn't behave as it appears. The GraphQL +spec mandates that list inputs [coerce non-list values into a list of one +value](https://spec.graphql.org/October2021/#sec-List.Input-Coercion). In this case, +that means that the `anyOf` expression is coerced into this: + +{% highlight graphql %} +query AnyOfGotcha { + artists(filter: { + bio: { + anyOf: [{ + yearFormed: {gt: 2000} + description: {matchesQuery: {query: "accordion"}} + }] + } + }) { + # ... + } +} +{% endhighlight %} + +Using `anyOf` with only a single sub-expression, as we have here, doesn't do anything; +the query is equivalent to: + +{% highlight graphql %} +query AnyOfGotcha { + artists(filter: { + bio: { + yearFormed: {gt: 2000} + description: {matchesQuery: {query: "accordion"}} + } + }) { + # ... + } +} +{% endhighlight %} diff --git a/config/site/src/query-api/filtering/date-time.md b/config/site/src/query-api/filtering/date-time.md new file mode 100644 index 00000000..924619d4 --- /dev/null +++ b/config/site/src/query-api/filtering/date-time.md @@ -0,0 +1,32 @@ +--- +layout: markdown +title: "ElasticGraph Query API: DateTime Filtering" +permalink: /query-api/filtering/date-time/ +subpage_title: "Date Time Filtering" +--- + +ElasticGraph supports three different date/time types: + +`Date` +: A date, represented as an [ISO 8601 date string](https://en.wikipedia.org/wiki/ISO_8601). + Example: `"2024-10-15"`. + +`DateTime` +: A timestamp, represented as an [ISO 8601 time string](https://en.wikipedia.org/wiki/ISO_8601). + Example: `"2024-10-15T07:23:15Z"`. + +`LocalTime` +: A local time such as `"23:59:33"` or `"07:20:47.454"` without a time zone or offset, + formatted based on the [partial-time portion of RFC3339](https://datatracker.ietf.org/doc/html/rfc3339#section-5.6). + +All three support the standard set of [equality]({% link query-api/filtering/equality.md %}) and +[comparison]({% link query-api/filtering/comparison.md %}) predicates. In addition, `DateTime` fields +support one more filtering operator: + +{% include filtering_predicate_definitions/time_of_day.md %} + +For example, you could use it to find shows that started between noon and 3 pm on any date: + +{% highlight graphql %} +{{ site.data.music_queries.filtering.FindEarlyAfternoonShows }} +{% endhighlight %} diff --git a/config/site/src/query-api/filtering/equality.md b/config/site/src/query-api/filtering/equality.md new file mode 100644 index 00000000..f9197f03 --- /dev/null +++ b/config/site/src/query-api/filtering/equality.md @@ -0,0 +1,22 @@ +--- +layout: markdown +title: "ElasticGraph Query API: Equality Filtering" +permalink: /query-api/filtering/equality/ +subpage_title: "Equality Filtering" +--- + +The most commonly used predicate supports equality filtering: + +{% include filtering_predicate_definitions/equality.md %} + +Here's a basic example: + +{% highlight graphql %} +{{ site.data.music_queries.filtering.EqualityFilter }} +{% endhighlight %} + +Unlike the SQL `IN` operator, you can find records with `null` values if you put `null` in the list: + +{% highlight graphql %} +{{ site.data.music_queries.filtering.EqualityFilterNull }} +{% endhighlight %} diff --git a/config/site/src/query-api/filtering/full-text-search.md b/config/site/src/query-api/filtering/full-text-search.md new file mode 100644 index 00000000..ad3ce689 --- /dev/null +++ b/config/site/src/query-api/filtering/full-text-search.md @@ -0,0 +1,41 @@ +--- +layout: markdown +title: "ElasticGraph Query API: Full Text Search" +permalink: /query-api/filtering/full-text-search/ +subpage_title: "Full Text Search" +--- + +ElasticGraph supports two full-text search filtering predicates: + +{% include filtering_predicate_definitions/fulltext.md %} + +### Matches Query + +`matchesQuery` is the more lenient of the two predicates. It's designed to match broadly. Here's an example: + +{% highlight graphql %} +{{ site.data.music_queries.filtering.AccordionOrViolinSearch }} +{% endhighlight %} + +This query will match artists with bios like: + +> Renowned for his mesmerizing performances, Luca "The Breeze" Fontana captivates audiences with his accordion, +> weaving intricate melodies that dance between the notes of traditional folk and modern jazz. + +> Sylvia Varela's avant-garde violin playing defies tradition, blending haunting dissonance with unexpected rhythms. + +Notably, the description needs `accordion` OR `violin`, but not both. In addition, it would match an artist bio that +mentioned "viola" since it supports fuzzy matching by default and "viola" is only 2 edits away from "violin". Arguments +are supported to control both aspects to make matching stricter: + +{% highlight graphql %} +{{ site.data.music_queries.filtering.AccordionAndViolinStrictSearch }} +{% endhighlight %} + +### Matches Phrase + +`matchesPhrase` is even stricter: it requires all terms _in the provided order_ (`matchesQuery` doesn't care about order). It's particularly useful when you want to search on a particular multi-word expression: + +{% highlight graphql %} +{{ site.data.music_queries.filtering.PhraseSearch }} +{% endhighlight %} diff --git a/config/site/src/query-api/filtering/geographic.md b/config/site/src/query-api/filtering/geographic.md new file mode 100644 index 00000000..18fafd2e --- /dev/null +++ b/config/site/src/query-api/filtering/geographic.md @@ -0,0 +1,14 @@ +--- +layout: markdown +title: "ElasticGraph Query API: Geographic Filtering" +permalink: /query-api/filtering/geographic/ +subpage_title: "Geographic Filtering" +--- + +The `GeoLocation` type supports a special predicate: + +{% include filtering_predicate_definitions/near.md %} + +{% highlight graphql %} +{{ site.data.music_queries.filtering.FindSeattleVenues }} +{% endhighlight %} diff --git a/config/site/src/query-api/filtering/list.md b/config/site/src/query-api/filtering/list.md new file mode 100644 index 00000000..fae3d013 --- /dev/null +++ b/config/site/src/query-api/filtering/list.md @@ -0,0 +1,34 @@ +--- +layout: markdown +title: "ElasticGraph Query API: List Filtering" +permalink: /query-api/filtering/list/ +subpage_title: "List Filtering" +--- + +ElasticGraph supports a couple predicates for filtering on list fields: + +{% include filtering_predicate_definitions/list.md %} + +### Filtering on list elements with `anySatisfy` + +When filtering on a list field, use `anySatisfy` to find records with matching list elements. +This query, for example, will find artists that released a platinum-selling album in the 1990s: + +{% highlight graphql %} +{{ site.data.music_queries.filtering.ArtistsWithPlatinum90sAlbum }} +{% endhighlight %} + +{% comment %}TODO: figure out a way to highlight this section as a warning.{% endcomment %} +One thing to bear in mind: this query is selecting which _artists_ to return, +not which _albums_ to return. You might expect that the returned `nodes.albums` would +all be platinum-selling 90s albums, but that's not how the filtering API works. Only artists +that had a platinum-selling 90s album will be returned, and for each returned artists, all +their albums will be returned--even ones that sold poorly or were released outside the 1990s. + +### Filtering on the list size with `count` + +If you'd rather filter on the _size_ of a list, use `count`: + +{% highlight graphql %} +{{ site.data.music_queries.filtering.FindProlificArtists }} +{% endhighlight %} diff --git a/config/site/src/query-api/filtering/negation.md b/config/site/src/query-api/filtering/negation.md new file mode 100644 index 00000000..c7d7e14f --- /dev/null +++ b/config/site/src/query-api/filtering/negation.md @@ -0,0 +1,22 @@ +--- +layout: markdown +title: "ElasticGraph Query API: Filter Negation" +permalink: /query-api/filtering/negation/ +subpage_title: "Negation" +--- + +ElasticGraph supports a negation predicate: + +{% include filtering_predicate_definitions/not.md %} + +One of the more common use cases is to filter to non-null values: + +{% highlight graphql %} +{{ site.data.music_queries.filtering.FindArtistsWithBios }} +{% endhighlight %} + +`not` is available at any level of a `filter`. All of these are equivalent: + +* `bio: {description: {not: {equalToAnyOf: [null]}}}` +* `bio: {not: {description: {equalToAnyOf: [null]}}}` +* `not: {bio: {description: {equalToAnyOf: [null]}}}` diff --git a/config/site/src/query-api/pagination.md b/config/site/src/query-api/pagination.md new file mode 100644 index 00000000..b8b2da9b --- /dev/null +++ b/config/site/src/query-api/pagination.md @@ -0,0 +1,37 @@ +--- +layout: markdown +title: "ElasticGraph Query API: Pagination" +permalink: /query-api/pagination/ +subpage_title: "Pagination" +--- + +To provide pagination, ElasticGraph implements the [Relay GraphQL Cursor Connections +Specification](https://relay.dev/graphql/connections.htm). Here's an example query showing +pagination in action: + +{% highlight graphql %} +{{ site.data.music_queries.pagination.PaginationExample }} +{% endhighlight %} + +In addition, ElasticGraph offers some additional features beyond the Relay spec. + +### Total Edge Count + +As an extension to the Relay spec, ElasticGraph offers a `totalEdgeCount` field alongside `edges` and `pageInfo`. +It can be used to get a total count of matching records: + +{% highlight graphql %} +{{ site.data.music_queries.pagination.Count21stCenturyArtists }} +{% endhighlight %} + +Note: `totalEdgeCount` is not available under an [aggregations]({% link query-api/aggregations.md %}) field. + +### Nodes + +As an alternative to `edges.node`, ElasticGraph offers `nodes`. This is recommended over `edges` except when you need +a per-node `cursor` (which is available under `edges`) since it removes an extra layer of nesting, providing a simpler +response structure: + +{% highlight graphql %} +{{ site.data.music_queries.pagination.PaginationNodes }} +{% endhighlight %} diff --git a/config/site/src/query-api/sorting.md b/config/site/src/query-api/sorting.md new file mode 100644 index 00000000..ab583377 --- /dev/null +++ b/config/site/src/query-api/sorting.md @@ -0,0 +1,14 @@ +--- +layout: markdown +title: "ElasticGraph Query API: Sorting" +permalink: /query-api/sorting/ +subpage_title: "Sorting" +--- + +Use `orderBy:` on a root query field to control how the results are sorted: + +{% highlight graphql %} +{{ site.data.music_queries.sorting.ListArtists }} +{% endhighlight %} + +This query, for example, would sort by `name` (ascending), with `bio.yearFormed` (descending) as a tie breaker. diff --git a/config/site/support/doctest_helper.rb b/config/site/support/doctest_helper.rb new file mode 100644 index 00000000..52848f08 --- /dev/null +++ b/config/site/support/doctest_helper.rb @@ -0,0 +1,156 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/apollo/schema_definition/api_extension" +require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" +require "elastic_graph/schema_definition/api" +require "elastic_graph/schema_definition/schema_artifact_manager" +require "tmpdir" + +module ElasticGraph + project_root = ::File.expand_path("../../..", __dir__) + + ::YARD::Doctest.configure do |doctest| + # Run each doctest in an empty temp dir. This ensures that the doc tests are not able + # to implicitly rely on the surrounding file system state, and allows doc test to interactive + # with the file system as needed without worrying about it corrupting our local file system. + doctest.before do + @original_pwd = ::Dir.pwd + @tmp_dir = ::Dir.mktmpdir + ::Dir.chdir(@tmp_dir) + end + + doctest.after do + ::Dir.chdir(@original_pwd) + ::FileUtils.rm_rf(@tmp_dir) + end + + # Many doc tests are for the schema definition API, and need to be run with a schema definition + # API instance being active. + descriptions_needing_schema_def_api_and_extension_modules = { + "ElasticGraph.define_schema" => [], + "ElasticGraph::Apollo::SchemaDefinition" => [ElasticGraph::Apollo::SchemaDefinition::APIExtension], + "ElasticGraph::SchemaDefinition" => [] + } + + descriptions_needing_schema_def_api_and_extension_modules.each do |description, extension_modules| + doctest.before(description) do + @api = SchemaDefinition::API.new( + SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: :camelCase, overrides: {}), + true, + extension_modules: extension_modules + ) + + # This is required in all schemas, but we don't want to have to put in all our examples, + # so we set it here. + @api.json_schema_version 1 + + # Store the api instance so that `ElasticGraph.define_schema` can access it. + ::Thread.current[:ElasticGraph_SchemaDefinition_API_instance] = @api + end + + doctest.after(description) do + ::Thread.current[:ElasticGraph_SchemaDefinition_API_instance] = nil + + artifacts_manager = SchemaDefinition::SchemaArtifactManager.new( + schema_definition_results: @api.results, + schema_artifacts_directory: "#{@tmp_dir}/schema_artifacts", + enforce_json_schema_version: true, + output: ::StringIO.new + ) + + # Dump the artifacts to surface any issues with the schema definition. + artifacts_manager.dump_artifacts + end + end + + doctest.before "ElasticGraph::SchemaDefinition::API#json_schema_version" do + ElasticGraph.define_schema do |schema| + # `schema.json_schema_version` raises an error when the version is set more than once. + # By default we set it above. Here we clear it to allow our example to set it. + schema.state.json_schema_version = nil + end + end + + doctest.before "ElasticGraph::SchemaDefinition::SchemaElements::ScalarType#coerce_with" do + ::FileUtils.mkdir_p "coercion_adapters" + ::File.write("coercion_adapters/phone_number.rb", <<~EOS) + module CoercionAdapters + class PhoneNumber + def self.coerce_input(value, ctx) + end + + def self.coerce_result(value, ctx) + end + end + end + EOS + end + + doctest.before "ElasticGraph::SchemaDefinition::SchemaElements::ScalarType#prepare_for_indexing_with" do + ::FileUtils.mkdir_p "indexing_preparers" + ::File.write("indexing_preparers/phone_number.rb", <<~EOS) + module IndexingPreparers + class PhoneNumber + def self.prepare_for_indexing(value) + end + end + end + EOS + end + + [ + "ElasticGraph::Apollo@Use elasticgraph-apollo in a project", + "ElasticGraph::Apollo::SchemaDefinition::APIExtension@Define local rake tasks with this extension module", + "ElasticGraph::Local::RakeTasks" + ].each do |description| + doctest.before description do + ::FileUtils.mkdir_p "config/settings" + ::FileUtils.cp "#{project_root}/config/settings/development.yaml", "config/settings/local.yaml" + end + end + + [ + "ElasticGraph::Rack::GraphiQL", + "ElasticGraph::Rack::GraphQLEndpoint" + ].each do |description| + doctest.before description do + require "elasticsearch" + ::FileUtils.ln_s "#{project_root}/config", "config" + + # These examples are the contents of a config.ru file which is evaluated in the context of a Rack::Builder + # instance. We need to define `run` to be compatible with the normal config.ru context. + def self.run(app) + end + end + end + end + + # Here we work around a bug in yard-doctest. Usually it evaluates examples in the context of the `YARD::Doctest::Example` instance. + # However, when the constant named by the example is defined, it instead evaluates the example in the context of the class itself. + # + # This creates ordering dependency bugs for us--specifically: + # + # * The example for `ElasticGraph::Rack::GraphiQL` and `ElasticGraph::Rack::GraphQLEndpoint` both depend on `run` being + # defined since they are `config.ru` examples. + # * We have a `before` hook above which defines the `run` method they need. + # * When `ElasticGraph::Rack::GraphiQL` is loaded it turns around and loads `ElasticGraph::Rack:::GraphQLEndpoint` since + # it depends on it. + # * That means that if the `GraphQLEndpoint` runs first everything works fine; but if the `GraphiQL` examples runs first, then + # when the `GraphQLEndpoint` example runs, its class is defined and it changes how yard-doctest evaluates the example. + # + # To ensure consistent, predictable evaluation, we override `evaluate` to _always_ use the binding of the example instance, + # avoiding this problem. + module YARDDoctestExampleBugFix + def evaluate(code, bind) + super(code, nil) + end + + ::YARD::Doctest::Example.prepend self + end +end diff --git a/config/site/tailwind.config.js b/config/site/tailwind.config.js new file mode 100644 index 00000000..bfcbe92a --- /dev/null +++ b/config/site/tailwind.config.js @@ -0,0 +1,9 @@ +/** @type {import('tailwindcss').Config} */ +module.exports = { + // Includes .yaml because we extract CSS classes to .yaml files like _config.yaml for reuse. + content: ["./src/**/*.html", "./src/**/*.md", "./src/**/*.svg", "./src/**/*.yaml"], + theme: { + extend: {}, + }, + plugins: [require("@tailwindcss/typography")], +}; diff --git a/config/site/yardopts b/config/site/yardopts new file mode 100644 index 00000000..a9975946 --- /dev/null +++ b/config/site/yardopts @@ -0,0 +1,9 @@ +--markup markdown +--no-private +--tag dynamic +--hide-tag dynamic +--tag implements +--hide-tag implements +--exclude bundle +--exclude spec +--exclude sig diff --git a/config/tested_datastore_versions.yaml b/config/tested_datastore_versions.yaml new file mode 120000 index 00000000..90d418e3 --- /dev/null +++ b/config/tested_datastore_versions.yaml @@ -0,0 +1 @@ +../elasticgraph-local/lib/elastic_graph/local/tested_datastore_versions.yaml \ No newline at end of file diff --git a/elasticgraph-admin/.rspec b/elasticgraph-admin/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-admin/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-admin/.yardopts b/elasticgraph-admin/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-admin/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-admin/Gemfile b/elasticgraph-admin/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-admin/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-admin/LICENSE.txt b/elasticgraph-admin/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-admin/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-admin/README.md b/elasticgraph-admin/README.md new file mode 100644 index 00000000..98720c39 --- /dev/null +++ b/elasticgraph-admin/README.md @@ -0,0 +1,3 @@ +# ElasticGraph::Admin + +Provides datastore administrative tasks for ElasticGraph. diff --git a/elasticgraph-admin/elasticgraph-admin.gemspec b/elasticgraph-admin/elasticgraph-admin.gemspec new file mode 100644 index 00000000..cc6fa5bb --- /dev/null +++ b/elasticgraph-admin/elasticgraph-admin.gemspec @@ -0,0 +1,23 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :core) do |spec, eg_version| + spec.summary = "ElasticGraph gem that provides datastore administrative tasks, to keep a datastore up-to-date with an ElasticGraph schema." + + spec.add_dependency "elasticgraph-datastore_core", eg_version + spec.add_dependency "elasticgraph-indexer", eg_version + spec.add_dependency "elasticgraph-schema_artifacts", eg_version + spec.add_dependency "elasticgraph-support", eg_version + spec.add_dependency "rake", "~> 13.2" + + spec.add_development_dependency "elasticgraph-elasticsearch", eg_version + spec.add_development_dependency "elasticgraph-opensearch", eg_version + spec.add_development_dependency "elasticgraph-schema_definition", eg_version +end diff --git a/elasticgraph-admin/lib/elastic_graph/admin.rb b/elasticgraph-admin/lib/elastic_graph/admin.rb new file mode 100644 index 00000000..c7fa33f9 --- /dev/null +++ b/elasticgraph-admin/lib/elastic_graph/admin.rb @@ -0,0 +1,97 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core" +require "elastic_graph/support/from_yaml_file" +require "time" + +module ElasticGraph + # The entry point into this library. Create an instance of this class to get access to + # the public interfaces provided by this library. + class Admin + extend Support::FromYamlFile + + # @dynamic datastore_core, schema_artifacts + attr_reader :datastore_core, :schema_artifacts + + # A factory method that builds an Admin instance from the given parsed YAML config. + # `from_yaml_file(file_name, &block)` is also available (via `Support::FromYamlFile`). + def self.from_parsed_yaml(parsed_yaml, &datastore_client_customization_block) + new(datastore_core: DatastoreCore.from_parsed_yaml(parsed_yaml, for_context: :admin, &datastore_client_customization_block)) + end + + def initialize(datastore_core:, monotonic_clock: nil, clock: ::Time) + @datastore_core = datastore_core + @monotonic_clock = monotonic_clock + @clock = clock + @schema_artifacts = @datastore_core.schema_artifacts + end + + def cluster_configurator + @cluster_configurator ||= begin + require "elastic_graph/admin/cluster_configurator" + ClusterConfigurator.new( + datastore_clients_by_name: @datastore_core.clients_by_name, + index_defs: @datastore_core.index_definitions_by_name.values, + index_configurations_by_name: schema_artifacts.indices, + index_template_configurations_by_name: schema_artifacts.index_templates, + scripts: schema_artifacts.datastore_scripts, + cluster_settings_manager: cluster_settings_manager, + clock: @clock + ) + end + end + + def cluster_settings_manager + @cluster_settings_manager ||= begin + require "elastic_graph/admin/cluster_configurator/cluster_settings_manager" + ClusterConfigurator::ClusterSettingsManager.new( + datastore_clients_by_name: @datastore_core.clients_by_name, + datastore_config: @datastore_core.config, + logger: @datastore_core.logger + ) + end + end + + def datastore_indexing_router + @datastore_indexing_router ||= begin + require "elastic_graph/indexer/datastore_indexing_router" + Indexer::DatastoreIndexingRouter.new( + datastore_clients_by_name: datastore_core.clients_by_name, + mappings_by_index_def_name: schema_artifacts.index_mappings_by_index_def_name, + monotonic_clock: monotonic_clock, + logger: datastore_core.logger + ) + end + end + + def monotonic_clock + @monotonic_clock ||= begin + require "elastic_graph/support/monotonic_clock" + Support::MonotonicClock.new + end + end + + # Returns an alternate `Admin` instance with the datastore clients replaced with + # alternate implementations that turn all write operations into no-ops. + def with_dry_run_datastore_clients + require "elastic_graph/admin/datastore_client_dry_run_decorator" + dry_run_clients_by_name = @datastore_core.clients_by_name.transform_values do |client| + DatastoreClientDryRunDecorator.new(client) + end + + Admin.new(datastore_core: DatastoreCore.new( + config: datastore_core.config, + logger: datastore_core.logger, + schema_artifacts: datastore_core.schema_artifacts, + clients_by_name: dry_run_clients_by_name, + client_customization_block: datastore_core.client_customization_block + )) + end + end +end diff --git a/elasticgraph-admin/lib/elastic_graph/admin/cluster_configurator.rb b/elasticgraph-admin/lib/elastic_graph/admin/cluster_configurator.rb new file mode 100644 index 00000000..12637d9c --- /dev/null +++ b/elasticgraph-admin/lib/elastic_graph/admin/cluster_configurator.rb @@ -0,0 +1,104 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin/cluster_configurator/script_configurator" +require "elastic_graph/admin/index_definition_configurator" +require "elastic_graph/errors" +require "stringio" + +module ElasticGraph + class Admin + # Facade responsible for overall cluster configuration. Delegates to other classes as + # necessary to configure different aspects of the cluster (such as index configuration, + # cluster settings, etc). + class ClusterConfigurator + def initialize( + datastore_clients_by_name:, + index_defs:, + index_configurations_by_name:, + index_template_configurations_by_name:, + scripts:, + cluster_settings_manager:, + clock: + ) + @datastore_clients_by_name = datastore_clients_by_name + @index_defs = index_defs + @index_configurations_by_name = index_configurations_by_name.merge(index_template_configurations_by_name) + @scripts_by_id = scripts + @cluster_settings_manager = cluster_settings_manager + @clock = clock + end + + # Attempts to configure all aspects of the datastore cluster. Known/expected failure + # cases are pre-validated so that an error can be raised before applying any changes to + # any indices, so that we hopefully don't wind up in a "partially configured" state. + def configure_cluster(output) + # Note: we do not want to cache `index_configurators_for` here in a variable, because it's important + # for our tests that different instances are used for `validate` vs `configure!`. That's the case because + # each `index_configurator` memoizes some datastore responses (e.g. when it fetches the settings or + # mappings for an index...). In our tests, we use different datastore clients that connect to the same + # datastore server, and that means that when we reuse the same `index_configurator`, the datastore + # index winds up being mutated (via another client) in between `validate` and `configure!` breaking assumptions + # of the datastore response memoization. By using different index configurators for the two steps it + # avoids some odd bugs. + script_configurators = script_configurators_for(output) + + errors = script_configurators.flat_map(&:validate) + index_definition_configurators_for(output).flat_map(&:validate) + + if errors.any? + error_descriptions = errors.map.with_index do |error, index| + "#{index + 1}): #{error}" + end.join("\n#{"=" * 80}\n\n") + + raise Errors::ClusterOperationError, "Got #{errors.size} validation error(s):\n\n#{error_descriptions}" + end + + script_configurators.each(&:configure!) + + @cluster_settings_manager.in_index_maintenance_mode(:all_clusters) do + index_definition_configurators_for(output).each(&:configure!) + end + end + + def accessible_index_definitions + @accessible_index_definitions ||= @index_defs.reject { |i| i.all_accessible_cluster_names.empty? } + end + + private + + def script_configurators_for(output) + # It's a bit tricky to know which datastore cluster a script is needed in (the script metadata + # doesn't store that), but storing a script in a cluster that doesn't need it causes no harm. The + # id of each script contains the hash of its contents so there's no possibility of different clusters + # needing a script with the same `id` to have different contents. So here we create a script configurator + # for each datastore client. + @datastore_clients_by_name.values.flat_map do |datastore_client| + @scripts_by_id.map do |id, payload| + ScriptConfigurator.new( + datastore_client: datastore_client, + script_context: payload.fetch("context"), + script_id: id, + script: payload.fetch("script"), + output: output + ) + end + end + end + + def index_definition_configurators_for(output) + @index_defs.flat_map do |index_def| + env_agnostic_config = @index_configurations_by_name.fetch(index_def.name) + + index_def.all_accessible_cluster_names.map do |cluster_name| + IndexDefinitionConfigurator.new(@datastore_clients_by_name.fetch(cluster_name), index_def, env_agnostic_config, output, @clock) + end + end + end + end + end +end diff --git a/elasticgraph-admin/lib/elastic_graph/admin/cluster_configurator/action_reporter.rb b/elasticgraph-admin/lib/elastic_graph/admin/cluster_configurator/action_reporter.rb new file mode 100644 index 00000000..8823ad35 --- /dev/null +++ b/elasticgraph-admin/lib/elastic_graph/admin/cluster_configurator/action_reporter.rb @@ -0,0 +1,23 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class Admin + class ClusterConfigurator + class ActionReporter + def initialize(output) + @output = output + end + + def report_action(message) + @output.puts "#{message.chomp}\n#{"=" * 80}\n" + end + end + end + end +end diff --git a/elasticgraph-admin/lib/elastic_graph/admin/cluster_configurator/cluster_settings_manager.rb b/elasticgraph-admin/lib/elastic_graph/admin/cluster_configurator/cluster_settings_manager.rb new file mode 100644 index 00000000..e4325678 --- /dev/null +++ b/elasticgraph-admin/lib/elastic_graph/admin/cluster_configurator/cluster_settings_manager.rb @@ -0,0 +1,99 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" + +module ElasticGraph + class Admin + class ClusterConfigurator + # Responsible for updating datastore cluster settings based on the mode EG is in, maintenance mode or indexing mode + class ClusterSettingsManager + def initialize(datastore_clients_by_name:, datastore_config:, logger:) + @datastore_clients_by_name = datastore_clients_by_name + @datastore_config = datastore_config + @logger = logger + end + + # Starts index maintenance mode, if it has not already been started. This method is idempotent. + # + # In index maintenance mode, you can safely delete or update the index configuration without + # worrying about indices being auto-created with dynamic mappings (e.g. due to an indexing + # race condition). While in this mode, indexing operations on documents that fall into new rollover + # indices may fail since the auto-creation of those indices is disabled. + # + # `cluster_spec` can be the name of a specific cluster (as a string) or `:all_clusters`. + def start_index_maintenance_mode!(cluster_spec) + cluster_names_for(cluster_spec).each do |cluster_name| + datastore_client_named(cluster_name).put_persistent_cluster_settings(desired_cluster_settings(cluster_name)) + end + end + + # Ends index maintenance mode, if it has not already ended. This method is idempotent. + # + # Outside of this mode, you cannot safely delete or update the index configuration. However, + # new rollover indices will correctly be auto-created as documents that fall in new months or + # years are indexed. + # + # `cluster_spec` can be the name of a specific cluster (as a string) or `:all_clusters`. + def end_index_maintenance_mode!(cluster_spec) + cluster_names_for(cluster_spec).each do |cluster_name| + datastore_client_named(cluster_name).put_persistent_cluster_settings( + desired_cluster_settings(cluster_name, auto_create_index_patterns: ["*#{ROLLOVER_INDEX_INFIX_MARKER}*"]) + ) + end + end + + # Runs a block in index maintenance mode. Should be used to wrap any code that updates your index configuration. + # + # `cluster_spec` can be the name of a specific cluster (as a string) or `:all_clusters`. + def in_index_maintenance_mode(cluster_spec) + start_index_maintenance_mode!(cluster_spec) + + begin + yield + rescue => e + @logger.warn "WARNING: ClusterSettingsManager#in_index_maintenance_mode is not able to exit index maintenance mode due to exception #{e}.\n A bit of manual cleanup may be required (although a re-try should be idempotent)." + raise # re-raise the same error + else + # Note: we intentionally do not end maintenance mode in an `ensure` block, because if an exception + # happens while we `yield`, we do _not_ want to exit maintenance mode. Exiting maintenance mode + # could put us in a state where indices are dynamically created when we do not want them to be. + end_index_maintenance_mode!(cluster_spec) + end + end + + private + + def desired_cluster_settings(cluster_name, auto_create_index_patterns: []) + { + # https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#index-creation + # + # We generally want to disable automatic index creation in order to require all indices to be properly + # defined and configured. However, we must allow kibana to create some indices for it to be usable + # (https://discuss.elastic.co/t/elasticsearchs-action-auto-create-index-setting-impact-on-kibana/117701). + "action.auto_create_index" => ([".kibana*"] + auto_create_index_patterns).map { |p| "+#{p}" }.join(",") + }.merge(@datastore_config.clusters.fetch(cluster_name).settings) + end + + def datastore_client_named(cluster_name) + @datastore_clients_by_name.fetch(cluster_name) do + raise Errors::ClusterOperationError, + "Unknown datastore cluster name: `#{cluster_name}`. Valid cluster names: #{@datastore_clients_by_name.keys}" + end + end + + def cluster_names_for(cluster_spec) + case cluster_spec + when :all_clusters then @datastore_clients_by_name.keys + else [cluster_spec] + end + end + end + end + end +end diff --git a/elasticgraph-admin/lib/elastic_graph/admin/cluster_configurator/script_configurator.rb b/elasticgraph-admin/lib/elastic_graph/admin/cluster_configurator/script_configurator.rb new file mode 100644 index 00000000..e860fbcc --- /dev/null +++ b/elasticgraph-admin/lib/elastic_graph/admin/cluster_configurator/script_configurator.rb @@ -0,0 +1,54 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin/cluster_configurator/action_reporter" +require "elastic_graph/errors" + +module ElasticGraph + class Admin + class ClusterConfigurator + class ScriptConfigurator + def initialize(datastore_client:, script_context:, script_id:, script:, output:) + @datastore_client = datastore_client + @script_context = script_context + @script_id = script_id + @script = script + @action_reporter = ActionReporter.new(output) + end + + def validate + case existing_datastore_script + when :not_found, @script + [] + else + [ + "#{@script_context} script #{@script_id} already exists in the datastore but has different contents. " \ + "\n\nScript in the datastore:\n#{::YAML.dump(existing_datastore_script)}" \ + "\n\nDesired script:\n#{::YAML.dump(@script)}" + ] + end + end + + def configure! + if existing_datastore_script == :not_found + @datastore_client.put_script(id: @script_id, body: {script: @script}, context: @script_context) + @action_reporter.report_action "Stored #{@script_context} script: #{@script_id}" + end + end + + private + + def existing_datastore_script + @existing_datastore_script ||= @datastore_client + .get_script(id: @script_id) + &.fetch("script") || :not_found + end + end + end + end +end diff --git a/elasticgraph-admin/lib/elastic_graph/admin/datastore_client_dry_run_decorator.rb b/elasticgraph-admin/lib/elastic_graph/admin/datastore_client_dry_run_decorator.rb new file mode 100644 index 00000000..0a494540 --- /dev/null +++ b/elasticgraph-admin/lib/elastic_graph/admin/datastore_client_dry_run_decorator.rb @@ -0,0 +1,76 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "forwardable" + +module ElasticGraph + class Admin + # Decorator that wraps a datastore client in order to implement dry run behavior. + # All write operations are implemented as no-ops, while read operations are passed through + # to the wrapped datastore client. + # + # We prefer this over having to check a `dry_run` flag in many places because that's + # easy to forget. One mistake and a dry run isn't truly a dry run! + # + # In contrast, this gives us a strong guarantee that dry run mode truly avoids mutating + # any datastore state. This decorator specifically picks and chooses which operations it + # allows. + # + # - Read operations are forwarded to the wrapped datastore client. + # - Write operations are implemented as no-ops. + # + # If/when the calling code evolves to call a new method on this, it'll trigger + # `NoMethodError`, giving us a good chance to evaluate how this decorator should + # support a particular API. This is also why this doesn't use Ruby's `delegate` library, + # because we don't want methods automatically delegated; we want to opt-in to only the read-only methods. + class DatastoreClientDryRunDecorator + extend Forwardable + + def initialize(wrapped_client) + @wrapped_client = wrapped_client + end + + # Cluster APIs + def_delegators :@wrapped_client, :get_flat_cluster_settings, :get_cluster_health + + def put_persistent_cluster_settings(*) = nil + + # Script APIs + def_delegators :@wrapped_client, :get_script + + def put_script(*) = nil + + def delete_script(*) = nil + + # Index Template APIs + def_delegators :@wrapped_client, :get_index_template + + def delete_index_template(*) = nil + + def put_index_template(*) = nil + + # Index APIs + def_delegators :@wrapped_client, :get_index, :list_indices_matching + + def delete_indices(*) = nil + + def create_index(*) = nil + + def put_index_mapping(*) = nil + + def put_index_settings(*) = nil + + # Document APIs + def_delegators :@wrapped_client, :get, :search, :msearch + + def delete_all_documents(*) = nil + + def bulk(*) = nil + end + end +end diff --git a/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator.rb b/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator.rb new file mode 100644 index 00000000..c41b56cc --- /dev/null +++ b/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator.rb @@ -0,0 +1,24 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin/index_definition_configurator/for_index" +require "elastic_graph/admin/index_definition_configurator/for_index_template" + +module ElasticGraph + class Admin + module IndexDefinitionConfigurator + def self.new(datastore_client, index_def, env_agnostic_index_config, output, clock) + if index_def.rollover_index_template? + ForIndexTemplate.new(datastore_client, _ = index_def, env_agnostic_index_config, output, clock) + else + ForIndex.new(datastore_client, _ = index_def, env_agnostic_index_config, output) + end + end + end + end +end diff --git a/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index.rb b/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index.rb new file mode 100644 index 00000000..f70d5b8a --- /dev/null +++ b/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index.rb @@ -0,0 +1,194 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin/cluster_configurator/action_reporter" +require "elastic_graph/datastore_core/index_config_normalizer" +require "elastic_graph/indexer/hash_differ" +require "elastic_graph/support/hash_util" + +module ElasticGraph + class Admin + module IndexDefinitionConfigurator + # Responsible for managing an index's configuration, including both mappings and settings. + class ForIndex + # @dynamic index + + attr_reader :index + + def initialize(datastore_client, index, env_agnostic_index_config, output) + @datastore_client = datastore_client + @index = index + @env_agnostic_index_config = env_agnostic_index_config + @reporter = ClusterConfigurator::ActionReporter.new(output) + end + + # Attempts to idempotently update the index configuration to the desired configuration + # exposed by the `IndexDefinition` object. Based on the configuration of the passed index + # and the state of the index in the datastore, does one of the following: + # + # - If the index did not already exist: creates the index with the desired mappings and settings. + # - If the desired mapping has fewer fields than what is in the index: raises an exception, + # because the datastore provides no way to remove fields from a mapping and it would be confusing + # for this method to silently ignore the issue. + # - If the settings have desired changes: updates the settings, restoring any setting that + # no longer has a desired value to its default. + # - If the mapping has desired changes: updates the mappings. + # + # Note that any of the writes to the index may fail. There are many things that cannot + # be changed on an existing index (such as static settings, field mapping types, etc). We do not attempt + # to validate those things ahead of time and instead rely on the datastore to fail if an invalid operation + # is attempted. + def configure! + return create_new_index unless index_exists? + + # Update settings before mappings, to front-load the API call that is more likely to fail. + # Our `validate` method guards against mapping changes that are known to be disallowed by + # the datastore, but it is much harder to validate that for settings, because there are so + # many settings, and there is not clear documentation that outlines all settings, which can + # be updated on existing indices, etc. + # + # If we get a failure, we'd rather it happen before any changes are applied to the index, instead + # of applying the mappings and then failing on the settings. + update_settings if settings_updates.any? + + update_mapping if has_mapping_updates? + end + + def validate + if index_exists? && mapping_type_changes.any? + [cannot_modify_mapping_field_type_error] + else + [] + end + end + + private + + def create_new_index + @datastore_client.create_index(index: @index.name, body: desired_config) + report_action "Created index: `#{@index.name}`" + end + + def update_mapping + @datastore_client.put_index_mapping(index: @index.name, body: desired_mapping) + action_description = "Updated mappings for index `#{@index.name}`:\n#{mapping_diff}" + + if mapping_removals.any? + action_description += "\n\nNote: the extra fields listed here will not actually get removed. " \ + "Mapping removals are unsupported (but ElasticGraph will leave them alone and they'll cause no problems)." + end + + report_action action_description + end + + def update_settings + @datastore_client.put_index_settings(index: @index.name, body: settings_updates) + report_action "Updated settings for index `#{@index.name}`:\n#{settings_diff}" + end + + def cannot_modify_mapping_field_type_error + "The datastore does not support modifying the type of a field from an existing index definition. " \ + "You are attempting to update type of fields (#{mapping_type_changes.inspect}) from the #{@index.name} index definition." + end + + def index_exists? + !current_config.empty? + end + + def mapping_removals + @mapping_removals ||= mapping_fields_from(current_mapping) - mapping_fields_from(desired_mapping) + end + + def mapping_type_changes + @mapping_type_changes ||= begin + flattened_current = Support::HashUtil.flatten_and_stringify_keys(current_mapping) + flattened_desired = Support::HashUtil.flatten_and_stringify_keys(desired_mapping) + + flattened_current.keys.select do |key| + key.end_with?(".type") && flattened_desired.key?(key) && flattened_desired[key] != flattened_current[key] + end + end + end + + def has_mapping_updates? + current_mapping != desired_mapping + end + + def settings_updates + @settings_updates ||= begin + # Updating a setting to null will cause the datastore to restore the default value of the setting. + restore_to_defaults = (current_settings.keys - desired_settings.keys).to_h { |key| [key, nil] } + desired_settings.select { |key, value| current_settings[key] != value }.merge(restore_to_defaults) + end + end + + def mapping_fields_from(mapping_hash, prefix = "") + (mapping_hash["properties"] || []).flat_map do |key, params| + field = prefix + key + if params.key?("properties") + [field] + mapping_fields_from(params, "#{field}.") + else + [field] + end + end + end + + def desired_mapping + desired_config.fetch("mappings") + end + + def desired_settings + @desired_settings ||= desired_config.fetch("settings") + end + + def desired_config + @desired_config ||= begin + # _meta is place where we can record state on the index mapping in the datastore. + # We want to maintain `_meta.ElasticGraph.sources` as an append-only set of all sources that have ever + # been configured to flow into an index, so that we can remember whether or not an index which currently + # has no `sourced_from` from fields ever did. This is necessary for our automatic filtering of multi-source + # indexes. + previously_recorded_sources = current_mapping.dig("_meta", "ElasticGraph", "sources") || [] + sources = previously_recorded_sources.union(@index.current_sources.to_a).sort + + DatastoreCore::IndexConfigNormalizer.normalize(Support::HashUtil.deep_merge(@env_agnostic_index_config, { + "mappings" => {"_meta" => {"ElasticGraph" => {"sources" => sources}}}, + "settings" => @index.flattened_env_setting_overrides + })) + end + end + + def current_mapping + current_config["mappings"] || {} + end + + def current_settings + @current_settings ||= current_config["settings"] + end + + def current_config + @current_config ||= DatastoreCore::IndexConfigNormalizer.normalize( + @datastore_client.get_index(@index.name) + ) + end + + def mapping_diff + @mapping_diff ||= Indexer::HashDiffer.diff(current_mapping, desired_mapping) || "(no diff)" + end + + def settings_diff + @settings_diff ||= Indexer::HashDiffer.diff(current_settings, desired_settings) || "(no diff)" + end + + def report_action(message) + @reporter.report_action(message) + end + end + end + end +end diff --git a/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index_template.rb b/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index_template.rb new file mode 100644 index 00000000..5b4d5c2a --- /dev/null +++ b/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index_template.rb @@ -0,0 +1,247 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin/cluster_configurator/action_reporter" +require "elastic_graph/admin/index_definition_configurator/for_index" +require "elastic_graph/datastore_core/index_config_normalizer" +require "elastic_graph/indexer/hash_differ" +require "elastic_graph/support/hash_util" + +module ElasticGraph + class Admin + module IndexDefinitionConfigurator + # Responsible for managing an index template's configuration, including both mappings and settings. + class ForIndexTemplate + # @dynamic index_template + + attr_reader :index_template + + def initialize(datastore_client, index_template, env_agnostic_index_config_parent, output, clock) + @datastore_client = datastore_client + @index_template = index_template + @env_agnostic_index_config_parent = env_agnostic_index_config_parent + @env_agnostic_index_config = env_agnostic_index_config_parent.fetch("template") + @reporter = ClusterConfigurator::ActionReporter.new(output) + @output = output + @clock = clock + end + + # Attempts to idempotently update the index configuration to the desired configuration + # exposed by the `IndexDefinition` object. Based on the configuration of the passed index + # and the state of the index in the datastore, does one of the following: + # + # - If the index did not already exist: creates the index with the desired mappings and settings. + # - If the desired mapping has fewer fields than what is in the index: raises an exception, + # because the datastore provides no way to remove fields from a mapping and it would be confusing + # for this method to silently ignore the issue. + # - If the settings have desired changes: updates the settings, restoring any setting that + # no longer has a desired value to its default. + # - If the mapping has desired changes: updates the mappings. + # + # Note that any of the writes to the index may fail. There are many things that cannot + # be changed on an existing index (such as static settings, field mapping types, etc). We do not attempt + # to validate those things ahead of time and instead rely on the datastore to fail if an invalid operation + # is attempted. + def configure! + related_index_configurators.each(&:configure!) + + # there is no partial update for index template config and the same API both creates and updates it + put_index_template if has_mapping_updates? || settings_updates.any? + end + + def validate + errors = related_index_configurators.flat_map(&:validate) + + return errors unless index_template_exists? + + errors << cannot_modify_mapping_field_type_error if mapping_type_changes.any? + + errors + end + + private + + def put_index_template + desired_template_config_payload = Support::HashUtil.deep_merge( + desired_config_parent, + {"template" => {"mappings" => merge_properties(desired_mapping, current_mapping)}} + ) + + action_description = "Updated index template: `#{@index_template.name}`:\n#{config_diff}" + + if mapping_removals.any? + action_description += "\n\nNote: the extra fields listed here will not actually get removed. " \ + "Mapping removals are unsupported (but ElasticGraph will leave them alone and they'll cause no problems)." + end + + @datastore_client.put_index_template(name: @index_template.name, body: desired_template_config_payload) + report_action action_description + end + + def cannot_modify_mapping_field_type_error + "The datastore does not support modifying the type of a field from an existing index definition. " \ + "You are attempting to update type of fields (#{mapping_type_changes.inspect}) from the #{@index_template.name} index definition." + end + + def index_template_exists? + !current_config_parent.empty? + end + + def mapping_removals + @mapping_removals ||= mapping_fields_from(current_mapping) - mapping_fields_from(desired_mapping) + end + + def mapping_type_changes + @mapping_type_changes ||= begin + flattened_current = Support::HashUtil.flatten_and_stringify_keys(current_mapping) + flattened_desired = Support::HashUtil.flatten_and_stringify_keys(desired_mapping) + + flattened_current.keys.select do |key| + key.end_with?(".type") && flattened_desired.key?(key) && flattened_desired[key] != flattened_current[key] + end + end + end + + def has_mapping_updates? + current_mapping != desired_mapping + end + + def settings_updates + @settings_updates ||= begin + # Updating a setting to null will cause the datastore to restore the default value of the setting. + restore_to_defaults = (current_settings.keys - desired_settings.keys).to_h { |key| [key, nil] } + desired_settings.select { |key, value| current_settings[key] != value }.merge(restore_to_defaults) + end + end + + def mapping_fields_from(mapping_hash, prefix = "") + (mapping_hash["properties"] || []).flat_map do |key, params| + field = prefix + key + if params.key?("properties") + [field] + mapping_fields_from(params, "#{field}.") + else + [field] + end + end + end + + def desired_mapping + desired_config_parent.fetch("template").fetch("mappings") + end + + def desired_settings + @desired_settings ||= desired_config_parent.fetch("template").fetch("settings") + end + + def desired_config_parent + @desired_config_parent ||= begin + # _meta is place where we can record state on the index mapping in the datastore. + # We want to maintain `_meta.ElasticGraph.sources` as an append-only set of all sources that have ever + # been configured to flow into an index, so that we can remember whether or not an index which currently + # has no `sourced_from` from fields ever did. This is necessary for our automatic filtering of multi-source + # indexes. + previously_recorded_sources = current_mapping.dig("_meta", "ElasticGraph", "sources") || [] + sources = previously_recorded_sources.union(@index_template.current_sources.to_a).sort + + env_agnostic_index_config_with_meta = + DatastoreCore::IndexConfigNormalizer.normalize(Support::HashUtil.deep_merge(@env_agnostic_index_config, { + "mappings" => {"_meta" => {"ElasticGraph" => {"sources" => sources}}}, + "settings" => @index_template.flattened_env_setting_overrides + })) + + @env_agnostic_index_config_parent.merge({"template" => env_agnostic_index_config_with_meta}) + end + end + + def current_mapping + current_config_parent.dig("template", "mappings") || {} + end + + def current_settings + @current_settings ||= current_config_parent.dig("template", "settings") + end + + def current_config_parent + @current_config_parent ||= begin + config = @datastore_client.get_index_template(@index_template.name) + if (template = config.dig("template")) + config.merge({"template" => DatastoreCore::IndexConfigNormalizer.normalize(template)}) + else + config + end + end + end + + def config_diff + @config_diff ||= Indexer::HashDiffer.diff(current_config_parent, desired_config_parent) || "(no diff)" + end + + def report_action(message) + @reporter.report_action(message) + end + + # Helper method used to merge properties between a _desired_ configuration and a _current_ configuration. + # This is used when we are figuring out how to update an index template. We do not want to delete existing + # fields from a template--while the datastore would allow it, our schema evolution strategy depends upon + # us not dropping old unused fields. The datastore doesn't allow it on indices, anyway (though it does allow + # it on index templates). We've ran into trouble (a near SEV) when allowing the logic here to delete an unused + # field from an index template. The indexer "mapping completeness" check started failing because an old version + # of the code (from back when the field in question was still used) noticed the expected field was missing and + # started failing on every event. + # + # This helps us avoid that problem by retaining any currently existing fields. + # + # Long term, if we want to support fully "garbage collecting" these old fields on templates, we will need + # to have them get dropped in a follow up step. We could have our `update_datastore_config` script notice that + # the deployed prod indexers are at a version that will tolerate the fields being dropped, or support it + # via an opt-in flag or something. + def merge_properties(desired_object, current_object) + desired_properties = desired_object.fetch("properties") { _ = {} } + current_properties = current_object.fetch("properties") { _ = {} } + + merged_properties = desired_properties.merge(current_properties) do |key, desired, current| + if current.is_a?(::Hash) && current.key?("properties") && desired.key?("properties") + merge_properties(desired, current) + else + desired + end + end + + desired_object.merge("properties" => merged_properties) + end + + def related_index_configurators + # Here we fan out and get a configurator for each related index. These are generally concrete + # index that are based on a template, either via being specified in our config YAML, or via + # auto creation at indexing time. + # + # Note that it should not matter whether the related indices are configured before of after + # its rollover template; our use of index maintenance mode below prevents new indidces from + # being auto-created while this configuration process runs. + @related_index_configurators ||= begin + rollover_indices = @index_template.related_rollover_indices(@datastore_client) + + # When we have a rollover index, it's important that we make at least one concrete index. Otherwise, if any + # queries come in before the first event is indexed, we won't have any concrete indices to search, and + # the datastore returns a response that differs from normal in that case. It particularly creates trouble + # for aggregation queries since the response format it expects is quite complex. + # + # Here we create a concrete index for the current timestamp if there are no concrete indices yet. + if rollover_indices.empty? + rollover_indices = [@index_template.related_rollover_index_for_timestamp(@clock.now.getutc.iso8601)].compact + end + + rollover_indices.map do |index| + IndexDefinitionConfigurator::ForIndex.new(@datastore_client, index, @env_agnostic_index_config, @output) + end + end + end + end + end + end +end diff --git a/elasticgraph-admin/lib/elastic_graph/admin/rake_tasks.rb b/elasticgraph-admin/lib/elastic_graph/admin/rake_tasks.rb new file mode 100644 index 00000000..fc0dffe0 --- /dev/null +++ b/elasticgraph-admin/lib/elastic_graph/admin/rake_tasks.rb @@ -0,0 +1,129 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin" +require "elastic_graph/support/from_yaml_file" +require "rake/tasklib" + +module ElasticGraph + class Admin + class RakeTasks < ::Rake::TaskLib + extend Support::FromYamlFile::ForRakeTasks.new(ElasticGraph::Admin) + + attr_reader :output, :prototype_index_names + + def initialize(prototype_index_names: [], output: $stdout, &load_admin) + @output = output + @prototype_index_names = prototype_index_names.to_set + @load_admin = load_admin + + define_tasks + end + + private + + def define_tasks + namespace :clusters do + namespace :configure do + desc "Performs the configuration of datastore clusters, including indices, settings, and scripts" + task :perform do + print_in_color "#{"=" * 80}\nNOTE: Performing datastore cluster updates for real!\n#{"=" * 80}", RED_COLOR_CODE + + index_defs = update_clusters_for(admin) + output.puts "Finished updating datastore clusters. Validating index consistency..." + admin.datastore_indexing_router.validate_mapping_completeness_of!(:all_accessible_cluster_names, *index_defs) + output.puts "Done." + end + + desc "Dry-runs the configuration of datastore clusters, including indices, settings, and scripts" + task :dry_run do + print_in_color "#{"=" * 80}\nNOTE: In dry-run mode. The updates reported below will actually be no-ops.\n#{"=" * 80}", GREEN_COLOR_CODE + update_clusters_for(admin.with_dry_run_datastore_clients) + print_in_color "#{"=" * 80}\nNOTE: This was dry-run mode. The updates reported above were actually no-ops.\n#{"=" * 80}", GREEN_COLOR_CODE + end + end + end + + namespace :indices do + desc "Drops all prototype index definitions on all datastore clusters" + task :drop_prototypes do + require "elastic_graph/support/threading" + + prototype_indices = admin + .datastore_core + .index_definitions_by_name.values + .select { |index| prototype_index_names.include?(index.name) } + .reject { |index| index.all_accessible_cluster_names.empty? } + + output.puts "Disabling rollover index auto creation for all clusters" + admin.cluster_settings_manager.start_index_maintenance_mode!(:all_clusters) + output.puts "Disabled rollover index auto creation for all clusters" + + output.puts "Dropping the following prototype index definitions: #{prototype_indices.map(&:name).join(",")}" + Support::Threading.parallel_map(prototype_indices) do |prototype_index_def| + delete_index_def_in_all_accessible_clusters(prototype_index_def) + end + + output.puts "Finished dropping all prototype index definitions" + end + + desc "Drops the specified index definition on the specified datastore cluster" + task :drop, :index_def_name, :cluster_name do |_, args| + index_def_name = args.fetch(:index_def_name) + cluster_name = args.fetch(:cluster_name) + datastore_client = admin.datastore_core.clients_by_name.fetch(cluster_name) do |key| + raise Errors::IndexOperationError, "Cluster named `#{key}` does not exist. Valid clusters: #{admin.datastore_core.clients_by_name.keys}." + end + + index_def = admin.datastore_core.index_definitions_by_name.fetch(index_def_name) + unless prototype_index_names.include?(index_def.name) + raise Errors::IndexOperationError, "Unable to drop live index #{index_def_name}. Deleting a live index is extremely dangerous. " \ + "Please ensure this is indeed intended, add the index name to the `prototype_index_names` list and retry." + end + + output.puts "Disabling rollover index auto creation for this cluster" + admin.cluster_settings_manager.in_index_maintenance_mode(cluster_name) do + output.puts "Disabled rollover index auto creation for this cluster" + output.puts "Dropping index #{index_def}" + index_def.delete_from_datastore(datastore_client) + output.puts "Dropped index #{index_def}" + end + output.puts "Re-enabled rollover index auto creation for this cluster" + end + end + end + + # See https://en.wikipedia.org/wiki/ANSI_escape_code#Colors for full list. + RED_COLOR_CODE = 31 + GREEN_COLOR_CODE = 32 + + def update_clusters_for(admin) + configurator = admin.cluster_configurator + + configurator.accessible_index_definitions.tap do |index_defs| + output.puts "The following index definitions will be configured:\n#{index_defs.map(&:name).join("\n")}" + configurator.configure_cluster(@output) + end + end + + def print_in_color(message, color_code) + @output.puts "\033[#{color_code}m#{message}\033[0m" + end + + def delete_index_def_in_all_accessible_clusters(index_def) + index_def.all_accessible_cluster_names.each do |cluster_name| + index_def.delete_from_datastore(admin.datastore_core.clients_by_name.fetch(cluster_name)) + end + end + + def admin + @admin ||= @load_admin.call + end + end + end +end diff --git a/elasticgraph-admin/sig/elastic_graph/admin.rbs b/elasticgraph-admin/sig/elastic_graph/admin.rbs new file mode 100644 index 00000000..945d5ed4 --- /dev/null +++ b/elasticgraph-admin/sig/elastic_graph/admin.rbs @@ -0,0 +1,30 @@ +module ElasticGraph + class Admin + extend _BuildableFromParsedYaml[Admin] + extend Support::FromYamlFile[Admin] + + def initialize: ( + datastore_core: DatastoreCore, + ?monotonic_clock: Support::MonotonicClock?, + ?clock: singleton(::Time) + ) -> void + + attr_reader datastore_core: DatastoreCore + attr_reader schema_artifacts: schemaArtifacts + @clock: singleton(::Time) + + @cluster_configurator: Admin::ClusterConfigurator? + def cluster_configurator: () -> Admin::ClusterConfigurator + + @cluster_settings_manager: Admin::ClusterConfigurator::ClusterSettingsManager? + def cluster_settings_manager: () -> Admin::ClusterConfigurator::ClusterSettingsManager + + @datastore_indexing_router: Indexer::DatastoreIndexingRouter? + def datastore_indexing_router: () -> Indexer::DatastoreIndexingRouter + + @monotonic_clock: Support::MonotonicClock? + def monotonic_clock: () -> Support::MonotonicClock + + def with_dry_run_datastore_clients: () -> Admin + end +end diff --git a/elasticgraph-admin/sig/elastic_graph/admin/cluster_configurator.rbs b/elasticgraph-admin/sig/elastic_graph/admin/cluster_configurator.rbs new file mode 100644 index 00000000..0ceee261 --- /dev/null +++ b/elasticgraph-admin/sig/elastic_graph/admin/cluster_configurator.rbs @@ -0,0 +1,32 @@ +module ElasticGraph + class Admin + class ClusterConfigurator + def initialize: ( + datastore_clients_by_name: ::Hash[::String, DatastoreCore::_Client], + index_defs: ::Array[DatastoreCore::indexDefinition], + index_configurations_by_name: ::Hash[::String, untyped], + index_template_configurations_by_name: ::Hash[::String, untyped], + scripts: datastoreScriptsByIdHash, + cluster_settings_manager: ClusterSettingsManager, + clock: singleton(::Time) + ) -> void + + def configure_cluster: (io) -> void + + @accessible_index_definitions: ::Array[DatastoreCore::indexDefinition]? + def accessible_index_definitions: () -> ::Array[DatastoreCore::indexDefinition] + + private + + @datastore_clients_by_name: ::Hash[::String, DatastoreCore::_Client] + @index_defs: ::Array[DatastoreCore::indexDefinition] + @index_configurations_by_name: DatastoreCore::indexConfigHash + @scripts_by_id: datastoreScriptsByIdHash + @cluster_settings_manager: ClusterSettingsManager + @clock: singleton(::Time) + + def script_configurators_for: (io) -> ::Array[ScriptConfigurator] + def index_definition_configurators_for: (io) -> ::Array[indexDefinitionConfigurator] + end + end +end diff --git a/elasticgraph-admin/sig/elastic_graph/admin/cluster_configurator/action_reporter.rbs b/elasticgraph-admin/sig/elastic_graph/admin/cluster_configurator/action_reporter.rbs new file mode 100644 index 00000000..6f3494fa --- /dev/null +++ b/elasticgraph-admin/sig/elastic_graph/admin/cluster_configurator/action_reporter.rbs @@ -0,0 +1,11 @@ +module ElasticGraph + class Admin + class ClusterConfigurator + class ActionReporter + def initialize: (io) -> void + def report_action: (::String) -> void + @output: io + end + end + end +end diff --git a/elasticgraph-admin/sig/elastic_graph/admin/cluster_configurator/cluster_settings_manager.rbs b/elasticgraph-admin/sig/elastic_graph/admin/cluster_configurator/cluster_settings_manager.rbs new file mode 100644 index 00000000..4b4368e3 --- /dev/null +++ b/elasticgraph-admin/sig/elastic_graph/admin/cluster_configurator/cluster_settings_manager.rbs @@ -0,0 +1,28 @@ +module ElasticGraph + class Admin + class ClusterConfigurator + class ClusterSettingsManager + type clusterSpec = ::String | :all_clusters + + def initialize: ( + datastore_clients_by_name: ::Hash[::String, DatastoreCore::_Client], + datastore_config: DatastoreCore::Config, + logger: ::Logger) -> void + + def start_index_maintenance_mode!: (clusterSpec) -> void + def end_index_maintenance_mode!: (clusterSpec) -> void + def in_index_maintenance_mode: (clusterSpec) { () -> void } -> void + + private + + @datastore_clients_by_name: ::Hash[::String, DatastoreCore::_Client] + @datastore_config: DatastoreCore::Config + @logger: ::Logger + + def desired_cluster_settings: (::String, ?auto_create_index_patterns: ::Array[::String]) -> ::Hash[::String, untyped] + def datastore_client_named: (::String) -> DatastoreCore::_Client + def cluster_names_for: (clusterSpec) -> ::Array[::String] + end + end + end +end diff --git a/elasticgraph-admin/sig/elastic_graph/admin/cluster_configurator/script_configurator.rbs b/elasticgraph-admin/sig/elastic_graph/admin/cluster_configurator/script_configurator.rbs new file mode 100644 index 00000000..3a913ab7 --- /dev/null +++ b/elasticgraph-admin/sig/elastic_graph/admin/cluster_configurator/script_configurator.rbs @@ -0,0 +1,27 @@ +module ElasticGraph + class Admin + class ClusterConfigurator + class ScriptConfigurator + def initialize: ( + datastore_client: DatastoreCore::_Client, + script_context: datastoreScriptContext, + script_id: ::String, + script: datastoreScriptScriptHash, + output: io) -> void + + def validate: () -> ::Array[::String] + def configure!: () -> void + + private + + @datastore_client: DatastoreCore::_Client + @script_context: datastoreScriptContext + @script_id: ::String + @script: datastoreScriptScriptHash + @action_reporter: ActionReporter + + attr_reader existing_datastore_script: :not_found | datastoreScriptScriptHash? + end + end + end +end diff --git a/elasticgraph-admin/sig/elastic_graph/admin/datastore_client_dry_run_decorator.rbs b/elasticgraph-admin/sig/elastic_graph/admin/datastore_client_dry_run_decorator.rbs new file mode 100644 index 00000000..10a36349 --- /dev/null +++ b/elasticgraph-admin/sig/elastic_graph/admin/datastore_client_dry_run_decorator.rbs @@ -0,0 +1,8 @@ +module ElasticGraph + class Admin + class DatastoreClientDryRunDecorator + include DatastoreCore::_Client + def initialize: (DatastoreCore::_Client) -> void + end + end +end \ No newline at end of file diff --git a/elasticgraph-admin/sig/elastic_graph/admin/index_definition_configurator.rbs b/elasticgraph-admin/sig/elastic_graph/admin/index_definition_configurator.rbs new file mode 100644 index 00000000..c41009db --- /dev/null +++ b/elasticgraph-admin/sig/elastic_graph/admin/index_definition_configurator.rbs @@ -0,0 +1,15 @@ +module ElasticGraph + class Admin + type indexDefinitionConfigurator = IndexDefinitionConfigurator::ForIndex | IndexDefinitionConfigurator::ForIndexTemplate + + module IndexDefinitionConfigurator + def self.new: ( + DatastoreCore::_Client, + DatastoreCore::indexDefinition, + ::Hash[::String, untyped], + io, + singleton(::Time) + ) -> indexDefinitionConfigurator + end + end +end diff --git a/elasticgraph-admin/sig/elastic_graph/admin/index_definition_configurator/for_index.rbs b/elasticgraph-admin/sig/elastic_graph/admin/index_definition_configurator/for_index.rbs new file mode 100644 index 00000000..54fbd0c4 --- /dev/null +++ b/elasticgraph-admin/sig/elastic_graph/admin/index_definition_configurator/for_index.rbs @@ -0,0 +1,70 @@ +module ElasticGraph + class Admin + module IndexDefinitionConfigurator + class ForIndex + attr_reader index: DatastoreCore::IndexDefinition::Index + + def initialize: ( + DatastoreCore::_Client, + DatastoreCore::IndexDefinition::Index, + ::Hash[::String, untyped], + io + ) -> void + + def configure!: () -> void + def validate: () -> ::Array[::String] + + private + + @datastore_client: DatastoreCore::_Client + @index: DatastoreCore::IndexDefinition::Index + @env_agnostic_index_config: ::Hash[::String, untyped] + @reporter: ClusterConfigurator::ActionReporter + @output: io + + def create_new_index: () -> void + def update_mapping: () -> void + def update_settings: () -> void + def cannot_modify_mapping_field_type_error: () -> ::String + def index_exists?: () -> bool + + @mapping_removals: ::Array[::String]? + def mapping_removals: () -> ::Array[::String] + + @mapping_type_changes: ::Array[::String]? + def mapping_type_changes: () -> ::Array[::String] + + def has_mapping_updates?: () -> bool + + @settings_updates: DatastoreCore::indexSettingsHash? + def settings_updates: () -> DatastoreCore::indexSettingsHash + + def mapping_fields_from: (DatastoreCore::indexMappingHash, ?::String) -> ::Array[::String] + + def desired_mapping: () -> DatastoreCore::indexMappingHash + + @desired_settings: DatastoreCore::indexSettingsHash? + def desired_settings: () -> DatastoreCore::indexSettingsHash + + @desired_config: DatastoreCore::indexConfigHash? + def desired_config: () -> DatastoreCore::indexConfigHash + + def current_mapping: () -> DatastoreCore::indexMappingHash + + @current_settings: DatastoreCore::indexSettingsHash? + def current_settings: () -> DatastoreCore::indexSettingsHash + + @current_config: DatastoreCore::indexConfigHash? + def current_config: () -> DatastoreCore::indexConfigHash + + @mapping_diff: ::String? + def mapping_diff: () -> ::String + + @settings_diff: ::String? + def settings_diff: () -> ::String + + def report_action: (::String) -> void + end + end + end +end diff --git a/elasticgraph-admin/sig/elastic_graph/admin/index_definition_configurator/for_index_template.rbs b/elasticgraph-admin/sig/elastic_graph/admin/index_definition_configurator/for_index_template.rbs new file mode 100644 index 00000000..68943db0 --- /dev/null +++ b/elasticgraph-admin/sig/elastic_graph/admin/index_definition_configurator/for_index_template.rbs @@ -0,0 +1,72 @@ +module ElasticGraph + class Admin + module IndexDefinitionConfigurator + class ForIndexTemplate + attr_reader index_template: DatastoreCore::IndexDefinition::RolloverIndexTemplate + + def initialize: ( + DatastoreCore::_Client, + DatastoreCore::IndexDefinition::RolloverIndexTemplate, + ::Hash[::String, untyped], + io, + singleton(::Time) + ) -> void + + def configure!: () -> void + def validate: () -> ::Array[::String] + + private + + @datastore_client: DatastoreCore::_Client + @index_template: DatastoreCore::IndexDefinition::RolloverIndexTemplate + @env_agnostic_index_config_parent: ::Hash[::String, untyped] + @env_agnostic_index_config: ::Hash[::String, untyped] + @reporter: ClusterConfigurator::ActionReporter + @output: io + @clock: singleton(::Time) + + def put_index_template: () -> void + def cannot_modify_mapping_field_type_error: () -> ::String + def index_template_exists?: () -> bool + + @mapping_removals: ::Array[::String]? + def mapping_removals: () -> ::Array[::String] + + @mapping_type_changes: ::Array[::String]? + def mapping_type_changes: () -> ::Array[::String] + + def has_mapping_updates?: () -> bool + + @settings_updates: DatastoreCore::indexSettingsHash? + def settings_updates: () -> DatastoreCore::indexSettingsHash + + def mapping_fields_from: (DatastoreCore::indexMappingHash, ?::String) -> ::Array[::String] + + def desired_mapping: () -> DatastoreCore::indexMappingHash + + @desired_settings: DatastoreCore::indexSettingsHash? + def desired_settings: () -> DatastoreCore::indexSettingsHash + + @desired_config_parent: ::Hash[::String, untyped] + def desired_config_parent: () -> ::Hash[::String, untyped] + + def current_mapping: () -> DatastoreCore::indexMappingHash + + @current_settings: DatastoreCore::indexSettingsHash? + def current_settings: () -> DatastoreCore::indexSettingsHash + + @current_config_parent: ::Hash[::String, untyped] + def current_config_parent: () -> ::Hash[::String, untyped] + + @config_diff: ::String + def config_diff: () -> ::String + + def report_action: (::String) -> void + def merge_properties: (::Hash[::String, untyped], ::Hash[::String, untyped]) -> ::Hash[::String, untyped] + + @related_index_configurators: ::Array[ForIndex]? + def related_index_configurators: () -> ::Array[ForIndex] + end + end + end +end diff --git a/elasticgraph-admin/sig/elastic_graph/admin/rake_tasks.rbs b/elasticgraph-admin/sig/elastic_graph/admin/rake_tasks.rbs new file mode 100644 index 00000000..de276231 --- /dev/null +++ b/elasticgraph-admin/sig/elastic_graph/admin/rake_tasks.rbs @@ -0,0 +1,7 @@ +module ElasticGraph + class Admin + class RakeTasks < ::Rake::TaskLib + def self.from_yaml_file: (::String | ::Pathname, ?output: io) -> RakeTasks + end + end +end diff --git a/elasticgraph-admin/spec/integration/elastic_graph/admin/cluster_configurator_spec.rb b/elasticgraph-admin/spec/integration/elastic_graph/admin/cluster_configurator_spec.rb new file mode 100644 index 00000000..253d92e8 --- /dev/null +++ b/elasticgraph-admin/spec/integration/elastic_graph/admin/cluster_configurator_spec.rb @@ -0,0 +1,240 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin/cluster_configurator" +require "elastic_graph/schema_definition/results" +require "stringio" +require "yaml" + +module ElasticGraph + class Admin + RSpec.describe ClusterConfigurator, :uses_datastore do + # Use a different index name than any other tests use, because most tests expect a specific index + # configuration (based on `config/schema.graphql`) and we do not want to mess with it here. + let(:index_definition_name) { unique_index_name } + let(:schema_def) do + lambda do |schema| + schema.object_type "WidgetOptions" do |t| + t.field "size", "Int" + t.field "color", "String" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "options", "WidgetOptions" + t.index unique_index_name + end + + schema.object_type "Widget2" do |t| + t.field "id", "ID!" + t.index "#{unique_index_name}2" + end + end + end + + it "configures the indices and the desired cluster-wide settings on the clusters configured on the indices" do + admin = admin_for(schema_def) do |config| + expect(config.clusters.keys).to include("main", "other1", "other2", "other3") + + config.with(index_definitions: { + unique_index_name => config_index_def_of( + query_cluster: "main", + index_into_clusters: ["other1", "other2"] # does not include `other3` + ), + "#{unique_index_name}2" => config_index_def_of( + query_cluster: "main", + index_into_clusters: ["other1"] # does not include `other2` or `other3` + ) + }) + end + + expect { + configure_cluster(admin) + }.to change { main_datastore_client.get_index(unique_index_name) }.from({}).to(a_hash_including("mappings")) + .and change { main_datastore_client.get_index("#{unique_index_name}2") }.from({}).to(a_hash_including("mappings")) + + # We expect 2 cluster settings calls to each cluster because of the need to start and end index maintenance mode. + # Then we expect the index-specific calls only to the specific clusters configured for those indices. + expect(tallied_datastore_calls("main")).to include("/_cluster/settings" => 2).and include("/#{unique_index_name}", "/#{unique_index_name}2") + expect(tallied_datastore_calls("other1")).to include("/_cluster/settings" => 2).and include("/#{unique_index_name}", "/#{unique_index_name}2") + expect(tallied_datastore_calls("other2")).to include("/_cluster/settings" => 2).and include("/#{unique_index_name}").and exclude("/#{unique_index_name}2") + expect(tallied_datastore_calls("other3")).to include("/_cluster/settings" => 2).and exclude("/#{unique_index_name}", "/#{unique_index_name}2") + + expect(admin.cluster_configurator.accessible_index_definitions.map(&:name)).to contain_exactly( + unique_index_name, + "#{unique_index_name}2" + ) + end + + it "makes no attempt to configure indices that are inaccessible due to residing on inaccessible clusters" do + admin = admin_for(schema_def) do |config| + expect(config.clusters.keys).to contain_exactly("main", "other1", "other2", "other3") + + config.with(index_definitions: { + unique_index_name => config_index_def_of( + query_cluster: "main", + index_into_clusters: ["other1"] + ), + "#{unique_index_name}2" => config_index_def_of( + query_cluster: "undefined", + index_into_clusters: ["undefined"] + ) + }) + end + + expect { + configure_cluster(admin) + }.to change { main_datastore_client.get_index(unique_index_name) }.from({}).to(a_hash_including("mappings")) + .and maintain { other3_datastore_client.get_index("#{unique_index_name}2") } + + # We expect 2 cluster settings calls to each cluster because of the need to start and end index maintenance mode. + # Then we expect the index-specific calls only to the specific clusters configured for those indices. + expect(tallied_datastore_calls("main")).to include("/_cluster/settings" => 2).and include("/#{unique_index_name}").and exclude("/#{unique_index_name}2") + expect(tallied_datastore_calls("other1")).to include("/_cluster/settings" => 2).and include("/#{unique_index_name}").and exclude("/#{unique_index_name}2") + expect(tallied_datastore_calls("other2")).to include("/_cluster/settings" => 2).and exclude("/#{unique_index_name}", "/#{unique_index_name}2") + + expect(admin.cluster_configurator.accessible_index_definitions.map(&:name)).to contain_exactly(unique_index_name) + end + + it "validates all index configurations before applying any of them, to prevent partial application of index configuration updates" do + # Setting up a situation that results in validation errors is quite complicated, so here we just + # intercept `validate` on `IndexDefinitionConfigurator` instances to force them to return errors. + # (But otherwise the `IndexDefinitionConfigurator`s behave just like real ones). + allow(IndexDefinitionConfigurator::ForIndex).to receive(:new).and_wrap_original do |original_impl, *args, &block| + original_impl.call(*args, &block).tap do |index_configurator| + allow(index_configurator).to receive(:validate).and_return(["Problem 1", "Problem 2"]) + end + end + + admin = admin_for(schema_def) + + expect { + configure_cluster(admin) + }.to maintain { main_datastore_client.get_index(unique_index_name) } + .and maintain { main_datastore_client.get_index("#{unique_index_name}2") } + .and raise_error(Errors::ClusterOperationError, a_string_including("Problem 1", "Problem 2")) + end + + context "when there is a schema artifact script" do + let(:standard_script_ids) { SchemaDefinition::Results::STATIC_SCRIPT_REPO.script_ids_by_scoped_name.values.to_set } + + let(:admin) do + admin_for(lambda do |schema| + schema.object_type "WidgetWorkspace" do |t| + t.field "id", "ID!" + t.field "widget_names", "[String!]!" + t.index "#{unique_index_name}_widget_workspaces" + end + + schema.object_type "Widget#{unique_index_name}" do |t| + t.field "id", "ID!" + t.field "#{unique_index_name}_name", "String" + t.field "workspace_id", "ID" + t.index unique_index_name + + t.derive_indexed_type_fields "WidgetWorkspace", from_id: "workspace_id" do |derive| + # Use `unique_index_name` in a field that goes into the script so that the script id is not used by any + # other tests; that way we can trust that the manipulations in these tests won't impact other tests. + derive.append_only_set "widget_names", from: "#{unique_index_name}_name" + end + end + end) + end + + let(:schema_specific_update_scripts) do + admin.schema_artifacts.datastore_scripts.select do |id| + id.start_with?("update_") && !standard_script_ids.include?(id) + end + end + + before do + expect(schema_specific_update_scripts).not_to be_empty # the tests below assume it is non-empty + delete_all_schema_specific_update_scripts + end + + it "idempotently stores the script in each datastore cluster" do + expect { + configure_cluster(admin) + }.to change { fetch_schema_specific_update_scripts }.from({}).to(schema_specific_update_scripts) + + script_path = "/_scripts/#{schema_specific_update_scripts.keys.first}" + expect(tallied_datastore_calls("main")).to include(script_path) + expect(tallied_datastore_calls("other1")).to include(script_path) + expect(tallied_datastore_calls("other2")).to include(script_path) + expect(tallied_datastore_calls("other3")).to include(script_path) + + expect { + configure_cluster(admin) + }.to maintain { fetch_schema_specific_update_scripts } + end + + it "correctly reports what it will do in dry run mode" do + output = configure_cluster(admin.with_dry_run_datastore_clients) + expect(output).to include("Stored update script: #{schema_specific_update_scripts.keys.first}") + expect(fetch_schema_specific_update_scripts).to eq({}) + + output = configure_cluster(admin) + expect(output).to include("Stored update script: #{schema_specific_update_scripts.keys.first}") + expect(fetch_schema_specific_update_scripts).to eq(schema_specific_update_scripts) + + output = configure_cluster(admin.with_dry_run_datastore_clients) + expect(output).to exclude("Stored update script") + end + + it "raises an error rather than mutating the script if a different script with that id already exists" do + first_script = schema_specific_update_scripts.values.first.fetch("script") + + main_datastore_client.put_script(id: schema_specific_update_scripts.keys.first, context: "update", body: { + script: first_script.merge( + "source" => "// a leading comment\n\n" + first_script.fetch("source") + ) + }) + + expect { + configure_cluster(admin) + }.to raise_error Errors::ClusterOperationError, a_string_including("already exists in the datastore but has different contents", "a leading comment") + + expect(main_datastore_client.get_index(unique_index_name)).to eq({}) + + expect { + configure_cluster(admin.with_dry_run_datastore_clients) + }.to raise_error Errors::ClusterOperationError, a_string_including("already exists in the datastore but has different contents", "a leading comment") + end + + def fetch_schema_specific_update_scripts + schema_specific_update_scripts.filter_map do |id, artifact_script| + if (fetched_script = main_datastore_client.get_script(id: id)) + [id, {"context" => artifact_script.fetch("context"), "script" => fetched_script.fetch("script")}] + end + end.to_h + end + + def delete_all_schema_specific_update_scripts + schema_specific_update_scripts.each do |id, script| + main_datastore_client.delete_script(id: id) + end + end + end + + def configure_cluster(admin) + output_io = StringIO.new + admin.cluster_configurator.configure_cluster(output_io) + output_io.string + end + + def admin_for(schema_def, &customize_datastore_config) + build_admin(schema_definition: schema_def, &customize_datastore_config) + end + + def tallied_datastore_calls(cluster_name) + datastore_requests(cluster_name).map { |r| r.url.path }.tally + end + end + end +end diff --git a/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_spec.rb b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_spec.rb new file mode 100644 index 00000000..57939892 --- /dev/null +++ b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_spec.rb @@ -0,0 +1,49 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "shared_examples" + +module ElasticGraph + class Admin + module IndexDefinitionConfigurator + RSpec.describe ForIndex do + include_examples IndexDefinitionConfigurator do + include ConcreteIndexAdapter + + it "raises an exception when attempting to change a static index setting (since the datastore disallows it)" do + configure_index_definition(schema_def) + + expect { + configure_index_definition(schema_def(number_of_shards: 47)) + }.to raise_error(Errors::BadDatastoreRequest, a_string_including("Can't update non dynamic settings", "index.number_of_shards")) + .and make_datastore_write_calls("main", "PUT /#{unique_index_name}/_settings") + .and log_warning(/Can't update non dynamic settings/) + end + + it "handles empty indexed types" do + schema = schema_def(define_no_widget_fields: true) + + configure_index_definition(schema) + + expect { + configure_index_definition(schema) + }.to make_no_datastore_write_calls("main") + end + + def make_datastore_calls_to_configure_index_def(index_name, subresource = nil) + make_datastore_write_calls("main", "PUT #{put_index_definition_url(index_name, subresource)}") + end + + def fetch_artifact_configuration(schema_artifacts, index_def_name) + schema_artifacts.indices.fetch(index_def_name) + end + end + end + end + end +end diff --git a/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_template_spec.rb b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_template_spec.rb new file mode 100644 index 00000000..88a86b96 --- /dev/null +++ b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_template_spec.rb @@ -0,0 +1,215 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "shared_examples" +require "elastic_graph/schema_definition/schema_artifact_manager" + +module ElasticGraph + class Admin + module IndexDefinitionConfigurator + RSpec.describe ForIndexTemplate do + include_examples IndexDefinitionConfigurator do + def concrete_index_name_for_now(base_index_name) + "#{base_index_name}_rollover__2024-03" # clock.now is at a time on 2024-03-20 + end + + prepend Module.new { + def schema_def(**options) + configure_index = ->(index) { index.rollover :monthly, "created_at" } + super(configure_index: configure_index, **options) + end + } + + def get_index_definition_configuration(index_definition_name) + index_template = main_datastore_client.get_index_template(index_definition_name) + expect(index_template).to include("index_patterns" => ["#{index_definition_name}_rollover__*"]).or be_empty + index_template["template"] || {} + end + alias_method :get_index_template_definition_configuration, :get_index_definition_configuration + + # Index templates don't have separate _settings or _mappings subresources, so we ignore it here. + def put_index_definition_url(index_definition_name, _subresource = nil) + "/_index_template/#{index_definition_name}" + end + alias_method :put_index_template_definition_url, :put_index_definition_url + + def make_datastore_calls_to_configure_index_def(index_name, subresource = nil) + # :nocov: -- when we are building against OpenSearch, one side of this conditional is not covered + subresource = :mapping if subresource == :mappings && datastore_backend == :elasticsearch + # :nocov: + + path_for_now_index = "/#{concrete_index_name_for_now(index_name)}" + path_for_now_index += "/_#{subresource}" if subresource + + make_datastore_write_calls( + "main", + "PUT #{put_index_definition_url(index_name, subresource)}", + "PUT #{path_for_now_index}" + ) + end + + def simulate_presence_of_extra_setting(admin, index_definition_name, name, value) + admin.datastore_core.clients_by_name.values.each do |client| + allow(client).to receive(:get_index_template).with(index_definition_name).and_wrap_original do |original, *args, **kwargs, &block| + original.call(*args, **kwargs, &block).tap do |result| + # Mutate the settings before we return them. + result["template"]["settings"][name] = value + end + end + end + end + + def fetch_artifact_configuration(schema_artifacts, index_def_name) + schema_artifacts.index_templates.fetch(index_def_name) + end + + # Allow index templates to change static index settings such as `index.number_of_shards` because the datastore allows it. + # Could also be beneficial for us if we need more shards as the data grows big for the new indices of a rollover index + it "allows changes to static index settings on an index template since the datastore allows it and it could be useful" do + configure_index_definition(schema_def(number_of_shards: 10)) + + expect { + configure_index_definition(schema_def) do |config| + config.with(index_definitions: { + "#{unique_index_name}_owners" => config_index_def_of, + unique_index_name => config_index_def_of( + setting_overrides: {"number_of_shards" => 47}, + setting_overrides_by_timestamp: { + clock.now.getutc.iso8601 => {"number_of_shards" => 10} + } + ) + }) + end + }.to change { get_index_definition_configuration(unique_index_name).fetch("settings") } + .from(a_hash_including("index.number_of_shards" => "10")) + .to(a_hash_including("index.number_of_shards" => "47")) + .and make_datastore_calls_to_configure_index_def(unique_index_name, :settings) + end + + it "creates concrete indices based on `setting_overrides_by_timestamp` configuration, and avoids creating an extra index for 'now'" do + jan_2020_index_name = unique_index_name + "_rollover__2020-01" + + expect { + configure_index_definition(schema_def(number_of_shards: 5)) do |config| + config.with(index_definitions: { + "#{unique_index_name}_owners" => config_index_def_of, + unique_index_name => config_index_def_of(setting_overrides_by_timestamp: { + "2020-01-01T00:00:00Z" => { + "number_of_shards" => 3 + } + }) + }) + end + }.to change { get_index_definition_configuration(unique_index_name)["settings"] } + .from(nil) + .to(a_hash_including("index.number_of_shards" => "5")) + .and change { main_datastore_client.get_index(jan_2020_index_name)["settings"] } + .from(nil) + .to(a_hash_including("index.number_of_shards" => "3")) + .and maintain { main_datastore_client.get_index(concrete_index_name_for_now(unique_index_name))["settings"] } + .from(nil) + + index_def_creation_order = datastore_write_requests("main").filter_map { |r| r.url.path.split("/").last if r.http_method == :put } + # the specific jan_2020 index must be created before the index template to guard against + # the jan_2020 index being generated from the template by another process concurrently indexing + # a jan 2020 document. + expect(index_def_creation_order).to eq([jan_2020_index_name, unique_index_name]) + end + + context "when the settings do not force the creation of any concrete indices" do + it "creates an index using the current time so that our search queries always have an index to hit" do + expect { + configure_index_definition(schema_def) + }.to change { get_index_definition_configuration(unique_index_name) } + .from({}) + .to(a_hash_including("mappings", "settings")) + .and change { main_datastore_client.get_index(concrete_index_name_for_now(unique_index_name)) } + .from({}) + .to(a_hash_including("mappings", "settings")) + end + end + + context "when a concrete index has been derived from the template", :factories do + include ConcreteIndexAdapter + + # our schema in the tests here is more limited than the main widget schema, so select only some fields. + let(:widget) { build(:widget).select { |k, v| k.start_with?("__") || %i[id name options created_at].include?(k) } } + + let(:concrete_index_name) do + admin_for(schema_def) + .datastore_core + .index_definitions_by_name + .fetch(unique_index_name) + .index_name_for_writes(Support::HashUtil.stringify_keys(widget)) + end + + it "propagates mapping changes to the derived concrete rollover indices, ignoring the fact that the derived indices are not in the dumped schema artifacts", :in_temp_dir do + configure_index_definition(schema_def) + index_into(indexer_for(schema_def), widget) + + updated_schema = schema_def(configure_widget: ->(t) { t.field "amount_cents", "Int" }) + + SchemaDefinition::SchemaArtifactManager.new( + schema_definition_results: generate_schema_artifacts(&updated_schema), + schema_artifacts_directory: Dir.pwd, + enforce_json_schema_version: true, + output: output_io + ).dump_artifacts + + expect { + configure_index_definition(updated_schema, schema_artifacts_directory: Dir.pwd) + }.to change { get_index_definition_configuration(concrete_index_name).dig("mappings", "properties").keys.sort } + .from([*index_meta_fields, "created_at", "id", "name", "options"]) + .to([*index_meta_fields, "amount_cents", "created_at", "id", "name", "options"]) + .and make_datastore_write_calls("main", + "PUT #{put_index_template_definition_url(unique_index_name)}", + "PUT #{put_index_definition_url(concrete_index_name_for_now(unique_index_name), :mappings)}", + "PUT #{put_index_definition_url(concrete_index_name, :mappings)}") + end + + it "propagates setting changes to the derived concrete rollover indices" do + configure_index_definition(schema_def) + index_into(indexer_for(schema_def), widget) + + expect { + configure_index_definition(schema_def(refresh_interval: "5s")) + }.to change { get_index_definition_configuration(concrete_index_name).fetch("settings").keys } + .from(a_collection_excluding("index.refresh_interval")) + .to(a_collection_including("index.refresh_interval")) + .and make_datastore_write_calls("main", + "PUT #{put_index_template_definition_url(unique_index_name)}", + "PUT #{put_index_definition_url(concrete_index_name_for_now(unique_index_name), :settings)}", + "PUT #{put_index_definition_url(concrete_index_name, :settings)}") + end + + it "fails before any changes are made if the changes can't be propagated to the concrete rollover indices" do + configure_index_definition(schema_def) + index_into(indexer_for(schema_def), widget) + + main_datastore_client.put_index_mapping(index: concrete_index_name, body: { + properties: { + amount_cents: {type: "keyword"} + } + }) + + expect { + configure_index_definition(schema_def(configure_widget: ->(t) { t.field "amount_cents", "Int" })) + }.to maintain { get_index_definition_configuration(concrete_index_name) } + .and maintain { get_index_template_definition_configuration(unique_index_name) } + .and raise_error(Errors::IndexOperationError, a_string_including(concrete_index_name, "properties.amount_cents.type")) + end + + def indexer_for(schema_def) + build_indexer(schema_definition: schema_def) + end + end + end + end + end + end +end diff --git a/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/shared_examples.rb b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/shared_examples.rb new file mode 100644 index 00000000..fa9a6804 --- /dev/null +++ b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/shared_examples.rb @@ -0,0 +1,363 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin/index_definition_configurator" +require "elastic_graph/errors" +require "stringio" + +module ElasticGraph + class Admin + module IndexDefinitionConfigurator + module ConcreteIndexAdapter + def get_index_definition_configuration(index_definition_name) + main_datastore_client.get_index(index_definition_name) + end + + def put_index_definition_url(index_definition_name, subresource = nil) + url = "/#{index_definition_name}" + # :nocov: -- when we are building against OpenSearch, one side of this conditional is not covered + subresource = :mapping if subresource == :mappings && datastore_backend == :elasticsearch + # :nocov: + subresource ? "#{url}/_#{subresource}" : url + end + + def simulate_presence_of_extra_setting(admin, index_definition_name, name, value) + admin.datastore_core.clients_by_name.values.each do |client| + allow(client).to receive(:get_index).with(index_definition_name).and_wrap_original do |original, *args, **kwargs, &block| + original.call(*args, **kwargs, &block).tap do |result| + # Mutate the settings before we return them. + result.fetch("settings")[name] = value + end + end + end + end + end + + RSpec.shared_examples_for IndexDefinitionConfigurator, :uses_datastore, :builds_indexer do + let(:output_io) { StringIO.new } + let(:clock) { class_double(::Time, now: ::Time.utc(2024, 3, 20, 12, 0, 0)) } + let(:mapping_removal_note_snippet) { "extra fields listed here will not actually get removed" } + let(:index_meta_fields) { ["__sources", "__versions"] } + + it "idempotently creates an index or index template, avoiding unneeded datastore write calls" do + expect { + configure_index_definition(schema_def) + }.to change { get_index_definition_configuration(unique_index_name) } + .from({}) + .to(a_hash_including( + "mappings" => a_hash_including("properties" => a_hash_including("id", "name", "options", "created_at")), + "settings" => a_kind_of(Hash) + )) + .and make_datastore_calls_to_configure_index_def(unique_index_name) + + expect { + configure_index_definition(schema_def) + }.to maintain { get_index_definition_configuration(unique_index_name) } + .and make_no_datastore_write_calls("main") + + expect(output_io.string).not_to include(mapping_removal_note_snippet) + end + + it "allows new top-level fields to be added to an existing index or index template" do + configure_index_definition(schema_def) + output_io.string = +"" # use `+` so it is not a frozen string literal. + + expect { + configure_index_definition(schema_def(configure_widget: ->(t) { t.field "amount_cents", "Int" })) + }.to change { get_index_definition_configuration(unique_index_name).dig("mappings", "properties").keys.sort } + .from([*index_meta_fields, "created_at", "id", "name", "options"]) + .to([*index_meta_fields, "amount_cents", "created_at", "id", "name", "options"]) + .and make_datastore_calls_to_configure_index_def(unique_index_name, :mappings) + + # The printed description of what was changed should not mention settings that are not being updated. + # (Requires us to normalize settings properly in the logic for this to be the case). + expect(output_io.string).to include("properties.amount_cents").and exclude("coerce", "ignore_malformed", "number_of_replicas", "number_of_shards") + expect(output_io.string).not_to include(mapping_removal_note_snippet) + end + + it "handles both `object` lists and `nested` lists" do + schema_def = schema_def(configure_widget: ->(t) { + t.field "object_list", "[WidgetOptions!]!" do |f| + f.mapping type: "object" + end + + t.field "nested_list", "[WidgetOptions!]!" do |f| + f.mapping type: "nested" + end + }) + + expect { + configure_index_definition(schema_def) + }.to change { get_index_definition_configuration(unique_index_name) } + .from({}) + .to(a_hash_including("mappings" => a_hash_including("properties" => a_hash_including("object_list", "nested_list")))) + .and make_datastore_calls_to_configure_index_def(unique_index_name) + + expect { + configure_index_definition(schema_def) + }.to maintain { get_index_definition_configuration(unique_index_name) } + .and make_no_datastore_write_calls("main") + end + + it "allows new fields on an embedded object to be added to an existing index or index template" do + configure_index_definition(schema_def) + + expect { + configure_index_definition(schema_def(configure_widget_options: ->(t) { t.field "weight", "Int" })) + }.to change { get_index_definition_configuration(unique_index_name).dig("mappings", "properties", "options", "properties").keys.sort } + .from(["color", "size"]) + .to(["color", "size", "weight"]) + .and make_datastore_calls_to_configure_index_def(unique_index_name, :mappings) + end + + it "does not support changing a field's mapping type on an existing index (since the datastore does not support it) or from an existing index template (for behavior consistency)" do + configure_index_definition(schema_def) + + expect { + configure_index_definition(schema_def( + avoid_defining_widget_fields: %w[name], + configure_widget: ->(t) { t.field "name", "Int" } + )) + }.to raise_error(Errors::IndexOperationError, /name/) + end + + it "supports adding a dynamic mapping param on an existing field on an existing index or index template" do + configure_index_definition(schema_def) + + expect { + configure_index_definition(schema_def( + avoid_defining_widget_fields: %w[name], + configure_widget: ->(t) { + t.field "name", "String" do |f| + f.mapping meta: {foo: "1"} + end + } + )) + }.to change { get_index_definition_configuration(unique_index_name).dig("mappings", "properties", "name") } + .from({"type" => "keyword"}) + .to({"type" => "keyword", "meta" => {"foo" => "1"}}) + .and make_datastore_calls_to_configure_index_def(unique_index_name, :mappings) + end + + it "supports removing a dynamic mapping param on an existing field on an existing index or index template" do + configure_index_definition(schema_def( + avoid_defining_widget_fields: %w[name], + configure_widget: ->(t) { + t.field "name", "String" do |f| + f.mapping meta: {foo: "1"} + end + } + )) + + expect { + configure_index_definition(schema_def) + }.to change { get_index_definition_configuration(unique_index_name).dig("mappings", "properties", "name") } + .from({"type" => "keyword", "meta" => {"foo" => "1"}}) + .to({"type" => "keyword"}) + .and make_datastore_calls_to_configure_index_def(unique_index_name, :mappings) + end + + it "allows some previously unset settings to be changed on an existing index or index template" do + configure_index_definition(schema_def) + + expect { + configure_index_definition(schema_def(refresh_interval: "5s")) + }.to change { get_index_definition_configuration(unique_index_name).fetch("settings").keys } + .from(a_collection_excluding("index.refresh_interval")) + .to(a_collection_including("index.refresh_interval")) + .and make_datastore_calls_to_configure_index_def(unique_index_name, :settings) + end + + it "allows some previously set index or index template settings to be restored to defaults" do + configure_index_definition(schema_def(refresh_interval: "5s")) + output_io.string = +"" # use `+` so it is not a frozen string literal. + + expect { + configure_index_definition(schema_def) + }.to change { get_index_definition_configuration(unique_index_name).fetch("settings") } + .from(a_collection_including("index.refresh_interval")) + .to(a_collection_excluding("index.refresh_interval")) + .and make_datastore_calls_to_configure_index_def(unique_index_name, :settings) + + # The printed description of what was changed should not mention settings that are not being updated. + # (Requires us to normalize settings properly in the logic for this to be the case). + expect(output_io.string).to include("index.refresh_interval").and exclude("coerce", "ignore_malformed", "number_of_replicas", "number_of_shards") + end + + it "ignores the `index.version.upgraded` read-only index or index template setting that can apparently be returned by `indices.get` on an upgraded cluster" do + configure_index_definition(schema_def) + + expect { + configure_index_definition(schema_def, customize_admin: lambda do |admin| + # Add in the weird index setting we are seeing on AWS but cannot find any documentation about + # what it means and when it is present. It appears to be related to upgrading a cluster, which + # is hard to setup in our tests, so we adding it in here. + simulate_presence_of_extra_setting(admin, unique_index_name, "index.version.upgraded", "7070099") + end) + }.to make_no_datastore_write_calls("main") + end + + it "is a no-op when we attempt to drop a field because the datastore doesn't support dropping mapping fields" do + configure_index_definition(schema_def) + + expect { + # Here we remove the `name` field and the `options.size` field to verify it works for both root and nested fields. + configure_index_definition(schema_def( + avoid_defining_widget_fields: %w[name], + avoid_defining_widget_options_fields: %w[size] + )) + }.to maintain { + props = get_index_definition_configuration(unique_index_name).dig("mappings", "properties") + [props.keys.sort, props.dig("options", "properties").keys.sort] + }.from([[*index_meta_fields, "created_at", "id", "name", "options"], ["color", "size"]]) + .and make_datastore_calls_to_configure_index_def(unique_index_name, :mappings) + + expect(output_io.string).to include(mapping_removal_note_snippet) + end + + it "maintains `_meta.ElasticGraph.sources` as a stateful append-only-set that remembers sources that were once active but we no longer have" do + expect { + configure_index_definition(schema_def( + configure_widget: lambda do |t| + t.relates_to_one "owner", "WidgetOwner", via: "widget_ids", dir: :in do |rel| + rel.equivalent_field "created_at" + end + + t.field "owner_name", "String" do |f| + f.sourced_from "owner", "name" + end + end + )) + }.to change { get_index_definition_configuration(unique_index_name).dig("mappings", "_meta") } + .from(nil) + .to({"ElasticGraph" => {"sources" => ["__self", "owner"]}}) + + expect { + configure_index_definition(schema_def( + configure_widget: lambda do |t| + t.relates_to_one "owner2", "WidgetOwner", via: "widget_ids", dir: :in do |rel| + rel.equivalent_field "created_at" + end + + t.field "owner_name", "String" do |f| + f.sourced_from "owner2", "name" + end + end + )) + }.to change { get_index_definition_configuration(unique_index_name).dig("mappings", "_meta") } + .from({"ElasticGraph" => {"sources" => ["__self", "owner"]}}) + .to({"ElasticGraph" => {"sources" => ["__self", "owner", "owner2"]}}) + end + + it "allows index sorting to be configured so long as there are no fields using the `nested` mapping type" do + expect { + configure_index_definition(schema_def(sort: {field: ["created_at"], order: ["asc"]})) + }.to change { + settings = get_index_definition_configuration(unique_index_name)["settings"] || {} + [settings["index.sort.field"], settings["index.sort.order"]] + }.from([nil, nil]).to([["created_at"], ["asc"]]) + end + + def schema_def( + configure_index: nil, + configure_widget: nil, + configure_widget_options: nil, + avoid_defining_widget_fields: [], + avoid_defining_widget_options_fields: [], + define_no_widget_fields: false, + **index_settings + ) + define_widget_field = lambda do |t, name, type| + return if avoid_defining_widget_fields.include?(name) + return if define_no_widget_fields + t.field name, type + end + + define_widget_options_field = lambda do |t, name, type| + return if avoid_defining_widget_options_fields.include?(name) + t.field name, type + end + + lambda do |schema| + schema.object_type "WidgetOwner" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "widget_ids", "[ID!]!" + t.field "created_at", "DateTime" + t.index "#{unique_index_name}_owners" + end + + schema.object_type "WidgetOptions" do |t| + define_widget_options_field.call(t, "size", "String") + define_widget_options_field.call(t, "color", "String") + + configure_widget_options&.call(t) + end + + schema.object_type "Widget" do |t| + t.field "id", "ID!" # can't be omitted. + + define_widget_field.call(t, "name", "String") + define_widget_field.call(t, "options", "WidgetOptions") + define_widget_field.call(t, "created_at", "DateTime!") + + configure_widget&.call(t) + + t.index unique_index_name, **index_settings, &configure_index + end + end + end + + def configure_index_definition(schema_def, schema_artifacts_directory: nil, customize_admin: nil, &customize_datastore_config) + admin = admin_for(schema_def, schema_artifacts_directory: schema_artifacts_directory, &customize_datastore_config) + configure_index_def_using_admin(admin, &customize_admin) + end + + def configure_index_def_using_admin(admin, try_dry_run: true, &customize_admin) + if try_dry_run + # To relieve us of the need of having to write manual test coverage for each possible configuration + # action, to verify that it's `dry_run` mode works correctly, here we test it automatically. + # Before performing the action with the normal datastore client, we first try it with the dry + # run client to prove that it makes no write calls to the datastore. + expect do + configure_index_def_using_admin(admin.with_dry_run_datastore_clients, try_dry_run: false, &customize_admin) + rescue Errors::IndexOperationError + # some tests trigger this intentionally; ignore it if so + end.to make_no_datastore_write_calls("main") + end + + customize_admin&.call(admin) + + index_definition = admin.datastore_core.index_definitions_by_name.fetch(unique_index_name) + artifact_configuration = fetch_artifact_configuration(admin.datastore_core.schema_artifacts, index_definition.name) + + configurator = IndexDefinitionConfigurator.new( + # Note: this MUST use a datastore client off the application so that a dry-run client is used + # when needed, rather than using `main_datastore_client` (provided by the `:uses_datastore` tag), which + # will not be a dry-run client ever. + admin.datastore_core.clients_by_name.fetch("main"), + index_definition, + artifact_configuration, + output_io, + clock + ) + + if (errors = configurator.validate).any? + raise Errors::IndexOperationError, errors.join("; ") + end + + configurator.configure! + end + + def admin_for(schema_def, **options, &customize_datastore_config) + build_admin(schema_definition: schema_def, **options, &customize_datastore_config) + end + end + end + end +end diff --git a/elasticgraph-admin/spec/integration/elastic_graph/admin/rake_tasks_spec.rb b/elasticgraph-admin/spec/integration/elastic_graph/admin/rake_tasks_spec.rb new file mode 100644 index 00000000..6d3927ce --- /dev/null +++ b/elasticgraph-admin/spec/integration/elastic_graph/admin/rake_tasks_spec.rb @@ -0,0 +1,110 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin/rake_tasks" +require "elastic_graph/constants" + +module ElasticGraph + class Admin + RSpec.describe RakeTasks, :rake_task do + describe "clusters:configure", :uses_datastore do + describe ":perform" do + it "updates the settings and mappings in the datastore, then verifies the index consistency" do + admin = build_admin + + expect { + output = run_rake(admin, "clusters:configure:perform") + expect(output.lines).to include a_string_including("Finished updating datastore clusters.") + }.to change { main_datastore_client.get_index(unique_index_name) }.from({}).to(a_hash_including("mappings")) + .and change { main_datastore_client.get_index("#{unique_index_name}2") }.from({}).to(a_hash_including("mappings")) + .and change { datastore_write_requests("main") } + + expect(admin.datastore_indexing_router).to have_received(:validate_mapping_completeness_of!).with( + :all_accessible_cluster_names, + an_object_having_attributes(name: unique_index_name), + an_object_having_attributes(name: "#{unique_index_name}2") + ) + end + + it "works when the cluster configuration has omitted a named cluster" do + admin = build_admin(index_definitions: { + unique_index_name => config_index_def_of, + "#{unique_index_name}2" => config_index_def_of( + # `undefined` is not in the `clusters` map + query_cluster: "undefined", + index_into_clusters: ["undefined"] + ) + }) + + expect { + output = run_rake(admin, "clusters:configure:perform") + expect(output.lines).to include a_string_including("Finished updating datastore clusters.") + }.to change { main_datastore_client.get_index(unique_index_name) }.from({}).to(a_hash_including("mappings")) + .and maintain { main_datastore_client.get_index("#{unique_index_name}2") }.from({}) + .and change { datastore_write_requests("main") } + + expect(admin.datastore_indexing_router).to have_received(:validate_mapping_completeness_of!).with( + :all_accessible_cluster_names, + an_object_having_attributes(name: unique_index_name) + ) + end + end + + describe ":dry_run" do + it "dry-runs the settings and mappings in the datastore, but does not verify the index consistency" do + admin = build_admin + + expect { + output = run_rake(admin, "clusters:configure:dry_run") + expect(output.lines.join("\n")).to include("dry-run", unique_index_name, "#{unique_index_name}2") + }.to maintain { main_datastore_client.get_index(unique_index_name) }.from({}) + .and maintain { main_datastore_client.get_index("#{unique_index_name}2") }.from({}) + .and make_no_datastore_write_calls("main") + + expect(admin.datastore_indexing_router).not_to have_received(:validate_mapping_completeness_of!) + end + end + + def build_admin(**config_overrides) + schema_def = lambda do |schema| + schema.object_type "Money" do |t| + t.field "currency", "String" + t.field "amount_cents", "Int" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "fees", "[Money!]!" do |f| + f.mapping type: "object" + end + t.index unique_index_name + end + + schema.object_type "Widget2" do |t| + t.field "id", "ID!" + t.field "fees", "[Money!]!" do |f| + f.mapping type: "nested" + end + t.index "#{unique_index_name}2" + end + end + + super(schema_definition: schema_def, **config_overrides).tap do |admin| + allow(admin.datastore_indexing_router).to receive(:validate_mapping_completeness_of!).and_call_original + end + end + end + + def run_rake(admin, *args) + super(*args) do |output| + RakeTasks.new(output: output) { admin } + end + end + end + end +end diff --git a/elasticgraph-admin/spec/spec_helper.rb b/elasticgraph-admin/spec/spec_helper.rb new file mode 100644 index 00000000..19de9337 --- /dev/null +++ b/elasticgraph-admin/spec/spec_helper.rb @@ -0,0 +1,17 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-admin`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. + +RSpec.configure do |config| + config.define_derived_metadata(absolute_file_path: %r{/elasticgraph-admin/}) do |meta| + meta[:builds_admin] = true + end +end diff --git a/elasticgraph-admin/spec/unit/elastic_graph/admin/cluster_configurator/cluster_setting_manager_spec.rb b/elasticgraph-admin/spec/unit/elastic_graph/admin/cluster_configurator/cluster_setting_manager_spec.rb new file mode 100644 index 00000000..01a27f55 --- /dev/null +++ b/elasticgraph-admin/spec/unit/elastic_graph/admin/cluster_configurator/cluster_setting_manager_spec.rb @@ -0,0 +1,151 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin" +require "elastic_graph/admin/cluster_configurator/cluster_settings_manager" + +module ElasticGraph + class Admin + class ClusterConfigurator + RSpec.describe ClusterSettingsManager, :stub_datastore_client do + let(:main_datastore_client) { new_datastore_client("main") } + let(:other1_datastore_client) { new_datastore_client("other1") } + let(:other2_datastore_client) { new_datastore_client("other2") } + let(:admin) { build_admin_with_stubbed_datastore_clients } + + describe "#start_index_maintenance_mode!" do + it "disables auto index creation on the named cluster" do + admin.cluster_settings_manager.start_index_maintenance_mode!("other1") + + expect(main_datastore_client).to have_left_cluster_auto_create_index_setting_unchanged + expect(other1_datastore_client).to change_cluster_auto_create_index_setting_to("+.kibana*") + end + + it "disables auto index creation on all clusters when passed `:all_clusters`" do + admin.cluster_settings_manager.start_index_maintenance_mode!(:all_clusters) + + expect(main_datastore_client).to change_cluster_auto_create_index_setting_to("+.kibana*") + expect(other1_datastore_client).to change_cluster_auto_create_index_setting_to("+.kibana*") + expect(other2_datastore_client).to change_cluster_auto_create_index_setting_to("+.kibana*") + end + end + + describe "#end_index_maintenance_mode!" do + it "enables auto index creation on the named cluster" do + admin.cluster_settings_manager.end_index_maintenance_mode!("other1") + + expect(main_datastore_client).to have_left_cluster_auto_create_index_setting_unchanged + expect(other1_datastore_client).to change_cluster_auto_create_index_setting_to("+.kibana*", "+*_rollover__*") + end + + it "enables auto index creation on all clusters when passed `:all_clusters`" do + admin.cluster_settings_manager.end_index_maintenance_mode!(:all_clusters) + + expect(main_datastore_client).to change_cluster_auto_create_index_setting_to("+.kibana*", "+*_rollover__*") + expect(other1_datastore_client).to change_cluster_auto_create_index_setting_to("+.kibana*", "+*_rollover__*") + expect(other2_datastore_client).to change_cluster_auto_create_index_setting_to("+.kibana*", "+*_rollover__*") + end + end + + describe "#in_index_maintenance_mode", :capture_logs do + it "runs the block while auto index creation is disabled on the named cluster, re-enabling it afterward" do + admin.cluster_settings_manager.in_index_maintenance_mode("other1") do + end + + expect(main_datastore_client).to have_left_cluster_auto_create_index_setting_unchanged + expect(other1_datastore_client).to change_cluster_auto_create_index_setting_to("+.kibana*").ordered + expect(other1_datastore_client).to change_cluster_auto_create_index_setting_to("+.kibana*", "+*_rollover__*").ordered + end + + it "leaves maintenance mode enabled if an exception occurs in the block, to guard against indices being created with the wrong settings" do + expect { + admin.cluster_settings_manager.in_index_maintenance_mode("other1") do + raise "boom" + end + }.to raise_error("boom").and log_warning(a_string_including("in_index_maintenance_mode is not able to exit index maintenance mode")) + + expect(main_datastore_client).to have_left_cluster_auto_create_index_setting_unchanged + expect(other1_datastore_client).to change_cluster_auto_create_index_setting_to("+.kibana*").ordered + expect(other1_datastore_client).not_to change_cluster_auto_create_index_setting_to("+.kibana*", "+*_rollover__*") + end + + it "applies to all clusters when given `:all_clusters`" do + admin.cluster_settings_manager.in_index_maintenance_mode(:all_clusters) do + end + + expect(main_datastore_client).to change_cluster_auto_create_index_setting_to("+.kibana*").ordered + expect(other1_datastore_client).to change_cluster_auto_create_index_setting_to("+.kibana*").ordered + expect(other2_datastore_client).to change_cluster_auto_create_index_setting_to("+.kibana*").ordered + + expect(main_datastore_client).to change_cluster_auto_create_index_setting_to("+.kibana*", "+*_rollover__*").ordered + expect(other1_datastore_client).to change_cluster_auto_create_index_setting_to("+.kibana*", "+*_rollover__*").ordered + expect(other2_datastore_client).to change_cluster_auto_create_index_setting_to("+.kibana*", "+*_rollover__*").ordered + end + end + + it "raises a clear error when given an unknown cluster name" do + expect { + admin.cluster_settings_manager.start_index_maintenance_mode!("unknown") + }.to raise_error Errors::ClusterOperationError, a_string_including("unknown", "main", "other1", "other2") + + expect { + admin.cluster_settings_manager.end_index_maintenance_mode!("unknown") + }.to raise_error Errors::ClusterOperationError, a_string_including("unknown", "main", "other1", "other2") + + expect { + admin.cluster_settings_manager.in_index_maintenance_mode("unknown") do + end + }.to raise_error Errors::ClusterOperationError, a_string_including("unknown", "main", "other1", "other2") + end + + it "favors cluster settings in app configuration over defaults" do + cluster_setting_overrides = { + "indices.recovery.max_concurrent_operations" => 2, # new setting + "search.allow_expensive_queries" => false # setting override + } + + admin = build_admin_with_stubbed_datastore_clients do |config| + config.with(clusters: config.clusters.merge( + "other1" => config.clusters.fetch("main").with(settings: cluster_setting_overrides) + )) + end + + admin.cluster_settings_manager.end_index_maintenance_mode!("main") + expect(main_datastore_client).to have_received(:put_persistent_cluster_settings).with( + a_hash_excluding("indices.recovery.max_concurrent_operations") + ) + + admin.cluster_settings_manager.end_index_maintenance_mode!("other1") + expect(other1_datastore_client).to have_received(:put_persistent_cluster_settings).with( + a_hash_including("indices.recovery.max_concurrent_operations" => 2, "search.allow_expensive_queries" => false) + ) + end + + def build_admin_with_stubbed_datastore_clients(**options, &block) + build_admin(clients_by_name: { + "main" => main_datastore_client, + "other1" => other1_datastore_client, + "other2" => other2_datastore_client + }, **options, &block) + end + + def new_datastore_client(name) + instance_double("ElasticGraph::Elasticsearch::Client", name, get_flat_cluster_settings: {"persistent" => {}}, put_persistent_cluster_settings: nil) + end + + def have_left_cluster_auto_create_index_setting_unchanged + have_never_received(:put_persistent_cluster_settings) + end + + def change_cluster_auto_create_index_setting_to(*expressions) + have_received(:put_persistent_cluster_settings).with(a_hash_including("action.auto_create_index" => expressions.join(","))) + end + end + end + end +end diff --git a/elasticgraph-admin/spec/unit/elastic_graph/admin/rake_tasks_spec.rb b/elasticgraph-admin/spec/unit/elastic_graph/admin/rake_tasks_spec.rb new file mode 100644 index 00000000..98211237 --- /dev/null +++ b/elasticgraph-admin/spec/unit/elastic_graph/admin/rake_tasks_spec.rb @@ -0,0 +1,210 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin/rake_tasks" + +module ElasticGraph + class Admin + RSpec.describe RakeTasks, :rake_task do + let(:main_datastore_client) { stubbed_datastore_client } + let(:other_datastore_client) { stubbed_datastore_client } + + it "evaluates the admin load block lazily so that if loading fails it only interferes with running tasks, not defining them" do + # Simulate schema artifacts being out of date. + load_admin = lambda { raise "schema artifacts are out of date" } + + # Printing tasks is unaffected... + output = run_rake("--tasks", &load_admin) + expect(output).to eq(<<~EOS) + rake clusters:configure:dry_run # Dry-runs the configuration of datastore clusters, including indices, settings, and scripts + rake clusters:configure:perform # Performs the configuration of datastore clusters, including indices, settings, and scripts + rake indices:drop[index_def_name,cluster_name] # Drops the specified index definition on the specified datastore cluster + rake indices:drop_prototypes # Drops all prototype index definitions on all datastore clusters + EOS + + # ...but when you run a task that needs the `Admin` instance the failure is surfaced. + expect { + run_rake("clusters:configure:dry_run", &load_admin) + }.to raise_error "schema artifacts are out of date" + end + + describe "indices:drop_prototypes" do + it "puts cluster in index maintenance mode, then drops all index definitions named in `prototype_index_names` but none others" do + admin = admin_with_schema do |schema| + schema.object_type "Widget1" do |t| + t.field "id", "ID!" + t.index "widgets1" + end + + schema.object_type "Widget2" do |t| + t.field "id", "ID!" + t.index "widgets2" + end + + schema.object_type "Widget3" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime!" + t.index "widgets3" do |i| + i.rollover :monthly, "created_at" + end + end + + schema.object_type "Widget4" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime!" + t.index "widgets4" do |i| + i.rollover :monthly, "created_at" + end + end + end + + # While we generally try to avoid mocking in the ElasticGraph test suite, here we are doing it because + # if we let the test delete indices for real, it deletes the index configuration that many other tests + # rely on. We could recreate that, but it's pretty slow (takes ~2 seconds) and the value of tests that + # cover this task are relatively low given this task is only ever run by an engineer as an admin + # task and this code is never in the path for a client request. + # + # So we have opted to mock it, as the "least bad" way of testing this. + call_order = [] + expect(admin.cluster_settings_manager).to receive(:start_index_maintenance_mode!) { |arg| call_order << [:start_index_maintenance_mode!, arg] } + expect(main_datastore_client).to receive(:delete_index_template).with("widgets4") { call_order << :delete_template } + expect(main_datastore_client).to receive(:delete_indices).with("widgets2") { call_order << :delete } + expect(other_datastore_client).to receive(:delete_index_template).with("widgets4") { call_order << :delete_template } + expect(other_datastore_client).to receive(:delete_indices).with("widgets2") { call_order << :delete } + + expect(main_datastore_client).not_to receive(:delete_index_template).with("widgets3") + expect(main_datastore_client).not_to receive(:delete_indices).with("widgets1") + expect(other_datastore_client).not_to receive(:delete_index_template).with("widgets3") + expect(other_datastore_client).not_to receive(:delete_indices).with("widgets1") + + output = run_rake("indices:drop_prototypes", prototype_index_names: ["widgets2", "widgets4"]) { admin } + + expect(output.lines).to include(a_string_including("Disabled rollover index auto creation for all clusters")) + expect(output.lines).to include(a_string_including("Dropping the following prototype index definitions", "widgets2", "widgets4")) + expect(output.lines).to include(a_string_including("Finished dropping all prototype index definitions")) + + expect(call_order).to start_with([:start_index_maintenance_mode!, :all_clusters]) + end + + it "ignores indices that do not reside on an accessible datastore cluster" do + admin = admin_with_schema( + clusters: {"main" => cluster_of}, + index_definitions: { + "widgets1" => config_index_def_of(query_cluster: "main", index_into_clusters: ["main", "other"]), + "widgets2" => config_index_def_of(query_cluster: "other", index_into_clusters: ["other"]) + } + ) do |schema| + schema.object_type "Widget1" do |t| + t.field "id", "ID!" + t.index "widgets1" + end + + schema.object_type "Widget2" do |t| + t.field "id", "ID!" + t.index "widgets2" + end + end + + allow(admin.cluster_settings_manager).to receive(:start_index_maintenance_mode!) + expect(main_datastore_client).to receive(:delete_indices) + expect(other_datastore_client).not_to receive(:delete_indices) + + output = run_rake("indices:drop_prototypes", prototype_index_names: ["widgets1", "widgets2"]) { admin } + + expect(output.lines).to include(a_string_including("Dropping the following prototype index definitions", "widgets1").and(excluding("widgets2"))) + end + end + + describe "indices:drop" do + it "does not drop the index if it is not listed in `prototype_index_names`" do + admin = admin_with_widgets + + expect(admin.cluster_settings_manager).not_to receive(:start_index_maintenance_mode!) + expect(main_datastore_client).not_to receive(:delete_indices).with("*") + expect(other_datastore_client).not_to receive(:delete_indices).with("*") + expect(admin.cluster_settings_manager).not_to receive(:end_index_maintenance_mode!) + + expect { + run_rake("indices:drop[widgets, other]", prototype_index_names: ["components"]) { admin } + }.to raise_error(Errors::IndexOperationError, a_string_including("widgets", "live index", "prototype_index_names")) + end + + it "drops the index on the specified datastore cluster if it is listed in `prototype_index_names`" do + admin = admin_with_widgets + + expect(admin.cluster_settings_manager).to receive(:start_index_maintenance_mode!).with("other").ordered + expect(main_datastore_client).not_to receive(:delete_indices).with("widgets") + expect(other_datastore_client).to receive(:delete_indices).with("widgets").ordered + expect(admin.cluster_settings_manager).to receive(:end_index_maintenance_mode!).with("other").ordered + + output = run_rake("indices:drop[widgets, other]", prototype_index_names: ["widgets"]) { admin } + expect(output.lines).to include(a_string_including("Disabled rollover index auto creation for this cluster")) + expect(output.lines).to include(a_string_including("Dropped index")) + end + + it "fails with a clear error if the specified cluster does not exist" do + admin = admin_with_widgets + + expect { + run_rake("indices:drop[widgets, typo]", prototype_index_names: ["widgets"]) { admin } + }.to raise_error Errors::IndexOperationError, a_string_including('Cluster named `typo` does not exist. Valid clusters: ["main", "other"]') + end + end + + def admin_with_schema( + clusters: {"main" => cluster_of, "other" => cluster_of}, + index_definitions: Hash.new do |h, k| + h[k] = config_index_def_of(query_cluster: "main", index_into_clusters: ["main", "other"]) + end, + &schema_definition + ) + build_admin( + schema_definition: schema_definition, + clients_by_name: {"main" => main_datastore_client, "other" => other_datastore_client} + ) do |config| + config.with( + clusters: clusters, + index_definitions: index_definitions + ) + end + end + + def admin_with_widgets + admin_with_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.index "widgets" + end + end + end + + def run_rake(*args, prototype_index_names: [], &load_admin) + super(*args) do |output| + RakeTasks.new(prototype_index_names: prototype_index_names, output: output, &load_admin) + end + end + end + + # This is a separate example group because it needs `run_rake` to be defined differently from the group above. + RSpec.describe RakeTasks, ".from_yaml_file", :rake_task do + it "loads the admin instance from the given yaml file" do + output = run_rake "--tasks" + + expect(output).to include("rake clusters") + end + + def run_rake(*args) + super(*args) do |output| + RakeTasks.from_yaml_file(CommonSpecHelpers.test_settings_file, output: output).tap do |tasks| + expect(tasks.send(:admin)).to be_a(Admin) + end + end + end + end + end +end diff --git a/elasticgraph-admin/spec/unit/elastic_graph/admin_spec.rb b/elasticgraph-admin/spec/unit/elastic_graph/admin_spec.rb new file mode 100644 index 00000000..e348d874 --- /dev/null +++ b/elasticgraph-admin/spec/unit/elastic_graph/admin_spec.rb @@ -0,0 +1,27 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin" + +module ElasticGraph + RSpec.describe Admin do + it "returns non-nil values from each attribute" do + expect_to_return_non_nil_values_from_all_attributes(build_admin) + end + + describe ".from_parsed_yaml" do + it "builds an Admin instance from the contents of a YAML settings file" do + customization_block = lambda { |conn| } + admin = Admin.from_parsed_yaml(parsed_test_settings_yaml, &customization_block) + + expect(admin).to be_a(Admin) + expect(admin.datastore_core.client_customization_block).to be(customization_block) + end + end + end +end diff --git a/elasticgraph-admin_lambda/.rspec b/elasticgraph-admin_lambda/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-admin_lambda/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-admin_lambda/.yardopts b/elasticgraph-admin_lambda/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-admin_lambda/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-admin_lambda/Gemfile b/elasticgraph-admin_lambda/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-admin_lambda/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-admin_lambda/LICENSE.txt b/elasticgraph-admin_lambda/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-admin_lambda/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-admin_lambda/README.md b/elasticgraph-admin_lambda/README.md new file mode 100644 index 00000000..1729c587 --- /dev/null +++ b/elasticgraph-admin_lambda/README.md @@ -0,0 +1,3 @@ +# ElasticGraph::AdminLambda + +This gem wraps `elasticgraph-admin` in order to run it from an AWS Lambda. diff --git a/elasticgraph-admin_lambda/elasticgraph-admin_lambda.gemspec b/elasticgraph-admin_lambda/elasticgraph-admin_lambda.gemspec new file mode 100644 index 00000000..238ba1d9 --- /dev/null +++ b/elasticgraph-admin_lambda/elasticgraph-admin_lambda.gemspec @@ -0,0 +1,19 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :lambda) do |spec, eg_version| + spec.summary = "ElasticGraph gem that wraps elasticgraph-admin in an AWS Lambda." + + spec.add_dependency "rake", "~> 13.2" + + spec.add_dependency "elasticgraph-admin", eg_version + spec.add_dependency "elasticgraph-lambda_support", eg_version + spec.add_development_dependency "httpx", "~> 1.3" +end diff --git a/elasticgraph-admin_lambda/lib/elastic_graph/admin_lambda.rb b/elasticgraph-admin_lambda/lib/elastic_graph/admin_lambda.rb new file mode 100644 index 00000000..c3ad86a7 --- /dev/null +++ b/elasticgraph-admin_lambda/lib/elastic_graph/admin_lambda.rb @@ -0,0 +1,20 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin" +require "elastic_graph/lambda_support" + +module ElasticGraph + # @private + module AdminLambda + # Builds an `ElasticGraph::Admin` instance from our lambda ENV vars. + def self.admin_from_env + LambdaSupport.build_from_env(Admin) + end + end +end diff --git a/elasticgraph-admin_lambda/lib/elastic_graph/admin_lambda/Rakefile b/elasticgraph-admin_lambda/lib/elastic_graph/admin_lambda/Rakefile new file mode 100644 index 00000000..a7657180 --- /dev/null +++ b/elasticgraph-admin_lambda/lib/elastic_graph/admin_lambda/Rakefile @@ -0,0 +1,13 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin/rake_tasks" +require "elastic_graph/admin_lambda" + +admin = ElasticGraph::AdminLambda.admin_from_env +ElasticGraph::Admin::RakeTasks.new { admin } diff --git a/elasticgraph-admin_lambda/lib/elastic_graph/admin_lambda/lambda_function.rb b/elasticgraph-admin_lambda/lib/elastic_graph/admin_lambda/lambda_function.rb new file mode 100644 index 00000000..8607ff82 --- /dev/null +++ b/elasticgraph-admin_lambda/lib/elastic_graph/admin_lambda/lambda_function.rb @@ -0,0 +1,36 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/lambda_support/lambda_function" + +module ElasticGraph + module AdminLambda + # @private + class LambdaFunction + prepend LambdaSupport::LambdaFunction + + def initialize + require "elastic_graph/admin_lambda/rake_adapter" + end + + def handle_request(event:, context:) + # @type var event: ::Hash[::String, untyped] + rake_output = RakeAdapter.run_rake(event.fetch("argv")) + + # Log the output of the rake task. We also want to return it so that when we invoke + # a lambda rake task from the terminal we can print the output there. + puts rake_output + + {"rake_output" => rake_output} + end + end + end +end + +# Lambda handler for `elasticgraph-admin_lambda`. +HandleAdminRequest = ElasticGraph::AdminLambda::LambdaFunction.new diff --git a/elasticgraph-admin_lambda/lib/elastic_graph/admin_lambda/rake_adapter.rb b/elasticgraph-admin_lambda/lib/elastic_graph/admin_lambda/rake_adapter.rb new file mode 100644 index 00000000..bb7ea280 --- /dev/null +++ b/elasticgraph-admin_lambda/lib/elastic_graph/admin_lambda/rake_adapter.rb @@ -0,0 +1,71 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "rake" +require "tempfile" + +module ElasticGraph + module AdminLambda + # @private + class RakeAdapter + RAKEFILE = File.expand_path("../Rakefile", __FILE__) + + def self.run_rake(argv) + capture_output do + # We need to instantiate a new application on each invocation, because Rake is normally + # designed to run once and exit. It keeps track of tasks that have already run and will + # no-op when you try to run a task a 2nd or 3rd time. Using a new application instance + # each time avoids this issue. + ::Rake.with_application(Application.new) do |application| + application.run(argv) + end + end + end + + # Captures stdout/stderr into a string so we can return it from the lambda. + # Inspired by a similar utility in RSpec: + # https://github.com/rspec/rspec-expectations/blob/v3.9.2/lib/rspec/matchers/built_in/output.rb#L172-L197 + def self.capture_output + original_stdout = $stdout.clone + original_stderr = $stderr.clone + captured_stream = Tempfile.new + + begin + captured_stream.sync = true + $stdout.reopen(captured_stream) + $stderr.reopen(captured_stream) + + yield + + captured_stream.rewind + captured_stream.read + ensure + $stdout.reopen(original_stdout) + $stderr.reopen(original_stderr) + captured_stream.close + captured_stream.unlink + end + end + + # A subclass that forces rake to use our desired Rakefile, and configures Rake to act + # a bit different. + class Application < ::Rake::Application + def initialize + super + @rakefiles = [RAKEFILE] + end + + # Rake defines this to catch exceptions and call `exit(false)`, but we do not want + # that behavior. We want to let lambda handle exceptions like normal. + def standard_exception_handling + yield + end + end + end + end +end diff --git a/elasticgraph-admin_lambda/sig/elastic_graph/admin_lambda.rbs b/elasticgraph-admin_lambda/sig/elastic_graph/admin_lambda.rbs new file mode 100644 index 00000000..ecb815fa --- /dev/null +++ b/elasticgraph-admin_lambda/sig/elastic_graph/admin_lambda.rbs @@ -0,0 +1,5 @@ +module ElasticGraph + module AdminLambda + def self.admin_from_env: () -> Admin + end +end diff --git a/elasticgraph-admin_lambda/sig/elastic_graph/admin_lambda/lambda_function.rbs b/elasticgraph-admin_lambda/sig/elastic_graph/admin_lambda/lambda_function.rbs new file mode 100644 index 00000000..77b70591 --- /dev/null +++ b/elasticgraph-admin_lambda/sig/elastic_graph/admin_lambda/lambda_function.rbs @@ -0,0 +1,10 @@ +module ElasticGraph + module AdminLambda + class LambdaFunction + include LambdaSupport::LambdaFunction[::Hash[::String, untyped]] + include LambdaSupport::_LambdaFunctionClass[::Hash[::String, untyped]] + end + end +end + +HandleAdminRequest: ElasticGraph::AdminLambda::LambdaFunction diff --git a/elasticgraph-admin_lambda/sig/elastic_graph/admin_lambda/rake_adapter.rbs b/elasticgraph-admin_lambda/sig/elastic_graph/admin_lambda/rake_adapter.rbs new file mode 100644 index 00000000..ef6d936e --- /dev/null +++ b/elasticgraph-admin_lambda/sig/elastic_graph/admin_lambda/rake_adapter.rbs @@ -0,0 +1,15 @@ +module ElasticGraph + module AdminLambda + class RakeAdapter + RAKEFILE: ::String + + def self.run_rake: (::Array[::String]) -> ::String? + def self.capture_output: () { () -> void } -> ::String? + + class Application < ::Rake::Application + # Steep 1.6 complains about our impl returning `nil` unless we define this as void + def initialize: () -> void + end + end + end +end diff --git a/elasticgraph-admin_lambda/sig/rake.rbs b/elasticgraph-admin_lambda/sig/rake.rbs new file mode 100644 index 00000000..724cbb82 --- /dev/null +++ b/elasticgraph-admin_lambda/sig/rake.rbs @@ -0,0 +1,10 @@ +module Rake + def self.with_application: (Application) { (Application) -> void } -> void + + class Application + @rakefiles: ::Array[::String] + + def run: (::Array[::String]) -> void + def standard_exception_handling: () { () -> void } -> void + end +end diff --git a/elasticgraph-admin_lambda/spec/integration/elastic_graph/admin_lambda/rake_adapter_spec.rb b/elasticgraph-admin_lambda/spec/integration/elastic_graph/admin_lambda/rake_adapter_spec.rb new file mode 100644 index 00000000..d2b65892 --- /dev/null +++ b/elasticgraph-admin_lambda/spec/integration/elastic_graph/admin_lambda/rake_adapter_spec.rb @@ -0,0 +1,53 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin_lambda/rake_adapter" +require "elastic_graph/spec_support/lambda_function" + +module ElasticGraph + module AdminLambda + RSpec.describe RakeAdapter do + include_context "lambda function" + + around do |ex| + # Ensure the `record_task_metadata` flag is restored to its original value after running the test. + # It impacts how rake task descriptions are processed inside rake, and `rake -T` causes the value + # to change. We have other tests in this repository that will fail if we allow this to leak this + # state change and if they run after this one. + orig = ::Rake::TaskManager.record_task_metadata + ex.run + ::Rake::TaskManager.record_task_metadata = orig + end + + it "runs the provided `argv` against our desired Rakefile, returning the printed output" do + with_lambda_env_vars do + output = RakeAdapter.run_rake(["-T"]) + + expect(rake_tasks_printed_to(output)).to include( + "clusters:configure:perform", + "indices:drop_prototypes" + ).and exclude( + # The `lambda:*` and `terraform:*` rake tasks are defined by elasticgraph-lambda only for + # local usage, and are not intended to be available to be run from the `admin` lambda. + a_string_starting_with("lambda:"), + a_string_starting_with("terraform:"), + # The `schema_artifacts:*` task are only intended for local usage, and should not be available in the lambda context. + a_string_starting_with("schema_artifacts:") + ) + end + end + + def rake_tasks_printed_to(string) + string + .split("\n") + .map { |line| line[/^rake (\S+)/, 1] } + .compact + end + end + end +end diff --git a/elasticgraph-admin_lambda/spec/spec_helper.rb b/elasticgraph-admin_lambda/spec/spec_helper.rb new file mode 100644 index 00000000..aa87fac3 --- /dev/null +++ b/elasticgraph-admin_lambda/spec/spec_helper.rb @@ -0,0 +1,11 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-admin_lambda`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. diff --git a/elasticgraph-admin_lambda/spec/unit/elastic_graph/admin_lambda/lambda_function_spec.rb b/elasticgraph-admin_lambda/spec/unit/elastic_graph/admin_lambda/lambda_function_spec.rb new file mode 100644 index 00000000..5ae08b8e --- /dev/null +++ b/elasticgraph-admin_lambda/spec/unit/elastic_graph/admin_lambda/lambda_function_spec.rb @@ -0,0 +1,23 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/lambda_function" + +RSpec.describe "Admin lambda function" do + include_context "lambda function" + + it "runs rake" do + expect_loading_lambda_to_define_constant( + lambda: "elastic_graph/admin_lambda/lambda_function.rb", + const: :HandleAdminRequest + ) do |lambda_function| + response = lambda_function.handle_request(event: {"argv" => ["-T"]}, context: {}) + expect(response["rake_output"]).to include("rake clusters:configure:dry_run") + end + end +end diff --git a/elasticgraph-admin_lambda/spec/unit/elastic_graph/admin_lambda_spec.rb b/elasticgraph-admin_lambda/spec/unit/elastic_graph/admin_lambda_spec.rb new file mode 100644 index 00000000..7a1aef73 --- /dev/null +++ b/elasticgraph-admin_lambda/spec/unit/elastic_graph/admin_lambda_spec.rb @@ -0,0 +1,23 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin_lambda" +require "elastic_graph/spec_support/lambda_function" + +module ElasticGraph + RSpec.describe AdminLambda do + describe ".admin_from_env" do + include_context "lambda function" + around { |ex| with_lambda_env_vars(&ex) } + + it "builds an admin instance" do + expect(AdminLambda.admin_from_env).to be_an(Admin) + end + end + end +end diff --git a/elasticgraph-apollo/.rspec b/elasticgraph-apollo/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-apollo/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-apollo/.yardopts b/elasticgraph-apollo/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-apollo/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-apollo/Gemfile b/elasticgraph-apollo/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-apollo/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-apollo/LICENSE.txt b/elasticgraph-apollo/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-apollo/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-apollo/README.md b/elasticgraph-apollo/README.md new file mode 100644 index 00000000..7a500108 --- /dev/null +++ b/elasticgraph-apollo/README.md @@ -0,0 +1,63 @@ +# ElasticGraph::Apollo + +Implements the [Apollo Federation Subgraph Spec](https://www.apollographql.com/docs/federation/subgraph-spec/), +allowing an ElasticGraph application to be plugged into an Apollo-powered GraphQL server as a subgraph. + +Note: this library only supports the v2 Federation specification. + +## Usage + +First, add `elasticgraph-apollo` to your `Gemfile`: + +``` ruby +gem "elasticgraph-apollo" +``` + +Finally, update your ElasticGraph schema artifact rake tasks in your `Rakefile` +so that `ElasticGraph::GraphQL::Apollo::SchemaDefinition::APIExtension` is +passed as one of the `extension_modules`: + +``` ruby +require "elastic_graph/schema_definition/rake_tasks" +require "elastic_graph/apollo/schema_definition/api_extension" + +ElasticGraph::SchemaDefinition::RakeTasks.new( + schema_element_name_form: :snake_case, + index_document_sizes: true, + path_to_schema: "config/schema.rb", + schema_artifacts_directory: artifacts_dir, + extension_modules: [ElasticGraph::Apollo::SchemaDefinition::APIExtension] +) +``` + +That's it! + +## Federation Version Support + +This library supports multiple versions of Apollo federation. As of Jan. 2024, it supports: + +* v2.0 +* v2.3 +* v2.5 +* v2.6 + +By default, the newest version is targeted. If you need an older version (e.g. because your organization is +running an older Apollo version), you can configure it in your schema definition with: + +```ruby +schema.target_apollo_federation_version "2.3" +``` + +## Testing Notes + +This project uses https://github.com/apollographql/apollo-federation-subgraph-compatibility +to verify compatibility with Apollo. Things to note: + +- Run `elasticgraph-apollo/script/test_compatibility` to run the compatibility tests (the CI build runs this). +- Run `elasticgraph-apollo/script/boot_eg_apollo_implementation` to boot the ElasticGraph compatibility test implementation (can be useful for debugging `test_compatibility` failures). +- These scripts require some additional dependencies to be installed (such as `docker`, `node`, and `npm`). +- To get that to pass locally on my Mac, I had to enable the `Use Docker Compose V2` flag in Docker Desktop (under "Preferences -> General"). Without that checked, I got errors like this: + +``` +ERROR: for apollo-federation-subgraph-compatibility_router_1 Cannot start service router: OCI runtime create failed: container_linux.go:380: starting container process caused: process_linux.go:545: container init caused: rootfs_linux.go:76: mounting "/host_mnt/Users/myron/Development/sq-elasticgraph-ruby/elasticgraph-apollo/vendor/apollo-federation-subgraph-compatibility/supergraph.graphql" to rootfs at "/etc/config/supergraph.graphql" caused: mount through procfd: not a directory: unknown: Are you trying to mount a directory onto a file (or vice-versa)? Check if the specified host path exists and is the expected type +``` diff --git a/elasticgraph-apollo/apollo_tests_implementation/Dockerfile b/elasticgraph-apollo/apollo_tests_implementation/Dockerfile new file mode 100644 index 00000000..558bc59e --- /dev/null +++ b/elasticgraph-apollo/apollo_tests_implementation/Dockerfile @@ -0,0 +1,66 @@ +ARG RUBY_VERSION +FROM ruby:${RUBY_VERSION} + +ARG TARGET_APOLLO_FEDERATION_VERSION + +WORKDIR /web + +# Each of the elasticgraph gems this implementation depends on must be +# copied into the container so it's available for inclusion in the bundle. +# Note: we've observed that hidden files (like .rspec) are not copied by COPY. +# gemspec_helper enforces the presence of `.rspec`/`.yardopts`, so we have to copy them. +COPY elasticgraph-admin /web/elasticgraph-admin +COPY elasticgraph-admin/.rspec /web/elasticgraph-admin/.rspec +COPY elasticgraph-admin/.yardopts /web/elasticgraph-admin/.yardopts + +COPY elasticgraph-apollo /web/elasticgraph-apollo +COPY elasticgraph-apollo/.rspec /web/elasticgraph-apollo/.rspec +COPY elasticgraph-apollo/.yardopts /web/elasticgraph-apollo/.yardopts + +COPY elasticgraph-datastore_core /web/elasticgraph-datastore_core +COPY elasticgraph-datastore_core/.rspec /web/elasticgraph-datastore_core/.rspec +COPY elasticgraph-datastore_core/.yardopts /web/elasticgraph-datastore_core/.yardopts + +COPY elasticgraph-elasticsearch /web/elasticgraph-elasticsearch +COPY elasticgraph-elasticsearch/.rspec /web/elasticgraph-elasticsearch/.rspec +COPY elasticgraph-elasticsearch/.yardopts /web/elasticgraph-elasticsearch/.yardopts + +COPY elasticgraph-graphql /web/elasticgraph-graphql +COPY elasticgraph-graphql/.rspec /web/elasticgraph-graphql/.rspec +COPY elasticgraph-graphql/.yardopts /web/elasticgraph-graphql/.yardopts + +COPY elasticgraph-indexer /web/elasticgraph-indexer +COPY elasticgraph-indexer/.rspec /web/elasticgraph-indexer/.rspec +COPY elasticgraph-indexer/.yardopts /web/elasticgraph-indexer/.yardopts + +COPY elasticgraph-json_schema /web/elasticgraph-json_schema +COPY elasticgraph-json_schema/.rspec /web/elasticgraph-json_schema/.rspec +COPY elasticgraph-json_schema/.yardopts /web/elasticgraph-json_schema/.yardopts + +COPY elasticgraph-rack /web/elasticgraph-rack +COPY elasticgraph-rack/.rspec /web/elasticgraph-rack/.rspec +COPY elasticgraph-rack/.yardopts /web/elasticgraph-rack/.yardopts + +COPY elasticgraph-schema_artifacts /web/elasticgraph-schema_artifacts +COPY elasticgraph-schema_artifacts/.rspec /web/elasticgraph-schema_artifacts/.rspec +COPY elasticgraph-schema_artifacts/.yardopts /web/elasticgraph-schema_artifacts/.yardopts + +COPY elasticgraph-schema_definition /web/elasticgraph-schema_definition +COPY elasticgraph-schema_definition/.rspec /web/elasticgraph-schema_definition/.rspec +COPY elasticgraph-schema_definition/.yardopts /web/elasticgraph-schema_definition/.yardopts + +COPY elasticgraph-support /web/elasticgraph-support +COPY elasticgraph-support/.rspec /web/elasticgraph-support/.rspec +COPY elasticgraph-support/.yardopts /web/elasticgraph-support/.yardopts + +# We also have to copy the implementation files (config, schema, etc) as well. +COPY elasticgraph-apollo/apollo_tests_implementation /web/ + +COPY gemspec_helper.rb /web/gemspec_helper.rb + +# We need to install the bundle and generate our schema artifacts. +RUN bundle install +RUN bundle exec rake schema_artifacts:dump TARGET_APOLLO_FEDERATION_VERSION=${TARGET_APOLLO_FEDERATION_VERSION} + +# Finally we can boot the app! +CMD ["bundle", "exec", "rackup", "--host", "0.0.0.0", "--port", "4001"] diff --git a/elasticgraph-apollo/apollo_tests_implementation/Gemfile b/elasticgraph-apollo/apollo_tests_implementation/Gemfile new file mode 100644 index 00000000..0b43ce65 --- /dev/null +++ b/elasticgraph-apollo/apollo_tests_implementation/Gemfile @@ -0,0 +1,27 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +source "https://rubygems.org" + +%w[ + admin + apollo + datastore_core + elasticsearch + graphql + indexer + json_schema + rack + schema_artifacts + schema_definition + support +].each do |suffix| + gem "elasticgraph-#{suffix}", path: "elasticgraph-#{suffix}" +end + +gem "rackup", "~> 2.1" diff --git a/elasticgraph-apollo/apollo_tests_implementation/Rakefile b/elasticgraph-apollo/apollo_tests_implementation/Rakefile new file mode 100644 index 00000000..034a8aec --- /dev/null +++ b/elasticgraph-apollo/apollo_tests_implementation/Rakefile @@ -0,0 +1,22 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_definition/rake_tasks" +require "elastic_graph/apollo/schema_definition/api_extension" +require "pathname" + +project_root = Pathname.new(__dir__) + +ElasticGraph::SchemaDefinition::RakeTasks.new( + schema_element_name_form: :camelCase, + index_document_sizes: false, + path_to_schema: project_root / "config/products_schema.rb", + schema_artifacts_directory: project_root / "config/schema/artifacts", + extension_modules: [ElasticGraph::Apollo::SchemaDefinition::APIExtension], + enforce_json_schema_version: false +) diff --git a/elasticgraph-apollo/apollo_tests_implementation/config.ru b/elasticgraph-apollo/apollo_tests_implementation/config.ru new file mode 100644 index 00000000..202807da --- /dev/null +++ b/elasticgraph-apollo/apollo_tests_implementation/config.ru @@ -0,0 +1,122 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin" +require "elastic_graph/graphql" +require "elastic_graph/indexer" +require "elastic_graph/rack/graphiql" +require "elastic_graph/indexer/test_support/converters" + +admin = ElasticGraph::Admin.from_yaml_file("config/settings.yaml") +graphql = ElasticGraph::GraphQL.from_yaml_file("config/settings.yaml") +indexer = ElasticGraph::Indexer.from_yaml_file("config/settings.yaml") + +admin.cluster_configurator.configure_cluster($stdout) + +# Example records expected by the apollo-federation-subgraph-compatibility test suite. based on: +# https://github.com/apollographql/apollo-federation-subgraph-compatibility/blob/2.1.0/COMPATIBILITY.md#expected-data-sets +dimension = { + size: "small", + weight: 1, + unit: "kg" +} + +user = { + id: "1", + averageProductsCreatedPerYear: 133, + email: "support@apollographql.com", + name: "Jane Smith", + totalProductsCreated: 1337, + yearsOfEmployment: 10 +} + +deprecated_product = { + id: "1", + sku: "apollo-federation-v1", + package: "@apollo/federation-v1", + reason: "Migrate to Federation V2", + createdBy: user +} + +products_research = [ + { + id: "1", + study: { + caseNumber: "1234", + description: "Federation Study" + }, + outcome: nil + }, + { + id: "2", + study: { + caseNumber: "1235", + description: "Studio Study" + }, + outcome: nil + } +] + +products = [ + { + id: "apollo-federation", + sku: "federation", + package: "@apollo/federation", + variation: { + id: "OSS" + }, + dimensions: dimension, + research: [products_research[0]], + createdBy: user, + notes: nil + }, + { + id: "apollo-studio", + sku: "studio", + package: "", + variation: { + id: "platform" + }, + dimensions: dimension, + research: [products_research[1]], + createdBy: user, + notes: nil + } +] + +inventory = { + id: "apollo-oss", + deprecatedProducts: [deprecated_product] +} + +records_by_type = { + "Product" => products, + "DeprecatedProduct" => [deprecated_product], + "ProductResearch" => products_research, + "User" => [user], + "Inventory" => [inventory] +} + +events = records_by_type.flat_map do |type_name, records| + records = records.map.with_index do |record, index| + { + __typename: type_name, + __version: 1, + __json_schema_version: 1 + }.merge(record) + end + + ElasticGraph::Indexer::TestSupport::Converters.upsert_events_for_records(records) +end + +indexer.processor.process(events, refresh_indices: true) + +puts "Elasticsearch bootstrapping done. Booting the GraphQL server." + +use Rack::ShowExceptions +run ElasticGraph::Rack::GraphiQL.new(graphql) diff --git a/elasticgraph-apollo/apollo_tests_implementation/config/products_schema.rb b/elasticgraph-apollo/apollo_tests_implementation/config/products_schema.rb new file mode 100644 index 00000000..703a0aa4 --- /dev/null +++ b/elasticgraph-apollo/apollo_tests_implementation/config/products_schema.rb @@ -0,0 +1,175 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# @private +module ApolloTestImpl + module GraphQLSDLEnumeratorExtension + # The `apollo-federation-subgraph-compatibility` project requires[^1] that each tested implementation provide + # specific `Query` fields: + # + # ```graphql + # type Query { + # product(id: ID!): Product + # deprecatedProduct(sku: String!, package: String!): DeprecatedProduct @deprecated(reason: "Use product query instead") + # } + # ``` + # + # ElasticGraph automatically provides plural fields for our indexed types (e.g. `products` and `deprecatedProducts`). + # For the Apollo tests we need to additionally provide the two fields above. This hooks into the generation of the + # `Query` type to add the required fields. + # + # [^1]: https://github.com/apollographql/apollo-federation-subgraph-compatibility/blob/2.0.0/COMPATIBILITY.md#products-schema-to-be-implemented-by-library-maintainers + def root_query_type + super.tap do |type| + type.field "product", "Product" do |f| + f.argument "id", "ID!" + end + + type.field "deprecatedProduct", "DeprecatedProduct" do |f| + f.argument "sku", "String!" + f.argument "package", "String!" + f.directive "deprecated", reason: "Use product query instead" + end + end + end + end + + # @private + module SchemaDefFactoryExtension + def new_graphql_sdl_enumerator(all_types_except_root_query_type) + super(all_types_except_root_query_type).tap do |enum| + enum.extend GraphQLSDLEnumeratorExtension + end + end + end + + federation_version = ENV["TARGET_APOLLO_FEDERATION_VERSION"] + + # Note: this includes many "manual" schema elements (directives, raw SDL, etc) that the + # `elasticgraph-apollo` library will generate on our behalf in the future. For now, this + # includes all these schema elements just to make it as close as possible to an apollo + # compatible schema without further changes to elasticgraph-apollo. + # + # https://github.com/apollographql/apollo-federation-subgraph-compatibility/blob/2.0.0/COMPATIBILITY.md#products-schema-to-be-implemented-by-library-maintainers + ElasticGraph.define_schema do |schema| + schema.factory.extend SchemaDefFactoryExtension + + schema.json_schema_version 1 + schema.target_apollo_federation_version(federation_version) if federation_version + + unless federation_version == "2.0" + schema.raw_sdl <<~EOS + extend schema + @link(url: "https://myspecs.dev/myCustomDirective/v1.0", import: ["@custom"]) + @composeDirective(name: "@custom") + + directive @custom on OBJECT + EOS + end + + schema.object_type "Product" do |t| + t.directive "custom" unless federation_version == "2.0" + t.apollo_key fields: "sku package" + t.apollo_key fields: "sku variation { id }" + + t.field "id", "ID!" + t.field "sku", "String" + t.field "package", "String" + t.field "variation", "ProductVariation" + t.field "dimensions", "ProductDimension" + t.field "createdBy", "User" do |f| + f.apollo_provides fields: "totalProductsCreated" + end + t.field "notes", "String" do |f| + f.tag_with "internal" + end + t.field "research", "[ProductResearch!]!" do |f| + f.mapping type: "object" + end + + t.index "products" + end + + schema.object_type "DeprecatedProduct" do |t| + t.apollo_key fields: "sku package" + t.field "id", "ID!", indexing_only: true + t.field "sku", "String!" + t.field "package", "String!" + t.field "reason", "String" + t.field "createdBy", "User" + + t.index "deprecated_products" + end + + schema.object_type "ProductVariation" do |t| + t.field "id", "ID!" + end + + schema.object_type "ProductResearch" do |t| + t.apollo_key fields: "study { caseNumber }" + t.field "id", "ID!", indexing_only: true + t.field "study", "CaseStudy!" + t.field "outcome", "String" + + t.index "product_research" + end + + schema.object_type "CaseStudy" do |t| + t.field "caseNumber", "ID!" + t.field "description", "String" + end + + schema.object_type "ProductDimension" do |t| + t.apollo_shareable + t.field "size", "String" + t.field "weight", "Float" + t.field "unit", "String" do |f| + f.apollo_inaccessible + end + end + + schema.object_type "User" do |t| + t.apollo_extends + t.apollo_key fields: "email" + t.field "id", "ID!", indexing_only: true + + t.field "averageProductsCreatedPerYear", "Int" do |f| + f.apollo_requires fields: "totalProductsCreated yearsOfEmployment" + end + + t.field "email", "ID!" do |f| + f.apollo_external + end + + t.field "name", "String" do |f| + f.apollo_override from: "users" + end + + t.field "totalProductsCreated", "Int" do |f| + f.apollo_external + end + + t.field "yearsOfEmployment", "Int!" do |f| + f.apollo_external + end + + t.index "users" + end + + unless federation_version == "2.0" + schema.object_type "Inventory" do |t| + t.apollo_interface_object + t.field "id", "ID!" + t.field "deprecatedProducts", "[DeprecatedProduct!]!" do |f| + f.mapping type: "object" + end + t.index "inventory" + end + end + end +end diff --git a/elasticgraph-apollo/apollo_tests_implementation/config/settings.yaml b/elasticgraph-apollo/apollo_tests_implementation/config/settings.yaml new file mode 100644 index 00000000..d34d7584 --- /dev/null +++ b/elasticgraph-apollo/apollo_tests_implementation/config/settings.yaml @@ -0,0 +1,34 @@ +datastore: + client_faraday_adapter: + name: net_http + clusters: + main: + url: http://elasticsearch:9200 + backend: elasticsearch + settings: {} + index_definitions: + products: &standard_index_settings + query_cluster: "main" + index_into_clusters: ["main"] + ignore_routing_values: [] + custom_timestamp_ranges: [] + setting_overrides: {} + setting_overrides_by_timestamp: {} + deprecated_products: *standard_index_settings + product_research: *standard_index_settings + users: *standard_index_settings + inventory: *standard_index_settings + log_traffic: true + max_client_retries: 3 +logger: + device: stdout +indexer: + latency_slo_thresholds_by_timestamp_in_ms: {} +graphql: + default_page_size: 50 + max_page_size: 500 + extension_modules: + - require_path: ./lib/test_implementation_extension + extension_name: ApolloTestImplementationExtension +schema_artifacts: + directory: config/schema/artifacts diff --git a/elasticgraph-apollo/apollo_tests_implementation/docker-compose.yaml b/elasticgraph-apollo/apollo_tests_implementation/docker-compose.yaml new file mode 100644 index 00000000..afcf81d6 --- /dev/null +++ b/elasticgraph-apollo/apollo_tests_implementation/docker-compose.yaml @@ -0,0 +1,18 @@ +include: + - ../../elasticgraph-local/lib/elastic_graph/local/elasticsearch/docker-compose.yaml +services: + products: + build: + context: ../.. + dockerfile: elasticgraph-apollo/apollo_tests_implementation/Dockerfile + args: + TARGET_APOLLO_FEDERATION_VERSION: ${TARGET_APOLLO_FEDERATION_VERSION} + RUBY_VERSION: ${RUBY_VERSION} + ports: + - 4001:4001 + depends_on: + - elasticsearch + command: ["./wait_for_datastore.sh", "elasticsearch:9200", "bundle", "exec", "rackup", "--host", "0.0.0.0", "--port", "4001"] +volumes: + data01: + driver: local diff --git a/elasticgraph-apollo/apollo_tests_implementation/lib/test_implementation_extension.rb b/elasticgraph-apollo/apollo_tests_implementation/lib/test_implementation_extension.rb new file mode 100644 index 00000000..5f1cdbe3 --- /dev/null +++ b/elasticgraph-apollo/apollo_tests_implementation/lib/test_implementation_extension.rb @@ -0,0 +1,60 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# The `apollo-federation-subgraph-compatibility` project requires that each tested +# implementation provide a `Query.product(id: ID!): Product` field. ElasticGraph provides +# `Query.products(...): ProductConnection!` automatically. To be able to pass the tests, +# we need to provide the `product` field, even though ElasticGraph doesn't natively provide +# it. +# +# This defines an extension that injects a custom resolver that supports the field. +# @private +module ApolloTestImplementationExtension + def graphql_resolvers + @graphql_resolvers ||= [product_field_resolver] + super + end + + def product_field_resolver + @product_field_resolver ||= ProductFieldResolver.new( + datastore_query_builder: datastore_query_builder, + product_index_def: datastore_core.index_definitions_by_name.fetch("products"), + datastore_router: datastore_search_router + ) + end + + # @private + class ProductFieldResolver + def initialize(datastore_query_builder:, product_index_def:, datastore_router:) + @datastore_query_builder = datastore_query_builder + @product_index_def = product_index_def + @datastore_router = datastore_router + end + + def can_resolve?(field:, object:) + field.parent_type.name == :Query && field.name == :product + end + + def resolve(field:, object:, args:, context:, lookahead:) + query = @datastore_query_builder.new_query( + search_index_definitions: [@product_index_def], + monotonic_clock_deadline: context[:monotonic_clock_deadline], + filter: {"id" => {"equalToAnyOf" => [args.fetch("id")]}}, + individual_docs_needed: true, + requested_fields: %w[ + id sku package notes + variation.id + dimensions.size dimensions.weight dimensions.unit + createdBy.averageProductsCreatedPerYear createdBy.email createdBy.name createdBy.totalProductsCreated createdBy.yearsOfEmployment + research.study.caseNumber research.study.description research.outcome + ] + ) + + @datastore_router.msearch([query]).fetch(query).documents.first + end + end +end diff --git a/elasticgraph-apollo/apollo_tests_implementation/wait_for_datastore.sh b/elasticgraph-apollo/apollo_tests_implementation/wait_for_datastore.sh new file mode 100755 index 00000000..d2297e6a --- /dev/null +++ b/elasticgraph-apollo/apollo_tests_implementation/wait_for_datastore.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Inspired by an example from the docker docks: +# https://docs.docker.com/compose/startup-order/ + +set -e + +host="$1" +shift + +until curl -f "$host"; do + >&2 echo "The datastore is unavailable - sleeping" + sleep 1 +done + +>&2 echo "The datastore is up - executing command" +exec "$@" diff --git a/elasticgraph-apollo/elasticgraph-apollo.gemspec b/elasticgraph-apollo/elasticgraph-apollo.gemspec new file mode 100644 index 00000000..b2ba1ee0 --- /dev/null +++ b/elasticgraph-apollo/elasticgraph-apollo.gemspec @@ -0,0 +1,27 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :extension) do |spec, eg_version| + spec.summary = "An ElasticGraph extension that implements the Apollo federation spec." + + spec.add_dependency "elasticgraph-graphql", eg_version + spec.add_dependency "elasticgraph-support", eg_version + spec.add_dependency "graphql", "~> 2.3.19" + spec.add_dependency "apollo-federation", "~> 3.8" + + # Note: technically, this is not purely a development dependency, but since `eg-schema_def` + # isn't intended to be used in production (or even included in a deployed bundle) we don't + # want to declare it as normal dependency here. + spec.add_development_dependency "elasticgraph-schema_definition", eg_version + spec.add_development_dependency "elasticgraph-admin", eg_version + spec.add_development_dependency "elasticgraph-elasticsearch", eg_version + spec.add_development_dependency "elasticgraph-opensearch", eg_version + spec.add_development_dependency "elasticgraph-indexer", eg_version +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/graphql/engine_extension.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/graphql/engine_extension.rb new file mode 100644 index 00000000..a9fbcb95 --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/graphql/engine_extension.rb @@ -0,0 +1,61 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Apollo + # Namespace for all Apollo GraphQL enging logic. + # + # @note This module provides no public types or APIs. It will be used automatically when you use + # {SchemaDefinition::APIExtension} as a schema definition extension module. + module GraphQL + # ElasticGraph application extension module designed to hook into the ElasticGraph + # GraphQL engine in order to support Apollo-specific fields. + # + # @private + module EngineExtension + # @private + def graphql_resolvers + @graphql_resolvers ||= begin + require "elastic_graph/apollo/graphql/entities_field_resolver" + require "elastic_graph/apollo/graphql/service_field_resolver" + + [ + EntitiesFieldResolver.new( + datastore_query_builder: datastore_query_builder, + schema_element_names: runtime_metadata.schema_element_names + ), + ServiceFieldResolver.new + ] + super + end + end + + # @private + def graphql_gem_plugins + @graphql_gem_plugins ||= begin + require "apollo-federation/tracing/proto" + require "apollo-federation/tracing/node_map" + require "apollo-federation/tracing/tracer" + require "apollo-federation/tracing" + + # @type var options: ::Hash[::Symbol, untyped] + options = {} + super.merge(ApolloFederation::Tracing => options) + end + end + + # @private + def graphql_http_endpoint + @graphql_http_endpoint ||= super.tap do |endpoint| + require "elastic_graph/apollo/graphql/http_endpoint_extension" + endpoint.extend HTTPEndpointExtension + end + end + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/graphql/entities_field_resolver.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/graphql/entities_field_resolver.rb new file mode 100644 index 00000000..51dcc2ed --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/graphql/entities_field_resolver.rb @@ -0,0 +1,312 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/graphql/query_adapter/requested_fields" +require "elastic_graph/graphql/resolvers/query_source" + +module ElasticGraph + module Apollo + module GraphQL + # GraphQL resolver for the Apollo `Query._entities` field. For details on this field, see: + # + # https://www.apollographql.com/docs/federation/subgraph-spec/#resolve-requests-for-entities + # + # @private + class EntitiesFieldResolver + def initialize(datastore_query_builder:, schema_element_names:) + @datastore_query_builder = datastore_query_builder + @schema_element_names = schema_element_names + end + + def can_resolve?(field:, object:) + field.parent_type.name == :Query && field.name == :_entities + end + + def resolve(field:, object:, args:, context:, lookahead:) + schema = context.fetch(:elastic_graph_schema) + + representations = args.fetch("representations").map.with_index do |rep, index| + try_parse_representation(rep, schema) do |error_description| + context.add_error(::GraphQL::ExecutionError.new("Representation at index #{index} #{error_description}.")) + end + end + + representations_by_adapter = representations.group_by { |rep| rep&.adapter } + + # The query attributes that are based on the requested subfields are the same across all representations, + # so we build the hash of those attributes once here. + query_attributes = ElasticGraph::GraphQL::QueryAdapter::RequestedFields + .new(schema) + .query_attributes_for(field: field, lookahead: lookahead) + .merge(monotonic_clock_deadline: context[:monotonic_clock_deadline]) + + # Build a separate query per adapter instance since each adapter instance is capable of building + # a single query that handles all representations assigned to it. + query_by_adapter = representations_by_adapter.to_h do |adapter, reps| + query = build_query(adapter, reps, query_attributes) if adapter + [adapter, query] + end + + responses_by_query = ElasticGraph::GraphQL::Resolvers::QuerySource.execute_many(query_by_adapter.values.compact, for_context: context) + indexed_search_hits_by_adapter = query_by_adapter.to_h do |adapter, query| + indexed_search_hits = query ? adapter.index_search_hits(responses_by_query.fetch(query)) : {} # : ::Hash[::String, ElasticGraph::GraphQL::DatastoreResponse::Document] + [adapter, indexed_search_hits] + end + + representations.map.with_index do |representation, index| + next unless (adapter = representation&.adapter) + + indexed_search_hits = indexed_search_hits_by_adapter.fetch(adapter) + adapter.identify_matching_hit(indexed_search_hits, representation, context: context, index: index) + end + end + + private + + # Builds a datastore query for the given specific representation. + def build_query(adapter, representations, query_attributes) + return nil unless adapter.indexed? + + type = adapter.type + query = @datastore_query_builder.new_query(search_index_definitions: type.search_index_definitions, **query_attributes) + adapter.customize_query(query, representations) + end + + # Helper method that parses an `_Any` representation of an entity into a `Representation` + # object that contains the GraphQL `type` and a query `filter`. + # + # Based on whether or not this is successful, one of two things will happen: + # + # - If we can't parse it, an error description will be yielded and `nil` will be return + # (to indicate we couldn't parse it). + # - If we can parse it, the representation will be returned (and nothing will be yielded). + def try_parse_representation(representation, schema) + notify_error = proc do |msg| + yield msg.to_s + return nil # returns `nil` from the `try_parse_representation` method. + end + + unless representation.is_a?(::Hash) + notify_error.call("is not a JSON object") + end + + unless (typename = representation["__typename"]) + notify_error.call("lacks a `__typename`") + end + + type = begin + schema.type_named(typename) + rescue ElasticGraph::Errors::NotFoundError + notify_error.call("has an unrecognized `__typename`: #{typename}") + end + + if (fields = representation.except("__typename")).empty? + notify_error.call("has only a `__typename` field") + end + + if !type.indexed_document? + RepresentationWithoutIndex.new( + type: type, + representation_hash: representation + ) + elsif (id = fields["id"]) + RepresentationWithId.new( + type: type, + id: id, + other_fields: translate_field_names(fields.except("id"), type), + schema_element_names: @schema_element_names + ) + else + RepresentationWithoutId.new( + type: type, + fields: translate_field_names(fields, type), + schema_element_names: @schema_element_names + ) + end + end + + def translate_field_names(fields_hash, type) + fields_hash.to_h do |public_field_name, value| + field = type.field_named(public_field_name) + field_name = field.name_in_index.to_s + + case value + when ::Hash + [field_name, translate_field_names(value, field.type.unwrap_fully)] + else + # TODO: Add support for array cases (e.g. when value is an array of hashes). + [field_name, value] + end + end + end + + # A simple value object containing a parsed form of an `_Any` representation when there's an `id` field. + # + # @private + class RepresentationWithId < ::Data.define(:type, :id, :other_fields, :schema_element_names, :adapter) + def initialize(type:, id:, other_fields:, schema_element_names:) + super( + type: type, id: id, other_fields: other_fields, schema_element_names: schema_element_names, + # All `RepresentationWithId` instances with the same `type` can be handled by the same adapter, + # since we can combine them into a single query filtering on `id`. + adapter: Adapter.new(type, schema_element_names) + ) + end + + Adapter = ::Data.define(:type, :schema_element_names) do + # @implements Adapter + + def customize_query(query, representations) + # Given a set of representations, builds a filter that will match all of them (and only them). + all_ids = representations.map(&:id).reject { |id| id.is_a?(::Array) or id.is_a?(::Hash) } + filter = {"id" => {schema_element_names.equal_to_any_of => all_ids}} + + query.merge_with( + document_pagination: {first: representations.length}, + requested_fields: additional_requested_fields_for(representations), + filter: filter + ) + end + + # Given a query response, indexes the search hits for easy `O(1)` retrieval by `identify_matching_hit`. + # This allows us to provide `O(N)` complexity in our resolver instead of `O(N^2)`. + def index_search_hits(response) + response.to_h { |hit| [hit.id, hit] } + end + + # Given some indexed search hits and a representation, identifies the search hit that matches the representation. + def identify_matching_hit(indexed_search_hits, representation, context:, index:) + hit = indexed_search_hits[representation.id] + hit if hit && match?(representation.other_fields, hit.payload) + end + + def indexed? + true + end + + private + + def additional_requested_fields_for(representations) + representations.flat_map do |representation| + fields_in(representation.other_fields) + end + end + + def fields_in(hash) + hash.flat_map do |field_name, value| + case value + when ::Hash + fields_in(value).map do |sub_field_name| + "#{field_name}.#{sub_field_name}" + end + else + # TODO: Add support for array cases. + [field_name] + end + end + end + + def match?(expected, actual) + expected.all? do |key, value| + case value + when ::Hash + match?(value, actual[key]) + when ::Array + # TODO: Add support for array filtering, instead of ignoring it. + true + else + value == actual[key] + end + end + end + end + end + + # A simple value object containing a parsed form of an `_Any` representation when there's no `id` field. + # + # @private + class RepresentationWithoutId < ::Data.define(:type, :fields, :schema_element_names) + # @dynamic type + + # Each `RepresentationWithoutId` instance needs to be handled by a separate adapter. We can't + # safely combine representations into a single datastore query, so we want each to handled + # by a separate adapter instance. So, we use the representation itself as the adapter. + def adapter + self + end + + def customize_query(query, representations) + query.merge_with( + # In the case of representations which don't query Id, we ask for 2 documents so that + # if something weird is going on and it matches more than 1, we can detect that and return an error. + document_pagination: {first: 2}, + filter: build_filter_for_hash(fields) + ) + end + + def index_search_hits(response) + {"search_hits" => response.to_a} + end + + def identify_matching_hit(indexed_search_hits, representation, context:, index:) + search_hits = indexed_search_hits.fetch("search_hits") + if search_hits.size > 1 + context.add_error(::GraphQL::ExecutionError.new("Representation at index #{index} matches more than one entity.")) + nil + else + search_hits.first + end + end + + def indexed? + true + end + + private + + def build_filter_for_hash(fields) + # We must exclude `Array` values because we'll get an exception from the datastore if we allow it here. + # Filtering it out just means that the representation will not match an entity. + fields.reject { |key, value| value.is_a?(::Array) }.transform_values do |value| + if value.is_a?(::Hash) + build_filter_for_hash(value) + else + {schema_element_names.equal_to_any_of => [value]} + end + end + end + end + + # @private + class RepresentationWithoutIndex < ::Data.define(:type, :representation_hash) + # @dynamic type + def adapter + self + end + + def customize_query(query, representations) + nil + end + + def index_search_hits(response) + nil + end + + def identify_matching_hit(indexed_search_hits, representation, context:, index:) + representation.representation_hash + end + + def indexed? + false + end + end + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/graphql/http_endpoint_extension.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/graphql/http_endpoint_extension.rb new file mode 100644 index 00000000..4eefef0c --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/graphql/http_endpoint_extension.rb @@ -0,0 +1,47 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "apollo-federation/tracing" + +module ElasticGraph + module Apollo + module GraphQL + # This extension is designed to hook `ElasticGraph::GraphQL::HTTPEndpoint` in order + # to provide Apollo Federated Tracing: + # + # https://www.apollographql.com/docs/federation/metrics/ + # + # Luckily, the apollo-federation gem supports this--we just need to: + # + # 1. Use the `ApolloFederation::Tracing` plugin (implemented via `EngineExtension#graphql_gem_plugins`). + # 2. Conditionally pass `tracing_enabled: true` into in `context`. + # + # This extension handles the latter requirement. For more info, see: + # https://github.com/Gusto/apollo-federation-ruby#tracing + # + # @private + module HTTPEndpointExtension + def with_context(request) + # Steep has an error here for some reason: + # UnexpectedError: undefined method `selector' for # + __skip__ = super(request) do |context| + # `ApolloFederation::Tracing.should_add_traces` expects the header to be in SCREAMING_SNAKE_CASE with an HTTP_ prefix: + # https://github.com/Gusto/apollo-federation-ruby/blob/v3.8.4/lib/apollo-federation/tracing.rb#L5 + normalized_headers = request.headers.transform_keys { |key| "HTTP_#{key.upcase.tr("-", "_")}" } + + if ApolloFederation::Tracing.should_add_traces(normalized_headers) + context = context.merge(tracing_enabled: true) + end + + yield context + end + end + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/graphql/service_field_resolver.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/graphql/service_field_resolver.rb new file mode 100644 index 00000000..97f47baa --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/graphql/service_field_resolver.rb @@ -0,0 +1,32 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Apollo + module GraphQL + # GraphQL resolver for the Apollo `Query._service` field. + # + # @private + class ServiceFieldResolver + def can_resolve?(field:, object:) + field.parent_type.name == :Query && field.name == :_service + end + + def resolve(field:, object:, args:, context:, lookahead:) + {"sdl" => service_sdl(context.fetch(:elastic_graph_schema).graphql_schema)} + end + + private + + def service_sdl(graphql_schema) + ::GraphQL::Schema::Printer.print_schema(graphql_schema) + end + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/api_extension.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/api_extension.rb new file mode 100644 index 00000000..03e35d7b --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/api_extension.rb @@ -0,0 +1,453 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/version" +require "elastic_graph/apollo/graphql/engine_extension" +require "elastic_graph/apollo/schema_definition/entity_type_extension" +require "elastic_graph/apollo/schema_definition/factory_extension" +require "elastic_graph/apollo/schema_definition/state_extension" + +module ElasticGraph + # ElasticGraph extension library that implements the [Apollo subgraph federation + # spec](https://www.apollographql.com/docs/federation/subgraph-spec/), turning + # any ElasticGraph instance into an Apollo subgraph. + # + # `ElasticGraph::Apollo` has two parts: + # + # * {Apollo::SchemaDefinition} is an extension used while defining an ElasticGraph schema. It includes all schema elements that are part + # of the Apollo spec, including `_Entity` and the various directives. + # * {Apollo::GraphQL} is an extension used by `elasticgraph-graphql` to support queries against Apollo's subgraph schema additions (e.g. + # `_service` and `_entities`). It includes [reference resolvers](https://www.apollographql.com/docs/federation/entities/#2-define-a-reference-resolver) + # for all indexed types in your schema. + # + # To use `elasticgraph-apollo`, simply use {Apollo::SchemaDefinition::APIExtension} as a schema definition extension module. The GraphQL + # extension module will get used by `elasticgraph-graphql` automatically. + # + # @example Use elasticgraph-apollo in a project + # require "elastic_graph/apollo/schema_definition/api_extension" + # + # ElasticGraph::Local::RakeTasks.new( + # local_config_yaml: "config/settings/local.yaml", + # path_to_schema: "config/schema.rb" + # ) do |tasks| + # tasks.schema_definition_extension_modules = [ElasticGraph::Apollo::SchemaDefinition::APIExtension] + # end + module Apollo + # Namespace for all Apollo schema definition support. + # + # {SchemaDefinition::APIExtension} is the primary entry point and should be used as a schema definition extension module. + module SchemaDefinition + # Module designed to be extended onto an {ElasticGraph::SchemaDefinition::API} instance + # to customize the schema artifacts based on the [Apollo Federation subgraph + # spec](https://www.apollographql.com/docs/federation/subgraph-spec/). + # + # To use this module, pass it in `schema_definition_extension_modules` when defining your {ElasticGraph::Local::RakeTasks}. + # + # @example Define local rake tasks with this extension module + # require "elastic_graph/apollo/schema_definition/api_extension" + # + # ElasticGraph::Local::RakeTasks.new( + # local_config_yaml: "config/settings/local.yaml", + # path_to_schema: "config/schema.rb" + # ) do |tasks| + # tasks.schema_definition_extension_modules = [ElasticGraph::Apollo::SchemaDefinition::APIExtension] + # end + module APIExtension + # @private + def results + register_graphql_extension GraphQL::EngineExtension, defined_at: "elastic_graph/apollo/graphql/engine_extension" + define_apollo_schema_elements + + super + end + + # Applies an apollo tag to built-in types so that they are included in the Apollo contract schema. + # + # @param name [String] tag name + # @param except [Array] built-in types not to tag + # @return [void] + # @see ApolloDirectives::Tag + # @see FieldExtension#tag_with + # + # @example Tag all built-in types (except two) for inclusion in the `public` schema + # ElasticGraph.define_schema do |schema| + # schema.tag_built_in_types_with "public", except: ["IntAggregatedValue", "FloatAggregatedValues"] + # end + def tag_built_in_types_with(name, except: []) + except_set = except.to_set + on_built_in_types do |type| + apollo_type = (_ = type) # : ApolloDirectives::Tag + apollo_type.apollo_tag(name: name) unless except_set.include?(type.name) + end + end + + # Picks which version of Apollo federation to target. By default, the latest supported version is + # targeted, but you can call this to pick an earlier version, which may be necessary if your + # organization is on an older version of Apollo federation. + # + # @param version [String] version number + # @return [void] + # + # @example Set the Apollo Federation Version + # ElasticGraph.define_schema do |schema| + # schema.target_apollo_federation_version "2.6" + # end + def target_apollo_federation_version(version) + # Allow the version to have the `v` prefix, but don't require it. + version = version.delete_prefix("v") + + state.apollo_directive_definitions = DIRECTIVE_DEFINITIONS_BY_FEDERATION_VERSION.fetch(version) do + supported_version_descriptions = DIRECTIVE_DEFINITIONS_BY_FEDERATION_VERSION.keys.map do |version_number| + "v#{version_number}" + end.join(", ") + + raise Errors::SchemaError, "elasticgraph-apollo v#{ElasticGraph::VERSION} does not support Apollo federation v#{version}. " \ + "Pick one of the supported versions (#{supported_version_descriptions}) instead." + end + end + + def self.extended(api) + api.factory.extend FactoryExtension + api.state.extend StateExtension + + latest_federation_version = DIRECTIVE_DEFINITIONS_BY_FEDERATION_VERSION + .keys + .max_by { |v| v.split(".").map(&:to_i) } # : ::String + + api.target_apollo_federation_version latest_federation_version + + api.on_built_in_types do |type| + # Built-in types like `PageInfo` need to be tagged with `@shareable` on Federation V2 since other subgraphs may + # have them and they aren't entity types. `Query`, as the root, is a special case that must be skipped. + (_ = type).apollo_shareable if type.respond_to?(:apollo_shareable) && type.name != "Query" + end + end + + private + + # These directive definitions come straight from the Apollo federation spec: + # https://github.com/apollographql/federation/blob/25beb382fff253ac38ef6d7a5454af60da0addbb/docs/source/subgraph-spec.mdx#L57-L70 + # https://github.com/apollographql/apollo-federation-subgraph-compatibility/blob/2.0.0/COMPATIBILITY.md#products-schema-to-be-implemented-by-library-maintainers + # + # I've updated them here to match the "canonical form" that the GraphQL + # gem dumps for directives (e.g. it sorts the `on` clauses alphabetically) so that + # we can use this from our tests to assert the resulting GraphQL SDL. + directives_for_fed_v2_6 = [ + <<~EOS.strip, + extend schema + @link(import: ["@authenticated", "@composeDirective", "@extends", "@external", "@inaccessible", "@interfaceObject", "@key", "@override", "@policy", "@provides", "@requires", "@requiresScopes", "@shareable", "@tag", "FieldSet"], url: "https://specs.apollo.dev/federation/v2.6") + EOS + "directive @authenticated on ENUM | FIELD_DEFINITION | INTERFACE | OBJECT | SCALAR", + "directive @composeDirective(name: String!) repeatable on SCHEMA", + "directive @extends on INTERFACE | OBJECT", + "directive @external on FIELD_DEFINITION | OBJECT", + "directive @inaccessible on ARGUMENT_DEFINITION | ENUM | ENUM_VALUE | FIELD_DEFINITION | INPUT_FIELD_DEFINITION | INPUT_OBJECT | INTERFACE | OBJECT | SCALAR | UNION", + "directive @interfaceObject on OBJECT", + "directive @key(fields: FieldSet!, resolvable: Boolean = true) repeatable on INTERFACE | OBJECT", + "directive @link(as: String, for: link__Purpose, import: [link__Import], url: String!) repeatable on SCHEMA", + "directive @override(from: String!) on FIELD_DEFINITION", + "directive @policy(policies: [[federation__Policy!]!]!) on ENUM | FIELD_DEFINITION | INTERFACE | OBJECT | SCALAR", + "directive @provides(fields: FieldSet!) on FIELD_DEFINITION", + "directive @requires(fields: FieldSet!) on FIELD_DEFINITION", + "directive @requiresScopes(scopes: [[federation__Scope!]!]!) on ENUM | FIELD_DEFINITION | INTERFACE | OBJECT | SCALAR", + "directive @shareable on FIELD_DEFINITION | OBJECT", + "directive @tag(name: String!) repeatable on ARGUMENT_DEFINITION | ENUM | ENUM_VALUE | FIELD_DEFINITION | INPUT_FIELD_DEFINITION | INPUT_OBJECT | INTERFACE | OBJECT | SCALAR | UNION" + ] + + # Differences between federation v2.5 and v2.6 + # + # - v2.5 has no @policy directive (v2.6 has this). + # - The link URL reflects the version + directives_for_fed_v2_5 = directives_for_fed_v2_6.filter_map do |directive| + if directive.include?("extend schema") + directive + .sub(', "@policy"', "") + .sub("v2.6", "v2.5") + elsif directive.include?("directive @policy") + nil + else + directive + end + end + + # Differences between federation v2.3 and v2.5 + # + # - v2.3 has no @authenticated directive (v2.5 has this). + # - v2.3 has no @requiresScopes directive (v2.5 has this). + # - The link URL reflects the version + directives_for_fed_v2_3 = directives_for_fed_v2_5.filter_map do |directive| + if directive.include?("extend schema") + directive + .sub('"@authenticated", ', "") + .sub(', "@requiresScopes"', "") + .sub("v2.5", "v2.3") + elsif directive.include?("directive @authenticated") || directive.include?("directive @requiresScopes") + nil + else + directive + end + end + + # Differences between federation v2.0 and v2.3 + # + # - v2.0 has no @composeDirective directive (v2.3 has this). + # - v2.0 has no @interfaceObject directive (v2.3 has this). + # - The link URL reflects the version + directives_for_fed_v2_0 = directives_for_fed_v2_3.filter_map do |directive| + if directive.include?("extend schema") + directive + .sub('"@composeDirective", ', "") + .sub(', "@interfaceObject"', "") + .sub("v2.3", "v2.0") + elsif directive.include?("directive @interfaceObject") || directive.include?("directive @composeDirective") + nil + else + directive + end + end + + DIRECTIVE_DEFINITIONS_BY_FEDERATION_VERSION = { + "2.6" => directives_for_fed_v2_6, + "2.5" => directives_for_fed_v2_5, + "2.3" => directives_for_fed_v2_3, + "2.0" => directives_for_fed_v2_0 + } + + def define_apollo_schema_elements + state.apollo_directive_definitions.each { |directive| raw_sdl directive } + + apollo_scalar_type "link__Import" do |t| + t.documentation "Scalar type used by the `@link` directive required for Apollo Federation V2." + # `scalar_type` requires we set these but this scalar type is only in GraphQL. + t.mapping type: nil + t.json_schema type: "null" + end + + apollo_scalar_type "federation__Scope" do |t| + t.documentation "Scalar type used by the `@requiresScopes` directive required for Apollo Federation V2.5+." + # `scalar_type` requires we set these but this scalar type is only in GraphQL. + t.mapping type: nil + t.json_schema type: "null" + end + + apollo_scalar_type "federation__Policy" do |t| + t.documentation "Scalar type used by the `@policy` directive required for Apollo Federation V2.6+." + # `scalar_type` requires we set these but this scalar type is only in GraphQL. + t.mapping type: nil + t.json_schema type: "null" + end + + # Copied from https://github.com/apollographql/federation/blob/b3a3cb84d8d67d1d6e817dc85b9ae0ecdd9908d1/docs/source/subgraph-spec.mdx#subgraph-schema-additions + apollo_enum_type "link__Purpose" do |t| + t.documentation "Enum type used by the `@link` directive required for Apollo Federation V2." + + t.value "SECURITY" do |v| + v.documentation "`SECURITY` features provide metadata necessary to securely resolve fields." + end + + t.value "EXECUTION" do |v| + v.documentation "`EXECUTION` features provide metadata necessary for operation execution." + end + end + + apollo_scalar_type "FieldSet" do |t| + t.documentation <<~EOS + A custom scalar type required by the [Apollo Federation subgraph + spec](https://www.apollographql.com/docs/federation/subgraph-spec/#scalar-fieldset): + + > This string-serialized scalar represents a set of fields that's passed to a federated directive, + > such as `@key`, `@requires`, or `@provides`. + > + > Grammatically, a `FieldSet` is a [selection set](http://spec.graphql.org/draft/#sec-Selection-Sets) + > minus the outermost curly braces. It can represent a single field (`"upc"`), multiple fields + > (`"id countryCode"`), and even nested selection sets (`"id organization { id }"`). + + Not intended for use by clients other than Apollo. + EOS + + # `scalar_type` requires we set these but this scalar type is only in GraphQL. + t.mapping type: nil + t.json_schema type: "null" + end + + apollo_scalar_type "_Any" do |t| + t.documentation <<~EOS + A custom scalar type required by the [Apollo Federation subgraph + spec](https://www.apollographql.com/docs/federation/subgraph-spec/#scalar-_any): + + > This scalar is the type used for entity **representations** that the graph router + > passes to the `Query._entities` field. An `_Any` scalar is validated by matching + > its `__typename` and `@key` fields against entities defined in the subgraph schema. + > + > An `_Any` is serialized as a JSON object, like so: + > + > ``` + > { + > "__typename": "Product", + > "upc": "abc123" + > } + > ``` + + Not intended for use by clients other than Apollo. + EOS + + # `scalar_type` requires we set these but this scalar type is only in GraphQL. + t.mapping type: nil + t.json_schema type: "null" + end + + apollo_object_type "_Service" do |t| + t.documentation <<~EOS + An object type required by the [Apollo Federation subgraph + spec](https://www.apollographql.com/docs/federation/subgraph-spec/#type-_service): + + > This object type must have an `sdl: String!` field, which returns the SDL of the subgraph schema as a string. + > + > - The returned schema string _must_ include all uses of federation-specific directives (`@key`, `@requires`, etc.). + > - **If supporting Federation 1,** the schema _must not_ include any definitions from [Subgraph schema additions](https://www.apollographql.com/docs/federation/subgraph-spec/#subgraph-schema-additions). + > + > For details, see [Enhanced introspection with `Query._service`](https://www.apollographql.com/docs/federation/subgraph-spec/#enhanced-introspection-with-query_service). + + Not intended for use by clients other than Apollo. + EOS + + t.field "sdl", "String", graphql_only: true do |f| + f.documentation <<~EOS + A field required by the [Apollo Federation subgraph + spec](https://www.apollographql.com/docs/federation/subgraph-spec/#required-resolvers-for-introspection): + + > The returned `sdl` string has the following requirements: + > + > - It must **include** all uses of all federation-specific directives, such as `@key`. + > - All of these directives are shown in [Subgraph schema additions](https://www.apollographql.com/docs/federation/subgraph-spec/#subgraph-schema-additions). + > - **If supporting Federation 1,** `sdl` must **omit** all automatically added definitions from + > [Subgraph schema additions](https://www.apollographql.com/docs/federation/subgraph-spec/#subgraph-schema-additions), + > such as `Query._service` and `_Service.sdl`! + > - If your library is _only_ supporting Federation 2, `sdl` can include these definitions. + + Not intended for use by clients other than Apollo. + EOS + end + end + + entity_types = state.object_types_by_name.values.select do |object_type| + object_type.directives.any? do |directive| + directive.name == "key" && directive.arguments.fetch(:resolvable, true) + end + end + + validate_entity_types_can_all_be_resolved(entity_types) + + entity_type_names = entity_types + # As per the GraphQL spec[1], only object types can be in a union, and interface + # types cannot be in a union. The GraphQL gem has validation[2] for this and will raise + # an error if we violate it, so we must filter to only object types here. + # + # [1] https://spec.graphql.org/October2021/#sec-Unions.Type-Validation + # [2] https://github.com/rmosolgo/graphql-ruby/pull/3024 + .grep(ElasticGraph::SchemaDefinition::SchemaElements::ObjectType) + .map(&:name) + + unless entity_type_names.empty? + apollo_union_type "_Entity" do |t| + t.extend EntityTypeExtension + t.documentation <<~EOS + A union type required by the [Apollo Federation subgraph + spec](https://www.apollographql.com/docs/federation/subgraph-spec/#union-_entity): + + > **⚠️ This union type is generated dynamically based on the input subgraph schema!** + > + > This union's possible types must include all entities that the subgraph defines. + > It's the return type of the `Query._entities` field, which the graph router uses + > to directly access a subgraph's entity fields. + > + > For details, see [Defining the `_Entity` union](https://www.apollographql.com/docs/federation/subgraph-spec/#defining-the-_entity-union). + + In an ElasticGraph schema, this is a union of all indexed types. + + Not intended for use by clients other than Apollo. + EOS + + t.subtypes(*entity_type_names) + end + end + end + + def apollo_object_type(name, &block) + object_type name do |type| + type.graphql_only true + yield type + end + end + + def apollo_union_type(name, &block) + union_type name do |type| + type.graphql_only true + yield type + end + end + + def apollo_scalar_type(name, &block) + scalar_type name do |type| + type.graphql_only true + yield type + end + end + + def apollo_enum_type(name, &block) + enum_type name do |type| + type.graphql_only true + yield type + end + end + + # state comes from object we extend with this module. + # @dynamic state + + def validate_entity_types_can_all_be_resolved(entity_types) + unresolvable_field_errors = + entity_types.reject(&:indexed?).filter_map do |object_type| + key_field_names = object_type.directives + .select { |dir| dir.name == "key" } + # https://rubular.com/r/JEuYKzqnyR712A + .flat_map { |dir| dir.arguments[:fields].to_s.gsub(/{.*}/, "").split(" ") } + .to_set + + unresolvable_fields = object_type.graphql_fields_by_name.values.reject do |field| + key_field_names.include?(field.name) || + field.relationship || + field.directives.any? { |directive| directive.name == "external" } + end + + if unresolvable_fields.any? + <<~EOS.strip + `#{object_type.name}` has fields that ElasticGraph will be unable to resolve when Apollo requests it as an entity: + + #{unresolvable_fields.map { |field| " * `#{field.name}`" }.join("\n")} + + On a resolvable non-indexed entity type like this, ElasticGraph can only resolve `@key` fields and + `relates_to_(one|many)` fields. To fix this, either add `resolvable: false` to the `apollo_key` or + do one of the following for each unresolvable field: + + * Add it to the `apollo_key` + * Redefine it as a relationship + * Use `field.apollo_external` so Apollo knows how to treat it + * Remove it + EOS + end + end + + if unresolvable_field_errors.any? + raise Errors::SchemaError, unresolvable_field_errors.join("\n#{"-" * 100}\n") + end + end + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/apollo_directives.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/apollo_directives.rb new file mode 100644 index 00000000..f6cf8ee8 --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/apollo_directives.rb @@ -0,0 +1,287 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Apollo + module SchemaDefinition + # Namespace for mixins that provide support for Apollo's [federation directives](https://www.apollographql.com/docs/federation/federated-schemas/federated-directives/). + # Each Apollo federation directive is offered via an API starting with `apollo`. For example, `apollo_key` can be used to define an + # Apollo `@key`. + module ApolloDirectives + # Supports Apollo's [`@authenticated` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#authenticated). + module Authenticated + # Adds the [`@authenticated` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#authenticated) + # to the schema element. + # + # @return [void] + # + # @example Add `@authenticated` to a type + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.apollo_authenticated + # end + # end + def apollo_authenticated + directive "authenticated" + end + end + + # Supports Apollo's [`@extends` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#extends). + module Extends + # Adds the [`@extends` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#extends) + # to the schema element. + # + # @return [void] + # + # @example Add `@extends` to a type + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.apollo_extends + # end + # end + def apollo_extends + directive "extends" + end + end + + # Supports Apollo's [`@external` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#external). + module External + # Adds the [`@external` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#external) + # to the schema element. + # + # @return [void] + # + # @example Add `@external` to a type + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.apollo_external + # end + # end + def apollo_external + directive "external" + end + end + + # Supports Apollo's [`@inaccessible` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#inaccessible). + module Inaccessible + # Adds the [`@inaccessible` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#inaccessible) + # to the schema element. + # + # @return [void] + # + # @example Add `@inaccessible` to a type + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.apollo_inaccessible + # end + # end + def apollo_inaccessible + directive "inaccessible" + end + end + + # Supports Apollo's [`@interfaceObject` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#interfaceobject). + module InterfaceObject + # Adds the [`@interfaceObject` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#interfaceobject) + # to the schema element. + # + # @return [void] + # + # @example Add `@interfaceObject` to a type + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.apollo_interface_object + # end + # end + def apollo_interface_object + directive "interfaceObject" + end + end + + # Supports Apollo's [`@key` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#key). + module Key + # Adds the [`@key` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#key) + # to the schema element. + # + # @param fields [String] A GraphQL selection set (provided as a string) of fields and subfields that contribute to the entity's + # unique key. + # @param resolvable [Boolean] If false, indicates to the Apollo router that this subgraph doesn't define a reference resolver for + # this entity. This means that router query plans can't "jump to" this subgraph to resolve fields that aren't defined in another + # subgraph. + # @return [void] + # + # @example Define a `@key` on a non-indexed type + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "organizationId", "ID" + # t.field "id", "ID" + # t.apollo_key fields: "id organizationId", resolvable: false + # end + # end + # + # @note ElasticGraph automatically defines an `apollo_key` of `id` for every indexed type. This API is only needed when defining + # additional keys on an indexed type, or defining a key for a non-indexed type. + def apollo_key(fields:, resolvable: true) + directive "key", fields: fields, resolvable: resolvable + end + end + + # Supports Apollo's [`@override` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#override). + module Override + # Adds the [`@override` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#override) + # to the schema element. + # + # @param from [String] The name of the other subgraph that no longer resolves the field. + # @return [void] + # + # @example Add `@override` to a field + # ElasticGraph.define_schema do |schema| + # schema.object_type "Product" do |t| + # t.field "inStock", "Boolean" do |f| + # f.apollo_override from: "Products" + # end + # end + # end + def apollo_override(from:) + directive "override", from: from + end + end + + # Supports Apollo's [`@policy` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#policy). + module Policy + # Adds the [`@policy` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#policy) + # to the schema element. + # + # @param policies [Array] List of authorization policies. + # @return [void] + # + # @example Add `@policy` to a type + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.apollo_policy policies: ["Policy1", "Policy2"] + # end + # end + def apollo_policy(policies:) + directive "policy", policies: policies + end + end + + # Supports Apollo's [`@provides` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#provides). + module Provides + # Adds the [`@provides` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#provides) + # to the schema element. + # + # @param fields [String] A GraphQL selection set (provided as a string) of object fields and subfields that the subgraph can + # resolve only at this query path. + # @return [void] + # + # @example Add `@provides` to a field + # ElasticGraph.define_schema do |schema| + # schema.object_type "Product" do |t| + # t.field "name", "String" + # end + # + # schema.object_type "StoreLocation" do |t| + # t.field "products", "[Product!]!" do |f| + # f.mapping type: "nested" + # f.apollo_provides fields: "name" + # end + # end + # end + def apollo_provides(fields:) + directive "provides", fields: fields + end + end + + # Supports Apollo's [`@requires` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#requires). + module Requires + # Adds the [`@requires` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#requires) + # to the schema element. + # + # @param fields [String] A GraphQL selection set (provided as a string) of object fields and subfields that the subgraph can + # resolve only at this query path. + # @return [void] + # + # @example Add `@requires` to a field + # ElasticGraph.define_schema do |schema| + # schema.object_type "Product" do |t| + # t.field "size", "Int" + # t.field "weight", "Int" + # t.field "shippingEstimate", "String" do |f| + # f.apollo_requires fields: "size weight" + # end + # end + # end + def apollo_requires(fields:) + directive "requires", fields: fields + end + end + + # Supports Apollo's [`@requiresScopes` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#requiresscopes). + module RequiresScopes + # Adds the [`@requiresScopes` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#requiresscopes) + # to the schema element. + # + # @param scopes [Array] List of JWT scopes that must be granted to the user in order to access the underlying element data. + # @return [void] + # + # @example Add `@requiresScopes` to a field + # ElasticGraph.define_schema do |schema| + # schema.object_type "Product" do |t| + # t.field "shippingEstimate", "String" do |f| + # f.apollo_requires_scopes scopes: "shipping" + # end + # end + # end + def apollo_requires_scopes(scopes:) + directive "requiresScopes", scopes: scopes + end + end + + # Supports Apollo's [`@shareable` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#shareable). + module Shareable + # Adds the [`@shareable` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#shareable) + # to the schema element. + # + # @return [void] + # + # @example Add `@shareable` to a type + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.apollo_shareable + # end + # end + def apollo_shareable + directive "shareable" + end + end + + # Supports Apollo's [`@tag` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#tag). + module Tag + # Adds the [`@tag` directive](https://www.apollographql.com/docs/federation/federated-types/federated-directives/#tag) + # to the schema element. + # + # @return [void] + # + # @example Add `@tag` to a type + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.apollo_tag name: "public" + # end + # end + # + # @see APIExtension#tag_built_in_types_with + # @see FieldExtension#tag_with + def apollo_tag(name:) + directive "tag", name: name + end + end + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/argument_extension.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/argument_extension.rb new file mode 100644 index 00000000..7e72c90f --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/argument_extension.rb @@ -0,0 +1,21 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/apollo/schema_definition/apollo_directives" + +module ElasticGraph + module Apollo + module SchemaDefinition + # Extends {ElasticGraph::SchemaDefinition::SchemaElements::Argument} to offer Apollo argument directives. + module ArgumentExtension + include ApolloDirectives::Inaccessible + include ApolloDirectives::Tag + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/entity_type_extension.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/entity_type_extension.rb new file mode 100644 index 00000000..164d2fcc --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/entity_type_extension.rb @@ -0,0 +1,32 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Apollo + module SchemaDefinition + # The Apollo `_Entity` type is a type union of _all_ entity subtypes in an ElasticGraph schema. + # However, unlike a normal union type: + # + # - `_Entity` is never an indexed type, and should not be treated as one (even though its subtypes are all indexed, which would + # usually cause it to be treated as indexed!). + # - A merged set of `graphql_fields_by_name` cannot be safely computed. That method raises errors if a field with the same name + # has conflicting definitions on different subtypes, but we must allow that on `_Entity` subtypes. + # + # @private + module EntityTypeExtension + def graphql_fields_by_name + {} + end + + def indexed? + false + end + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/enum_type_extension.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/enum_type_extension.rb new file mode 100644 index 00000000..09f9193d --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/enum_type_extension.rb @@ -0,0 +1,24 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/apollo/schema_definition/apollo_directives" + +module ElasticGraph + module Apollo + module SchemaDefinition + # Extends {ElasticGraph::SchemaDefinition::SchemaElements::EnumType} to offer Apollo enum type directives. + module EnumTypeExtension + include ApolloDirectives::Authenticated + include ApolloDirectives::Inaccessible + include ApolloDirectives::Policy + include ApolloDirectives::RequiresScopes + include ApolloDirectives::Tag + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/enum_value_extension.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/enum_value_extension.rb new file mode 100644 index 00000000..12bf4fc4 --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/enum_value_extension.rb @@ -0,0 +1,21 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/apollo/schema_definition/apollo_directives" + +module ElasticGraph + module Apollo + module SchemaDefinition + # Extends {ElasticGraph::SchemaDefinition::SchemaElements::EnumValue} to offer Apollo enum value directives. + module EnumValueExtension + include ApolloDirectives::Inaccessible + include ApolloDirectives::Tag + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/factory_extension.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/factory_extension.rb new file mode 100644 index 00000000..5891d8a8 --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/factory_extension.rb @@ -0,0 +1,106 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/apollo/schema_definition/argument_extension" +require "elastic_graph/apollo/schema_definition/enum_type_extension" +require "elastic_graph/apollo/schema_definition/enum_value_extension" +require "elastic_graph/apollo/schema_definition/field_extension" +require "elastic_graph/apollo/schema_definition/graphql_sdl_enumerator_extension" +require "elastic_graph/apollo/schema_definition/input_type_extension" +require "elastic_graph/apollo/schema_definition/interface_type_extension" +require "elastic_graph/apollo/schema_definition/object_type_extension" +require "elastic_graph/apollo/schema_definition/scalar_type_extension" +require "elastic_graph/apollo/schema_definition/union_type_extension" + +module ElasticGraph + module Apollo + module SchemaDefinition + # Extension module applied to `ElasticGraph::SchemaDefinition::Factory` to add Apollo tagging support. + # + # @private + module FactoryExtension + # Steep has a hard type with the arg splats here. + __skip__ = def new_field(**kwargs) + super(**kwargs) do |field| + field.extend FieldExtension + yield field if block_given? + end + end + + def new_graphql_sdl_enumerator(all_types_except_root_query_type) + super.tap do |enum| + enum.extend GraphQLSDLEnumeratorExtension + end + end + + def new_argument(field, name, value_type) + super(field, name, value_type) do |type| + type.extend ArgumentExtension + yield type if block_given? + end + end + + def new_enum_type(name) + super(name) do |type| + type.extend EnumTypeExtension + yield type + end + end + + def new_enum_value(name, original_name) + super(name, original_name) do |type| + type.extend EnumValueExtension + yield type if block_given? + end + end + + def new_input_type(name) + super(name) do |type| + type.extend InputTypeExtension + yield type + end + end + + def new_interface_type(name) + super(name) do |type| + type.extend InterfaceTypeExtension + yield type + end + end + + # Here we override `object_type` in order to automatically add the apollo `@key` directive to indexed types. + def new_object_type(name) + super(name) do |raw_type| + raw_type.extend ObjectTypeExtension + type = raw_type # : ElasticGraph::SchemaDefinition::SchemaElements::ObjectType & ObjectTypeExtension + + yield type if block_given? + + if type.indexed? && type.graphql_fields_by_name.key?("id") + type.apollo_key fields: "id" + end + end + end + + def new_scalar_type(name) + super(name) do |type| + type.extend ScalarTypeExtension + yield type + end + end + + def new_union_type(name) + super(name) do |type| + type.extend UnionTypeExtension + yield type + end + end + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/field_extension.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/field_extension.rb new file mode 100644 index 00000000..a8a4d661 --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/field_extension.rb @@ -0,0 +1,76 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/apollo/schema_definition/apollo_directives" + +module ElasticGraph + module Apollo + module SchemaDefinition + # Extends {ElasticGraph::SchemaDefinition::SchemaElements::Field} to offer Apollo field directives. + module FieldExtension + include ApolloDirectives::Authenticated + include ApolloDirectives::External + include ApolloDirectives::Inaccessible + include ApolloDirectives::Override + include ApolloDirectives::Policy + include ApolloDirectives::Provides + include ApolloDirectives::Requires + include ApolloDirectives::RequiresScopes + include ApolloDirectives::Shareable + include ApolloDirectives::Tag + + # Extension method designed to support Apollo's [contract variant tagging](https://www.apollographql.com/docs/studio/contracts/). + # + # Calling this method on a field will cause the field and every schema element derived from the field (e.g. the filter field, + # he aggregation field, etc), to be tagged with the given `tag_name`, ensuring that all capabilities related to the field are + # available in the contract variant. + # + # @param tag_name [String] tag to add to schema elements generated for this field + # @return [void] + # + # @example Tag a field (and its derived elements) with "public" + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "name", "String" do |f| + # f.tag_with "public" + # end + # end + # end + # + # @see ApolloDirectives::Tag + # @see APIExtension#tag_built_in_types_with + def tag_with(tag_name) + on_each_generated_schema_element do |element| + needs_tagging = + if element.is_a?(ElasticGraph::SchemaDefinition::SchemaElements::SortOrderEnumValue) + # Each sort order enum value is based on a full field path (e.g. `parentField_subField_furtherNestedField_ASC`). + # We must only tag the enum if each part of the full field path is also tagged. In this example, we should only + # tag the enum value if `parentField`, `subField`, and `furtherNestedField` are all tagged. + element.sort_order_field_path.all? { |f| FieldExtension.tagged_with?(f, tag_name) } + else + true + end + + if needs_tagging && !FieldExtension.tagged_with?(element, tag_name) + (_ = element).apollo_tag name: tag_name + end + end + end + + # Helper method that indicates if the given schema element has a specific tag. + # + # @param element [Object] element to check + # @param tag_name [String] tag to check + # @return [Boolean] + def self.tagged_with?(element, tag_name) + element.directives.any? { |dir| dir.name == "tag" && dir.arguments == {name: tag_name} } + end + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/graphql_sdl_enumerator_extension.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/graphql_sdl_enumerator_extension.rb new file mode 100644 index 00000000..107f0187 --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/graphql_sdl_enumerator_extension.rb @@ -0,0 +1,71 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Apollo + module SchemaDefinition + # Module designed to be extended onto an `ElasticGraph::SchemaDefinition::GraphQLSDLEnumerator` + # instance to customize the schema artifacts to support Apollo. + # + # @private + module GraphQLSDLEnumeratorExtension + def root_query_type + super.tap do |type_or_nil| + # @type var type: ElasticGraph::SchemaDefinition::SchemaElements::ObjectType + type = _ = type_or_nil + + if schema_def_state.object_types_by_name.values.any?(&:indexed?) + type.field "_entities", "[_Entity]!" do |f| + f.documentation <<~EOS + A field required by the [Apollo Federation subgraph + spec](https://www.apollographql.com/docs/federation/subgraph-spec/#query_entities): + + > The graph router uses this root-level `Query` field to directly fetch fields of entities defined by a subgraph. + > + > This field must take a `representations` argument of type `[_Any!]!` (a non-nullable list of non-nullable + > [`_Any` scalars](https://www.apollographql.com/docs/federation/subgraph-spec/#scalar-_any)). Its return type must be `[_Entity]!` (a non-nullable list of _nullable_ + > objects that belong to the [`_Entity` union](https://www.apollographql.com/docs/federation/subgraph-spec/#union-_entity)). + > + > Each entry in the `representations` list must be validated with the following rules: + > + > - A representation must include a `__typename` string field. + > - A representation must contain all fields included in the fieldset of a `@key` directive applied to the corresponding entity definition. + > + > For details, see [Resolving entity fields with `Query._entities`](https://www.apollographql.com/docs/federation/subgraph-spec/#resolving-entity-fields-with-query_entities). + + Not intended for use by clients other than Apollo. + EOS + + f.argument "representations", "[_Any!]!" do |a| + a.documentation <<~EOS + A list of entity data blobs from other apollo subgraphs. For more information (and + to see an example of what form this argument takes), see the [Apollo Federation subgraph + spec](https://www.apollographql.com/docs/federation/subgraph-spec/#resolve-requests-for-entities). + EOS + end + end + end + + type.field "_service", "_Service!" do |f| + f.documentation <<~EOS + A field required by the [Apollo Federation subgraph + spec](https://www.apollographql.com/docs/federation/subgraph-spec/#query_service): + + > This field of the root `Query` type must return a non-nullable [`_Service` type](https://www.apollographql.com/docs/federation/subgraph-spec/#type-_service). + + > For details, see [Enhanced introspection with `Query._service`](https://www.apollographql.com/docs/federation/subgraph-spec/#enhanced-introspection-with-query_service). + + Not intended for use by clients other than Apollo. + EOS + end + end + end + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/input_type_extension.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/input_type_extension.rb new file mode 100644 index 00000000..5ee2b1e7 --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/input_type_extension.rb @@ -0,0 +1,21 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/apollo/schema_definition/apollo_directives" + +module ElasticGraph + module Apollo + module SchemaDefinition + # Extends {ElasticGraph::SchemaDefinition::SchemaElements::InputType} to offer Apollo input type directives. + module InputTypeExtension + include ApolloDirectives::Inaccessible + include ApolloDirectives::Tag + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/interface_type_extension.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/interface_type_extension.rb new file mode 100644 index 00000000..83ad2215 --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/interface_type_extension.rb @@ -0,0 +1,26 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/apollo/schema_definition/apollo_directives" + +module ElasticGraph + module Apollo + module SchemaDefinition + # Extends {ElasticGraph::SchemaDefinition::SchemaElements::InterfaceType} to offer Apollo interface type directives. + module InterfaceTypeExtension + include ApolloDirectives::Authenticated + include ApolloDirectives::Extends + include ApolloDirectives::Inaccessible + include ApolloDirectives::Key + include ApolloDirectives::Policy + include ApolloDirectives::RequiresScopes + include ApolloDirectives::Tag + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/object_type_extension.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/object_type_extension.rb new file mode 100644 index 00000000..54b5ddcf --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/object_type_extension.rb @@ -0,0 +1,29 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/apollo/schema_definition/apollo_directives" + +module ElasticGraph + module Apollo + module SchemaDefinition + # Extends {ElasticGraph::SchemaDefinition::SchemaElements::ObjectType} to offer Apollo object type directives. + module ObjectTypeExtension + include ApolloDirectives::Authenticated + include ApolloDirectives::Extends + include ApolloDirectives::External + include ApolloDirectives::Inaccessible + include ApolloDirectives::InterfaceObject + include ApolloDirectives::Key + include ApolloDirectives::Policy + include ApolloDirectives::RequiresScopes + include ApolloDirectives::Shareable + include ApolloDirectives::Tag + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/scalar_type_extension.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/scalar_type_extension.rb new file mode 100644 index 00000000..f17d441e --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/scalar_type_extension.rb @@ -0,0 +1,24 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/apollo/schema_definition/apollo_directives" + +module ElasticGraph + module Apollo + module SchemaDefinition + # Extends {ElasticGraph::SchemaDefinition::SchemaElements::ScalarType} to offer Apollo scalar type directives. + module ScalarTypeExtension + include ApolloDirectives::Authenticated + include ApolloDirectives::Inaccessible + include ApolloDirectives::Policy + include ApolloDirectives::RequiresScopes + include ApolloDirectives::Tag + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/state_extension.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/state_extension.rb new file mode 100644 index 00000000..e8dcb952 --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/state_extension.rb @@ -0,0 +1,25 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Apollo + module SchemaDefinition + # Extension module applied to `ElasticGraph::SchemaDefinition::State` to support extra Apollo state. + # + # @private + module StateExtension + # @dynamic apollo_directive_definitions, apollo_directive_definitions= + attr_accessor :apollo_directive_definitions + + def self.extended(state) + state.apollo_directive_definitions = [] # Ensure it's never `nil`. + end + end + end + end +end diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/union_type_extension.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/union_type_extension.rb new file mode 100644 index 00000000..b521a36b --- /dev/null +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/union_type_extension.rb @@ -0,0 +1,21 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/apollo/schema_definition/apollo_directives" + +module ElasticGraph + module Apollo + module SchemaDefinition + # Extends {ElasticGraph::SchemaDefinition::SchemaElements::UnionType} to offer Apollo union type directives. + module UnionTypeExtension + include ApolloDirectives::Inaccessible + include ApolloDirectives::Tag + end + end + end +end diff --git a/elasticgraph-apollo/script/boot_eg_apollo_implementation b/elasticgraph-apollo/script/boot_eg_apollo_implementation new file mode 100755 index 00000000..28d1f9ff --- /dev/null +++ b/elasticgraph-apollo/script/boot_eg_apollo_implementation @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# Abort script at first error, when a command exits with non-zero status. +# Verbose form of `set -e`. +set -o errexit + +# Attempt to use undefined variable outputs error message, and forces an exit +# Verbose form of `set -u`. +set -o nounset + +# If set, the return value of a pipeline is the value of the last (rightmost) +# command to exit with a non-zero status, or zero if all commands in the +# pipeline exit successfully. +set -o pipefail + +# Print a trace of simple commands. +# Verbose form of `set -x`. +set -o xtrace + +script_dir=$(dirname $0) +source $script_dir/export_docker_env_vars.sh +docker compose -f $script_dir/../apollo_tests_implementation/docker-compose.yaml up --build diff --git a/elasticgraph-apollo/script/export_docker_env_vars.sh b/elasticgraph-apollo/script/export_docker_env_vars.sh new file mode 100644 index 00000000..0d1ba73e --- /dev/null +++ b/elasticgraph-apollo/script/export_docker_env_vars.sh @@ -0,0 +1,15 @@ +script_dir=$(dirname $0) + +# Use the current max Elasticsearch version we test against. +export VERSION=$(ruby -ryaml -e "puts YAML.load_file('$script_dir/../../config/tested_datastore_versions.yaml').fetch('elasticsearch').max_by { |v| Gem::Version.new(v) }") + +# Use the same Ruby version in the docker container as what we are currently using. +export RUBY_VERSION=$(ruby -e "puts RUBY_VERSION") + +# Call the ENV "apollo" instead of "test" or "local" to avoid interference with +# the Elasticsearch container booted for those envs. +export ENV=apollo + +# Apollo federation version used to generate the schema artifacts. +# TODO: Move to v2.6 once it is supported by the test suite. +export TARGET_APOLLO_FEDERATION_VERSION=2.3 diff --git a/elasticgraph-apollo/script/test_compatibility b/elasticgraph-apollo/script/test_compatibility new file mode 100755 index 00000000..a3db71a5 --- /dev/null +++ b/elasticgraph-apollo/script/test_compatibility @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Abort script at first error, when a command exits with non-zero status. +# Verbose form of `set -e`. +set -o errexit + +# Attempt to use undefined variable outputs error message, and forces an exit +# Verbose form of `set -u`. +set -o nounset + +# If set, the return value of a pipeline is the value of the last (rightmost) +# command to exit with a non-zero status, or zero if all commands in the +# pipeline exit successfully. +set -o pipefail + +# Print a trace of simple commands. +# Verbose form of `set -x`. +set -o xtrace + +script_dir=$(dirname $0) + +# Default to the Apollo federation version 2.6, as higher versions are not supported by the test suite properly. +apollo_federation_version=${1:-2.6} + +# Latest version as of 2024-05-08. Gotten from: +# https://www.npmjs.com/package/@apollo/federation-subgraph-compatibility?activeTab=versions +apollo_subgraph_tests_version=2.2.0 + +TARGET_APOLLO_FEDERATION_VERSION=$apollo_federation_version bundle exec rake \ + --rakefile $script_dir/../apollo_tests_implementation/Rakefile schema_artifacts:dump + +source $script_dir/export_docker_env_vars.sh + +# Running the tests produces some artifacts we don't want at our project root, so we run it from a tmp directory. +rm -rf $script_dir/../../tmp/apollo_compat_test +mkdir -p $script_dir/../../tmp/apollo_compat_test + +pushd $script_dir/../../tmp/apollo_compat_test + +# The latest apollo/federation-subgraph-compatibility tests target federation v2.3, and +# when we target lower federation versions (e.g. v2.0), we get warnings from some of the tests. +# So we only want to `--failOnWarning` when targeting federation versions higher versions +# (e.g. v2.3, v2.5, v2.6). +additional_flags=$([ "$apollo_federation_version" == "2.0" ] && echo "" || echo "--failOnWarning" ) + +TARGET_APOLLO_FEDERATION_VERSION=2.3 npx --yes \ + @apollo/federation-subgraph-compatibility@${apollo_subgraph_tests_version} docker \ + --compose ../../elasticgraph-apollo/apollo_tests_implementation/docker-compose.yaml \ + --path /graphql \ + --schema ../../elasticgraph-apollo/apollo_tests_implementation/config/schema/artifacts/schema.graphql \ + --debug \ + --failOnRequired $additional_flags + +popd diff --git a/elasticgraph-apollo/sig/apollo_federation.rbs b/elasticgraph-apollo/sig/apollo_federation.rbs new file mode 100644 index 00000000..7b4ddad0 --- /dev/null +++ b/elasticgraph-apollo/sig/apollo_federation.rbs @@ -0,0 +1,5 @@ +module ApolloFederation + class Tracing + def self.should_add_traces: (::Hash[::String, ::String]) -> bool + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/graphql/engine_extension.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/graphql/engine_extension.rbs new file mode 100644 index 00000000..2a00d4cc --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/graphql/engine_extension.rbs @@ -0,0 +1,8 @@ +module ElasticGraph + module Apollo + module GraphQL + module EngineExtension: ElasticGraph::GraphQL + end + end + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/graphql/entities_field_resolver.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/graphql/entities_field_resolver.rbs new file mode 100644 index 00000000..a14df1b2 --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/graphql/entities_field_resolver.rbs @@ -0,0 +1,148 @@ +module ElasticGraph + module Apollo + module GraphQL + class EntitiesFieldResolver + include ::ElasticGraph::GraphQL::Resolvers::_Resolver + + def initialize: ( + datastore_query_builder: ::ElasticGraph::GraphQL::DatastoreQuery::Builder, + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + ) -> void + + private + + @datastore_query_builder: ::ElasticGraph::GraphQL::DatastoreQuery::Builder + @schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + + def build_query: ( + _Adapter[_Representation, untyped], + ::Array[representation], + ::Hash[::Symbol, untyped] + ) -> ::ElasticGraph::GraphQL::DatastoreQuery? + + def try_parse_representation: ( + ::Hash[::String, untyped], + ElasticGraph::GraphQL::Schema + ) { (::String) -> void } -> representation? + + def translate_field_names: ( + ::Hash[::String, untyped], + ElasticGraph::GraphQL::Schema::Type + ) -> ::Hash[::String, untyped] + + interface _Representation + end + + interface _Adapter[R < _Representation, I] + def type: () -> ElasticGraph::GraphQL::Schema::Type + + def customize_query: ( + ElasticGraph::GraphQL::DatastoreQuery, + ::Array[R] + ) -> ElasticGraph::GraphQL::DatastoreQuery? + + def index_search_hits: ( + ElasticGraph::GraphQL::DatastoreResponse::SearchResponse + ) -> I + + def identify_matching_hit: ( + I, + R, + context: ::GraphQL::Query::Context, + index: ::Integer + ) -> (ElasticGraph::GraphQL::DatastoreResponse::Document | ::Hash[::String, untyped])? + + def indexed? : -> boolish + end + + class RepresentationWithIdSupertype + attr_reader type: ElasticGraph::GraphQL::Schema::Type + attr_reader id: ::String + attr_reader other_fields: ::Hash[::String, untyped] + attr_reader schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + attr_reader adapter: RepresentationWithId::Adapter + + def initialize: ( + type: ElasticGraph::GraphQL::Schema::Type, + id: ::String, + other_fields: ::Hash[::String, untyped], + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + adapter: RepresentationWithId::Adapter + ) -> void + end + + class RepresentationWithId < RepresentationWithIdSupertype + def initialize: ( + type: ElasticGraph::GraphQL::Schema::Type, + id: ::String, + other_fields: ::Hash[::String, untyped], + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + ) -> void + + class Adapter + include _Adapter[RepresentationWithId, ::Hash[::String, ElasticGraph::GraphQL::DatastoreResponse::Document]] + attr_reader schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + + def initialize: ( + type: ElasticGraph::GraphQL::Schema::Type, + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + ) -> void + + def self.new: ( + type: ElasticGraph::GraphQL::Schema::Type, + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + ) -> instance | ( + ElasticGraph::GraphQL::Schema::Type, + SchemaArtifacts::RuntimeMetadata::SchemaElementNames + ) -> instance + + private + + def additional_requested_fields_for: (::Array[RepresentationWithId]) -> ::Array[::String] + def fields_in: (::Hash[::String, untyped]) -> ::Array[::String] + def match?: (::Hash[::String, untyped], ::Hash[::String, untyped]) -> bool + end + end + + class RepresentationWithoutIdSupertype + attr_reader type: ElasticGraph::GraphQL::Schema::Type + attr_reader fields: ::Hash[::String, untyped] + attr_reader schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + + def initialize: ( + type: ElasticGraph::GraphQL::Schema::Type, + fields: ::Hash[::String, untyped], + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + ) -> void + end + + class RepresentationWithoutId < RepresentationWithoutIdSupertype + include _Adapter[RepresentationWithoutId, ::Hash[::String, ::Array[ElasticGraph::GraphQL::DatastoreResponse::Document]]] + def adapter: () -> self + + private + + def build_filter_for_hash: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + end + + class RepresentationWithoutIndexSupertype + attr_reader type: ElasticGraph::GraphQL::Schema::Type + attr_reader representation_hash: Hash[::String, untyped] + + def initialize: ( + type: ElasticGraph::GraphQL::Schema::Type, + representation_hash: Hash[::String, untyped] + ) -> void + end + + class RepresentationWithoutIndex < RepresentationWithoutIndexSupertype + include _Adapter[RepresentationWithoutIndex, ::Hash[::String, untyped]?] + def adapter: () -> self + + end + + type representation = RepresentationWithId | RepresentationWithoutId | RepresentationWithoutIndex + end + end + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/graphql/http_endpoint_extension.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/graphql/http_endpoint_extension.rbs new file mode 100644 index 00000000..f09c2bdc --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/graphql/http_endpoint_extension.rbs @@ -0,0 +1,8 @@ +module ElasticGraph + module Apollo + module GraphQL + module HTTPEndpointExtension: ElasticGraph::GraphQL::HTTPEndpoint + end + end + end +end \ No newline at end of file diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/graphql/service_field_resolver.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/graphql/service_field_resolver.rbs new file mode 100644 index 00000000..3c378679 --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/graphql/service_field_resolver.rbs @@ -0,0 +1,13 @@ +module ElasticGraph + module Apollo + module GraphQL + class ServiceFieldResolver + include ::ElasticGraph::GraphQL::Resolvers::_Resolver + + private + + def service_sdl: (::GraphQL::Schema) -> ::String + end + end + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/api_extension.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/api_extension.rbs new file mode 100644 index 00000000..e8ba2eb5 --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/api_extension.rbs @@ -0,0 +1,25 @@ +module ElasticGraph + module Apollo + module SchemaDefinition + module APIExtension: ElasticGraph::SchemaDefinition::API + def tag_built_in_types_with: (::String, ?except: ::Array[::String]) -> void + def target_apollo_federation_version: (::String) -> void + + def state: () -> (ElasticGraph::SchemaDefinition::State & StateExtension) + + private + + DIRECTIVE_DEFINITIONS_BY_FEDERATION_VERSION: ::Hash[::String, ::Array[::String]] + + def define_apollo_schema_elements: () -> void + def apollo_object_type: (::String) { (ElasticGraph::SchemaDefinition::SchemaElements::ObjectType) -> void } -> void + def apollo_union_type: (::String) { (ElasticGraph::SchemaDefinition::SchemaElements::UnionType) -> void } -> void + def apollo_scalar_type: (::String) { (ElasticGraph::SchemaDefinition::SchemaElements::ScalarType) -> void } -> void + def apollo_enum_type: (::String) { (ElasticGraph::SchemaDefinition::SchemaElements::EnumType) -> void } -> void + def validate_entity_types_can_all_be_resolved: (::Array[ElasticGraph::SchemaDefinition::indexableType]) -> void + + def self.extended: (ElasticGraph::SchemaDefinition::API & APIExtension) -> void + end + end + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/apollo_directives.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/apollo_directives.rbs new file mode 100644 index 00000000..9e7bd440 --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/apollo_directives.rbs @@ -0,0 +1,59 @@ +module ElasticGraph + module Apollo + module SchemaDefinition + module ApolloDirectives + module Authenticated: ElasticGraph::SchemaDefinition::Mixins::HasDirectives + def apollo_authenticated: -> void + end + + module Extends: ElasticGraph::SchemaDefinition::Mixins::HasDirectives + def apollo_extends: -> void + end + + module External: ElasticGraph::SchemaDefinition::Mixins::HasDirectives + def apollo_external: -> void + end + + module Inaccessible: ElasticGraph::SchemaDefinition::Mixins::HasDirectives + def apollo_inaccessible: -> void + end + + module InterfaceObject: ElasticGraph::SchemaDefinition::Mixins::HasDirectives + def apollo_interface_object: -> void + end + + module Key: ElasticGraph::SchemaDefinition::Mixins::HasDirectives + def apollo_key: (fields: String, ?resolvable: bool) -> void + end + + module Override: ElasticGraph::SchemaDefinition::Mixins::HasDirectives + def apollo_override: (from: String) -> void + end + + module Policy: ElasticGraph::SchemaDefinition::Mixins::HasDirectives + def apollo_policy: (policies: ::Array[::Array[String]]) -> void + end + + module Provides: ElasticGraph::SchemaDefinition::Mixins::HasDirectives + def apollo_provides: (fields: String) -> void + end + + module Requires: ElasticGraph::SchemaDefinition::Mixins::HasDirectives + def apollo_requires: (fields: String) -> void + end + + module RequiresScopes: ElasticGraph::SchemaDefinition::Mixins::HasDirectives + def apollo_requires_scopes: (scopes: ::Array[::Array[String]]) -> void + end + + module Shareable: ElasticGraph::SchemaDefinition::Mixins::HasDirectives + def apollo_shareable: -> void + end + + module Tag: ElasticGraph::SchemaDefinition::Mixins::HasDirectives + def apollo_tag: (name: String) -> void + end + end + end + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/argument_extension.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/argument_extension.rbs new file mode 100644 index 00000000..5f84fd5a --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/argument_extension.rbs @@ -0,0 +1,10 @@ +module ElasticGraph + module Apollo + module SchemaDefinition + module ArgumentExtension: ElasticGraph::SchemaDefinition::SchemaElements::Argument + include ApolloDirectives::Inaccessible + include ApolloDirectives::Tag + end + end + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/entity_type_extension.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/entity_type_extension.rbs new file mode 100644 index 00000000..bc183a7e --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/entity_type_extension.rbs @@ -0,0 +1,8 @@ +module ElasticGraph + module Apollo + module SchemaDefinition + module EntityTypeExtension: ::ElasticGraph::SchemaDefinition::SchemaElements::UnionType + end + end + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/enum_type_extension.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/enum_type_extension.rbs new file mode 100644 index 00000000..b61c743c --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/enum_type_extension.rbs @@ -0,0 +1,13 @@ +module ElasticGraph + module Apollo + module SchemaDefinition + module EnumTypeExtension: ElasticGraph::SchemaDefinition::SchemaElements::EnumType + include ApolloDirectives::Authenticated + include ApolloDirectives::Inaccessible + include ApolloDirectives::Policy + include ApolloDirectives::RequiresScopes + include ApolloDirectives::Tag + end + end + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/enum_value_extension.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/enum_value_extension.rbs new file mode 100644 index 00000000..5489c0bd --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/enum_value_extension.rbs @@ -0,0 +1,10 @@ +module ElasticGraph + module Apollo + module SchemaDefinition + module EnumValueExtension: ElasticGraph::SchemaDefinition::SchemaElements::EnumValue + include ApolloDirectives::Inaccessible + include ApolloDirectives::Tag + end + end + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/factory_extension.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/factory_extension.rbs new file mode 100644 index 00000000..2e3acdbd --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/factory_extension.rbs @@ -0,0 +1,8 @@ +module ElasticGraph + module Apollo + module SchemaDefinition + module FactoryExtension: ElasticGraph::SchemaDefinition::Factory + end + end + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/field_extension.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/field_extension.rbs new file mode 100644 index 00000000..cda320ec --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/field_extension.rbs @@ -0,0 +1,20 @@ +module ElasticGraph + module Apollo + module SchemaDefinition + module FieldExtension: ElasticGraph::SchemaDefinition::SchemaElements::Field + include ApolloDirectives::Authenticated + include ApolloDirectives::External + include ApolloDirectives::Inaccessible + include ApolloDirectives::Override + include ApolloDirectives::Policy + include ApolloDirectives::Provides + include ApolloDirectives::Requires + include ApolloDirectives::RequiresScopes + include ApolloDirectives::Shareable + include ApolloDirectives::Tag + def tag_with: (::String) -> void + def self.tagged_with?: (ElasticGraph::SchemaDefinition::Mixins::HasDirectives, ::String) -> bool + end + end + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/graphql_sdl_enumerator_extension.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/graphql_sdl_enumerator_extension.rbs new file mode 100644 index 00000000..07ad5c2d --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/graphql_sdl_enumerator_extension.rbs @@ -0,0 +1,8 @@ +module ElasticGraph + module Apollo + module SchemaDefinition + module GraphQLSDLEnumeratorExtension: ElasticGraph::SchemaDefinition::SchemaElements::GraphQLSDLEnumerator + end + end + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/input_type_extension.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/input_type_extension.rbs new file mode 100644 index 00000000..0ad3b253 --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/input_type_extension.rbs @@ -0,0 +1,10 @@ +module ElasticGraph + module Apollo + module SchemaDefinition + module InputTypeExtension: ElasticGraph::SchemaDefinition::SchemaElements::InputType + include ApolloDirectives::Inaccessible + include ApolloDirectives::Tag + end + end + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/interface_type_extension.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/interface_type_extension.rbs new file mode 100644 index 00000000..f325c1c4 --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/interface_type_extension.rbs @@ -0,0 +1,15 @@ +module ElasticGraph + module Apollo + module SchemaDefinition + module InterfaceTypeExtension: ElasticGraph::SchemaDefinition::SchemaElements::InterfaceType + include ApolloDirectives::Authenticated + include ApolloDirectives::Extends + include ApolloDirectives::Inaccessible + include ApolloDirectives::Key + include ApolloDirectives::Policy + include ApolloDirectives::RequiresScopes + include ApolloDirectives::Tag + end + end + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/object_type_extension.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/object_type_extension.rbs new file mode 100644 index 00000000..96ef22f3 --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/object_type_extension.rbs @@ -0,0 +1,18 @@ +module ElasticGraph + module Apollo + module SchemaDefinition + module ObjectTypeExtension: ElasticGraph::SchemaDefinition::SchemaElements::ObjectType + include ApolloDirectives::Authenticated + include ApolloDirectives::Extends + include ApolloDirectives::External + include ApolloDirectives::Inaccessible + include ApolloDirectives::InterfaceObject + include ApolloDirectives::Key + include ApolloDirectives::Policy + include ApolloDirectives::RequiresScopes + include ApolloDirectives::Shareable + include ApolloDirectives::Tag + end + end + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/scalar_type_extension.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/scalar_type_extension.rbs new file mode 100644 index 00000000..20a37f00 --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/scalar_type_extension.rbs @@ -0,0 +1,13 @@ +module ElasticGraph + module Apollo + module SchemaDefinition + module ScalarTypeExtension: ElasticGraph::SchemaDefinition::SchemaElements::ScalarType + include ApolloDirectives::Authenticated + include ApolloDirectives::Inaccessible + include ApolloDirectives::Policy + include ApolloDirectives::RequiresScopes + include ApolloDirectives::Tag + end + end + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/state_extension.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/state_extension.rbs new file mode 100644 index 00000000..ac8223a3 --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/state_extension.rbs @@ -0,0 +1,11 @@ +module ElasticGraph + module Apollo + module SchemaDefinition + module StateExtension: ElasticGraph::SchemaDefinition::State + attr_accessor apollo_directive_definitions: ::Array[::String] + + def self.extended: (ElasticGraph::SchemaDefinition::State & StateExtension) -> void + end + end + end +end diff --git a/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/union_type_extension.rbs b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/union_type_extension.rbs new file mode 100644 index 00000000..dbc3e5a6 --- /dev/null +++ b/elasticgraph-apollo/sig/elastic_graph/apollo/schema_definition/union_type_extension.rbs @@ -0,0 +1,10 @@ +module ElasticGraph + module Apollo + module SchemaDefinition + module UnionTypeExtension: ElasticGraph::SchemaDefinition::SchemaElements::UnionType + include ApolloDirectives::Inaccessible + include ApolloDirectives::Tag + end + end + end +end diff --git a/elasticgraph-apollo/spec/integration/elastic_graph/apollo/graphql/entities_field_resolver_spec.rb b/elasticgraph-apollo/spec/integration/elastic_graph/apollo/graphql/entities_field_resolver_spec.rb new file mode 100644 index 00000000..2fab2ab7 --- /dev/null +++ b/elasticgraph-apollo/spec/integration/elastic_graph/apollo/graphql/entities_field_resolver_spec.rb @@ -0,0 +1,628 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/apollo/graphql/engine_extension" +require "elastic_graph/apollo/graphql/entities_field_resolver" +require "elastic_graph/graphql" + +module ElasticGraph + module Apollo + module GraphQL + RSpec.describe EntitiesFieldResolver, :uses_datastore, :factories, :builds_graphql, :builds_indexer do + let(:graphql) do + build_graphql( + schema_artifacts_directory: "config/schema/artifacts_with_apollo", + extension_modules: [EngineExtension] + ) + end + + let(:indexer) { build_indexer(datastore_core: graphql.datastore_core) } + + before do + # Perform any cached calls to the datastore to happen before our `query_datastore` + # matcher below which tries to assert which specific requests get made, since index definitions + # have caching behavior that can make the presence or absence of that request slightly non-deterministic. + pre_cache_index_state(graphql) + end + + context "with an empty array of representations" do + it "returns an empty array" do + data = execute_expecting_no_errors(<<~QUERY) + query { + _entities(representations: []) { + ... on Widget { + id + } + } + } + QUERY + + expect(data).to eq("_entities" => []) + + # It should perform 0 msearch request... + expect(datastore_msearch_requests("main").size).to eq(0) + # ...with 0 searches + expect(performed_search_metadata("main")).to have_total_widget_searches(0) + end + end + + context "with a non-empty array of representations" do + it "looks up each representation by the given fields, returning each in the response (or nil if not found)" do + index_records( + build(:widget, id: "w1", name: "widget1"), + build(:widget, id: "w2", name: "widget2") + ) + + data = execute_expecting_no_errors(<<~QUERY) + query { + _entities(representations: [ + {__typename: "Widget", id: "w1"} + {__typename: "Widget", id: "w2"} + {__typename: "Widget", id: "w3"} + {__typename: "Widget", name: "widget1"} + ]) { + ... on Widget { + id + name + } + } + } + QUERY + + expect(data).to eq("_entities" => [ + {"id" => "w1", "name" => "widget1"}, + {"id" => "w2", "name" => "widget2"}, + nil, + {"id" => "w1", "name" => "widget1"} + ]) + + # It should perform 1 msearch request... + expect(datastore_msearch_requests("main").size).to eq(1) + # ...with 2 searches (1 for the `id` queries, one for the `name`) + expect(performed_search_metadata("main")).to have_total_widget_searches(2) + end + + it "supports lookups on nested representation fields" do + index_records( + build(:widget, id: "w1", name: "widget1", options: build(:widget_options, size: "SMALL")), + build(:widget, id: "w2", name: "widget2", options: build(:widget_options, size: "LARGE")) + ) + + data = execute_expecting_no_errors(<<~QUERY) + query { + _entities(representations: [ + {__typename: "Widget", options: {size: "SMALL"}} + {__typename: "Widget", options: {size: "MEDIUM"}} + {__typename: "Widget", options: {size: "LARGE"}} + ]) { + ... on Widget { + id + name + } + } + } + QUERY + + expect(data).to eq("_entities" => [ + {"id" => "w1", "name" => "widget1"}, + nil, + {"id" => "w2", "name" => "widget2"} + ]) + + # It should perform 1 msearch request... + expect(datastore_msearch_requests("main").size).to eq(1) + # ...with 3 searches (1 for the `id` queries, one for the `name` and one for `options.size`) + expect(performed_search_metadata("main")).to have_total_widget_searches(3) + end + + it "resolves non-indexed entities, including a relation" do + index_records( + build(:team, id: "t1", current_name: "team1", country_code: "US"), + build(:team, id: "t2", current_name: "team2", country_code: "MX"), + build(:team, id: "t3", current_name: "team3", country_code: "US"), + build(:team, id: "t4", current_name: "team4", country_code: "CA") + ) + + data = execute_expecting_no_errors(<<~QUERY) + query { + _entities(representations: [ + {__typename: "Country", id: "US"}, + {__typename: "Country", id: "MX"}, + {__typename: "Country", id: "CA"} + ]) { + ... on Country { + id + teams { + nodes { + current_name + } + } + } + } + } + QUERY + + expect(data).to eq("_entities" => [ + {"id" => "US", "teams" => {"nodes" => [{"current_name" => "team1"}, {"current_name" => "team3"}]}}, + {"id" => "MX", "teams" => {"nodes" => [{"current_name" => "team2"}]}}, + {"id" => "CA", "teams" => {"nodes" => [{"current_name" => "team4"}]}} + ]) + end + + it "returns empty values (nil or empty list) when asked to resolve non-relation/non-key fields on a non-indexed entity" do + data = execute_expecting_no_errors(<<~QUERY) + query { + _entities(representations: [ + {__typename: "Country", id: "US"}, + {__typename: "Country", id: "MX"}, + {__typename: "Country", id: "CA"} + ]) { + ... on Country { + id + currency + # These two cases (nodes vs edges) have manifested different issues, so we check both here. + name_nodes: names { nodes } + name_edges: names { edges { node } } + } + } + } + QUERY + + expect(data).to eq("_entities" => [ + {"id" => "US", "name_nodes" => {"nodes" => []}, "name_edges" => {"edges" => []}, "currency" => nil}, + {"id" => "MX", "name_nodes" => {"nodes" => []}, "name_edges" => {"edges" => []}, "currency" => nil}, + {"id" => "CA", "name_nodes" => {"nodes" => []}, "name_edges" => {"edges" => []}, "currency" => nil} + ]) + end + + it "supports lookups involving fields that have an alternate `name_in_index`" do + index_records( + build(:widget, id: "w1", name: "widget1", the_options: build(:widget_options, the_size: "SMALL")) + ) + + data = execute_expecting_no_errors(<<~QUERY) + query { + _entities(representations: [ + {__typename: "Widget", id: "w1", the_options: {the_size: "SMALL"}} + {__typename: "Widget", name: "widget1", the_options: {the_size: "SMALL"}} + ]) { + ... on Widget { + id + name + } + } + } + QUERY + + expect(data).to eq("_entities" => [ + {"id" => "w1", "name" => "widget1"}, + {"id" => "w1", "name" => "widget1"} + ]) + + # It should perform 1 msearch request... + expect(datastore_msearch_requests("main").size).to eq(1) + # ...with 1 searches (1 for the `id` queries) + expect(performed_search_metadata("main")).to have_total_widget_searches(2) + end + + it "supports array and non-nullable fields" do + index_records( + build(:widget, id: "w1", name: "widget1", fees: [build(:money, currency: "USD", amount_cents: 100)]) + ) + + data = execute_expecting_no_errors(<<~QUERY) + query { + _entities(representations: [ + {__typename: "Widget", id: "w1", fees: [{currency: "USD"}]} + ]) { + ... on Widget { + id + name + } + } + } + QUERY + + expect(data).to eq("_entities" => [ + {"id" => "w1", "name" => "widget1"} + ]) + + # It should perform 1 msearch request... + expect(datastore_msearch_requests("main").size).to eq(1) + # ...with 1 searches (1 for the `id` queries) + expect(performed_search_metadata("main")).to have_total_widget_searches(1) + end + + it "supports lookups on compound representation fields" do + index_records( + build(:widget, id: "w1", name: "foo", options: build(:widget_options, size: "SMALL")), + build(:widget, id: "w2", name: "foo", options: build(:widget_options, size: "LARGE")), + build(:widget, id: "w3", name: "bar", options: build(:widget_options, size: "LARGE")) + ) + + data = execute_expecting_no_errors(<<~QUERY) + query { + _entities(representations: [ + {__typename: "Widget", name: "foo", options: {size: "SMALL"}} + {__typename: "Widget", name: "bar", options: {size: "SMALL"}} + {__typename: "Widget", name: "foo", options: {size: "LARGE"}} + {__typename: "Widget", name: "bar", options: {size: "LARGE"}} + ]) { + ... on Widget { + id + name + } + } + } + QUERY + + expect(data).to eq("_entities" => [ + {"id" => "w1", "name" => "foo"}, + nil, + {"id" => "w2", "name" => "foo"}, + {"id" => "w3", "name" => "bar"} + ]) + + # It should perform 1 msearch request... + expect(datastore_msearch_requests("main").size).to eq(1) + # ...with 4 searches + expect(performed_search_metadata("main")).to have_total_widget_searches(4) + end + + it "supports lookups on compound representation fields with id" do + index_records( + build(:widget, id: "w1", name: "foo", options: build(:widget_options, size: "SMALL")), + build(:widget, id: "w2", name: "foo", options: build(:widget_options, size: "LARGE")), + build(:widget, id: "w3", name: "bar", options: build(:widget_options, size: "LARGE")) + ) + + data = execute_expecting_no_errors(<<~QUERY) + query { + _entities(representations: [ + {__typename: "Widget", id: "w1", name: "foo", options: {size: "SMALL"}} + {__typename: "Widget", id: "w3", name: "bar", options: {size: "SMALL"}} + {__typename: "Widget", id: "w2", name: "foo", options: {size: "LARGE"}} + {__typename: "Widget", id: "w3", name: "bar", options: {size: "LARGE"}} + ]) { + ... on Widget { + id + name + } + } + } + QUERY + + expect(data).to eq("_entities" => [ + {"id" => "w1", "name" => "foo"}, + nil, + {"id" => "w2", "name" => "foo"}, + {"id" => "w3", "name" => "bar"} + ]) + + # It should perform 1 msearch request... + expect(datastore_msearch_requests("main").size).to eq(1) + # ...with 1 searches (1 for the `id` queries) + expect(performed_search_metadata("main")).to have_total_widget_searches(1) + end + + it "ignores array filtering (for now, will be improved later)" do + index_records( + build(:widget, id: "w1", name: "foo", options: build(:widget_options, size: "SMALL")), + build(:widget, id: "w2", name: "bar", options: build(:widget_options, size: "LARGE")) + ) + + response = execute(<<~QUERY) + query { + _entities(representations: [ + {__typename: "Widget", id: "w1", name: ["foo"]} + {__typename: "Widget", id: "w1", name: ["bar"]} + {__typename: "Widget", name: ["foo"], options: {size: "LARGE"}} + {__typename: "Widget", name: ["bar"], options: {size: "LARGE"}} + ]) { + ... on Widget { + id + name + } + } + } + QUERY + + expect(response.dig("data", "_entities")).to eq [ + {"id" => "w1", "name" => "foo"}, + {"id" => "w1", "name" => "foo"}, + {"id" => "w2", "name" => "bar"}, + {"id" => "w2", "name" => "bar"} + ] + + # It should perform 1 msearch request... + expect(datastore_msearch_requests("main").size).to eq(1) + # ...with 2 searches (1 for the `id` queries, one for the `name`) + expect(performed_search_metadata("main")).to have_total_widget_searches(2) + end + + it "returns `nil` for representations with unexpected field types" do + data = execute_expecting_no_errors(<<~QUERY) + query { + _entities(representations: [ + {__typename: "Widget", id: nil} + {__typename: "Widget", id: 3} + {__typename: "Widget", id: true} + {__typename: "Widget", id: []} + {__typename: "Widget", id: {}} + ]) { + ... on Widget { + id + name + } + } + } + QUERY + + expect(data).to eq("_entities" => [ + nil, + nil, + nil, + nil, + nil + ]) + + # It should perform 1 msearch request... + expect(datastore_msearch_requests("main").size).to eq(1) + # ...with 1 searches (1 for the `id` queries) + expect(performed_search_metadata("main")).to have_total_widget_searches(1) + end + + it "returns an error for each representation that is not a hash as expected, while still returning the entities that it can" do + index_records( + build(:widget, id: "w1", name: "widget1") + ) + + expect { + response = execute(<<~QUERY) + query { + _entities(representations: [ + true + 3 + "foo" + {__typename: "Widget", id: "w1"} + 2.5 + [{__typename: "Widget", id: "w1"}] + ]) { + ... on Widget { + id + name + } + } + } + QUERY + + expect(response.dig("data", "_entities")).to eq [ + nil, + nil, + nil, + {"id" => "w1", "name" => "widget1"}, + nil, + nil + ] + + expect(response["errors"]).to eq [ + {"message" => "Representation at index 0 is not a JSON object."}, + {"message" => "Representation at index 1 is not a JSON object."}, + {"message" => "Representation at index 2 is not a JSON object."}, + {"message" => "Representation at index 4 is not a JSON object."}, + {"message" => "Representation at index 5 is not a JSON object."} + ] + + # It should perform 1 msearch request... + expect(datastore_msearch_requests("main").size).to eq(1) + # ...with 1 searches (1 for the `id` queries) + expect(performed_search_metadata("main")).to have_total_widget_searches(1) + }.to log_warning(a_string_including("is not a JSON object")) + end + + it "returns an error if the representations lacks a `__typename`, while still returning the entities that it can" do + index_records( + build(:widget, id: "w1", name: "widget1") + ) + + expect { + response = execute(<<~QUERY) + query { + _entities(representations: [ + {id: "w1"}, + {__typename: "Widget", id: "w1"} + ]) { + ... on Widget { + id + name + } + } + } + QUERY + + expect(response.dig("data", "_entities")).to eq [ + nil, + {"id" => "w1", "name" => "widget1"} + ] + + expect(response["errors"]).to eq [ + {"message" => "Representation at index 0 lacks a `__typename`."} + ] + + # It should perform 1 msearch request... + expect(datastore_msearch_requests("main").size).to eq(1) + # ...with 1 searches (1 for the `id` queries) + expect(performed_search_metadata("main")).to have_total_widget_searches(1) + }.to log_warning(a_string_including("lacks a `__typename`")) + end + + it "returns an error if the `__typename` is unknown, while still returning the entities that it can" do + index_records( + build(:widget, id: "w1", name: "widget1") + ) + + expect { + response = execute(<<~QUERY) + query { + _entities(representations: [ + {__typename: "Fidget", id: "w1"} + {__typename: "Widget", id: "w1"} + ]) { + ... on Widget { + id + name + } + } + } + QUERY + + expect(response.dig("data", "_entities")).to eq [ + nil, + {"id" => "w1", "name" => "widget1"} + ] + + expect(response["errors"]).to eq [ + {"message" => "Representation at index 0 has an unrecognized `__typename`: Fidget."} + ] + + # It should perform 1 msearch request... + expect(datastore_msearch_requests("main").size).to eq(1) + # ...with 1 searches (1 for the `id` queries) + expect(performed_search_metadata("main")).to have_total_widget_searches(1) + }.to log_warning(a_string_including("has an unrecognized `__typename`: Fidget")) + end + + it "returns an error if the only field in the representation is `__typename`, while still returning the entities that it can" do + index_records( + build(:widget, id: "w1", name: "widget1") + ) + + expect { + response = execute(<<~QUERY) + query { + _entities(representations: [ + {__typename: "Widget"} + {__typename: "Widget", id: "w1"} + ]) { + ... on Widget { + id + name + } + } + } + QUERY + + expect(response.dig("data", "_entities")).to eq [ + nil, + {"id" => "w1", "name" => "widget1"} + ] + + expect(response["errors"]).to eq [ + {"message" => "Representation at index 0 has only a `__typename` field."} + ] + + # It should perform 1 msearch request... + expect(datastore_msearch_requests("main").size).to eq(1) + # ...with 1 searches (1 for the `id` queries) + expect(performed_search_metadata("main")).to have_total_widget_searches(1) + }.to log_warning(a_string_including("has only a `__typename` field")) + end + end + + it "returns an error if a representation matches more than one document" do + index_records( + build(:widget, id: "w1", name: "foo"), + build(:widget, id: "w2", name: "foo"), + build(:widget, id: "w3", name: "bar") + ) + + expect { + response = execute(<<~QUERY) + query { + _entities(representations: [ + {__typename: "Widget", name: "foo"} + {__typename: "Widget", name: "bar"} + ]) { + ... on Widget { + id + name + } + } + } + QUERY + + expect(response.dig("data", "_entities")).to eq [ + nil, + {"id" => "w3", "name" => "bar"} + ] + + expect(response["errors"]).to eq [ + {"message" => "Representation at index 0 matches more than one entity."} + ] + + # It should perform 1 msearch request... + expect(datastore_msearch_requests("main").size).to eq(1) + # ...with 2 searches (1 for the `id` queries, one for the `name`) + expect(performed_search_metadata("main")).to have_total_widget_searches(2) + }.to log_warning(a_string_including("matches more than one entity")) + end + + it "does not interfere with other fields on the `Query` type, and batches datastore queries when possible" do + expect { + data = execute_expecting_no_errors(<<~EOS) + query { + _entities(representations: [ + {__typename: "Widget", id: "w1"} + ]) { + ... on Widget { + id + name + } + } + + widgets { + total_edge_count + } + } + EOS + + expect(data).to eq( + "_entities" => [nil], + "widgets" => {"total_edge_count" => 0} + ) + + # It should perform 1 msearch request... + expect(datastore_msearch_requests("main").size).to eq(1) + # ...with 2 searches (1 for the `id` queries, one for the `total_edge_count`) + expect(performed_search_metadata("main")).to have_total_widget_searches(2) + }.to query_datastore("main", 1).time + end + + def execute(query, **options) + response = nil + + expect { + response = graphql.graphql_query_executor.execute(query, **options) + }.to query_datastore("main", 1).time.or query_datastore("main", 0).times + + response + end + + def execute_expecting_no_errors(query, **options) + response = execute(query, **options) + expect(response["errors"]).to be nil + response.fetch("data") + end + + def have_total_widget_searches(count) + eq([{"index" => "widgets_rollover__*"}] * count) + end + end + end + end +end diff --git a/elasticgraph-apollo/spec/spec_helper.rb b/elasticgraph-apollo/spec/spec_helper.rb new file mode 100644 index 00000000..b54fc2fe --- /dev/null +++ b/elasticgraph-apollo/spec/spec_helper.rb @@ -0,0 +1,11 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-apollo`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. diff --git a/elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb new file mode 100644 index 00000000..ba22afc6 --- /dev/null +++ b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb @@ -0,0 +1,564 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/apollo/schema_definition/api_extension" +require "elastic_graph/spec_support/schema_definition_helpers" + +module ElasticGraph + module Apollo + RSpec.describe SchemaDefinition do + include_context "SchemaDefinitionHelpers" + + def self.with_both_casing_forms(&block) + context "with schema elements configured to use camelCase" do + let(:schema_element_name_form) { :camelCase } + module_exec(&block) + end + + context "with schema elements configured to use snake_case" do + let(:schema_element_name_form) { :snake_case } + module_exec(&block) + end + end + + with_both_casing_forms do + let(:schema_elements) { SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: schema_element_name_form) } + + it "adds an `@authenticated` directive when `apollo_authenticated` is called on a schema element" do + schema_string = graphql_schema_string do |schema| + schema.enum_type "Size" do |e| + e.apollo_authenticated + e.value "SMALL" + end + + schema.interface_type "Identifiable" do |t| + t.apollo_authenticated + t.field "size", "Size" + end + + schema.object_type "Widget" do |t| + t.implements "Identifiable" + t.apollo_authenticated + + t.field "name", "String" do |f| + f.apollo_authenticated + end + + t.field "size", "Size" + t.field "url", "Url" + end + + schema.scalar_type "Url" do |f| + f.apollo_authenticated + f.mapping type: "keyword" + f.json_schema type: "string" + end + end + + expect(type_def_from(schema_string, "Size")).to eq(<<~EOS.strip) + enum Size @authenticated { + SMALL + } + EOS + + expect(type_def_from(schema_string, "Identifiable")).to eq(<<~EOS.strip) + interface Identifiable @authenticated { + size: Size + } + EOS + + expect(type_def_from(schema_string, "Widget")).to eq(<<~EOS.strip) + type Widget implements Identifiable @authenticated { + name: String @authenticated + size: Size + url: Url + } + EOS + + expect(type_def_from(schema_string, "Url")).to eq(<<~EOS.strip) + scalar Url @authenticated + EOS + end + + it "adds an `@extends` directive when `apollo_extends` is called on a schema element" do + schema_string = graphql_schema_string do |schema| + schema.interface_type "Identifiable" do |t| + t.apollo_extends + t.field "name", "String" + end + + schema.object_type "Widget" do |t| + t.implements "Identifiable" + + t.apollo_extends + t.field "name", "String" + end + end + + expect(type_def_from(schema_string, "Identifiable")).to eq(<<~EOS.strip) + interface Identifiable @extends { + name: String + } + EOS + + expect(type_def_from(schema_string, "Widget")).to eq(<<~EOS.strip) + type Widget implements Identifiable @extends { + name: String + } + EOS + end + + it "adds an `@external` directive when `apollo_external` is called on a schema element" do + schema_string = graphql_schema_string do |schema| + schema.object_type "Widget" do |t| + t.apollo_external + + t.field "name", "String" do |f| + f.apollo_external + end + end + end + + expect(type_def_from(schema_string, "Widget")).to eq(<<~EOS.strip) + type Widget @external { + name: String @external + } + EOS + end + + it "adds an `@inaccessible` directive when `apollo_inaccessible` is called on a schema element" do + schema_string = graphql_schema_string do |schema| + schema.enum_type "Size" do |e| + e.apollo_inaccessible + + e.value "SMALL" do |v| + v.apollo_inaccessible + end + end + + schema.interface_type "Identifiable" do |t| + t.apollo_inaccessible + t.field "size", "Size" + end + + schema.object_type "Widget" do |t| + t.implements "Identifiable" + + t.apollo_inaccessible + + t.field "name", "String" do |f| + f.apollo_inaccessible + + f.argument "some_arg", "String" do |a| + a.apollo_inaccessible + end + end + + t.field "size", "Size" + t.field "url", "Url" + end + + schema.scalar_type "Url" do |f| + f.apollo_inaccessible + f.mapping type: "keyword" + f.json_schema type: "string" + + f.customize_derived_types "UrlFilterInput" do |dt| + dt.apollo_inaccessible + dt.field "host", "String" do |dtf| + dtf.apollo_inaccessible + end + end + end + + schema.union_type "Thing" do |t| + t.apollo_inaccessible + t.subtype "Widget" + end + end + + expect(type_def_from(schema_string, "Size")).to eq(<<~EOS.strip) + enum Size @inaccessible { + SMALL @inaccessible + } + EOS + + expect(type_def_from(schema_string, "Identifiable")).to eq(<<~EOS.strip) + interface Identifiable @inaccessible { + size: Size + } + EOS + + expect(type_def_from(schema_string, "Widget")).to eq(<<~EOS.strip) + type Widget implements Identifiable @inaccessible { + name( + some_arg: String @inaccessible): String @inaccessible + size: Size + url: Url + } + EOS + + expect(type_def_from(schema_string, "Thing")).to eq(<<~EOS.strip) + union Thing @inaccessible = Widget + EOS + + expect(type_def_from(schema_string, "Url")).to eq(<<~EOS.strip) + scalar Url @inaccessible + EOS + + expect(type_def_from(schema_string, "UrlFilterInput")).to eq(<<~EOS.strip) + input UrlFilterInput @inaccessible { + #{schema_elements.any_of}: [UrlFilterInput!] + #{schema_elements.not}: UrlFilterInput + #{schema_elements.equal_to_any_of}: [Url] + host: String @inaccessible + } + EOS + end + + it "adds an `@interfaceObject` directive when `apollo_interface_object` is called on a schema element" do + schema_string = graphql_schema_string do |schema| + schema.object_type "Widget" do |t| + t.apollo_interface_object + t.field "name", "String" + end + end + + expect(type_def_from(schema_string, "Widget")).to eq(<<~EOS.strip) + type Widget @interfaceObject { + name: String + } + EOS + end + + it "adds a `@key` directive when `apollo_key` is called on a schema element" do + schema_string = graphql_schema_string do |schema| + schema.interface_type "Identifiable" do |t| + t.apollo_key(fields: "age") + t.field "age", "Int" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.implements "Identifiable" + + t.apollo_key(fields: "first_name last_name") + t.apollo_key(fields: "address", resolvable: false) + + t.field "first_name", "String" + t.field "last_name", "String" + t.field "address", "String" + t.field "age", "Int" + t.index "widgets" + end + end + + expect(type_def_from(schema_string, "Identifiable")).to eq(<<~EOS.strip) + interface Identifiable @key(fields: "age", resolvable: true) { + age: Int + } + EOS + + expect(type_def_from(schema_string, "Widget")).to eq(<<~EOS.strip) + type Widget implements Identifiable @key(fields: "first_name last_name", resolvable: true) @key(fields: "address", resolvable: false) @key(fields: "id", resolvable: true) { + id: ID! + first_name: String + last_name: String + address: String + age: Int + } + EOS + end + + it "adds an `@override` directive when `apollo_override` is called on a schema element" do + schema_string = graphql_schema_string do |schema| + schema.object_type "Widget" do |t| + t.field "name", "String" do |f| + f.apollo_override(from: "AnotherGraph") + end + end + end + + expect(type_def_from(schema_string, "Widget")).to eq(<<~EOS.strip) + type Widget { + name: String @override(from: "AnotherGraph") + } + EOS + end + + it "adds a `@policy` directive when `apollo_policy` is called on a schema element" do + schema_string = graphql_schema_string do |schema| + schema.enum_type "Size" do |e| + e.apollo_policy(policies: [["Policy1", "Policy2"], ["Policy3"]]) + e.value "SMALL" + end + + schema.interface_type "Identifiable" do |t| + t.apollo_policy(policies: [["Policy1", "Policy2"], ["Policy3"]]) + t.field "size", "Size" + end + + schema.object_type "Widget" do |t| + t.implements "Identifiable" + t.apollo_policy(policies: [["Policy1", "Policy2"], ["Policy3"]]) + + t.field "name", "String" do |f| + f.apollo_policy(policies: [["Policy1", "Policy2"], ["Policy3"]]) + end + + t.field "size", "Size" + t.field "url", "Url" + end + + schema.scalar_type "Url" do |f| + f.apollo_policy(policies: [["Policy1", "Policy2"], ["Policy3"]]) + f.mapping type: "keyword" + f.json_schema type: "string" + end + end + + expect(type_def_from(schema_string, "Size")).to eq(<<~EOS.strip) + enum Size @policy(policies: [["Policy1", "Policy2"], ["Policy3"]]) { + SMALL + } + EOS + + expect(type_def_from(schema_string, "Identifiable")).to eq(<<~EOS.strip) + interface Identifiable @policy(policies: [["Policy1", "Policy2"], ["Policy3"]]) { + size: Size + } + EOS + + expect(type_def_from(schema_string, "Widget")).to eq(<<~EOS.strip) + type Widget implements Identifiable @policy(policies: [["Policy1", "Policy2"], ["Policy3"]]) { + name: String @policy(policies: [["Policy1", "Policy2"], ["Policy3"]]) + size: Size + url: Url + } + EOS + + expect(type_def_from(schema_string, "Url")).to eq(<<~EOS.strip) + scalar Url @policy(policies: [["Policy1", "Policy2"], ["Policy3"]]) + EOS + end + + it "adds a `@provides` directive when `apollo_provides` is called on a schema element" do + schema_string = graphql_schema_string do |schema| + schema.object_type "Widget" do |t| + t.field "name", "String" do |f| + f.apollo_provides(fields: "name") + end + end + end + + expect(type_def_from(schema_string, "Widget")).to eq(<<~EOS.strip) + type Widget { + name: String @provides(fields: "name") + } + EOS + end + + it "adds a `@requires` directive when `apollo_requires` is called on a schema element" do + schema_string = graphql_schema_string do |schema| + schema.object_type "Widget" do |t| + t.field "name", "String" do |f| + f.apollo_requires(fields: "name") + end + end + end + + expect(type_def_from(schema_string, "Widget")).to eq(<<~EOS.strip) + type Widget { + name: String @requires(fields: "name") + } + EOS + end + + it "adds a `@requiresScopes` directive when `apollo_requires_scopes` is called on a schema element" do + schema_string = graphql_schema_string do |schema| + schema.enum_type "Size" do |e| + e.apollo_requires_scopes(scopes: [["Scope1", "Scope2"], ["Scope3"]]) + e.value "SMALL" + end + + schema.interface_type "Identifiable" do |t| + t.apollo_requires_scopes(scopes: [["Scope1", "Scope2"], ["Scope3"]]) + t.field "size", "Size" + end + + schema.object_type "Widget" do |t| + t.implements "Identifiable" + t.apollo_requires_scopes(scopes: [["Scope1", "Scope2"], ["Scope3"]]) + + t.field "name", "String" do |f| + f.apollo_requires_scopes(scopes: [["Scope1", "Scope2"], ["Scope3"]]) + end + + t.field "size", "Size" + t.field "url", "Url" + end + + schema.scalar_type "Url" do |f| + f.apollo_requires_scopes(scopes: [["Scope1", "Scope2"], ["Scope3"]]) + f.mapping type: "keyword" + f.json_schema type: "string" + end + end + + expect(type_def_from(schema_string, "Size")).to eq(<<~EOS.strip) + enum Size @requiresScopes(scopes: [["Scope1", "Scope2"], ["Scope3"]]) { + SMALL + } + EOS + + expect(type_def_from(schema_string, "Identifiable")).to eq(<<~EOS.strip) + interface Identifiable @requiresScopes(scopes: [["Scope1", "Scope2"], ["Scope3"]]) { + size: Size + } + EOS + + expect(type_def_from(schema_string, "Widget")).to eq(<<~EOS.strip) + type Widget implements Identifiable @requiresScopes(scopes: [["Scope1", "Scope2"], ["Scope3"]]) { + name: String @requiresScopes(scopes: [["Scope1", "Scope2"], ["Scope3"]]) + size: Size + url: Url + } + EOS + + expect(type_def_from(schema_string, "Url")).to eq(<<~EOS.strip) + scalar Url @requiresScopes(scopes: [["Scope1", "Scope2"], ["Scope3"]]) + EOS + end + + it "adds a `@shareable` directive when `apollo_shareable` is called on a schema element" do + schema_string = graphql_schema_string do |schema| + schema.object_type "Widget" do |t| + t.apollo_shareable + + t.field "name", "String" do |f| + f.apollo_shareable + end + end + end + + expect(type_def_from(schema_string, "Widget")).to eq(<<~EOS.strip) + type Widget @shareable { + name: String @shareable + } + EOS + end + + it "adds a `@tag` directive when `apollo_tag` is called on a schema element" do + schema_string = graphql_schema_string do |schema| + schema.enum_type "Size" do |e| + e.apollo_tag name: "test" + + e.value "SMALL" do |v| + v.apollo_tag name: "test" + end + + e.value "MEDIUM" + end + + schema.interface_type "Identifiable" do |t| + t.apollo_tag name: "test" + t.field "size", "Size" + end + + schema.object_type "Widget" do |t| + t.implements "Identifiable" + + t.apollo_tag name: "test" + + t.field "name", "String" do |f| + f.apollo_tag name: "test" + + f.argument "some_arg", "String" do |a| + a.apollo_tag name: "test" + end + + f.argument "empty_argument", "String" + end + + t.field "size", "Size" + t.field "url", "Url" + end + + schema.scalar_type "Url" do |f| + f.apollo_tag name: "test" + f.mapping type: "keyword" + f.json_schema type: "string" + + f.customize_derived_types "UrlFilterInput" do |dt| + dt.apollo_tag name: "test" + dt.field "host", "String" do |dtf| + dtf.apollo_tag name: "test" + end + end + end + + schema.union_type "Thing" do |t| + t.apollo_tag name: "test" + t.subtype "Widget" + end + end + + expect(type_def_from(schema_string, "Size")).to eq(<<~EOS.strip) + enum Size @tag(name: "test") { + SMALL @tag(name: "test") + MEDIUM + } + EOS + + expect(type_def_from(schema_string, "Identifiable")).to eq(<<~EOS.strip) + interface Identifiable @tag(name: "test") { + size: Size + } + EOS + + expect(type_def_from(schema_string, "Widget")).to eq(<<~EOS.strip) + type Widget implements Identifiable @tag(name: "test") { + name( + some_arg: String @tag(name: "test") + empty_argument: String): String @tag(name: "test") + size: Size + url: Url + } + EOS + + expect(type_def_from(schema_string, "Thing")).to eq(<<~EOS.strip) + union Thing @tag(name: "test") = Widget + EOS + + expect(type_def_from(schema_string, "Url")).to eq(<<~EOS.strip) + scalar Url @tag(name: "test") + EOS + + expect(type_def_from(schema_string, "UrlFilterInput")).to eq(<<~EOS.strip) + input UrlFilterInput @tag(name: "test") { + #{schema_elements.any_of}: [UrlFilterInput!] + #{schema_elements.not}: UrlFilterInput + #{schema_elements.equal_to_any_of}: [Url] + host: String @tag(name: "test") + } + EOS + end + + def define_schema(&block) + extension_modules = [SchemaDefinition::APIExtension] + super(schema_element_name_form: schema_element_name_form, extension_modules: extension_modules, &block) + end + end + + def graphql_schema_string(&block) + schema = define_schema(&block) + schema.graphql_schema_string + end + end + end +end diff --git a/elasticgraph-apollo/spec/unit/elastic_graph/apollo/graphql/http_endpoint_extension_spec.rb b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/graphql/http_endpoint_extension_spec.rb new file mode 100644 index 00000000..7c4177a6 --- /dev/null +++ b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/graphql/http_endpoint_extension_spec.rb @@ -0,0 +1,71 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/apollo/graphql/engine_extension" +require "elastic_graph/apollo/graphql/http_endpoint_extension" +require "elastic_graph/graphql" +require "elastic_graph/graphql/http_endpoint" + +module ElasticGraph + module Apollo + module GraphQL + RSpec.describe HTTPEndpointExtension, :builds_graphql do + let(:graphql) do + build_graphql( + schema_artifacts_directory: "config/schema/artifacts_with_apollo", + extension_modules: [EngineExtension] + ) + end + + let(:query) { "query { __typename }" } + + it "does not add Apollo tracing by default" do + response_body = execute_expecting_no_errors(query) + + expect(response_body).to eq("data" => {"__typename" => "Query"}) + end + + it "adds Apollo tracing if the `Apollo-Federation-Include-Trace` header is set to `ftv1`, regardless of its casing" do + r1 = execute_expecting_no_errors(query, headers: {"Apollo-Federation-Include-Trace" => "ftv1"}) + r2 = execute_expecting_no_errors(query, headers: {"Apollo_Federation_Include_Trace" => "ftv1"}) + r3 = execute_expecting_no_errors(query, headers: {"apollo-federation-include-trace" => "ftv1"}) + r4 = execute_expecting_no_errors(query, headers: {"apollo_federation_include_trace" => "ftv1"}) + r5 = execute_expecting_no_errors(query, headers: {"APOLLO-FEDERATION-INCLUDE-TRACE" => "ftv1"}) + r6 = execute_expecting_no_errors(query, headers: {"APOLLO_FEDERATION_INCLUDE_TRACE" => "ftv1"}) + + expect([r1, r2, r3, r4, r5, r6]).to all match( + "data" => {"__typename" => "Query"}, + "extensions" => {"ftv1" => /\w+/} + ) + end + + it "does not add Apollo tracing if the `Apollo-Federation-Include-Trace` header is set another value, regardless of its casing" do + response_body = execute_expecting_no_errors(query, headers: {"Apollo-Federation-Include-Trace" => "ftv2"}) + + expect(response_body).to eq("data" => {"__typename" => "Query"}) + end + + def execute_expecting_no_errors(query, headers: {}) + request = ElasticGraph::GraphQL::HTTPRequest.new( + http_method: :post, + url: "/", + headers: headers.merge("Content-Type" => "application/json"), + body: ::JSON.generate("query" => query) + ) + + response = graphql.graphql_http_endpoint.process(request) + expect(response.status_code).to eq(200) + + response_body = ::JSON.parse(response.body) + expect(response_body["errors"]).to be nil + response_body + end + end + end + end +end diff --git a/elasticgraph-apollo/spec/unit/elastic_graph/apollo/graphql/service_field_resolver_spec.rb b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/graphql/service_field_resolver_spec.rb new file mode 100644 index 00000000..002aade8 --- /dev/null +++ b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/graphql/service_field_resolver_spec.rb @@ -0,0 +1,56 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/apollo/graphql/engine_extension" +require "elastic_graph/apollo/graphql/service_field_resolver" +require "elastic_graph/graphql" +require "elastic_graph/graphql/datastore_response/search_response" + +module ElasticGraph + module Apollo + module GraphQL + RSpec.describe ServiceFieldResolver, :builds_graphql do + before do + allow(datastore_client).to receive(:msearch).and_return( + {"responses" => [ElasticGraph::GraphQL::DatastoreResponse::SearchResponse::RAW_EMPTY]} + ) + end + + let(:graphql) do + build_graphql( + schema_artifacts_directory: "config/schema/artifacts_with_apollo", + extension_modules: [EngineExtension] + ) + end + + it "returns the SDL string of the schema as per the Apollo federation v2 spec" do + data = execute_expecting_no_errors("query { _service { sdl } }") + expect(data).to match("_service" => {"sdl" => an_instance_of(::String)}) + + returned_schema = ::GraphQL::Schema.from_definition(data.fetch("_service").fetch("sdl")) + full_schema = graphql.schema.graphql_schema + + expect(full_schema.types.keys).to match_array(returned_schema.types.keys) + expect(full_schema.types.fetch("Query").fields.keys).to match_array(returned_schema.types.fetch("Query").fields.keys) + end + + it "does not interfere with other fields on the `Query` type" do + data = execute_expecting_no_errors("query { widgets { total_edge_count } }") + + expect(data).to eq("widgets" => {"total_edge_count" => 0}) + end + + def execute_expecting_no_errors(query, **options) + response = graphql.graphql_query_executor.execute(query, **options) + expect(response["errors"]).to be nil + response.fetch("data") + end + end + end + end +end diff --git a/elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb new file mode 100644 index 00000000..90e6d4db --- /dev/null +++ b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb @@ -0,0 +1,882 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/apollo/schema_definition/api_extension" +require "elastic_graph/spec_support/schema_definition_helpers" +require "graphql" + +module ElasticGraph + module Apollo + RSpec.describe SchemaDefinition do + include_context "SchemaDefinitionHelpers" + + def self.with_both_casing_forms(&block) + context "with schema elements configured to use camelCase" do + let(:schema_element_name_form) { :camelCase } + module_exec(&block) + end + + context "with schema elements configured to use snake_case" do + let(:schema_element_name_form) { :snake_case } + module_exec(&block) + end + end + + with_both_casing_forms do + let(:schema_elements) { SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: schema_element_name_form) } + + it "defines the static schema elements that must be present in every apollo subgraph schema" do + schema_string = graphql_schema_string { |s| define_some_types_on(s) } + expect(schema_string).to include(*SchemaDefinition::APIExtension::DIRECTIVE_DEFINITIONS_BY_FEDERATION_VERSION.fetch("2.6")) + + # Verify the 2.6 vs 2.0 differences. + expect(schema_string).to include("federation/v2.6") + expect(schema_string).to exclude("federation/v2.5") + expect(schema_string).to exclude("federation/v2.3") + expect(schema_string).to exclude("federation/v2.0") + expect(schema_string).to include("@authenticated") + expect(schema_string).to include("@composeDirective") + expect(schema_string).to include("@interfaceObject") + expect(schema_string).to include("@policy") + expect(schema_string).to include("@requiresScope") + + expect(type_def_from(schema_string, "federation__Scope")).to eq("scalar federation__Scope") + expect(type_def_from(schema_string, "federation__Policy")).to eq("scalar federation__Policy") + expect(type_def_from(schema_string, "FieldSet")).to eq("scalar FieldSet") + expect(type_def_from(schema_string, "link__Import")).to eq("scalar link__Import") + + expect(type_def_from(schema_string, "link__Purpose")).to eq(<<~EOS.strip) + enum link__Purpose { + EXECUTION + SECURITY + } + EOS + + expect(type_def_from(schema_string, "_Any")).to eq("scalar _Any") + expect(type_def_from(schema_string, "_Service")).to eq(<<~EOS.strip) + type _Service { + sdl: String + } + EOS + end + + it "allows the older v2.5 apollo federation directives to be defined instead of the v2.6 ones" do + schema_string = graphql_schema_string do |schema| + schema.target_apollo_federation_version "2.5" + define_some_types_on(schema) + end + + expect(schema_string).to include(*SchemaDefinition::APIExtension::DIRECTIVE_DEFINITIONS_BY_FEDERATION_VERSION.fetch("2.5")) + + # verify the 2.5 vs 2.6 differences + expect(schema_string).to include("federation/v2.5") + expect(schema_string).to exclude("federation/v2.6") + expect(schema_string).to exclude("federation/v2.3") + expect(schema_string).to exclude("federation/v2.0") + expect(schema_string).to exclude("@policy") + + # If an unsupported version is passed, the error includes a `v` in the formatting, so + # users might try to configure it with that. Here we verify that that is supported. + schema_string_with_v = graphql_schema_string do |schema| + schema.target_apollo_federation_version "v2.5" + define_some_types_on(schema) + end + + expect(schema_string_with_v).to eq(schema_string) + end + + it "allows the older v2.3 apollo federation directives to be defined instead of the v2.6 ones" do + schema_string = graphql_schema_string do |schema| + schema.target_apollo_federation_version "2.3" + define_some_types_on(schema) + end + + expect(schema_string).to include(*SchemaDefinition::APIExtension::DIRECTIVE_DEFINITIONS_BY_FEDERATION_VERSION.fetch("2.3")) + + # verify the 2.3 vs 2.6 differences + expect(schema_string).to include("federation/v2.3") + expect(schema_string).to exclude("federation/v2.6") + expect(schema_string).to exclude("federation/v2.5") + expect(schema_string).to exclude("federation/v2.0") + expect(schema_string).to exclude("@authenticated") + expect(schema_string).to exclude("@policy") + expect(schema_string).to exclude("@requiresScope") + + # If an unsupported version is passed, the error includes a `v` in the formatting, so + # users might try to configure it with that. Here we verify that that is supported. + schema_string_with_v = graphql_schema_string do |schema| + schema.target_apollo_federation_version "v2.3" + define_some_types_on(schema) + end + + expect(schema_string_with_v).to eq(schema_string) + end + + it "allows the older v2.0 apollo federation directives to be defined instead of the v2.6 ones" do + schema_string = graphql_schema_string do |schema| + schema.target_apollo_federation_version "2.0" + define_some_types_on(schema) + end + + expect(schema_string).to include(*SchemaDefinition::APIExtension::DIRECTIVE_DEFINITIONS_BY_FEDERATION_VERSION.fetch("2.0")) + + # verify the 2.0 vs 2.6 differences + expect(schema_string).to include("federation/v2.0") + expect(schema_string).to exclude("federation/v2.6") + expect(schema_string).to exclude("federation/v2.5") + expect(schema_string).to exclude("federation/v2.3") + expect(schema_string).to exclude("@authenticated") + expect(schema_string).to exclude("@composeDirective") + expect(schema_string).to exclude("@interfaceObject") + expect(schema_string).to exclude("@policy") + expect(schema_string).to exclude("@requiresScope") + + # If an unsupported version is passed, the error includes a `v` in the formatting, so + # users might try to configure it with that. Here we verify that that is supported. + schema_string_with_v = graphql_schema_string do |schema| + schema.target_apollo_federation_version "v2.0" + define_some_types_on(schema) + end + + expect(schema_string_with_v).to eq(schema_string) + end + + it "raises a clear error if the user tries to target an unsupported apollo federation version" do + expect { + graphql_schema_string do |schema| + schema.target_apollo_federation_version "1.75" + end + }.to raise_error Errors::SchemaError, a_string_including("does not support Apollo federation v1.75. Pick one of the supported versions") + + expect { + graphql_schema_string do |schema| + schema.target_apollo_federation_version "v1.75" + end + }.to raise_error Errors::SchemaError, a_string_including("does not support Apollo federation v1.75. Pick one of the supported versions") + end + + it 'adds a `@key(fields: "id")` directive to each indexed type and includes them in the `_Entity` union (but not to embedded object types)' do + schema_string = graphql_schema_string { |s| define_some_types_on(s) } + + expect(type_def_from(schema_string, "IndexedType1")).to eq(<<~EOS.strip) + type IndexedType1 @key(fields: "id") { + embedded: EmbeddedObjectType1 + graphql: String + id: ID! + num: Int + } + EOS + + expect(type_def_from(schema_string, "IndexedType2")).to eq(<<~EOS.strip) + type IndexedType2 @key(fields: "id") { + id: ID! + } + EOS + + expect(type_def_from(schema_string, "EmbeddedObjectType1")).to eq(<<~EOS.strip) + type EmbeddedObjectType1 { + id: ID! + } + EOS + + expect(type_def_from(schema_string, "_Entity")).to eq("union _Entity = IndexedType1 | IndexedType2") + end + + it 'omits the `@key(fields: "id")` directive when the `id` field is indexing-only, since key fields must be GraphQL fields' do + schema_string = graphql_schema_string { |s| define_some_types_on(s, id_is_indexing_only: ["IndexedType2"]) } + + expect(type_def_from(schema_string, "IndexedType1").lines.first.strip).to eq("type IndexedType1 @key(fields: \"id\") {") + expect(type_def_from(schema_string, "IndexedType2").lines.first.strip).to eq("type IndexedType2 {") + + expect(type_def_from(schema_string, "_Entity")).to eq("union _Entity = IndexedType1") + end + + it "adds object types that have resolvable apollo keys to the Entity union" do + schema_string = graphql_schema_string do |schema| + schema.object_type "IndexedType1" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index "index1" + end + + schema.object_type "UnindexedType1" do |t| + t.field "id", "ID!" + t.field "key", "KeyType!" + t.apollo_key fields: "id key { nestedKey { id } }", resolvable: true + t.field "name", "String" do |f| + f.apollo_external + end + end + + schema.object_type "UnindexedType2" do |t| + t.field "id", "ID!" + t.apollo_key fields: "id", resolvable: false + end + + schema.object_type "UnindexedType3" do |t| + t.field "id", "ID!" + end + + schema.object_type "UnindexedType4" do |t| + t.field "id", "ID!" + t.directive "key", fields: "id" + end + + schema.object_type "KeyType" do |t| + t.field "nestedKey", "NestedKeyType" + end + schema.object_type "NestedKeyType" do |t| + t.field "id", "ID!" + end + end + + expect(type_def_from(schema_string, "UnindexedType1")).to eq(<<~EOS.strip) + type UnindexedType1 @key(fields: "id key { nestedKey { id } }") { + id: ID! + key: KeyType! + name: String @external + } + EOS + + expect(type_def_from(schema_string, "_Entity")).to eq("union _Entity = IndexedType1 | UnindexedType1 | UnindexedType4") + end + + it "raises a clear error when an unindexed resolvable entity types have fields that aren't key fields, relationships, or apollo_external fields" do + expect { + define_unindexed_types do |t| + t.apollo_key fields: "id key { keyType { field1 } }" + end + }.to raise_error Errors::SchemaError, a_string_including( + "`UnindexedType1` has fields", + "`UnindexedType2` has fields", + "unable to resolve", + "* `field1`", "* `field2`" + ) + + expect { + define_unindexed_types do |t| + t.directive "key", fields: "id key { keyType { field1 } }" + end + }.to raise_error Errors::SchemaError, a_string_including( + "`UnindexedType1` has fields", + "`UnindexedType2` has fields", + "unable to resolve", + "* `field1`", "* `field2`" + ) + end + + it "allows non-key, non-relationship, non-external fields on an unindexed entity type when it has `resolvable: false`" do + expect { + define_unindexed_types do |t| + t.apollo_key fields: "id key { keyType { field1 } }", resolvable: false + end + }.not_to raise_error + end + + it "avoids including indexed interfaces in the `_Entity` union (and does not add `@key` to it) since unions can't include interfaces" do + schema_string = graphql_schema_string do |schema| + schema.object_type "IndexedType1" do |t| + t.implements "NamedEntity" + t.field "graphql", "String", name_in_index: "index" + t.field "id", "ID!" + t.field "name", "String" + t.index "index1" + end + + schema.object_type "IndexedType2" do |t| + t.implements "NamedEntity" + t.field "id", "ID!" + t.field "name", "String" + t.index "index1" + end + + schema.interface_type "NamedEntity" do |t| + t.field "id", "ID!" + t.field "name", "String" + end + end + + expect(type_def_from(schema_string, "IndexedType1")).to eq(<<~EOS.strip) + type IndexedType1 implements NamedEntity @key(fields: "id") { + graphql: String + id: ID! + name: String + } + EOS + + expect(type_def_from(schema_string, "IndexedType2")).to eq(<<~EOS.strip) + type IndexedType2 implements NamedEntity @key(fields: "id") { + id: ID! + name: String + } + EOS + + expect(type_def_from(schema_string, "NamedEntity")).to eq(<<~EOS.strip) + interface NamedEntity { + id: ID! + name: String + } + EOS + + expect(type_def_from(schema_string, "_Entity")).to eq("union _Entity = IndexedType1 | IndexedType2") + end + + it "has no problem with the different entity subtypes having fields with the same names and different types, mappings, etc" do + schema_string = graphql_schema_string do |schema| + schema.object_type "IndexedType1" do |t| + t.field "id", "ID!" do |f| + f.mapping null_value: "" + end + t.index "index1" + end + + schema.object_type "IndexedType2" do |t| + t.field "id", "String!" do |f| + f.mapping null_value: "(missing)" + end + t.index "index2" + end + end + + expect(type_def_from(schema_string, "IndexedType1")).to eq(<<~EOS.strip) + type IndexedType1 @key(fields: "id") { + id: ID! + } + EOS + + expect(type_def_from(schema_string, "IndexedType2")).to eq(<<~EOS.strip) + type IndexedType2 @key(fields: "id") { + id: String! + } + EOS + + expect(type_def_from(schema_string, "_Entity")).to eq("union _Entity = IndexedType1 | IndexedType2") + end + + it "defines the fields required by apollo on the `Query` type" do + schema_string = graphql_schema_string { |s| define_some_types_on(s) } + + expect(type_def_from(schema_string, "Query")[/\A.*_Service!/m]).to eq(<<~EOS.strip) + type Query { + _entities( + representations: [_Any!]! + ): [_Entity]! + _service: _Service! + EOS + end + + it "avoids defining `_Entity` and `Query._entities` if there are no indexed types (as per the apollo spec)" do + schema_string = graphql_schema_string { |s| define_some_types_on(s, define_indexed_types: false) } + + # As per https://www.apollographql.com/docs/federation/subgraph-spec/#resolve-requests-for-entities: + # + # > If no types are annotated with the key directive, then the `_Entity` union and `Query._entities` + # > field should be removed from the schema. + + expect(schema_string).to exclude("_Entity") + expect(type_def_from(schema_string, "Query")).to eq(<<~EOS.strip) + type Query { + _service: _Service! + } + EOS + end + + it "avoids defining unneeded derived schema elements (filters, aggregations, query fields) for apollo types" do + schema_state = nil + + schema_string = graphql_schema_string do |schema| + schema_state = schema.state + define_some_types_on(schema) + end + + all_type_names = ::GraphQL::Schema.from_definition(schema_string).types.keys + + # Verify that the typical ElasticGraph derived types were not generated for the Apollo `FieldSet`/`_Entity` types. + expect(all_type_names.grep(/FieldSet/)).to eq ["FieldSet"] + expect(all_type_names.grep(/_Entity/)).to eq ["_Entity"] + + # Ensure `Query` doesn't have a typical ElasticGraph query field that it includes for all indexed types (including union indexed types) + expect(type_def_from(schema_string, "Query")).to exclude("__entitys", "_EntityConnection") + end + + it "has minimal impact on schema artifacts that are not used by the ElasticGraph GraphQL engine" do + with_apollo_results = define_schema(with_apollo: true) { |s| define_some_types_on(s) } + without_apollo_results = define_schema(with_apollo: false) { |s| define_some_types_on(s) } + + expect(with_apollo_results.datastore_scripts).to eq(without_apollo_results.datastore_scripts) + expect(with_apollo_results.json_schemas_for(1)).to eq(without_apollo_results.json_schemas_for(1)) + expect(with_apollo_results.indices).to eq(without_apollo_results.indices) + expect(with_apollo_results.index_templates).to eq(without_apollo_results.index_templates) + + with_apollo_runtime_metadata = SchemaArtifacts::RuntimeMetadata::Schema.from_hash(with_apollo_results.runtime_metadata.to_dumpable_hash, for_context: :graphql) + without_apollo_runtime_metadata = SchemaArtifacts::RuntimeMetadata::Schema.from_hash(without_apollo_results.runtime_metadata.to_dumpable_hash, for_context: :graphql) + expect(with_apollo_runtime_metadata.enum_types_by_name).to eq(without_apollo_runtime_metadata.enum_types_by_name) + expect(with_apollo_runtime_metadata.object_types_by_name.except("_Entity", "_Service")).to eq(without_apollo_runtime_metadata.object_types_by_name) + end + + # We use `dont_validate_graphql_schema` here because the validation triggers the example exceptions we assert on from + # the `derive_schema` call instead of happening when we expect them. + it "has no problems with `_Entity` subtypes that have conflicting field definitions (even though a normal `union` type would not allow that)", :dont_validate_graphql_schema do + define_types = lambda do |schema, define_manual_union_type:| + schema.object_type "Component" do |t| + t.field "id", "ID" + t.field "number", "String" + t.index "components" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "number", "Int" + t.index "widgets" + end + + if define_manual_union_type + schema.union_type "ComponentOrWidget" do |t| + t.subtypes "Component", "Widget" + end + end + end + + results = define_schema(with_apollo: false) do |schema| + define_types.call(schema, define_manual_union_type: true) + end + + # Demonstrate that this is usually a problem... + expect { results.datastore_config }.to raise_error a_string_including("Conflicting definitions for field `number` on the subtypes of `ComponentOrWidget`.") + expect { results.runtime_metadata }.to raise_error a_string_including("Conflicting definitions for field `number` on the subtypes of `ComponentOrWidget`.") + + results = define_schema(with_apollo: true) do |schema| + define_types.call(schema, define_manual_union_type: false) + end + + # ...but it's not a problem for the `_Entity` union type. + results.runtime_metadata + results.datastore_config + + # Demonstrate that the `_Entity` union type has `Component` and `Widget` as subtypes. + expect(type_def_from(results.graphql_schema_string, "_Entity")).to eq "union _Entity = Component | Widget" + end + + it "registers the GraphQL extension since the GraphQL endpoint will be buggy/broken if the extension is not loaded given the custom schema elements that have been added" do + runtime_metadata = define_schema(with_apollo: true) { |s| define_some_types_on(s) }.runtime_metadata + + expect(runtime_metadata.graphql_extension_modules).to include( + SchemaArtifacts::RuntimeMetadata::Extension.new(GraphQL::EngineExtension, "elastic_graph/apollo/graphql/engine_extension", {}) + ) + end + + it "marks the built-in types (such as `PageInfo`) as being shareable since they are identically defined in every ElasticGraph schema and must be shareable to compose multiple ElasticGraph sub-graphs" do + schema_string = graphql_schema_string do |schema| + define_some_types_on(schema) + + schema.object_type "AnotherType" do |t| + t.field "id", "ID!" + t.paginated_collection_field "tags", "String" + t.field "location", "GeoLocation" + t.index "another_type" + end + end + + expect(type_def_from(schema_string, "PageInfo")).to start_with("type PageInfo @shareable {") + expect(type_def_from(schema_string, "IntAggregatedValues")).to start_with("type IntAggregatedValues @shareable {") + expect(type_def_from(schema_string, "StringConnection")).to start_with("type StringConnection @shareable {") + expect(type_def_from(schema_string, "StringEdge")).to start_with("type StringEdge @shareable {") + expect(type_def_from(schema_string, "GeoLocation")).to start_with("type GeoLocation @shareable {") + + # ...but `Query` must not be marked as shareable, since it's the root type and is not shared between sub-graphs. + expect(type_def_from(schema_string, "Query")).to exclude("@shareable") + end + + it "adds tags to built in types when no exceptions are given" do + result = graphql_schema_string do |schema| + schema.object_type "NotABuiltInType" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index "widgets" + end + + schema.tag_built_in_types_with("tag1") + schema.tag_built_in_types_with("tag2") + end + + all_type_names = ::GraphQL::Schema.from_definition(result).types.keys + categorized_type_names = all_type_names.group_by do |type_name| + if type_name.start_with?("__") || STOCK_GRAPHQL_SCALARS.include?(type_name) + :not_explicitly_defined + elsif type_name.include?("NotABuiltInType") || + type_name.start_with?("_", "link__", "federation__") || + %w[FieldSet].include?(type_name) + :expect_no_tags + else + :expect_tags + end + end + + # Verify that we have types in all 3 categories as expected. + expect(categorized_type_names).to include(:expect_no_tags, :expect_tags, :not_explicitly_defined) + expect(categorized_type_names[:expect_no_tags]).not_to be_empty + expect(categorized_type_names[:expect_tags]).not_to be_empty + expect(categorized_type_names[:not_explicitly_defined]).not_to be_empty + + type_defs_by_name = all_type_names.to_h { |type| [type, type_def_from(result, type)] } + expect(type_defs_by_name.select { |k, type_def| type_def.nil? }.keys).to match_array(categorized_type_names[:not_explicitly_defined]) + + categorized_type_names[:expect_tags].each do |type| + expect(type_defs_by_name[type]).to include("@tag(name: \"tag1\")") + expect(type_defs_by_name[type]).to include("@tag(name: \"tag1\")", "@tag(name: \"tag2\")") + end + + categorized_type_names[:expect_no_tags].each do |type| + expect(type_defs_by_name[type]).not_to include("@tag") + end + end + + it "does not add tags to built in types when they are listed in `except: []`" do + tag = "any-tag-name" + schema_string = graphql_schema_string do |schema| + define_some_types_on(schema) + schema.tag_built_in_types_with(tag, except: ["IntAggregatedValues", "AggregatedIntValues"]) + end + + result = type_def_from(schema_string, "IntAggregatedValues") + expect(result).to start_with("type IntAggregatedValues @shareable {") + expect(result).to_not include("@tag") + end + + it "adds its extension methods in a way that does not leak into a schema definition that lacks the apollo extension" do + schema_extensions = [:tag_built_in_types_with] + field_extensions = [:tag_with] + + graphql_schema_string(with_apollo: true) do |schema| + expect(schema).to respond_to(*schema_extensions) + + schema.object_type "T1" do |t| + t.field "id", "ID" do |f| + expect(f).to respond_to(*field_extensions) + end + end + + schema.interface_type "T2" do |t| + t.field "id", "ID" do |f| + expect(f).to respond_to(*field_extensions) + end + end + end + + graphql_schema_string(with_apollo: false) do |schema| + expect(schema).not_to respond_to(*schema_extensions) + + schema.object_type "T1" do |t| + t.field "id", "ID" do |f| + expect(f).not_to respond_to(*field_extensions) + end + end + + schema.interface_type "T2" do |t| + t.field "id", "ID" do |f| + expect(f).not_to respond_to(*field_extensions) + end + end + end + end + + it "provides an API on `object_type` and `interface_type` to make it easy to tag a field and all derived schema elements for inclusion in an apollo contract variant" do + schema_string = graphql_schema_string do |schema| + # For full branch test coverage, verify that these overridden methods still work when no block is given. + schema.object_type "EmptyObject" + + schema.object_type "WidgetOptions" do |t| + t.field "color", "String" do |f| + f.tag_with "public" + end + + t.field "size", "Int" + end + + schema.interface_type "Identifiable" do |t| + t.field "id", "ID!", groupable: false + t.field "token", "String" do |f| + f.tag_with "public-interface" + end + end + + schema.object_type "Widget" do |t| + t.implements "Identifiable" + + t.field "id", "ID!", groupable: false + + t.field "token", "String" + + t.field "options1", "WidgetOptions" do |f| + f.tag_with "public" + end + + t.field "options2", "WidgetOptions" do |f| + f.tag_with "internal" + end + + t.field "name", "String" do |f| + f.tag_with "public" + end + + # Verify we can use it on `relates_to_*` fields as well (originally this didn't work!) + t.relates_to_one "parent_widget", "Widget", via: "parent_id", dir: :out do |f| + f.tag_with "public" + end + + t.index "widgets" + end + end + + considered_types = [] + + expect_widget_type_tagging_of_name_and_option1_color do |type_name| + considered_types << type_name + type_def_from(schema_string, type_name) + end + + expect_identifiable_type_tagging_of_token do |type_name| + considered_types << type_name + type_def_from(schema_string, type_name) + end + + all_types = ::GraphQL::Schema.from_definition(schema_string).types.keys + widget_type_names = all_types.grep(/Widget/) + identifiable_type_names = all_types.grep(/Identifiable/) + + # Here we are verifying that we properly verified all related types. If a new ElasticGraph feature causes new types + # to be generated for the `Widget` and `Identifiable` source types, we will want this test to be updated to cover them. + # This expectation should fail, notifying us of the need to cover the new type. + expect(widget_type_names + identifiable_type_names).to match_array( + considered_types + [ + # We do not look at these types because the fields on them are based on the relay spec and not based on + # the fields of the Widget/Identifiable source types. + "WidgetConnection", "WidgetEdge", "WidgetAggregationConnection", "WidgetAggregationEdge", + "IdentifiableConnection", "IdentifiableEdge", "IdentifiableAggregationConnection", "IdentifiableAggregationEdge" + ] + [ + # We do not look at these types because the fields on them are static (`groupedBy`/`count`/`aggregatedValues`) and + # are not derived fields from the Widget/Identifiable source types. + "WidgetAggregation", "IdentifiableAggregation" + ] + ) + end + + def expect_widget_type_tagging_of_name_and_option1_color(&type_def_for) + expect(type_def_for.call("Widget")).to eq(<<~EOS.strip) + type Widget implements Identifiable @key(fields: "id") { + id: ID! + name: String @tag(name: "public") + options1: WidgetOptions @tag(name: "public") + options2: WidgetOptions @tag(name: "internal") + parent_widget: Widget @tag(name: "public") + token: String + } + EOS + + expect(type_def_for.call("WidgetFilterInput")).to eq(<<~EOS.strip) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + id: IDFilterInput + name: StringFilterInput @tag(name: "public") + not: WidgetFilterInput + options1: WidgetOptionsFilterInput @tag(name: "public") + options2: WidgetOptionsFilterInput @tag(name: "internal") + token: StringFilterInput + } + EOS + + expect(type_def_for.call("WidgetGroupedBy")).to eq(<<~EOS.strip) + type WidgetGroupedBy { + name: String @tag(name: "public") + options1: WidgetOptionsGroupedBy @tag(name: "public") + options2: WidgetOptionsGroupedBy @tag(name: "internal") + token: String + } + EOS + + expect(type_def_for.call("WidgetAggregatedValues")).to eq(<<~EOS.strip) + type WidgetAggregatedValues { + id: NonNumericAggregatedValues + name: NonNumericAggregatedValues @tag(name: "public") + options1: WidgetOptionsAggregatedValues @tag(name: "public") + options2: WidgetOptionsAggregatedValues @tag(name: "internal") + token: NonNumericAggregatedValues + } + EOS + + expect(type_def_for.call("WidgetOptions")).to eq(<<~EOS.strip) + type WidgetOptions { + color: String @tag(name: "public") + size: Int + } + EOS + + expect(type_def_for.call("WidgetOptionsFilterInput")).to eq(<<~EOS.strip) + input WidgetOptionsFilterInput { + #{schema_elements.any_of}: [WidgetOptionsFilterInput!] + color: StringFilterInput @tag(name: "public") + not: WidgetOptionsFilterInput + size: IntFilterInput + } + EOS + + expect(type_def_for.call("WidgetOptionsGroupedBy")).to eq(<<~EOS.strip) + type WidgetOptionsGroupedBy { + color: String @tag(name: "public") + size: Int + } + EOS + + expect(type_def_for.call("WidgetOptionsAggregatedValues")).to eq(<<~EOS.strip) + type WidgetOptionsAggregatedValues { + color: NonNumericAggregatedValues @tag(name: "public") + size: IntAggregatedValues + } + EOS + + expect(type_def_for.call("WidgetSortOrderInput")).to eq(<<~EOS.strip) + enum WidgetSortOrderInput { + id_ASC + id_DESC + name_ASC @tag(name: "public") + name_DESC @tag(name: "public") + options1_color_ASC @tag(name: "public") + options1_color_DESC @tag(name: "public") + options1_size_ASC + options1_size_DESC + options2_color_ASC + options2_color_DESC + options2_size_ASC + options2_size_DESC + token_ASC + token_DESC + } + EOS + end + + def expect_identifiable_type_tagging_of_token(&type_def_for) + expect(type_def_for.call("Identifiable")).to eq(<<~EOS.strip) + interface Identifiable { + id: ID! + token: String @tag(name: "public-interface") + } + EOS + + # For these 3 types, the generated types uses the union of fields of all subtypes, and automatically inherits + # the tagging of those source fields. That's why `name`, `options1`, etc are tagged with `public` below. + expect(type_def_for.call("IdentifiableFilterInput")).to eq(<<~EOS.strip) + input IdentifiableFilterInput { + #{schema_elements.any_of}: [IdentifiableFilterInput!] + id: IDFilterInput + name: StringFilterInput @tag(name: "public") + not: IdentifiableFilterInput + options1: WidgetOptionsFilterInput @tag(name: "public") + options2: WidgetOptionsFilterInput @tag(name: "internal") + token: StringFilterInput @tag(name: "public-interface") + } + EOS + + expect(type_def_for.call("IdentifiableGroupedBy")).to eq(<<~EOS.strip) + type IdentifiableGroupedBy { + name: String @tag(name: "public") + options1: WidgetOptionsGroupedBy @tag(name: "public") + options2: WidgetOptionsGroupedBy @tag(name: "internal") + token: String @tag(name: "public-interface") + } + EOS + + expect(type_def_for.call("IdentifiableAggregatedValues")).to eq(<<~EOS.strip) + type IdentifiableAggregatedValues { + id: NonNumericAggregatedValues + name: NonNumericAggregatedValues @tag(name: "public") + options1: WidgetOptionsAggregatedValues @tag(name: "public") + options2: WidgetOptionsAggregatedValues @tag(name: "internal") + token: NonNumericAggregatedValues @tag(name: "public-interface") + } + EOS + + expect(type_def_for.call("IdentifiableSortOrderInput")).to eq(<<~EOS.strip) + enum IdentifiableSortOrderInput { + id_ASC + id_DESC + name_ASC @tag(name: "public") + name_DESC @tag(name: "public") + options1_color_ASC @tag(name: "public") + options1_color_DESC @tag(name: "public") + options1_size_ASC + options1_size_DESC + options2_color_ASC + options2_color_DESC + options2_size_ASC + options2_size_DESC + token_ASC @tag(name: "public-interface") + token_DESC @tag(name: "public-interface") + } + EOS + end + + def define_schema(with_apollo: true, &block) + extension_modules = with_apollo ? [SchemaDefinition::APIExtension] : [] + super(schema_element_name_form: schema_element_name_form, extension_modules: extension_modules, &block) + end + end + + def graphql_schema_string(with_apollo: true, &block) + schema_string = define_schema(with_apollo: with_apollo, &block).graphql_schema_string + ::GraphQL::Schema.from_definition(schema_string).to_definition.strip + end + + def define_some_types_on(schema, define_indexed_types: true, id_is_indexing_only: []) + schema.object_type "IndexedType1" do |t| + t.field "embedded", "EmbeddedObjectType1" + t.field "graphql", "String", name_in_index: "index" + t.field "id", "ID!", indexing_only: id_is_indexing_only.include?("IndexedType1") + t.field "num", "Int" + t.index "index1" if define_indexed_types + end + + schema.object_type "IndexedType2" do |t| + t.field "id", "ID!", indexing_only: id_is_indexing_only.include?("IndexedType2") + + # Ensure there's at least one field defined on the GraphQL type--if `id` is indexing-only, we need another field defined. + t.field "name", "String" if id_is_indexing_only.include?("IndexedType2") + t.index "index1" if define_indexed_types + end + + schema.object_type "EmbeddedObjectType1" do |t| + t.field "id", "ID!" + end + end + + def define_unindexed_types + graphql_schema_string do |schema| + %w[UnindexedType1 UnindexedType2].each do |type| + schema.object_type type do |t| + t.field "id", "ID!" + t.field "key", "KeyType1!" + t.field "field1", "String" + t.field "field2", "String" + t.field "field3", "String" do |f| + f.apollo_external + end + + yield t + end + end + + schema.object_type "KeyType1" do |t| + t.field "keyType", "KeyType2" + end + schema.object_type "KeyType2" do |t| + t.field "field1", "ID!" + end + end + end + end + end +end diff --git a/elasticgraph-datastore_core/.rspec b/elasticgraph-datastore_core/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-datastore_core/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-datastore_core/.yardopts b/elasticgraph-datastore_core/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-datastore_core/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-datastore_core/Gemfile b/elasticgraph-datastore_core/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-datastore_core/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-datastore_core/LICENSE.txt b/elasticgraph-datastore_core/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-datastore_core/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-datastore_core/README.md b/elasticgraph-datastore_core/README.md new file mode 100644 index 00000000..65f87ad9 --- /dev/null +++ b/elasticgraph-datastore_core/README.md @@ -0,0 +1,3 @@ +# ElasticGraph::DatastoreCore + +Contains the core datastore logic used by the rest of ElasticGraph. diff --git a/elasticgraph-datastore_core/elasticgraph-datastore_core.gemspec b/elasticgraph-datastore_core/elasticgraph-datastore_core.gemspec new file mode 100644 index 00000000..ff49d0d5 --- /dev/null +++ b/elasticgraph-datastore_core/elasticgraph-datastore_core.gemspec @@ -0,0 +1,21 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :core) do |spec, eg_version| + spec.summary = "ElasticGraph gem containing the core datastore support types and logic." + + spec.add_dependency "elasticgraph-schema_artifacts", eg_version + spec.add_dependency "elasticgraph-support", eg_version + + spec.add_development_dependency "elasticgraph-admin", eg_version + spec.add_development_dependency "elasticgraph-elasticsearch", eg_version + spec.add_development_dependency "elasticgraph-opensearch", eg_version + spec.add_development_dependency "elasticgraph-schema_definition", eg_version +end diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core.rb new file mode 100644 index 00000000..3c9df570 --- /dev/null +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core.rb @@ -0,0 +1,100 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core/config" +require "elastic_graph/schema_artifacts/from_disk" +require "elastic_graph/support/logger" + +module ElasticGraph + # The entry point into this library. Create an instance of this class to get access to + # the public interfaces provided by this library. + class DatastoreCore + # @dynamic config, schema_artifacts, logger, client_customization_block + attr_reader :config, :schema_artifacts, :logger, :client_customization_block + + def self.from_parsed_yaml(parsed_yaml, for_context:, &client_customization_block) + new( + config: DatastoreCore::Config.from_parsed_yaml(parsed_yaml), + logger: Support::Logger.from_parsed_yaml(parsed_yaml), + schema_artifacts: SchemaArtifacts.from_parsed_yaml(parsed_yaml, for_context: for_context), + client_customization_block: client_customization_block + ) + end + + def initialize( + config:, + logger:, + schema_artifacts:, + clients_by_name: nil, + client_customization_block: nil + ) + @config = config + @logger = logger + @schema_artifacts = schema_artifacts + @clients_by_name = clients_by_name + @client_customization_block = client_customization_block + end + + # Exposes the datastore index definitions as a map, keyed by index definition name. + def index_definitions_by_name + @index_definitions_by_name ||= begin + require "elastic_graph/datastore_core/index_definition" + schema_artifacts.runtime_metadata.index_definitions_by_name.to_h do |name, index_def_metadata| + index_def = IndexDefinition.with( + name: name, + runtime_metadata: index_def_metadata, + config: config, + datastore_clients_by_name: clients_by_name + ) + + [name, index_def] + end + end + end + + # Exposes the datastore index definitions as a map, keyed by GraphQL type. + # Note: the GraphQL type name is also used in non-GraphQL contexts (e.g. it is + # used in events processed by elasticgraph-indexer), so we expose this hear instead + # of from elasticgraph-graphql. + def index_definitions_by_graphql_type + @index_definitions_by_graphql_type ||= schema_artifacts + .runtime_metadata + .object_types_by_name + .transform_values do |metadata| + metadata.index_definition_names.map do |name| + index_definitions_by_name.fetch(name) + end + end + end + + # Exposes the datastore clients in a map, keyed by cluster name. + def clients_by_name + @clients_by_name ||= begin + if (adapter_lib = config.client_faraday_adapter&.require) + require adapter_lib + end + + adapter_name = config.client_faraday_adapter&.name + client_logger = config.log_traffic ? logger : nil + + config.clusters.to_h do |name, cluster_def| + client = cluster_def.backend_client_class.new( + name, + faraday_adapter: adapter_name, + url: cluster_def.url, + logger: client_logger, + retry_on_failure: config.max_client_retries, + &@client_customization_block + ) + + [name, client] + end + end + end + end +end diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/config.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/config.rb new file mode 100644 index 00000000..6be18870 --- /dev/null +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/config.rb @@ -0,0 +1,58 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core/configuration/client_faraday_adapter" +require "elastic_graph/datastore_core/configuration/cluster_definition" +require "elastic_graph/datastore_core/configuration/index_definition" +require "elastic_graph/errors" + +module ElasticGraph + class DatastoreCore + # Defines the configuration related to datastores. + class Config < ::Data.define( + # Configuration of the faraday adapter to use with the datastore client. + :client_faraday_adapter, + # Map of datastore cluster definitions, keyed by cluster name. The names will be referenced within + # `index_definitions` by `query_cluster` and `index_into_clusters` to identify + # datastore clusters. Each definition has a `url` and `settings`. `settings` contains datastore + # settings in the flattened name form, e.g. `"cluster.max_shards_per_node": 2000`. + :clusters, + # Map of index definition names to `IndexDefinition` objects containing customizations + # for the named index definitions for this environment. + :index_definitions, + # Determines if we log requests/responses to/from the datastore. + # Defaults to `false`. + :log_traffic, + # Passed down to the datastore client, controls the number of times ElasticGraph attempts a call against + # the datastore before failing. Retrying a handful of times is generally advantageous, since some sporadic + # failures are expected during the course of operation, and better to retry than fail the entire call. + # Defaults to 3. + :max_client_retries + ) + # Helper method to build an instance from parsed YAML config. + def self.from_parsed_yaml(parsed_yaml) + parsed_yaml = parsed_yaml.fetch("datastore") + extra_keys = parsed_yaml.keys - EXPECTED_KEYS + + unless extra_keys.empty? + raise Errors::ConfigError, "Unknown `datastore` config settings: #{extra_keys.join(", ")}" + end + + new( + client_faraday_adapter: Configuration::ClientFaradayAdapter.from_parsed_yaml(parsed_yaml), + clusters: Configuration::ClusterDefinition.definitions_by_name_hash_from(parsed_yaml.fetch("clusters")), + index_definitions: Configuration::IndexDefinition.definitions_by_name_hash_from(parsed_yaml.fetch("index_definitions")), + log_traffic: parsed_yaml.fetch("log_traffic", false), + max_client_retries: parsed_yaml.fetch("max_client_retries", 3) + ) + end + + EXPECTED_KEYS = members.map(&:to_s) + end + end +end diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/configuration/client_faraday_adapter.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/configuration/client_faraday_adapter.rb new file mode 100644 index 00000000..16d78c2b --- /dev/null +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/configuration/client_faraday_adapter.rb @@ -0,0 +1,38 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class DatastoreCore + module Configuration + class ClientFaradayAdapter < ::Data.define( + # The faraday adapter to use with the datastore client, such as `httpx` or `typhoeus`. + # For more info, see: + # https://github.com/elastic/elasticsearch-ruby/commit/a7bbdbf2a96168c1b33dca46ee160d2d4d75ada0 + :name, + # A Ruby library to require which provides the named adapter (optional). + :require + ) + def self.from_parsed_yaml(parsed_yaml) + parsed_yaml = parsed_yaml.fetch("client_faraday_adapter") || {} + extra_keys = parsed_yaml.keys - EXPECTED_KEYS + + unless extra_keys.empty? + raise Errors::ConfigError, "Unknown `datastore.client_faraday_adapter` config settings: #{extra_keys.join(", ")}" + end + + new( + name: parsed_yaml["name"]&.to_sym, + require: parsed_yaml["require"] + ) + end + + EXPECTED_KEYS = members.map(&:to_s) + end + end + end +end diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/configuration/cluster_definition.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/configuration/cluster_definition.rb new file mode 100644 index 00000000..21daf0c4 --- /dev/null +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/configuration/cluster_definition.rb @@ -0,0 +1,52 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" + +module ElasticGraph + class DatastoreCore + module Configuration + class ClusterDefinition < ::Data.define(:url, :backend_client_class, :settings) + def self.from_hash(hash) + extra_keys = hash.keys - EXPECTED_KEYS + + unless extra_keys.empty? + raise Errors::ConfigError, "Unknown `datastore.clusters` config settings: #{extra_keys.join(", ")}" + end + + backend_name = hash["backend"] + backend_client_class = + case backend_name + when "elasticsearch" + require "elastic_graph/elasticsearch/client" + Elasticsearch::Client + when "opensearch" + require "elastic_graph/opensearch/client" + OpenSearch::Client + else + raise Errors::ConfigError, "Unknown `datastore.clusters` backend: `#{backend_name}`. Valid backends are `elasticsearch` and `opensearch`." + end + + new( + url: hash.fetch("url"), + backend_client_class: backend_client_class, + settings: hash.fetch("settings") + ) + end + + def self.definitions_by_name_hash_from(cluster_def_hash_by_name) + cluster_def_hash_by_name.transform_values do |cluster_def_hash| + from_hash(cluster_def_hash) + end + end + + EXPECTED_KEYS = members.map(&:to_s) - ["backend_client_class"] + ["backend"] + end + end + end +end diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/configuration/index_definition.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/configuration/index_definition.rb new file mode 100644 index 00000000..b6dcdf7c --- /dev/null +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/configuration/index_definition.rb @@ -0,0 +1,110 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/time_set" +require "elastic_graph/errors" +require "time" + +module ElasticGraph + class DatastoreCore + module Configuration + # Defines environment-specific customizations for an index definition. + # + # - ignore_routing_values: routing values for which we will ignore routing as configured on the index. + # This is intended to be used when a single routing value contains such a large portion of the dataset that it creates lopsided shards. + # By including that routing value in this config setting, it'll spread that value's data across all shards instead of concentrating it on a single shard. + # - query_cluster: named search cluster to be used for queries on this index. + # - index_into_cluster: named search clusters to index data into. + # - setting_overrides: overrides for index (or index template) settings. + # - setting_overrides_by_timestamp: overrides for index template settings for specific dates, + # allowing us to have different settings than the template for some timestamp. + # - custom_timestamp_ranges: defines indices for a custom timestamp range (rather than relying + # on the configured rollover frequency). + # - use_updates_for_indexing: when `true`, opts the index into using the `update` API instead of the `index` API for indexing. + # (Defaults to `true`). + class IndexDefinition < ::Data.define( + :ignore_routing_values, + :query_cluster, + :index_into_clusters, + :setting_overrides, + :setting_overrides_by_timestamp, + :custom_timestamp_ranges, + :use_updates_for_indexing + ) + def initialize(ignore_routing_values:, **rest) + __skip__ = super(ignore_routing_values: ignore_routing_values.to_set, **rest) + + # Verify the custom ranges are disjoint. + # Yeah, this is O(N^2), which isn't great, but we expect a _very_ small number of custom + # ranges (0-2) so this should be ok. + return if custom_timestamp_ranges + .map(&:time_set) + .combination(2) + .none? do |s1_s2| + s1, s2 = s1_s2 + s1.intersect?(s2) + end + + raise Errors::ConfigError, "Your configured `custom_timestamp_ranges` are not disjoint, as required." + end + + def without_env_overrides + with(setting_overrides: {}, setting_overrides_by_timestamp: {}, custom_timestamp_ranges: []) + end + + def custom_timestamp_range_for(timestamp) + custom_timestamp_ranges.find do |range| + range.time_set.member?(timestamp) + end + end + + def self.definitions_by_name_hash_from(index_def_hash_by_name) + index_def_hash_by_name.transform_values do |index_def_hash| + __skip__ = from(**index_def_hash.transform_keys(&:to_sym)) + end + end + + def self.from(custom_timestamp_ranges:, use_updates_for_indexing: true, **rest) + __skip__ = new( + custom_timestamp_ranges: CustomTimestampRange.ranges_from(custom_timestamp_ranges), + use_updates_for_indexing: use_updates_for_indexing, + **rest + ) + end + + # Represents an index definition that is based on a custom timestamp range. + class CustomTimestampRange < ::Data.define(:index_name_suffix, :setting_overrides, :time_set) + def initialize(index_name_suffix:, setting_overrides:, time_set:) + super + + if time_set.empty? + raise Errors::ConfigError, "Custom timestamp range with suffix `#{index_name_suffix}` is invalid: no timestamps exist in it." + end + end + + def self.ranges_from(range_hashes) + range_hashes.map do |range_hash| + __skip__ = from(**range_hash.transform_keys(&:to_sym)) + end + end + + private_class_method def self.from(index_name_suffix:, setting_overrides:, **predicates_hash) + if predicates_hash.empty? + raise Errors::ConfigSettingNotSetError, "Custom timestamp range with suffix `#{index_name_suffix}` lacks boundary definitions." + end + + range_options = predicates_hash.transform_values { |iso8601_string| ::Time.iso8601(iso8601_string) } + time_set = Support::TimeSet.of_range(**range_options) + + new(index_name_suffix: index_name_suffix, setting_overrides: setting_overrides, time_set: time_set) + end + end + end + end + end +end diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_config_normalizer.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_config_normalizer.rb new file mode 100644 index 00000000..dbb21430 --- /dev/null +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_config_normalizer.rb @@ -0,0 +1,79 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class DatastoreCore + module IndexConfigNormalizer + # These are settings that the datastore exposes when you fetch an index, but that you can + # never set. We need to ignore them when figuring out what settings to update. + # + # Note: `index.routing.allocation.include._tier_preference` is not a read-only setting, but + # we want to treat it as one, because (1) Elasticsearch 7.10+ sets it and (2) we do not want + # to ever write it at this time. + # + # Note: `index.history.uuid` is a weird setting that sometimes shows up in managed AWS OpenSearch + # clusters, but only on _some_ indices. It's not documented and we don't want to mess with it here, + # so we want to treat it as a read only setting. + READ_ONLY_SETTINGS = %w[ + index.creation_date + index.history.uuid + index.provided_name + index.replication.type + index.routing.allocation.include._tier_preference + index.uuid + index.version.created + index.version.upgraded + ] + + # Normalizes the provided index configuration so that it is in a stable form that we can compare to what + # the datastore returns when we query it for the configuration of an index. This includes: + # + # - Dropping read-only settings that we never interact with but that the datastore automatically sets on an index. + # Omitting them makes it easier for us to compare our desired configuration to what is in the datastore. + # - Converting setting values to a normalized string form. The datastore oddly returns setting values as strings + # (e.g. `"false"` or `"7"` instead of `false` or `7`), so this matches that behavior. + # - Drops `type: object` from a mapping when there are `properties` because the datastore omits it in that + # situation, treating it as the default type. + def self.normalize(index_config) + if (settings = index_config["settings"]) + index_config = index_config.merge("settings" => normalize_settings(settings)) + end + + if (mappings = index_config["mappings"]) + index_config = index_config.merge("mappings" => normalize_mappings(mappings)) + end + + index_config + end + + def self.normalize_mappings(mappings) + return mappings unless (properties = mappings["properties"]) + + mappings = mappings.except("type") if mappings["type"] == "object" + mappings.merge("properties" => properties.transform_values { |prop| normalize_mappings(prop) }) + end + + def self.normalize_settings(settings) + settings + .except(*READ_ONLY_SETTINGS) + .to_h { |name, value| [name, normalize_setting_value(value)] } + end + + private_class_method def self.normalize_setting_value(value) + case value + when nil + nil + when ::Array + value.map { |v| normalize_setting_value(v) } + else + value.to_s + end + end + end + end +end diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition.rb new file mode 100644 index 00000000..639f757e --- /dev/null +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition.rb @@ -0,0 +1,51 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core/index_definition/index" +require "elastic_graph/datastore_core/index_definition/rollover_index_template" +require "elastic_graph/errors" + +module ElasticGraph + class DatastoreCore + # Represents the definition of a datastore index (or rollover template). + # Intended to be an entry point for working with datastore indices. + # + # This module contains common implementation logic for both the rollover and non-rollover + # case, as well as a `with` factory method. + module IndexDefinition + def self.with(name:, runtime_metadata:, config:, datastore_clients_by_name:) + if (env_index_config = config.index_definitions[name]).nil? + raise Errors::ConfigError, "Configuration does not provide an index definition for `#{name}`, " \ + "but it is required so we can identify the datastore cluster(s) to query and index into." + end + + common_args = { + name: name, + route_with: runtime_metadata.route_with, + default_sort_clauses: runtime_metadata.default_sort_fields.map(&:to_query_clause), + current_sources: runtime_metadata.current_sources, + fields_by_path: runtime_metadata.fields_by_path, + env_index_config: env_index_config, + defined_clusters: config.clusters.keys.to_set, + datastore_clients_by_name: datastore_clients_by_name + } + + if (rollover = runtime_metadata.rollover) + RolloverIndexTemplate.new( + timestamp_field_path: rollover.timestamp_field_path, + frequency: rollover.frequency, + index_args: common_args, + **common_args + ) + else + Index.new(**common_args) + end + end + end + end +end diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/base.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/base.rb new file mode 100644 index 00000000..bd2841ee --- /dev/null +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/base.rb @@ -0,0 +1,162 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core/index_config_normalizer" +require "elastic_graph/errors" +require "elastic_graph/support/hash_util" + +module ElasticGraph + class DatastoreCore + module IndexDefinition + # This module contains common implementation logic for both the rollover and non-rollover + # implementations of the common IndexDefinition type. + module Base + # Returns any setting overrides for this index from the environment-specific config file, + # after flattening it so that it can be directly used in a create index request. + def flattened_env_setting_overrides + @flattened_env_setting_overrides ||= Support::HashUtil.flatten_and_stringify_keys( + env_index_config.setting_overrides, + prefix: "index" + ) + end + + # Gets the routing value for the given `prepared_record`. Notably, `prepared_record` must be previously + # prepared with an `Indexer::RecordPreparer` in order to ensure that it uses internal index + # field names (to align with `route_with_path`/`route_with` which also use the internal name) rather + # than the public field name (which can differ). + def routing_value_for_prepared_record(prepared_record, route_with_path: route_with, id_path: "id") + return nil unless has_custom_routing? + + unless route_with_path + raise Errors::ConfigError, "`#{self}` uses custom routing, but `route_with_path` is misconfigured (was `nil`)" + end + + config_routing_value = Support::HashUtil.fetch_value_at_path(prepared_record, route_with_path).to_s + return config_routing_value unless ignored_values_for_routing.include?(config_routing_value) + + Support::HashUtil.fetch_value_at_path(prepared_record, id_path).to_s + end + + def has_custom_routing? + route_with != "id" + end + + # Indicates if a search on this index definition may hit incomplete documents. An incomplete document + # can occur when multiple event types flow into the same index. An index that has only one source type + # can never have incomplete documents, but an index that has 2 or more sources can have incomplete + # documents when the "primary" event type hasn't yet been received for a document. + # + # This case is notable because we need to apply automatic filtering in order to hide documents that are + # not yet complete. + # + # Note: determining this value sometimes requires that we query the datastore for the record of all + # sources that an index has ever had. This value changes very, very rarely, and we don't want to slow + # down every GraphQL query by adding the extra query against the datastore, so we cache the value here. + def searches_could_hit_incomplete_docs? + return @searches_could_hit_incomplete_docs if defined?(@searches_could_hit_incomplete_docs) + + if current_sources.size > 1 + # We know that incomplete docs are possible, without needing to check sources recorded in `_meta`. + @searches_could_hit_incomplete_docs = true + else + # While our current configuration can't produce incomplete documents, some may already exist in the index + # if we previously had some `sourced_from` fields (but no longer have them). Here we check for the sources + # we've recorded in `_meta` to account for that. + client = datastore_clients_by_name.fetch(cluster_to_query) + recorded_sources = mappings_in_datastore(client).dig("_meta", "ElasticGraph", "sources") || [] + sources = recorded_sources.union(current_sources.to_a) + + @searches_could_hit_incomplete_docs = sources.size > 1 + end + end + + def cluster_to_query + env_index_config.query_cluster + end + + def clusters_to_index_into + env_index_config.index_into_clusters.tap do |clusters_to_index_into| + raise Errors::ConfigError, "No `index_into_clusters` defined for #{self} in env_index_config" unless clusters_to_index_into + end + end + + def use_updates_for_indexing? + env_index_config.use_updates_for_indexing + end + + def ignored_values_for_routing + env_index_config.ignore_routing_values + end + + # Returns a list of all defined datastore clusters this index resides within. + def all_accessible_cluster_names + @all_accessible_cluster_names ||= + # Using `_` because steep doesn't understand that `compact` removes nils. + (clusters_to_index_into + [_ = cluster_to_query]).compact.uniq.select do |name| + defined_clusters.include?(name) + end + end + + def accessible_cluster_names_to_index_into + @accessible_cluster_names_to_index_into ||= clusters_to_index_into.select do |name| + defined_clusters.include?(name) + end + end + + # Indicates whether not the index is be accessible from GraphQL queries, by virtue of + # the `cluster_to_query` being a defined cluster or not. This will be used to + # hide GraphQL schema elements that can't be queried when our config omits the means + # to query an index (e.g. due to lacking a configured URL). + def accessible_from_queries? + return false unless (cluster = cluster_to_query) + defined_clusters.include?(cluster) + end + + # Returns a list of indices related to this template in the datastore cluster this + # index definition is configured to query. Note that for performance reasons, this method + # memoizes the result of querying the datastore for its current list of indices, and as + # a result the return value may be out of date. If it is absolutely essential that you get + # an up-to-date list of related indices, use `related_rollover_indices(datastore_client`) instead of + # this method. + # + # Note, however, that indices generally change *very* rarely (say, monthly or yearly) and as such + # this will very rarely be out of date, even with the memoization. + def known_related_query_rollover_indices + @known_related_query_rollover_indices ||= cluster_to_query&.then do |name| + # For query purposes, we only want indices that exist. If we return a query that is defined in our configuration + # but does not exist, and that gets used in a search index expression (even for the purposes of excluding it!), + # the datastore will return an error. + related_rollover_indices(datastore_clients_by_name.fetch(name), only_if_exists: true) + end || [] + end + + # Returns a set of all of the field paths to subfields of the special `LIST_COUNTS_FIELD` + # that contains the element counts of all list fields. The returned set is filtered based + # on the provided `source` to only contain the paths of fields that are populated by the + # given source. + def list_counts_field_paths_for_source(source) + @list_counts_field_paths_for_source ||= {} # : ::Hash[::String, ::Set[::String]] + @list_counts_field_paths_for_source[source] ||= identify_list_counts_field_paths_for_source(source) + end + + def to_s + "#<#{self.class.name} #{name}>" + end + alias_method :inspect, :to_s + + private + + def identify_list_counts_field_paths_for_source(source) + fields_by_path.filter_map do |path, field| + path if field.source == source && path.split(".").include?(LIST_COUNTS_FIELD) + end.to_set + end + end + end + end +end diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/index.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/index.rb new file mode 100644 index 00000000..04df17b6 --- /dev/null +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/index.rb @@ -0,0 +1,64 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core/index_config_normalizer" +require "elastic_graph/datastore_core/index_definition/base" +require "elastic_graph/support/memoizable_data" + +module ElasticGraph + class DatastoreCore + module IndexDefinition + class Index < Support::MemoizableData.define( + :name, :route_with, :default_sort_clauses, :current_sources, :fields_by_path, + :env_index_config, :defined_clusters, :datastore_clients_by_name + ) + # `Data.define` provides all these methods: + # @dynamic name, route_with, default_sort_clauses, current_sources, fields_by_path, env_index_config, defined_clusters, datastore_clients_by_name, initialize + + # `include IndexDefinition::Base` provides all these methods. Steep should be able to detect it + # but can't for some reason so we have to declare them with `@dynamic`. + # @dynamic flattened_env_setting_overrides, routing_value_for_prepared_record, has_custom_routing?, cluster_to_query, use_updates_for_indexing? + # @dynamic clusters_to_index_into, all_accessible_cluster_names, ignored_values_for_routing, searches_could_hit_incomplete_docs? + # @dynamic accessible_cluster_names_to_index_into, accessible_from_queries?, known_related_query_rollover_indices, list_counts_field_paths_for_source + include IndexDefinition::Base + + def mappings_in_datastore(datastore_client) + IndexConfigNormalizer.normalize_mappings(datastore_client.get_index(name)["mappings"] || {}) + end + + # `ignore_unavailable: true` is needed to prevent errors when we delete non-existing non-rollover indices + def delete_from_datastore(datastore_client) + datastore_client.delete_indices(name) + end + + # Indicates if this is a rollover index definition. + # + # Use of this is considered a mild code smell. When feasible, it's generally better to + # implement a new polymorphic API on the IndexDefinition interface, rather + # then branching on the value of this predicate. + def rollover_index_template? + false + end + + def index_expression_for_search + name + end + + # Returns an index name to use for write operations. + def index_name_for_writes(record, timestamp_field_path: nil) + name + end + + # A concrete index has no related indices (really only rollover indices do). + def related_rollover_indices(datastore_client, only_if_exists: false) + [] + end + end + end + end +end diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/rollover_index.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/rollover_index.rb new file mode 100644 index 00000000..5b69d39c --- /dev/null +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/rollover_index.rb @@ -0,0 +1,48 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" +require "elastic_graph/datastore_core/index_definition/index" + +module ElasticGraph + class DatastoreCore + module IndexDefinition + # Represents a concrete index for specific time range, derived from a RolloverIndexTemplate. + class RolloverIndex < DelegateClass(Index) + # @dynamic time_set + attr_reader :time_set + + def initialize(index, time_set) + super(index) + @time_set = time_set + end + + # We need to override `==` so that two `RolloverIndex` objects that wrap the same `Index` object are + # considered equal. Oddly enough, the `DelegateClass` implementation of `==` returns `true` if `other` + # is the wrapped object, but not if it's another instance of the same `DelegateClass` wrapping the same + # instance. + # + # https://github.com/ruby/ruby/blob/v3_0_3/lib/delegate.rb#L156-L159 + # + # We need this because we want two `RolloverIndex` instances that wrap the same + # underlying `Index` instance to be considered equal (something a test relies upon, + # but also generally useful and expected). + def ==(other) + if RolloverIndex === other + __getobj__ == other.__getobj__ && time_set == other.time_set + else + # :nocov: -- this method isn't explicitly covered by tests (not worth writing a test just to cover this line). + super + # :nocov: + end + end + alias_method :eql?, :== + end + end + end +end diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/rollover_index_template.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/rollover_index_template.rb new file mode 100644 index 00000000..29971a4d --- /dev/null +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/rollover_index_template.rb @@ -0,0 +1,232 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "date" +require "elastic_graph/datastore_core/index_config_normalizer" +require "elastic_graph/datastore_core/index_definition/base" +require "elastic_graph/datastore_core/index_definition/index" +require "elastic_graph/datastore_core/index_definition/rollover_index" +require "elastic_graph/errors" +require "elastic_graph/support/memoizable_data" +require "elastic_graph/support/time_set" +require "elastic_graph/support/time_util" +require "time" + +module ElasticGraph + class DatastoreCore + module IndexDefinition + class RolloverIndexTemplate < Support::MemoizableData.define( + :name, :route_with, :default_sort_clauses, :current_sources, :fields_by_path, :env_index_config, + :index_args, :defined_clusters, :datastore_clients_by_name, :timestamp_field_path, :frequency + ) + # `Data.define` provides all these methods: + # @dynamic name, route_with, default_sort_clauses, current_sources, fields_by_path, env_index_config, + # @dynamic index_args, defined_clusters, datastore_clients_by_name, timestamp_field_path, frequency, initialize + + # `include IndexDefinition::Base` provides all these methods. Steep should be able to detect it + # but can't for some reason so we have to declare them with `@dynamic`. + # @dynamic flattened_env_setting_overrides, routing_value_for_prepared_record, has_custom_routing?, cluster_to_query, use_updates_for_indexing? + # @dynamic clusters_to_index_into, all_accessible_cluster_names, ignored_values_for_routing, searches_could_hit_incomplete_docs? + # @dynamic accessible_cluster_names_to_index_into, accessible_from_queries?, known_related_query_rollover_indices, list_counts_field_paths_for_source + include IndexDefinition::Base + + def mappings_in_datastore(datastore_client) + IndexConfigNormalizer.normalize_mappings( + datastore_client.get_index_template(name).dig("template", "mappings") || {} + ) + end + + # We need to delete both the template and the actual indices for rollover indices + def delete_from_datastore(datastore_client) + datastore_client.delete_index_template(name) + datastore_client.delete_indices(index_expression_for_search) + end + + # Indicates if this is a rollover index definition. + # + # Use of this is considered a mild code smell. When feasible, it's generally better to + # implement a new polymorphic API on the IndexDefinition interface, rather + # then branching on the value of this predicate. + def rollover_index_template? + true + end + + # Two underscores used to avoid collisions + # with other types (e.g. payments_2020 and payments_xyz_2020), though regardless shouldn't + # happen if types follow naming conventions. + def index_expression_for_search + index_name_with_suffix("*") + end + + # Returns an index name to use for write operations. The index_definition selection is a function of + # the index_definition's rollover configuration and the record's timestamp. + def index_name_for_writes(record, timestamp_field_path: nil) + index_name_with_suffix(rollover_index_suffix_for_record( + record, + timestamp_field_path: timestamp_field_path || self.timestamp_field_path + )) + end + + # Returns a list of indices related to this template. This includes both indices that are + # specified in our configuration settings (e.g. via `setting_overrides_by_timestamp` and + # `custom_time_sets`) and also indices that have been auto-created from the template. + # + # Note that there can be discrepancies between the configuration settings and the indices in + # the datastore. Sometimes this is planned/expected (e.g. such as when invoking `elasticgraph-admin` + # to configure an index newly defined in configuration) and in other cases it's not. + # + # The `only_if_exists` argument controls how a discrepancy is treated. + # + # - When `false` (the default), indices that are defined in config but do not exist in the datastore are still returned. + # This is generally what we want for indexing and cluster administration. + # - When `true`, any indices in our configuration that do not exist are ignored, and not included in the returned list. + # This is appropriate for searching the datastore: if we attempt to exclude an index which is defined in config but does + # not exist (e.g. via `-[index_name]` in the search index expression), the datastore will return an error, but we can + # safely ignore the index. Likewise, if we have an index in the datastore which we cannot infer a timestamp range, we + # need to ignore it to avoid getting errors. Ignoring an index is safe when searching because our search logic uses a + # wildcard to match _all_ indices with the same prefix, and then excludes certain known indices that it can safely + # exclude based on their timestamp range. Ignored indices which exist will still be searched. + # + # In addition, any indices which exist, but which are not controlled by our current configuration, are ignored. Examples: + # + # - An index with a custom suffix (e.g. `__before_2019`) which has no corresponding configuration. We have no way to guess + # what the timestamp range is for such an index, and we want to completely ignore it. + # - An index with for a different rollover frequency than our current configuration. For example, a `__2019-03` index, + # which must rollover monthly, would be ignored if our current rollover frequency is yearly or daily. + # + # These latter cases are quite rare but can happen when we are dealing with indices defined before an update to our + # configuration. Our searches will continue to search these indices so long as their name matches the pattern, and + # we otherwise want to ignore these indices (e.g. we don't want admin to attempt to configure them, or want our + # indexer to attempt to write to them). + def related_rollover_indices(datastore_client, only_if_exists: false) + config_indices_by_name = rollover_indices_to_pre_create.to_h { |i| [i.name, i] } + + db_indices_by_name = datastore_client.list_indices_matching(index_expression_for_search).filter_map do |name| + index = concrete_rollover_index_for(name, {}, config_indices_by_name[name]&.time_set) + [name, index] if index + end.to_h + + config_indices_by_name = config_indices_by_name.slice(*db_indices_by_name.keys) if only_if_exists + + db_indices_by_name.merge(config_indices_by_name).values + end + + # Gets a single related `RolloverIndex` for a given timestamp. + def related_rollover_index_for_timestamp(timestamp, setting_overrides = {}) + # @type var record: ::Hash[::String, untyped] + # We need to use `__skip__` here because `inner_value` has different types on different + # block iterations: initially, it's a string, then it becomes a hash. Steep has trouble + # with this but it works fine. + __skip__ = record = timestamp_field_path.split(".").reverse.reduce(timestamp) do |inner_value, field_name| + {field_name => inner_value} + end + + concrete_rollover_index_for(index_name_for_writes(record), setting_overrides) + end + + private + + def after_initialize + unless timestamp_field_path && ROLLOVER_SUFFIX_FORMATS_BY_FREQUENCY.key?(frequency) + raise Errors::SchemaError, "Rollover index config 'timestamp_field' or 'frequency' is invalid." + end + end + + # Returns a list of indices that must be pre-created (rather than allowing them to be + # created lazily based on the template). This is done so that we can use different + # index settings for some indices. For example, you might want your template to be + # configured to use 5 shards, but for old months with a small data set you may only + # want to use 1 shard. + def rollover_indices_to_pre_create + @rollover_indices_to_pre_create ||= begin + indices_with_overrides = setting_overrides_by_timestamp.filter_map do |(timestamp, setting_overrides)| + related_rollover_index_for_timestamp(timestamp, setting_overrides) + end + + indices_for_custom_timestamp_ranges = custom_timestamp_ranges.filter_map do |range| + concrete_rollover_index_for( + index_name_with_suffix(range.index_name_suffix), + range.setting_overrides, + range.time_set + ) + end + + indices_with_overrides + indices_for_custom_timestamp_ranges + end + end + + def setting_overrides_by_timestamp + env_index_config.setting_overrides_by_timestamp + end + + def custom_timestamp_ranges + env_index_config.custom_timestamp_ranges + end + + def index_name_with_suffix(suffix) + "#{name}#{ROLLOVER_INDEX_INFIX_MARKER}#{suffix}" + end + + ROLLOVER_SUFFIX_FORMATS_BY_FREQUENCY = {hourly: "%Y-%m-%d-%H", daily: "%Y-%m-%d", monthly: "%Y-%m", yearly: "%Y"} + ROLLOVER_TIME_ELEMENT_COUNTS_BY_FREQUENCY = ROLLOVER_SUFFIX_FORMATS_BY_FREQUENCY.transform_values { |format| format.split("-").size } + TIME_UNIT_BY_FREQUENCY = {hourly: :hour, daily: :day, monthly: :month, yearly: :year} + + def rollover_index_suffix_for_record(record, timestamp_field_path:) + timestamp_value = ::DateTime.iso8601( + Support::HashUtil.fetch_value_at_path(record, timestamp_field_path) + ).to_time + + if (matching_custom_range = env_index_config.custom_timestamp_range_for(timestamp_value)) + return matching_custom_range.index_name_suffix + end + + timestamp_value.strftime(ROLLOVER_SUFFIX_FORMATS_BY_FREQUENCY[frequency]) + end + + def concrete_rollover_index_for(index_name, setting_overrides, time_set = nil) + time_set ||= infer_time_set_from_index_name(index_name) + return nil if time_set.nil? + + args = index_args.merge({ + name: index_name, + env_index_config: env_index_config.without_env_overrides.with( + setting_overrides: env_index_config.setting_overrides.merge(setting_overrides) + ) + }) + + RolloverIndex.new(Index.new(**args), time_set) + end + + def infer_time_set_from_index_name(index_name) + time_args = index_name.split(ROLLOVER_INDEX_INFIX_MARKER).last.to_s.split("-") + + # Verify that the index is for the same rollover frequency as we are currently configured to use. + # If not, return `nil` because we can't accurately infer the time set without the frequency aligning + # with the index itself. + # + # This can happen when we are migrating from one index frequency to another. + return nil unless time_args.size == ROLLOVER_TIME_ELEMENT_COUNTS_BY_FREQUENCY.fetch(frequency) + + # Verify that the args are all numeric. If not, return `nil` because we have no idea what the + # time set for the index is. + # + # This can happen when we are migrating from one index configuration to another while also using + # custom timestamp ranges (e.g. to have a `__before_2020` index). + return nil if time_args.any? { |arg| /\A\d+\z/ !~ arg } + + # Steep can't type the dynamic nature of `*time_args` so we have to use `__skip__` here. + # @type var lower_bound: ::Time + __skip__ = lower_bound = ::Time.utc(*time_args) + upper_bound = Support::TimeUtil.advance_one_unit(lower_bound, TIME_UNIT_BY_FREQUENCY.fetch(frequency)) + + Support::TimeSet.of_range(gte: lower_bound, lt: upper_bound) + end + end + end + end +end diff --git a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core.rbs b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core.rbs new file mode 100644 index 00000000..2a6941ed --- /dev/null +++ b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core.rbs @@ -0,0 +1,37 @@ +module ElasticGraph + class DatastoreCore + attr_reader config: Config + attr_reader logger: ::Logger + attr_reader schema_artifacts: schemaArtifacts + attr_reader client_customization_block: (^(untyped) -> void)? + + def self.from_parsed_yaml: ( + parsedYamlSettings, + for_context: SchemaArtifacts::context + ) ?{ (untyped) -> void } -> DatastoreCore + + def initialize: ( + config: Config, + logger: ::Logger, + schema_artifacts: schemaArtifacts, + ?clients_by_name: ::Hash[::String, _Client]?, + ?client_customization_block: (^(untyped) -> void)? + ) -> void + + @index_definitions_by_name: ::Hash[::String, indexDefinition]? + def index_definitions_by_name: () -> ::Hash[::String, indexDefinition] + + @index_definitions_by_graphql_type: ::Hash[::String, ::Array[indexDefinition]]? + def index_definitions_by_graphql_type: () -> ::Hash[::String, ::Array[indexDefinition]] + + @clients_by_name: ::Hash[::String, _Client]? + def clients_by_name: () -> ::Hash[::String, _Client] + + private + + @config: Config + @logger: ::Logger + @schema_artifacts: schemaArtifacts + @client_customization_block: (^(untyped) -> void)? + end +end diff --git a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/client.rbs b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/client.rbs new file mode 100644 index 00000000..e5b94e3e --- /dev/null +++ b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/client.rbs @@ -0,0 +1,48 @@ +module ElasticGraph + class DatastoreCore + type indexMappingHash = ::Hash[::String, untyped] + type indexSettingsHash = ::Hash[::String, untyped] + type indexConfigHash = ::Hash[::String, untyped] + + interface _Client + def cluster_name: () -> ::String + + def get_cluster_health: () -> ::Hash[::String, untyped] + def get_node_os_stats: () -> ::Hash[::String, untyped] + def get_flat_cluster_settings: () -> ::Hash[::String, untyped] + def put_persistent_cluster_settings: (::Hash[::Symbol | ::String, untyped]) -> void + + def get_script: (id: ::String) -> ::Hash[::String, untyped]? + def put_script: (id: ::String, body: ::Hash[::Symbol, untyped], context: ::String) -> void + def delete_script: (id: ::String) -> void + + def get_index_template: (::String) -> DatastoreCore::indexConfigHash + def put_index_template: (name: ::String, body: DatastoreCore::indexConfigHash) -> void + def delete_index_template: (::String) -> void + + def get_index: (::String) -> DatastoreCore::indexConfigHash + def list_indices_matching: (::String) -> ::Array[::String] + def create_index: (index: ::String, body: ::Hash[::String, untyped]) -> void + def put_index_mapping: (index: ::String, body: ::Hash[::String, untyped]) -> void + def put_index_settings: (index: ::String, body: ::Hash[::String, untyped]) -> void + def delete_indices: (*::String) -> void + + def msearch: (body: ::Array[::Hash[::String | ::Symbol, untyped]], ?headers: ::Hash[::String, untyped]?) -> ::Hash[::String, untyped] + def bulk: ( + body: ::Array[::Hash[::String | ::Symbol, untyped]], + ?refresh: bool + ) -> ::Hash[::String, untyped] + def delete_all_documents: (?index: ::String) -> void + end + + interface _ClientClass + def new: ( + ::String, + url: ::String, + ?faraday_adapter: ::Symbol?, + ?retry_on_failure: ::Integer, + ?logger: ::Logger? + ) ?{ (::Faraday::RackBuilder) -> void } -> _Client + end + end +end diff --git a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/config.rbs b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/config.rbs new file mode 100644 index 00000000..1a176d98 --- /dev/null +++ b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/config.rbs @@ -0,0 +1,34 @@ +module ElasticGraph + class DatastoreCore + class ConfigSupertype + attr_reader client_faraday_adapter: Configuration::ClientFaradayAdapter + attr_reader clusters: ::Hash[::String, Configuration::ClusterDefinition] + attr_reader index_definitions: ::Hash[::String, Configuration::IndexDefinition] + attr_reader log_traffic: bool + attr_reader max_client_retries: ::Integer + + def initialize: ( + client_faraday_adapter: Configuration::ClientFaradayAdapter, + clusters: ::Hash[::String, Configuration::ClusterDefinition], + index_definitions: ::Hash[::String, Configuration::IndexDefinition], + log_traffic: bool, + max_client_retries: ::Integer + ) -> void + + def with: ( + ?client_faraday_adapter: Configuration::ClientFaradayAdapter, + ?clusters: ::Hash[::String, Configuration::ClusterDefinition], + ?index_definitions: ::Hash[::String, Configuration::IndexDefinition], + ?log_traffic: bool, + ?max_client_retries: ::Integer + ) -> Config + + def self.members: () -> ::Array[::Symbol] + end + + class Config < ConfigSupertype + extend _BuildableFromParsedYaml[Config] + EXPECTED_KEYS: ::Array[::String] + end + end +end diff --git a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/configuration/client_faraday_adapter.rbs b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/configuration/client_faraday_adapter.rbs new file mode 100644 index 00000000..9cab11e8 --- /dev/null +++ b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/configuration/client_faraday_adapter.rbs @@ -0,0 +1,30 @@ +module ElasticGraph + class DatastoreCore + module Configuration + class ClientFaradayAdapterSupertypee + attr_reader name: ::Symbol? + attr_reader require: ::String? + + def initialize: (name: ::Symbol?, require: ::String?) -> void + + def with: ( + ?name: ::Symbol?, + ?require: ::String?) -> ClientFaradayAdapter + + def self.with: ( + name: ::Symbol?, + require: ::String?) -> ClientFaradayAdapter + + def self.members: () -> ::Array[::Symbol] + end + + class ClientFaradayAdapter < ClientFaradayAdapterSupertypee + def self.from_parsed_yaml: ( + ::Hash[::String, ::Hash[::String, untyped]] + ) -> ClientFaradayAdapter + + EXPECTED_KEYS: ::Array[::String] + end + end + end +end diff --git a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/configuration/cluster_definition.rbs b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/configuration/cluster_definition.rbs new file mode 100644 index 00000000..12808e8b --- /dev/null +++ b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/configuration/cluster_definition.rbs @@ -0,0 +1,35 @@ +module ElasticGraph + class DatastoreCore + module Configuration + class ClusterDefinitionSupertype + attr_reader url: ::String + attr_reader backend_client_class: _ClientClass + attr_reader settings: ::Hash[::String, untyped] + + def initialize: ( + url: ::String, + backend_client_class: _ClientClass, + settings: ::Hash[::String, untyped] + ) -> void + + def with: ( + ?url: ::String, + ?backend_client_class: _ClientClass, + ?settings: ::Hash[::String, untyped] + ) -> ClusterDefinition + + def self.members: () -> ::Array[::Symbol] + end + + class ClusterDefinition < ClusterDefinitionSupertype + def self.from_hash: (::Hash[::String, untyped]) -> ClusterDefinition + + def self.definitions_by_name_hash_from: ( + ::Hash[::String, ::Hash[::String, untyped]] + ) -> ::Hash[::String, ClusterDefinition] + + EXPECTED_KEYS: ::Array[::String] + end + end + end +end diff --git a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/configuration/index_definition.rbs b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/configuration/index_definition.rbs new file mode 100644 index 00000000..f6795471 --- /dev/null +++ b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/configuration/index_definition.rbs @@ -0,0 +1,73 @@ +module ElasticGraph + class DatastoreCore + module Configuration + class IndexDefinitionSupertype + attr_reader ignore_routing_values: ::Set[::String] + attr_reader query_cluster: ::String + attr_reader index_into_clusters: ::Array[::String] + attr_reader setting_overrides: ::Hash[::String, untyped] + attr_reader setting_overrides_by_timestamp: ::Hash[::String, ::Hash[::String, untyped]] + attr_reader custom_timestamp_ranges: ::Array[IndexDefinition::CustomTimestampRange] + attr_reader use_updates_for_indexing: bool + + def initialize: ( + ignore_routing_values: ::Set[::String], + query_cluster: ::String, + index_into_clusters: ::Array[::String], + setting_overrides: ::Hash[::String, untyped], + setting_overrides_by_timestamp: ::Hash[::String, ::Hash[::String, untyped]], + custom_timestamp_ranges: ::Array[IndexDefinition::CustomTimestampRange], + use_updates_for_indexing: bool) -> void + + def with: ( + ?ignore_routing_values: ::Set[::String], + ?query_cluster: ::String, + ?index_into_clusters: ::Array[::String], + ?setting_overrides: ::Hash[::String, untyped], + ?setting_overrides_by_timestamp: ::Hash[::String, ::Hash[::String, untyped]], + ?custom_timestamp_ranges: ::Array[IndexDefinition::CustomTimestampRange], + ?use_updates_for_indexing: bool) -> IndexDefinition + end + + class IndexDefinition < IndexDefinitionSupertype + def initialize: (ignore_routing_values: ::Array[::String], **untyped) -> void + def without_env_overrides: () -> IndexDefinition + def custom_timestamp_range_for: (::Time) -> CustomTimestampRange? + def self.definitions_by_name_hash_from: (::Hash[::String, ::Hash[::String, untyped]]) -> ::Hash[::String, IndexDefinition] + + def self.from: ( + custom_timestamp_ranges: ::Array[::Hash[::String, untyped]], + ?use_updates_for_indexing: bool, + **untyped) -> IndexDefinition + + class CustomTimestampRangeSupertype + attr_reader index_name_suffix: ::String + attr_reader setting_overrides: ::Hash[::String, untyped] + attr_reader time_set: Support::TimeSet + + def initialize: ( + index_name_suffix: ::String, + setting_overrides: ::Hash[::String, untyped], + time_set: Support::TimeSet) -> void + + def with: ( + ?index_name_suffix: ::String, + ?setting_overrides: ::Hash[::String, untyped], + ?time_set: Support::TimeSet) -> CustomTimestampRange + end + + class CustomTimestampRange < CustomTimestampRangeSupertype + def self.ranges_from: (::Array[::Hash[::String, untyped]]) -> ::Array[CustomTimestampRange] + + private + + def self.from: ( + index_name_suffix: ::String, + setting_overrides: ::Hash[::String, untyped], + **::String + ) -> CustomTimestampRange + end + end + end + end +end diff --git a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_config_normalizer.rbs b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_config_normalizer.rbs new file mode 100644 index 00000000..056ecb4b --- /dev/null +++ b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_config_normalizer.rbs @@ -0,0 +1,15 @@ +module ElasticGraph + class DatastoreCore + module IndexConfigNormalizer + READ_ONLY_SETTINGS: ::Array[::String] + + def self.normalize: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + def self.normalize_mappings: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + + private + + def self.normalize_settings: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + def self.normalize_setting_value: (untyped) -> untyped + end + end +end diff --git a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition.rbs b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition.rbs new file mode 100644 index 00000000..1d087a4b --- /dev/null +++ b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition.rbs @@ -0,0 +1,55 @@ +module ElasticGraph + class DatastoreCore + # Defines methods of the _IndexDefinition interface that have a common implementation + # provided by the `IndexDefinition::Base` module. + interface _IndexDefinitionCommonMethods + def flattened_env_setting_overrides: () -> ::Hash[::String, untyped] + def use_updates_for_indexing?: () -> bool + def routing_value_for_prepared_record: (::Hash[::String, untyped], ?route_with_path: ::String?, ?id_path: ::String) -> ::String? + def has_custom_routing?: () -> bool + def searches_could_hit_incomplete_docs?: () -> bool + def cluster_to_query: () -> ::String + def clusters_to_index_into: () -> ::Array[::String] + def ignored_values_for_routing: () -> ::Set[::String] + def all_accessible_cluster_names: () -> ::Array[::String] + def accessible_cluster_names_to_index_into: () -> ::Array[::String] + def accessible_from_queries?: () -> bool + def known_related_query_rollover_indices: () -> ::Array[IndexDefinition::RolloverIndex] + def list_counts_field_paths_for_source: (::String) -> ::Set[::String] + end + + # Defines methods of the _IndexDefinition interface that each specific implementation must provide. + interface _IndexDefinitionImplementationMethods + def name: () -> ::String + def route_with: () -> ::String + def default_sort_clauses: () -> ::Array[::Hash[::String, ::String]] + def current_sources: () -> ::Set[::String] + def fields_by_path: () -> ::Hash[::String, SchemaArtifacts::RuntimeMetadata::IndexField] + def env_index_config: () -> Configuration::IndexDefinition + def defined_clusters: () -> ::Set[::String] + def datastore_clients_by_name: () -> ::Hash[::String, DatastoreCore::_Client] + def rollover_index_template?: () -> bool + def index_expression_for_search: () -> ::String + def index_name_for_writes: (::Hash[::String, untyped], ?timestamp_field_path: ::String?) -> ::String + def mappings_in_datastore: (DatastoreCore::_Client) -> ::Hash[::String, untyped] + def related_rollover_indices: (DatastoreCore::_Client, ?only_if_exists: bool) -> ::Array[IndexDefinition::RolloverIndex] + end + + # Defines the full `_IndexDefinition` interface. + interface _IndexDefinition + include _IndexDefinitionImplementationMethods + include _IndexDefinitionCommonMethods + end + + type indexDefinition = _IndexDefinition & (IndexDefinition::Index | IndexDefinition::RolloverIndexTemplate) + + module IndexDefinition + def self.with: ( + name: ::String, + runtime_metadata: SchemaArtifacts::RuntimeMetadata::IndexDefinition, + config: DatastoreCore::Config, + datastore_clients_by_name: ::Hash[::String, DatastoreCore::_Client] + ) -> indexDefinition + end + end +end diff --git a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition/base.rbs b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition/base.rbs new file mode 100644 index 00000000..669ec894 --- /dev/null +++ b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition/base.rbs @@ -0,0 +1,20 @@ +module ElasticGraph + class DatastoreCore + module IndexDefinition + module Base: _IndexDefinitionImplementationMethods + include _IndexDefinitionCommonMethods + + @flattened_env_setting_overrides: ::Hash[::String, untyped] + @all_accessible_cluster_names: ::Array[::String]? + @accessible_cluster_names_to_index_into: ::Array[::String]? + @known_related_query_rollover_indices: ::Array[RolloverIndex]? + @searches_could_hit_incomplete_docs: bool + @list_counts_field_paths_for_source: ::Hash[::String, ::Set[::String]] + + private + + def identify_list_counts_field_paths_for_source: (::String) -> ::Set[::String] + end + end + end +end diff --git a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition/index.rbs b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition/index.rbs new file mode 100644 index 00000000..52f545e8 --- /dev/null +++ b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition/index.rbs @@ -0,0 +1,11 @@ +module ElasticGraph + class DatastoreCore + module IndexDefinition + class Index + include IndexDefinition::Base + include _IndexDefinition + def initialize: (**untyped) -> void + end + end + end +end diff --git a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition/rollover_index.rbs b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition/rollover_index.rbs new file mode 100644 index 00000000..76155f15 --- /dev/null +++ b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition/rollover_index.rbs @@ -0,0 +1,16 @@ +module ElasticGraph + class DatastoreCore + module IndexDefinition + class RolloverIndexSupertype < Index + include _IndexDefinition + def initialize: (Index) -> void + attr_reader __getobj__: Index + end + + class RolloverIndex < RolloverIndexSupertype + def initialize: (Index, Support::TimeSet) -> void + attr_reader time_set: Support::TimeSet + end + end + end +end diff --git a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition/rollover_index_template.rbs b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition/rollover_index_template.rbs new file mode 100644 index 00000000..858da6bf --- /dev/null +++ b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition/rollover_index_template.rbs @@ -0,0 +1,33 @@ +module ElasticGraph + class DatastoreCore + module IndexDefinition + class RolloverIndexTemplate + include IndexDefinition::Base + include _IndexDefinition + + def initialize: (**untyped) -> void + + attr_reader timestamp_field_path: ::String + attr_reader frequency: ::Symbol + attr_reader index_args: ::Hash[::Symbol, untyped] + + def related_rollover_index_for_timestamp: (::String, ?::Hash[::String, untyped]) -> RolloverIndex? + + private + + @rollover_indices_to_pre_create: ::Array[RolloverIndex]? + def rollover_indices_to_pre_create: () -> ::Array[RolloverIndex] + def setting_overrides_by_timestamp: () -> ::Hash[::String, indexSettingsHash] + def custom_timestamp_ranges: () -> ::Array[Configuration::IndexDefinition::CustomTimestampRange] + def index_name_with_suffix: (::String) -> ::String + def rollover_index_suffix_for_record: (::Hash[::String, untyped], timestamp_field_path: ::String) -> ::String + def concrete_rollover_index_for: (::String, indexSettingsHash, ?Support::TimeSet?) -> RolloverIndex? + def infer_time_set_from_index_name: (::String) -> Support::TimeSet? + + ROLLOVER_SUFFIX_FORMATS_BY_FREQUENCY: ::Hash[::Symbol, ::String] + ROLLOVER_TIME_ELEMENT_COUNTS_BY_FREQUENCY: ::Hash[::Symbol, ::Integer] + TIME_UNIT_BY_FREQUENCY: ::Hash[::Symbol, Support::TimeUtil::advancementUnit] + end + end + end +end diff --git a/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/implementation_shared_examples.rb b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/implementation_shared_examples.rb new file mode 100644 index 00000000..af9217e4 --- /dev/null +++ b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/implementation_shared_examples.rb @@ -0,0 +1,120 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core/index_definition" +require "elastic_graph/errors" + +module ElasticGraph + class DatastoreCore + module IndexDefinition + RSpec.shared_examples_for "an IndexDefinition implementation (integration specs)", :builds_admin do + describe "#searches_could_hit_incomplete_docs?" do + it "returns `false` on an index that has no `sourced_from` fields" do + index = define_index + + expect { + expect(index.searches_could_hit_incomplete_docs?).to be false + }.to change { datastore_requests("main").count }.by(1) + + # Demonstrate that we cache the value by showing the datastore request count doesn't change + # when we call `searches_could_hit_incomplete_docs?` with the same client again. + expect { + expect(index.searches_could_hit_incomplete_docs?).to be false + }.not_to change { datastore_requests("main").count } + end + + it "returns `true` on an index that has `sourced_from` fields, without hitting the datastore (since it is not needed!)" do + index = define_index do |t| + t.field "owner_name", "String" do |f| + f.sourced_from "owner", "name" + end + end + + expect { + expect(index.searches_could_hit_incomplete_docs?).to be true + }.not_to change { datastore_requests("main").count } + + # Demonstrate that we cache the value by showing the datastore request count doesn't change + # when we call `searches_could_hit_incomplete_docs?` with the same client again. + expect { + expect(index.searches_could_hit_incomplete_docs?).to be true + }.not_to change { datastore_requests("main").count } + end + + it "returns `true` on an index that no longer has `sourced_from` fields but used to" do + define_index do |t| + t.field "owner_name", "String" do |f| + f.sourced_from "owner", "name" + end + end + + index = define_index + + expect(index.searches_could_hit_incomplete_docs?).to be true + end + + context "when there are no sources recorded in `_meta` on the index" do + it "uses the `current_sources` to determine the value" do + index = define_index(skip_configure_datastore: true) + expect(index.searches_could_hit_incomplete_docs?).to be false + + index = define_index(skip_configure_datastore: true) do |t| + t.field "owner_name", "String" do |f| + f.sourced_from "owner", "name" + end + end + expect(index.searches_could_hit_incomplete_docs?).to be true + end + end + + describe "#mappings_in_datastore" do + it "returns the mappings in normalized form" do + index = define_index + allow(IndexConfigNormalizer).to receive(:normalize_mappings).and_call_original + + mappings = index.mappings_in_datastore(main_datastore_client) + + expect(IndexConfigNormalizer).to have_received(:normalize_mappings).at_least(:once) + expect(mappings).to eq(IndexConfigNormalizer.normalize_mappings(mappings)) + end + end + + def define_index(skip_configure_datastore: false, &schema_definition) + datastore_core = build_datastore_core(schema_definition: lambda do |schema| + schema.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime" + t.relates_to_one "owner", "Owner", via: "my_type_id", dir: :in do |rel| + rel.equivalent_field "created_at" + end + schema_definition&.call(t) + t.index unique_index_name do |i| + configure_index(i) + end + end + + schema.object_type "Owner" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "created_at", "DateTime" + t.field "my_type_id", "ID" + t.index "#{unique_index_name}_owners" + end + end) + + unless skip_configure_datastore + build_admin(datastore_core: datastore_core).cluster_configurator.configure_cluster(output_io) + end + + datastore_core.index_definitions_by_name.fetch(unique_index_name) + end + end + end + end + end +end diff --git a/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/index_spec.rb b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/index_spec.rb new file mode 100644 index 00000000..3cd05f83 --- /dev/null +++ b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/index_spec.rb @@ -0,0 +1,99 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core/index_definition" +require "stringio" +require_relative "implementation_shared_examples" + +module ElasticGraph + class DatastoreCore + module IndexDefinition + RSpec.describe Index, :uses_datastore do + # Use different index names than any other tests use, because most tests expect a specific index + # configuration (based on `config/schema.graphql`) and we do not want to mess with it here. + let(:index_prefix) { unique_index_name } + let(:widgets_index_name) { "#{index_prefix}_widgets" } + let(:components_index_name) { "#{index_prefix}_components" } + let(:output_io) { StringIO.new } + let(:schema_definition) do + lambda do |schema| + schema.object_type "Component" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index components_index_name + end + end + end + + include_examples "an IndexDefinition implementation (integration specs)" do + def configure_index(index) + end + end + + describe "#delete_from_datastore", :builds_admin do + let(:datastore_core) do + build_datastore_core(schema_definition: schema_definition) + end + + before do + build_admin(datastore_core: datastore_core).cluster_configurator.configure_cluster(output_io) + end + + it "deletes non-rollover index definition" do + index_definition = datastore_core.index_definitions_by_name.fetch(components_index_name) + + expect { + index_definition.delete_from_datastore(main_datastore_client) + }.to change { main_datastore_client.get_index(index_definition.name) } + .from(a_hash_including( + "mappings" => a_hash_including("properties" => a_hash_including("id", "name")), + "settings" => a_kind_of(Hash) + )) + .to({}) + end + + it "ignores non-existing index" do + index_def_not_exist = index_def_named("does_not_exist") + + expect { + index_def_not_exist.delete_from_datastore(main_datastore_client) + }.not_to raise_error + end + end + + describe "related indices" do + it "returns an empty list as it never has any related indices" do + datastore_core = build_datastore_core(schema_definition: schema_definition) + index_definition = datastore_core.index_definitions_by_name.fetch(components_index_name) + + expect(index_definition.rollover_index_template?).to be false + expect(index_definition.related_rollover_indices(main_datastore_client)).to eq [] + expect(index_definition.known_related_query_rollover_indices).to eq [] + end + end + + def index_def_named(name, rollover: nil) + runtime_metadata = SchemaArtifacts::RuntimeMetadata::IndexDefinition.new( + route_with: nil, + rollover: nil, + default_sort_fields: [], + current_sources: [SELF_RELATIONSHIP_NAME], + fields_by_path: {} + ) + + DatastoreCore::IndexDefinition.with( + name: name, + config: datastore_core.config, + runtime_metadata: runtime_metadata, + datastore_clients_by_name: datastore_core.clients_by_name + ) + end + end + end + end +end diff --git a/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb new file mode 100644 index 00000000..3ec9da05 --- /dev/null +++ b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb @@ -0,0 +1,582 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/datastore_core/index_definition" +require "elastic_graph/support/hash_util" +require "stringio" +require_relative "implementation_shared_examples" + +module ElasticGraph + class DatastoreCore + module IndexDefinition + RSpec.describe RolloverIndexTemplate, :uses_datastore, :builds_indexer do + # Use different index names than any other tests use, because most tests expect a specific index + # configuration (based on `config/schema.graphql`) and we do not want to mess with it here. + let(:index_prefix) { unique_index_name } + let(:widgets_index_name) { "#{index_prefix}_widgets" } + let(:components_index_name) { "#{index_prefix}_components" } + let(:output_io) { StringIO.new } + let(:schema_definition) do + lambda do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime" + t.index widgets_index_name do |i| + i.rollover :monthly, "created_at" + end + end + end + end + + include_examples "an IndexDefinition implementation (integration specs)" do + def configure_index(index) + index.rollover :monthly, "created_at" + end + end + + describe "#delete_from_datastore", :builds_admin do + let(:datastore_core) { build_datastore_core(schema_definition: schema_definition) } + + before do + build_admin(datastore_core: datastore_core).cluster_configurator.configure_cluster(output_io) + end + + it "deletes the rollover index definition" do + index_definition = datastore_core.index_definitions_by_name.fetch(widgets_index_name) + record = { + "id" => "1234", + "created_at" => "2019-06-02T12:00:00Z", + "__typename" => "Widget", + "__version" => 1, + "__json_schema_version" => 1 + } + index_name_for_writes = index_definition.index_name_for_writes(record) + derive_index_from_template(record, datastore_core) + + expect { + index_definition.delete_from_datastore(main_datastore_client) + }.to change { main_datastore_client.get_index_template(index_definition.name)["template"] || {} } + .from(a_hash_including( + "mappings" => a_hash_including("properties" => a_hash_including("id", "created_at")), + "settings" => a_kind_of(Hash) + )) + .to({}) + .and change { main_datastore_client.get_index(index_name_for_writes) } + .from(a_hash_including( + "mappings" => a_hash_including("properties" => a_hash_including("id", "created_at")), + "settings" => a_kind_of(Hash) + )) + .to({}) + end + + it "ignores non-existing index template and index" do + index_def_not_exist = index_def_named("does_not_exist", rollover: { + timestamp_field_path: "created_at", frequency: :monthly + }) + + expect { + index_def_not_exist.delete_from_datastore(main_datastore_client) + }.not_to raise_error + end + end + + describe "related indices", :factories do + it "returns an `Index` for each entry in `setting_overrides_by_timestamp` or `custom_timestamp_ranges` in config, with the name and normalized settings overridden" do + schema_definition = lambda do |s| + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime" + t.index "my_type", number_of_shards: 5, some: {other_setting: false} do |i| + i.rollover :monthly, "created_at" + end + end + end + + datastore_core = build_datastore_core(schema_definition: schema_definition) do |config| + config.with(index_definitions: { + "my_type" => config_index_def_of( + setting_overrides: { + "common" => "override", # a setting not overridden by other setting overrides below. + "number_of_shards" => 12 # ...whereas this one is overridden below. + }, + query_cluster: "other2", + setting_overrides_by_timestamp: { + "2020-01-01T00:00:00Z" => { + "number_of_shards" => 1, + "yet" => {"another" => {"setting" => true}} + }, + "2020-02-01T00:00:00Z" => { + "number_of_shards" => 2, + "yet" => {"another" => {"setting" => true}} + } + }, + custom_timestamp_ranges: [ + { + "index_name_suffix" => "before_2015", + "lt" => "2015-01-01T00:00:00Z", + "setting_overrides" => { + "number_of_shards" => 3, + "yet" => {"another" => {"setting" => true}} + } + }, + { + "index_name_suffix" => "2016_and_2017", + "gte" => "2016-01-01T00:00:00Z", + "lt" => "2018-01-01T00:00:00Z", + "setting_overrides" => { + "number_of_shards" => 4, + "yet" => {"another" => {"setting" => true}} + } + } + ] + ) + }) + end + + index = datastore_core.index_definitions_by_name.fetch("my_type") + related_rollover_indices = index.related_rollover_indices(main_datastore_client) + # No indexes exist to query yet, so `known_related_query_rollover_indices` should be empty. + expect(index.known_related_query_rollover_indices).to eq([]) + + expect(related_rollover_indices.size).to eq 4 + + expect(related_rollover_indices[0]).to be_a RolloverIndex + expect(related_rollover_indices[0].name).to eq "my_type_rollover__2020-01" + expect(related_rollover_indices[0].cluster_to_query).to eq "other2" + expect(related_rollover_indices[0].time_set).to eq(Support::TimeSet.of_range( + gte: ::Time.iso8601("2020-01-01T00:00:00Z"), + lt: ::Time.iso8601("2020-02-01T00:00:00Z") + )) + expect(related_rollover_indices[0].flattened_env_setting_overrides).to include( + "index.common" => "override", + "index.number_of_shards" => 1, + "index.yet.another.setting" => true + ) + + expect(related_rollover_indices[1]).to be_a RolloverIndex + expect(related_rollover_indices[1].name).to eq "my_type_rollover__2020-02" + expect(related_rollover_indices[1].cluster_to_query).to eq "other2" + expect(related_rollover_indices[1].time_set).to eq(Support::TimeSet.of_range( + gte: ::Time.iso8601("2020-02-01T00:00:00Z"), + lt: ::Time.iso8601("2020-03-01T00:00:00Z") + )) + expect(related_rollover_indices[1].flattened_env_setting_overrides).to include( + "index.common" => "override", + "index.number_of_shards" => 2, + "index.yet.another.setting" => true + ) + + expect(related_rollover_indices[2]).to be_a RolloverIndex + expect(related_rollover_indices[2].name).to eq "my_type_rollover__before_2015" + expect(related_rollover_indices[2].cluster_to_query).to eq "other2" + expect(related_rollover_indices[2].time_set).to eq(Support::TimeSet.of_range( + lt: ::Time.iso8601("2015-01-01T00:00:00Z") + )) + expect(related_rollover_indices[2].flattened_env_setting_overrides).to include( + "index.common" => "override", + "index.number_of_shards" => 3, + "index.yet.another.setting" => true + ) + + expect(related_rollover_indices[3]).to be_a RolloverIndex + expect(related_rollover_indices[3].name).to eq "my_type_rollover__2016_and_2017" + expect(related_rollover_indices[3].cluster_to_query).to eq "other2" + expect(related_rollover_indices[3].time_set).to eq(Support::TimeSet.of_range( + gte: ::Time.iso8601("2016-01-01T00:00:00Z"), + lt: ::Time.iso8601("2018-01-01T00:00:00Z") + )) + expect(related_rollover_indices[3].flattened_env_setting_overrides).to include( + "index.common" => "override", + "index.number_of_shards" => 4, + "index.yet.another.setting" => true + ) + end + + it "supports nested rollover timestamp fields" do + schema_definition = lambda do |s| + s.object_type "NestedFields" do |t| + t.field "created_at", "DateTime" + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "nested_fields", "NestedFields" + t.index "my_type", number_of_shards: 5, some: {other_setting: false} do |i| + i.rollover :monthly, "nested_fields.created_at" + end + end + end + + datastore_core = build_datastore_core(schema_definition: schema_definition) do |config| + config.with(index_definitions: { + "my_type" => config_index_def_of( + setting_overrides: { + "common" => "override", # a setting not overridden by other setting overrides below. + "number_of_shards" => 12 # ...whereas this one is overridden below. + }, + query_cluster: "other2", + setting_overrides_by_timestamp: { + "2020-01-01T00:00:00Z" => { + "number_of_shards" => 1, + "yet" => {"another" => {"setting" => true}} + }, + "2020-02-01T00:00:00Z" => { + "number_of_shards" => 2, + "yet" => {"another" => {"setting" => true}} + } + }, + custom_timestamp_ranges: [] + ) + }) + end + + index = datastore_core.index_definitions_by_name.fetch("my_type") + related_rollover_indices = index.related_rollover_indices(main_datastore_client) + # No indexes exist to query yet, so `known_related_query_rollover_indices` should be empty. + expect(index.known_related_query_rollover_indices).to eq([]) + + expect(related_rollover_indices.size).to eq 2 + + expect(related_rollover_indices[0]).to be_a RolloverIndex + expect(related_rollover_indices[0].name).to eq "my_type_rollover__2020-01" + expect(related_rollover_indices[0].cluster_to_query).to eq "other2" + expect(related_rollover_indices[0].time_set).to eq(Support::TimeSet.of_range( + gte: ::Time.iso8601("2020-01-01T00:00:00Z"), + lt: ::Time.iso8601("2020-02-01T00:00:00Z") + )) + expect(related_rollover_indices[0].flattened_env_setting_overrides).to include( + "index.common" => "override", + "index.number_of_shards" => 1, + "index.yet.another.setting" => true + ) + + expect(related_rollover_indices[1]).to be_a RolloverIndex + expect(related_rollover_indices[1].name).to eq "my_type_rollover__2020-02" + expect(related_rollover_indices[1].cluster_to_query).to eq "other2" + expect(related_rollover_indices[1].time_set).to eq(Support::TimeSet.of_range( + gte: ::Time.iso8601("2020-02-01T00:00:00Z"), + lt: ::Time.iso8601("2020-03-01T00:00:00Z") + )) + expect(related_rollover_indices[1].flattened_env_setting_overrides).to include( + "index.common" => "override", + "index.number_of_shards" => 2, + "index.yet.another.setting" => true + ) + end + + it "returns any indices that have been auto-created from the rollover template, while ignoring other indices" do + datastore_core = build_datastore_core(schema_definition: schema_definition) + record = Support::HashUtil.stringify_keys(build(:widget)) + derive_index_from_template(record, datastore_core) + + index_def = datastore_core.index_definitions_by_name.fetch(widgets_index_name) + related_rollover_indices = index_def.related_rollover_indices(main_datastore_client) + expect(index_def.known_related_query_rollover_indices).to eq(related_rollover_indices) + + expect(related_rollover_indices.size).to eq 1 + expect(related_rollover_indices.first).to be_a(RolloverIndex) + expect(related_rollover_indices.first.name).to eq index_def.index_name_for_writes(record) + expect(main_datastore_client.list_indices_matching("*").size).to be > 1 # demonstrate there were other indices that were ignored + end + + it "prefers the config settings defined in `setting_overrides_by_timestamp`/`custom_timestamp_ranges` over the existing settings on auto-created indices" do + datastore_core = build_datastore_core(schema_definition: schema_definition) + record = Support::HashUtil.stringify_keys(build(:widget)) + derive_index_from_template(record, datastore_core) + + datastore_core = build_datastore_core(schema_definition: schema_definition) do |config| + config.with(index_definitions: config.index_definitions.merge(widgets_index_name => config_index_def_of(setting_overrides_by_timestamp: { + record.fetch("created_at") => { + "number_of_shards" => 7, + "yet" => {"another" => {"setting" => true}} + } + }))) + end + + index_def = datastore_core.index_definitions_by_name.fetch(widgets_index_name) + related_rollover_indices = index_def.related_rollover_indices(main_datastore_client) + expect(index_def.known_related_query_rollover_indices).to eq(related_rollover_indices) + + expect(related_rollover_indices.size).to eq 1 + expect(related_rollover_indices.first).to be_a(RolloverIndex) + expect(related_rollover_indices.first.name).to eq index_def.index_name_for_writes(record) + expect(related_rollover_indices.first.flattened_env_setting_overrides).to include( + "index.number_of_shards" => 7, + "index.yet.another.setting" => true + ) + end + + it "returns an empty array, if no indices have been autocreated and config has no overrides" do + schema_definition = lambda do |s| + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime" + t.index "my_type" do |i| + i.rollover :monthly, "created_at" + end + end + end + + datastore_core = build_datastore_core(schema_definition: schema_definition) do |config| + config.with(index_definitions: config.index_definitions.dup.clear) + end + + index = datastore_core.index_definitions_by_name.fetch("my_type") + + expect(index.related_rollover_indices(main_datastore_client)).to be_empty + expect(index.known_related_query_rollover_indices).to be_empty + end + + it "memoizes the `#known_related_query_rollover_indices` result so we only ever query the datastore once for that info" do + index = build_datastore_core(schema_definition: schema_definition).index_definitions_by_name.fetch(widgets_index_name) + + expect { + index.known_related_query_rollover_indices + }.to make_datastore_calls("main").to include(a_string_starting_with("GET /_cat/indices/#{widgets_index_name}_rollover__%2A")) + + expect { + index.known_related_query_rollover_indices + index.known_related_query_rollover_indices + }.to make_no_datastore_calls("main") + end + + it "returns `[]` from `#known_related_query_rollover_indices` when there is no configured datastore cluster to query" do + index = build_datastore_core(schema_definition: schema_definition) do |config| + config.with(index_definitions: { + widgets_index_name => config_index_def_of(query_cluster: nil), + components_index_name => config_index_def_of(query_cluster: nil) + }) + end.index_definitions_by_name.fetch(widgets_index_name) + + expect(index.known_related_query_rollover_indices).to be_empty + end + + context "when the index definition configuration disagrees with what the indices we have in the datastore" do + let(:index_name) { "#{index_prefix}_my_type" } + + it "ignores (from `#known_related_query_rollover_indices` but not `#related_rollover_indices`) config-defined indices that do not exist in the datastore" do + datastore_core = build_datastore_core_for(rollover_frequency: :yearly, index_config: config_index_def_of( + setting_overrides_by_timestamp: { + "2020-01-01T00:00:00Z" => {} + }, + custom_timestamp_ranges: [ + { + "index_name_suffix" => "before_2015", + "lt" => "2015-01-01T00:00:00Z", + "setting_overrides" => {} + } + ] + )) + + record = Support::HashUtil.stringify_keys(build(:widget, created_at: "2019-08-01T00:00:00Z")) + derive_index_from_template(record, datastore_core) + + index = datastore_core.index_definitions_by_name.fetch(index_name) + + expect(index.related_rollover_indices(main_datastore_client).map(&:name)).to contain_exactly("#{index_name}_rollover__2019", "#{index_name}_rollover__2020", "#{index_name}_rollover__before_2015") + expect(index.known_related_query_rollover_indices.map(&:name)).to contain_exactly("#{index_name}_rollover__2019") + end + + it "includes indices with open ranges in both `#known_related_query_rollover_indices` and `#related_rollover_indices` when the index is defined in config and exists in the datastore" do + datastore_core = build_datastore_core_for(rollover_frequency: :yearly, index_config: config_index_def_of( + custom_timestamp_ranges: [ + { + "index_name_suffix" => "before_2015", + "lt" => "2015-01-01T00:00:00Z", + "setting_overrides" => {} + } + ] + )) + + record = Support::HashUtil.stringify_keys(build(:widget, created_at: "2014-08-01T00:00:00Z")) + derive_index_from_template(record, datastore_core) + + index = datastore_core.index_definitions_by_name.fetch(index_name) + + expect(index.related_rollover_indices(main_datastore_client).map(&:name)).to contain_exactly("#{index_name}_rollover__before_2015") + expect(index.known_related_query_rollover_indices.map(&:name)).to contain_exactly("#{index_name}_rollover__before_2015") + end + + it "ignores (from both `#known_related_query_rollover_indices` and `#related_rollover_indices`) datastore indices with custom suffixes that have no corresponding definition in config" do + datastore_core1 = build_datastore_core_for(rollover_frequency: :yearly, index_config: config_index_def_of( + setting_overrides_by_timestamp: { + "2020-01-01T00:00:00Z" => {} + }, + custom_timestamp_ranges: [ + { + "index_name_suffix" => "before_2015", + "lt" => "2015-01-01T00:00:00Z", + "setting_overrides" => {} + } + ] + )) + + record = Support::HashUtil.stringify_keys(build(:widget, created_at: "2014-08-01T00:00:00Z")) + derive_index_from_template(record, datastore_core1) + + record = Support::HashUtil.stringify_keys(build(:widget, created_at: "2020-08-01T00:00:00Z")) + derive_index_from_template(record, datastore_core1) + + datastore_core2 = build_datastore_core_for(rollover_frequency: :yearly, index_config: config_index_def_of( + setting_overrides_by_timestamp: { + "2020-01-01T00:00:00Z" => {} + } + )) + + index = datastore_core2.index_definitions_by_name.fetch(index_name) + + expect(index.related_rollover_indices(main_datastore_client).map(&:name)).to contain_exactly("#{index_name}_rollover__2020") + expect(index.known_related_query_rollover_indices.map(&:name)).to contain_exactly("#{index_name}_rollover__2020") + end + + it "ignores (from both `#known_related_query_rollover_indices` and `#related_rollover_indices`) datastore indices created for a different rollover granularity than what is currently defined" do + datastore_core1 = build_datastore_core_for(rollover_frequency: :yearly) + record = Support::HashUtil.stringify_keys(build(:widget, created_at: "2017-08-01T00:00:00Z")) + derive_index_from_template(record, datastore_core1) + + datastore_core2 = build_datastore_core_for(rollover_frequency: :monthly) + index = datastore_core2.index_definitions_by_name.fetch(index_name) + + expect(index.related_rollover_indices(main_datastore_client).map(&:name)).to be_empty + expect(index.known_related_query_rollover_indices.map(&:name)).to be_empty + end + + def build_datastore_core_for(rollover_frequency:, index_config: config_index_def_of) + schema_definition = lambda do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime" + t.index index_name do |i| + i.rollover rollover_frequency, "created_at" + end + end + end + + build_datastore_core(schema_definition: schema_definition) do |config| + config.with(index_definitions: {index_name => index_config}) + end + end + end + + describe "rollover frequencies" do + example_ranges_by_frequency = { + yearly: ["2020-01-01T00:00:00Z".."2021-01-01T00:00:00Z", "2021-01-01T00:00:00Z".."2022-01-01T00:00:00Z"], + monthly: ["2020-09-01T00:00:00Z".."2020-10-01T00:00:00Z", "2021-10-01T00:00:00Z".."2021-11-01T00:00:00Z"], + daily: ["2020-09-09T00:00:00Z".."2020-09-10T00:00:00Z", "2021-10-12T00:00:00Z".."2021-10-13T00:00:00Z"], + hourly: ["2020-09-09T09:00:00Z".."2020-09-09T10:00:00Z", "2021-10-12T16:00:00Z".."2021-10-12T17:00:00Z"] + } + + after(:context) do + # verify `example_ranges_by_frequency` covers all supported frequencies + expect(example_ranges_by_frequency.keys).to match_array(RolloverIndexTemplate::TIME_UNIT_BY_FREQUENCY.keys) + end + + example_ranges_by_frequency.each do |frequency, (range_for_single_digits, range_for_double_digits)| + context "when the index is configured with #{frequency} rollover" do + let(:indexer) { build_indexer(datastore_core: datastore_core) } + let(:datastore_core) do + schema_definition = lambda do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime" + t.index widgets_index_name do |i| + i.rollover frequency, "created_at" + end + end + end + + build_datastore_core(schema_definition: schema_definition) do |config| + config.with(index_definitions: config.index_definitions.dup.clear) + end + end + + before do + # The tests below use a slightly different schema from our main test schema, in order to + # exercise different relationships (e.g. with foreign keys pointing both directions for + # any kind of relationship). As a result, calls to `validate_mapping_completeness_of!` will + # raise exceptions. Since the mapping difference is intentional in these tests, we want + # to silence the exception here, which we can do by stubbing it to be a no-op. + allow(indexer.datastore_router).to receive(:validate_mapping_completeness_of!) + end + + it "is correctly able to infer the timestamp range from indices created from timestamps with both single and double digit numbers" do + index_into( + indexer, + # Here we use all zero-leading single digit numbers in the time. We do that so that our + # test covers an odd Ruby edge case: in some contexts, Ruby interprets a leading `0` to mean + # a numeric string is an octal string: + # + # 2.7.4 :005 > Integer("07") + # => 7 + # 2.7.4 :006 > Integer("011") + # => 9 + # 2.7.4 :007 > Integer("09") + # Traceback (most recent call last): + # 5: from /Users/myron/.rvm/rubies/ruby-2.7.4/bin/irb:23:in `
' + # 4: from /Users/myron/.rvm/rubies/ruby-2.7.4/bin/irb:23:in `load' + # 3: from /Users/myron/.rvm/rubies/ruby-2.7.4/lib/ruby/gems/2.7.0/gems/irb-1.2.6/exe/irb:11:in `' + # 2: from (irb):7 + # 1: from (irb):7:in `Integer' + # ArgumentError (invalid value for Integer(): "09") + # + # As shown here, `09` is particularly problematic because it isn't a valid octal string, and blows up. + # So here we use that for all numbers (except year), to verify that our logic handles it fine. + build(:widget, created_at: "2020-09-09T09:09:09Z"), + # ...vs here we use double digit numbers in the time. + build(:widget, created_at: "2021-10-12T16:37:52Z") + ) + + index = datastore_core.index_definitions_by_name.fetch(widgets_index_name) + rollover_indices = index.known_related_query_rollover_indices.sort_by(&:name) + + expect(rollover_indices.size).to eq(2) + expect(rollover_indices.first.time_set).to eq(Support::TimeSet.of_range( + gte: ::Time.iso8601(range_for_single_digits.begin), + lt: ::Time.iso8601(range_for_single_digits.end) + )) + expect(rollover_indices.last.time_set).to eq(Support::TimeSet.of_range( + gte: ::Time.iso8601(range_for_double_digits.begin), + lt: ::Time.iso8601(range_for_double_digits.end) + )) + end + end + end + end + end + + def derive_index_from_template(record, datastore_core) + indexer = build_indexer(datastore_core: datastore_core) + # The tests above use a slightly different schema definition than the main test schema definition, + # and as a result calls to `validate_mapping_completeness_of!` will fail. This is intentional, and + # we want to allow it, so here we stub it to be a no-op. + allow(indexer.datastore_router).to receive(:validate_mapping_completeness_of!) + + index_into(indexer, record) + end + + def index_def_named(name, rollover: nil) + runtime_metadata = SchemaArtifacts::RuntimeMetadata::IndexDefinition.new( + route_with: nil, + rollover: SchemaArtifacts::RuntimeMetadata::IndexDefinition::Rollover.new(**rollover), + default_sort_fields: [], + current_sources: [SELF_RELATIONSHIP_NAME], + fields_by_path: {} + ) + + DatastoreCore::IndexDefinition.with( + name: name, + config: datastore_core.config, + runtime_metadata: runtime_metadata, + datastore_clients_by_name: datastore_core.clients_by_name + ) + end + end + end + end +end diff --git a/elasticgraph-datastore_core/spec/spec_helper.rb b/elasticgraph-datastore_core/spec/spec_helper.rb new file mode 100644 index 00000000..58479dc2 --- /dev/null +++ b/elasticgraph-datastore_core/spec/spec_helper.rb @@ -0,0 +1,31 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-datastore_core`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. + +require "elastic_graph/spec_support/builds_datastore_core" + +module ElasticGraph + module DatastoreCoreSpecHelpers + include BuildsDatastoreCore + + def build_datastore_core(**options, &block) + # Default `for_context` to :admin since it is a more limited context. + options = {for_context: :admin}.merge(options) + super(**options, &block) + end + end + + RSpec.configure do |config| + config.include DatastoreCoreSpecHelpers, absolute_file_path: %r{/elasticgraph-datastore_core/} + end +end + +RSpec::Matchers.define_negated_matcher :differ_from, :eq diff --git a/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/config_spec.rb b/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/config_spec.rb new file mode 100644 index 00000000..a8df926a --- /dev/null +++ b/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/config_spec.rb @@ -0,0 +1,409 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core/config" +require "elastic_graph/elasticsearch/client" +require "elastic_graph/opensearch/client" +require "yaml" + +module ElasticGraph + class DatastoreCore + RSpec.describe Config do + it "populates every config field" do + config = config_from_yaml(<<~YAML) + client_faraday_adapter: + name: net_http + clusters: + main1: + url: http://example.com/1234 + backend: elasticsearch + settings: + foo: 23 + index_definitions: + widgets: + use_updates_for_indexing: true + query_cluster: "main" + index_into_clusters: ["main"] + ignore_routing_values: [] + setting_overrides: {} + setting_overrides_by_timestamp: {} + custom_timestamp_ranges: {} + log_traffic: true + max_client_retries: 3 + YAML + + expect(config.client_faraday_adapter).to eq Configuration::ClientFaradayAdapter.new( + name: :net_http, + require: nil + ) + expect(config.clusters).to eq("main1" => Configuration::ClusterDefinition.new( + url: "http://example.com/1234", + backend_client_class: Elasticsearch::Client, + settings: {"foo" => 23} + )) + expect(config.index_definitions).to eq("widgets" => Configuration::IndexDefinition.new( + ignore_routing_values: [], + query_cluster: "main", + index_into_clusters: ["main"], + setting_overrides: {}, + setting_overrides_by_timestamp: {}, + custom_timestamp_ranges: [], + use_updates_for_indexing: true + )) + expect(config.log_traffic).to eq true + expect(config.max_client_retries).to eq 3 + end + + it "provides useful defaults for config settings that rarely need to be set" do + config = config_from_yaml(<<~YAML) + client_faraday_adapter: + clusters: + main1: + url: http://example.com/1234 + backend: opensearch + settings: + foo: 23 + index_definitions: + widgets: + query_cluster: "main" + index_into_clusters: ["main"] + ignore_routing_values: [] + setting_overrides: {} + setting_overrides_by_timestamp: {} + custom_timestamp_ranges: {} + YAML + + expect(config.client_faraday_adapter).to eq Configuration::ClientFaradayAdapter.new( + name: nil, + require: nil + ) + expect(config.clusters).to eq("main1" => Configuration::ClusterDefinition.new( + url: "http://example.com/1234", + backend_client_class: OpenSearch::Client, + settings: {"foo" => 23} + )) + expect(config.index_definitions).to eq("widgets" => Configuration::IndexDefinition.new( + ignore_routing_values: [], + query_cluster: "main", + index_into_clusters: ["main"], + setting_overrides: {}, + setting_overrides_by_timestamp: {}, + custom_timestamp_ranges: [], + use_updates_for_indexing: true + )) + expect(config.log_traffic).to eq false + expect(config.max_client_retries).to eq 3 + end + + it "surfaces misspellings in `backend`" do + expect { + config_from_yaml(<<~YAML) + client_faraday_adapter: + clusters: + main1: + url: http://example.com/1234 + backend: opensaerch + settings: + foo: 23 + index_definitions: + widgets: + query_cluster: "main" + index_into_clusters: ["main"] + ignore_routing_values: [] + setting_overrides: {} + setting_overrides_by_timestamp: {} + custom_timestamp_ranges: {} + YAML + }.to raise_error Errors::ConfigError, a_string_including("Unknown `datastore.clusters` backend: `opensaerch`. Valid backends are `elasticsearch` and `opensearch`.") + end + + it "surfaces any unknown root config settings" do + expect { + config_from_yaml(<<~YAML) + not_a_real_setting: 23 + client_faraday_adapter: + name: net_http + clusters: + main1: + url: http://example.com/1234 + backend: elasticsearch + settings: + foo: 23 + index_definitions: + widgets: + query_cluster: "main" + index_into_clusters: ["main"] + ignore_routing_values: [] + setting_overrides: {} + setting_overrides_by_timestamp: {} + custom_timestamp_ranges: {} + YAML + }.to raise_error Errors::ConfigError, a_string_including("not_a_real_setting") + end + + it "surfaces unknown clusters config settings" do + expect { + config_from_yaml(<<~YAML) + client_faraday_adapter: + name: net_http + clusters: + main1: + url: http://example.com/1234 + backend: elasticsearch + not_a_real_setting: 23 + settings: + foo: 23 + index_definitions: + widgets: + query_cluster: "main" + index_into_clusters: ["main"] + ignore_routing_values: [] + setting_overrides: {} + setting_overrides_by_timestamp: {} + custom_timestamp_ranges: {} + YAML + }.to raise_error Errors::ConfigError, a_string_including("not_a_real_setting") + end + + it "surfaces unknown index definition config settings" do + expect { + config_from_yaml(<<~YAML) + client_faraday_adapter: + name: net_http + clusters: + main1: + url: http://example.com/1234 + backend: opensearch + settings: + foo: 23 + index_definitions: + widgets: + not_a_real_setting: 23 + query_cluster: "main" + index_into_clusters: ["main"] + ignore_routing_values: [] + setting_overrides: {} + setting_overrides_by_timestamp: {} + custom_timestamp_ranges: {} + YAML + }.to raise_error ArgumentError, a_string_including("not_a_real_setting") + end + + it "surfaces unknown client_faraday_adapter config settings" do + expect { + config_from_yaml(<<~YAML) + client_faraday_adapter: + name: net_http + not_a_real_setting: foo + clusters: + main1: + url: http://example.com/1234 + backend: elasticsearch + settings: + foo: 23 + index_definitions: + widgets: + query_cluster: "main" + index_into_clusters: ["main"] + ignore_routing_values: [] + setting_overrides: {} + setting_overrides_by_timestamp: {} + custom_timestamp_ranges: {} + YAML + }.to raise_error Errors::ConfigError, a_string_including("not_a_real_setting") + end + + describe "index_definitions.custom_timestamp_ranges" do + it "builds a `CustomTimestampRange` object from the provided YAML" do + range = only_custom_range_from(<<~YAML) + - index_name_suffix: "before_2015" + lt: "2015-01-01T00:00:00Z" + setting_overrides: + number_of_shards: 17 + YAML + + expect(range.index_name_suffix).to eq "before_2015" + expect(range.setting_overrides).to eq({"number_of_shards" => 17}) + expect(range.time_set).to eq(Support::TimeSet.of_range(lt: ::Time.iso8601("2015-01-01T00:00:00Z"))) + end + + it "correctly supports `lt`" do + range = only_custom_range_from(<<~YAML) + - index_name_suffix: "before_2015" + lt: "2015-01-01T00:00:00Z" + setting_overrides: + number_of_shards: 17 + YAML + + expect(range.time_set).to eq(Support::TimeSet.of_range(lt: ::Time.iso8601("2015-01-01T00:00:00Z"))) + end + + it "correctly supports `lte`" do + range = only_custom_range_from(<<~YAML) + - index_name_suffix: "before_2015" + lte: "2015-01-01T00:00:00Z" + setting_overrides: + number_of_shards: 17 + YAML + + expect(range.time_set).to eq(Support::TimeSet.of_range(lte: ::Time.iso8601("2015-01-01T00:00:00Z"))) + end + + it "correctly supports `gt`" do + range = only_custom_range_from(<<~YAML) + - index_name_suffix: "before_2015" + gt: "2015-01-01T00:00:00Z" + setting_overrides: + number_of_shards: 17 + YAML + + expect(range.time_set).to eq(Support::TimeSet.of_range(gt: ::Time.iso8601("2015-01-01T00:00:00Z"))) + end + + it "correctly supports `gte`" do + range = only_custom_range_from(<<~YAML) + - index_name_suffix: "before_2015" + gte: "2015-01-01T00:00:00Z" + setting_overrides: + number_of_shards: 17 + YAML + + expect(range.time_set).to eq(Support::TimeSet.of_range(gte: ::Time.iso8601("2015-01-01T00:00:00Z"))) + end + + it "supports ranges having multiple boundary conditions" do + range = only_custom_range_from(<<~YAML) + - index_name_suffix: "before_2015" + gte: "2015-01-01T00:00:00Z" + lt: "2020-01-01T00:00:00Z" + setting_overrides: + number_of_shards: 17 + YAML + + expect(range.time_set).to eq(Support::TimeSet.of_range( + gte: ::Time.iso8601("2015-01-01T00:00:00Z"), + lt: ::Time.iso8601("2020-01-01T00:00:00Z") + )) + end + + it "supports identifying the range a timestamp is covered by" do + index_def = index_definition_for(<<~YAML) + - index_name_suffix: "before_2015" + lt: "2015-01-01T00:00:00Z" + setting_overrides: + number_of_shards: 17 + - index_name_suffix: "2016_and_2017" + gte: "2016-01-01T00:00:00Z" + lt: "2018-01-01T00:00:00Z" + setting_overrides: + number_of_shards: 17 + YAML + + expect(index_def.custom_timestamp_range_for(Time.iso8601("2014-01-01T00:00:00Z")).index_name_suffix).to eq "before_2015" + expect(index_def.custom_timestamp_range_for(Time.iso8601("2015-01-01T00:00:00Z"))).to eq nil + expect(index_def.custom_timestamp_range_for(Time.iso8601("2016-01-01T00:00:00Z")).index_name_suffix).to eq "2016_and_2017" + expect(index_def.custom_timestamp_range_for(Time.iso8601("2017-01-01T00:00:00Z")).index_name_suffix).to eq "2016_and_2017" + expect(index_def.custom_timestamp_range_for(Time.iso8601("2018-01-01T00:00:00Z"))).to eq nil + end + + it "raises an error when a boundary timestamp cannot be parsed" do + expect { + only_custom_range_from(<<~YAML) + - index_name_suffix: "before_2015" + gte: "2015-13-01T00:00:00Z" + setting_overrides: + number_of_shards: 17 + YAML + }.to raise_error ArgumentError, a_string_including("out of range") + end + + it "raises an error when a range is invalid due to no timestamps being covered by it" do + expect { + only_custom_range_from(<<~YAML) + - index_name_suffix: "before_2015" + gte: "2020-01-01T00:00:00Z" + lt: "2015-01-01T00:00:00Z" + setting_overrides: + number_of_shards: 17 + YAML + }.to raise_error Errors::ConfigError, a_string_including("is invalid") + end + + it "raises an error if a custom range lacks any boundaries" do + expect { + index_definition_for(<<~YAML) + - index_name_suffix: "before_2015" + setting_overrides: + number_of_shards: 17 + YAML + }.to raise_error Errors::ConfigSettingNotSetError, a_string_including("before_2015", "lacks boundary definitions") + end + + it "raises an error if custom timestamp ranges overlap" do + expect { + index_definition_for(<<~YAML) + - index_name_suffix: "before_2015" + lte: "2015-01-01T00:00:00Z" + setting_overrides: + number_of_shards: 17 + - index_name_suffix: "2015_and_2016" + gte: "2015-01-01T00:00:00Z" + lt: "2016-01-01T00:00:00Z" + setting_overrides: + number_of_shards: 17 + YAML + }.to raise_error Errors::ConfigError, a_string_including("are not disjoint") + end + + it "raises an error when given an unrecognized config setting" do + expect { + only_custom_range_from(<<~YAML) + - index_name_suffix: "before_2015" + gtf: "2020-01-01T00:00:00Z" + lt: "2015-01-01T00:00:00Z" + setting_overrides: + number_of_shards: 17 + YAML + }.to raise_error ArgumentError, a_string_including("gtf") + end + + def only_custom_range_from(yaml_section) + ranges = index_definition_for(yaml_section).custom_timestamp_ranges + expect(ranges.size).to eq 1 + ranges.first + end + + def index_definition_for(yaml_section) + yaml = <<~YAML + client_faraday_adapter: + name: net_http + clusters: {} + log_traffic: false + max_client_retries: 3 + index_definitions: + widgets: + query_cluster: "main" + index_into_clusters: ["main"] + ignore_routing_values: [] + setting_overrides: {} + setting_overrides_by_timestamp: {} + custom_timestamp_ranges: + #{yaml_section.split("\n").join("\n" + (" " * 6))} + YAML + + config = config_from_yaml(yaml) + config.index_definitions.fetch("widgets") + end + end + + def config_from_yaml(yaml_string) + Config.from_parsed_yaml("datastore" => ::YAML.safe_load(yaml_string)) + end + end + end +end diff --git a/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/index_config_normalizer_spec.rb b/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/index_config_normalizer_spec.rb new file mode 100644 index 00000000..4982537b --- /dev/null +++ b/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/index_config_normalizer_spec.rb @@ -0,0 +1,208 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core/index_config_normalizer" +require "json" + +module ElasticGraph + class DatastoreCore + RSpec.describe IndexConfigNormalizer do + it "returns an empty hash unchanged" do + normalized = IndexConfigNormalizer.normalize({}) + + expect(normalized).to eq({}) + end + + it "filters out read-only settings" do + index_config = { + "settings" => { + "index.creation_date" => "2020-07-20", + "index.uuid" => "abcdefg", + "index.history.uuid" => "98765", + "index.random.setting" => "random" + } + } + + normalized = IndexConfigNormalizer.normalize(index_config) + + expect(normalized).to eq({ + "settings" => { + "index.random.setting" => "random" + } + }) + end + + it "converts non-string setting primitives to strings to mirror Elasticsearch and OpenSearch behavior, which do this when fetching the settings for an index" do + index_config = { + "settings" => { + "index.random.setting" => 123 + } + } + + normalized = IndexConfigNormalizer.normalize(index_config) + + expect(normalized).to eq({ + "settings" => { + "index.random.setting" => "123" + } + }) + end + + it "leaves nil settings alone" do + index_config = { + "settings" => { + "index.random.setting" => nil + } + } + + normalized = IndexConfigNormalizer.normalize(index_config) + + expect(normalized).to eq({ + "settings" => { + "index.random.setting" => nil + } + }) + end + + context "when a setting is a list" do + it "converts the individual list elements to strings instead of converting the list as a whole to a string" do + index_config = { + "settings" => { + "index.numbers" => [1, 10, 20], + "index.strings" => ["a", "b", "c"] + } + } + + normalized = IndexConfigNormalizer.normalize(index_config) + + expect(normalized).to eq({ + "settings" => { + "index.numbers" => ["1", "10", "20"], + "index.strings" => ["a", "b", "c"] + } + }) + end + end + + it "drops `type: object` when it is along side `properties` since the datastore treats that as the default type when `properties` are used and omits it" do + index_config = { + "mappings" => { + "type" => "object", + "properties" => {} + } + } + + normalized = IndexConfigNormalizer.normalize(index_config) + + expect(normalized).to eq({ + "mappings" => { + "properties" => {} + } + }) + end + + it "leaves `type: object` unchanged when there are no `properties`" do + index_config = { + "mappings" => { + "type" => "object" + } + } + + normalized = IndexConfigNormalizer.normalize(index_config) + + expect(normalized).to eq({ + "mappings" => { + "type" => "object" + } + }) + end + + it "leaves other types unchanged" do + index_config = { + "mappings" => { + "properties" => { + "options" => {"type" => "nested", "properties" => {}}, + "name" => {"type" => "keyword"} + } + } + } + + normalized = IndexConfigNormalizer.normalize(index_config) + + expect(normalized).to eq({ + "mappings" => { + "properties" => { + "options" => {"type" => "nested", "properties" => {}}, + "name" => {"type" => "keyword"} + } + } + }) + end + + it "applies the removal of `type: object` recursively" do + index_config = { + "mappings" => { + "type" => "object", + "properties" => { + "options" => { + "type" => "object", + "properties" => { + "name" => {"type" => "keyword"}, + "suboptions" => {"type" => "object", "properties" => {}} + } + } + } + } + } + + normalized = IndexConfigNormalizer.normalize(index_config) + + expect(normalized).to eq({ + "mappings" => { + "properties" => { + "options" => { + "properties" => { + "name" => {"type" => "keyword"}, + "suboptions" => {"properties" => {}} + } + } + } + } + }) + end + + it "avoids mutating the passed config hash" do + index_config = { + "mappings" => { + "type" => "object", + "properties" => { + "options" => { + "type" => "object", + "properties" => { + "name" => {"type" => "keyword"}, + "suboptions" => {"type" => "object"} + } + } + } + }, + "settings" => { + "index.creation_date" => "2020-07-20", + "index.uuid" => "abcdefg", + "index.history.uuid" => "98765", + "index.random.setting" => 123 + } + } + + as_json = ::JSON.pretty_generate(index_config) + + expect(IndexConfigNormalizer.normalize(index_config)).not_to eq(index_config) + expect(::JSON.pretty_generate(index_config)).to eq(as_json) + end + end + end +end diff --git a/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/index_definition/implementation_shared_examples.rb b/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/index_definition/implementation_shared_examples.rb new file mode 100644 index 00000000..824d952a --- /dev/null +++ b/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/index_definition/implementation_shared_examples.rb @@ -0,0 +1,552 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core/index_definition" +require "elastic_graph/errors" +require "elastic_graph/spec_support/runtime_metadata_support" + +module ElasticGraph + class DatastoreCore + module IndexDefinition + RSpec.shared_examples_for "an IndexDefinition implementation (unit specs)" do + include SchemaArtifacts::RuntimeMetadata::RuntimeMetadataSupport + + context "when instantiated without a config index definition" do + it "raises an error" do + datastore_core = define_datastore_core_with_index "my_type", config_overrides: {index_definitions: {}} + + expect { + datastore_core.index_definitions_by_name + }.to raise_error Errors::ConfigError, a_string_including("does not provide an index definition for `my_type`") + end + end + + it "exposes `use_updates_for_indexing?` based on index config" do + index = define_index("my_type", config_overrides: { + index_definitions: {"my_type" => config_index_def_of(use_updates_for_indexing: true)} + }) + + expect(index.use_updates_for_indexing?).to be true + + index = define_index("my_type", config_overrides: { + index_definitions: {"my_type" => config_index_def_of(use_updates_for_indexing: false)} + }) + + expect(index.use_updates_for_indexing?).to be false + end + + describe "#index_expression_for_search" do + it "returns the index expression to search" do + index_def = define_index("items") + + expect(index_def.index_expression_for_search).to start_with("items") + end + end + + describe "#flattened_env_setting_overrides" do + it "returns flattened environment-specific overrides from config" do + config_overrides = { + index_definitions: { + "my_type" => config_index_def_of(setting_overrides: { + "number_of_replicas" => 7, + "yet" => {"another" => {"setting" => true}} + }) + } + } + + datastore_core = define_datastore_core_with_index "my_type", + number_of_replicas: 2, + mapping: {coerce: true}, + some: {other_setting: false}, + config_overrides: config_overrides + + index_def = datastore_core.index_definitions_by_name.fetch("my_type") + + expect(index_def.flattened_env_setting_overrides).to include( + "index.number_of_replicas" => 7, + "index.yet.another.setting" => true + ) + end + end + + describe "#cluster_to_query" do + it "references query_cluster from config" do + index = define_index("my_type", config_overrides: {index_definitions: { + "my_type" => config_index_def_of(query_cluster: "my_type_cluster") + }}) + + expect(index.cluster_to_query).to eq("my_type_cluster") + end + + it "returns `nil` when query_cluster is nil" do + index = define_index("my_type", config_overrides: {index_definitions: { + "my_type" => config_index_def_of(query_cluster: nil) + }}) + + expect(index.cluster_to_query).to eq(nil) + end + end + + describe "#all_accessible_cluster_names" do + it "includes all cluster names from both `index_into_clusters` and `query_cluster`" do + cluster_names = all_accessible_cluster_names_for_config( + clusters: { + "a" => cluster_of, + "b" => cluster_of, + "c" => cluster_of + }, + index_definitions: { + "my_type" => config_index_def_of( + index_into_clusters: ["a", "b"], + query_cluster: "c" + ) + } + ) + + expect(cluster_names).to contain_exactly("a", "b", "c") + end + + it "removes duplicates" do + cluster_names = all_accessible_cluster_names_for_config( + clusters: { + "a" => cluster_of, + "b" => cluster_of + }, + index_definitions: { + "my_type" => config_index_def_of( + index_into_clusters: ["a", "b"], + query_cluster: "a" + ) + } + ) + + expect(cluster_names).to contain_exactly("a", "b") + end + + it "ignores `query_cluster` when it is nil" do + cluster_names = all_accessible_cluster_names_for_config( + clusters: { + "a" => cluster_of, + "b" => cluster_of + }, + index_definitions: { + "my_type" => config_index_def_of( + index_into_clusters: ["a", "b"], + query_cluster: nil + ) + } + ) + + expect(cluster_names).to contain_exactly("a", "b") + end + + it "excludes cluster names that are referenced from an index definition but undefined as a cluster" do + cluster_names = all_accessible_cluster_names_for_config( + clusters: { + "a" => cluster_of, + "c" => cluster_of + }, + index_definitions: { + "my_type" => config_index_def_of( + index_into_clusters: ["a", "b"], + query_cluster: "c" + ) + } + ) + + expect(cluster_names).to contain_exactly("a", "c") + end + + def all_accessible_cluster_names_for_config(clusters:, index_definitions:) + define_index("my_type", config_overrides: { + clusters: clusters, + index_definitions: index_definitions, + clients_by_name: clusters.transform_values { stubbed_datastore_client } + }).all_accessible_cluster_names + end + end + + describe "#accessible_cluster_names_to_index_into" do + it "includes cluster names from `index_into_clusters` but not from `query_cluster`" do + cluster_names = accessible_cluster_names_to_index_into_for_config( + clusters: { + "a" => cluster_of, + "b" => cluster_of, + "c" => cluster_of + }, + index_definitions: { + "my_type" => config_index_def_of( + index_into_clusters: ["a", "b"], + query_cluster: "c" + ) + } + ) + + expect(cluster_names).to contain_exactly("a", "b") + end + + it "excludes cluster names that are referenced from an index definition but undefined as a cluster" do + cluster_names = accessible_cluster_names_to_index_into_for_config( + clusters: { + "a" => cluster_of, + "c" => cluster_of + }, + index_definitions: { + "my_type" => config_index_def_of( + index_into_clusters: ["a", "b"], + query_cluster: "c" + ) + } + ) + + expect(cluster_names).to contain_exactly("a") + end + + def accessible_cluster_names_to_index_into_for_config(clusters:, index_definitions:) + define_index("my_type", config_overrides: { + clusters: clusters, + index_definitions: index_definitions, + clients_by_name: clusters.transform_values { stubbed_datastore_client } + }).accessible_cluster_names_to_index_into + end + end + + describe "#clusters_to_index_into" do + it "references index_into_clusters from config" do + index = define_index("my_type", config_overrides: { + index_definitions: { + "my_type" => config_index_def_of(index_into_clusters: ["a", "b"]) + } + }) + + expect(index.clusters_to_index_into).to contain_exactly("a", "b") + end + + it "raises an error when index_into_clusters is nil" do + index = define_index("my_type", config_overrides: { + index_definitions: { + "my_type" => config_index_def_of(index_into_clusters: nil) + } + }) + + expect { + index.clusters_to_index_into + }.to raise_error(Errors::ConfigError, a_string_including("index_into_clusters")) + end + + it "allows it to be set to an empty list as a way to disable the application from being able to index that type" do + index = define_index("my_type", config_overrides: { + index_definitions: { + "my_type" => config_index_def_of(index_into_clusters: []) + } + }) + + expect(index.clusters_to_index_into).to eq [] + end + end + + describe "#ignored_values_for_routing" do + it "returns a list of ignored routing_values from the index config" do + index = define_index("my_type", config_overrides: { + index_definitions: { + "my_type" => config_index_def_of(ignore_routing_values: ["value"]) + } + }) + + expect(index.ignored_values_for_routing).to eq Set.new(["value"]) + end + end + + describe "#route_with" do + it "returns the field used for routing" do + index = define_index do |i| + i.route_with "name" + end + + expect(index.route_with).to eq "name" + end + end + + describe "#current_sources" do + it "returns the current sources that flow into an index" do + index = define_index + + expect(index.current_sources).to contain_exactly SELF_RELATIONSHIP_NAME + end + end + + describe "#fields_by_path" do + it "returns the `fields_by_path` from the runtime metadata" do + index = define_index + + expect(index.fields_by_path).to include({ + "created_at" => index_field_with, + "id" => index_field_with, + "name" => index_field_with, + "nested_fields.nested_id" => index_field_with + }) + end + end + + describe "#routing_value_for_prepared_record" do + it "returns `nil` if the index does not use custom routing" do + index = define_index + + expect(index.routing_value_for_prepared_record({"id" => 17, "name" => "Joe"})).to eq nil + end + + it "returns the value of the field used for custom routing when custom routing is used" do + index = define_index do |i| + i.route_with "name" + end + + expect(index.routing_value_for_prepared_record({"id" => 17, "name" => "Joe"})).to eq "Joe" + end + + it "resolves a nested routing field to a nested value" do + index = define_index do |i| + i.route_with "nested_fields.nested_id" + end + + expect(index.routing_value_for_prepared_record({"id" => 17, "name" => "Joe", "nested_fields" => {"nested_id" => "12"}})).to eq "12" + end + + it "returns the value as a string even if it wasn't originally since the datastore routing uses string values" do + index = define_index do |i| + i.route_with "nested_fields.nested_id" + end + + expect(index.routing_value_for_prepared_record({"id" => 17, "name" => "Joe", "nested_fields" => {"nested_id" => 12}})).to eq "12" + end + + it "returns the `id` value (as a string) if the custom routing value is configured as an ignored routing value" do + index = define_index("my_type", config_overrides: {index_definitions: { + "my_type" => config_index_def_of(ignore_routing_values: ["Joe"]) + }}) do |i| + i.route_with "name" + end + + expect(index.routing_value_for_prepared_record({"id" => 17, "name" => "Joe"})).to eq "17" + expect(index.routing_value_for_prepared_record({"id" => 17, "name" => "Bob"})).to eq "Bob" + end + + it "allows the `route_with_path` to be provided by the caller to support cases where the source event is of a different type" do + index = define_index do |i| + i.route_with "name" + end + + record = {"id" => 17, "name" => "Joe", "nested" => {"alternate_name" => "Joseph"}} + expect(index.routing_value_for_prepared_record(record, route_with_path: "nested.alternate_name")).to eq "Joseph" + end + + it "allows the `id_path` to be provided by the caller to support cases where the source event is of a different type" do + index = define_index("my_type", config_overrides: {index_definitions: { + "my_type" => config_index_def_of(ignore_routing_values: ["Joseph"]) + }}) do |i| + i.route_with "name" + end + + record = {"id" => 17, "name" => "Joe", "nested" => {"alternate_name" => "Joseph", "alt_id" => 12}} + expect(index.routing_value_for_prepared_record(record, route_with_path: "nested.alternate_name", id_path: "nested.alt_id")).to eq "12" + end + + it "raises an error if `route_with_path` is `nil` while there is custom routing configured" do + index = define_index do |i| + i.route_with "name" + end + + record = {"id" => 17, "name" => "Joe", "nested" => {"alternate_name" => "Joseph"}} + expect { + index.routing_value_for_prepared_record(record, route_with_path: nil) + }.to raise_error a_string_including("my_type", "`route_with_path` is misconfigured (was `nil`)") + end + + it "ignores `route_with_path: nil` when custom routing is not used" do + index = define_index + + expect(index.routing_value_for_prepared_record({"id" => 17, "name" => "Joe"}, route_with_path: nil)).to eq nil + end + end + + describe "#default_sort_clauses" do + it "returns the configured default sort fields in datastore sort clause form" do + index = define_index do |i| + i.default_sort "name", :asc, "created_at", :desc + end + + expect(index.default_sort_clauses).to eq [ + {"name" => {"order" => "asc"}}, + {"created_at" => {"order" => "desc"}} + ] + end + end + + describe "#accessible_from_queries?" do + it "returns `true` when `cluster_to_query` references a main cluster" do + index = define_index("my_type", config_overrides: { + clusters: {"main" => cluster_of}, + index_definitions: {"my_type" => config_index_def_of(query_cluster: "main")} + }) + + expect(index.accessible_from_queries?).to be true + end + + it "returns `false` when `cluster_to_query` references an undefined cluster" do + index = define_index("my_type", config_overrides: { + clusters: {"main" => cluster_of}, + index_definitions: {"my_type" => config_index_def_of(query_cluster: "undefined")} + }) + + expect(index.accessible_from_queries?).to be false + end + + it "returns `false` when `cluster_to_query` is nil" do + index = define_index("my_type", config_overrides: { + clusters: {"main" => cluster_of}, + index_definitions: {"my_type" => config_index_def_of(query_cluster: nil)} + }) + + expect(index.accessible_from_queries?).to be false + end + end + + describe "#list_counts_field_paths_for_source" do + it "returns an empty set for an index definition that has no list fields" do + index = define_index + + expect(index.list_counts_field_paths_for_source(SELF_RELATIONSHIP_NAME)).to eq ::Set.new + end + + it "returns the paths to the `#{LIST_COUNTS_FIELD}` subfields" do + index = define_index(schema_def: lambda do |schema| + update_type_for_index(schema) do |t| + t.field "ints", "[Int]" + t.field "strings", "[String]" + end + end) + + expect(index.list_counts_field_paths_for_source(SELF_RELATIONSHIP_NAME)).to eq [ + "#{LIST_COUNTS_FIELD}.ints", + "#{LIST_COUNTS_FIELD}.strings" + ].to_set + end + + it "omits list count field paths that are populated by a different source" do + index = define_index(schema_def: lambda do |schema| + schema.object_type "RelatedThing" do |t| + t.field "id", "ID" + t.field "created_at", "DateTime" + t.field "strings", "[String]" + t.index "related_things" + end + + update_type_for_index(schema) do |t| + t.relates_to_one "related_thing", "RelatedThing", via: "id", dir: :in do |r| + r.equivalent_field "created_at" + end + t.field "ints", "[Int]" + t.field "strings", "[String]" do |f| + f.sourced_from "related_thing", "strings" + end + end + end) + + expect(index.list_counts_field_paths_for_source(SELF_RELATIONSHIP_NAME)).to eq [ + "#{LIST_COUNTS_FIELD}.ints" + ].to_set + + expect(index.list_counts_field_paths_for_source("related_thing")).to eq [ + "#{LIST_COUNTS_FIELD}.strings" + ].to_set + end + + it "includes the paths to nested `#{LIST_COUNTS_FIELD}` subfields" do + index = define_index(schema_def: lambda do |schema| + schema.object_type "OtherType" do |t| + t.field "strings", "[String]" + end + + update_type_for_index(schema) do |t| + t.field "ints", "[Int]" + t.field "nesteds", "[OtherType]" do |f| + f.mapping type: "nested" + end + t.field "objects", "[OtherType]" do |f| + f.mapping type: "object" + end + end + end) + + expect(index.list_counts_field_paths_for_source(SELF_RELATIONSHIP_NAME)).to eq [ + "#{LIST_COUNTS_FIELD}.ints", + "#{LIST_COUNTS_FIELD}.nesteds", + "#{LIST_COUNTS_FIELD}.objects", + "#{LIST_COUNTS_FIELD}.objects|strings", + "nesteds.#{LIST_COUNTS_FIELD}.strings" + ].to_set + end + + it "omits fields that happen to contain `#{LIST_COUNTS_FIELD}` within their name" do + index = define_index(schema_def: lambda do |schema| + update_type_for_index(schema) do |t| + t.field "ints", "[Int]" + t.field "string#{LIST_COUNTS_FIELD}", "String" + end + end) + + expect(index.list_counts_field_paths_for_source(SELF_RELATIONSHIP_NAME)).to eq [ + "#{LIST_COUNTS_FIELD}.ints" + ].to_set + end + + it "memoizes the result to not waste time recomputing the same set later" do + index = define_index(schema_def: lambda do |schema| + schema.object_type "RelatedThing" do |t| + t.field "id", "ID" + t.field "strings", "[String]" + t.field "created_at", "DateTime" + t.index "related_things" + end + + update_type_for_index(schema) do |t| + t.relates_to_one "related_thing", "RelatedThing", via: "id", dir: :in do |r| + r.equivalent_field "created_at" + end + t.field "ints", "[Int]" + t.field "strings", "[String]" do |f| + f.sourced_from "related_thing", "strings" + end + end + end) + + expect(index.list_counts_field_paths_for_source(SELF_RELATIONSHIP_NAME)).to be( + index.list_counts_field_paths_for_source(SELF_RELATIONSHIP_NAME) + ).and differ_from( + index.list_counts_field_paths_for_source("related_thing") + ) + + expect(index.list_counts_field_paths_for_source("related_thing")).to be( + index.list_counts_field_paths_for_source("related_thing") + ).and differ_from( + index.list_counts_field_paths_for_source(SELF_RELATIONSHIP_NAME) + ) + end + + def update_type_for_index(schema) + yield schema.state.object_types_by_name.fetch("MyType") + end + end + + def define_index(name = "my_type", **options, &block) + define_datastore_core_with_index(name, **options, &block) + .index_definitions_by_name.fetch(name) + end + end + end + end +end diff --git a/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/index_definition/index_spec.rb b/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/index_definition/index_spec.rb new file mode 100644 index 00000000..9c5316ac --- /dev/null +++ b/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/index_definition/index_spec.rb @@ -0,0 +1,61 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core/index_definition/index" +require_relative "implementation_shared_examples" + +module ElasticGraph + class DatastoreCore + module IndexDefinition + RSpec.describe Index do + include_examples "an IndexDefinition implementation (unit specs)" do + def define_datastore_core_with_index(index_name, config_overrides: {}, schema_def: nil, **index_options, &block) + build_datastore_core(schema_definition: lambda do |s| + s.object_type "NestedFields" do |t| + t.field "nested_id", "ID" + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "created_at", "DateTime!" + t.field "nested_fields", "NestedFields" + t.index(index_name, **index_options, &block) + end + + schema_def&.call(s) + end, **config_overrides) + end + + it "inspects well" do + index = define_index("colors") + + expect(index.inspect).to eq "#" + end + + describe "#index_name_for_writes" do + it "returns the configured index name" do + index = define_index("things") + test_record = {"id" => "1", "created_at" => "2020-04-23T18:25:43.511Z"} + + expect(index.index_name_for_writes(test_record)).to eq("things") + end + + it "ignores the `timestamp_field_path` argument if passed" do + index = define_index("things") + test_record = {"id" => "1", "created_at" => "2020-04-23T18:25:43.511Z"} + + expect(index.index_name_for_writes(test_record, timestamp_field_path: nil)).to eq("things") + expect(index.index_name_for_writes(test_record, timestamp_field_path: "created_at")).to eq("things") + end + end + end + end + end + end +end diff --git a/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb b/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb new file mode 100644 index 00000000..9866843f --- /dev/null +++ b/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb @@ -0,0 +1,211 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core/index_definition/rollover_index_template" +require "elastic_graph/errors" +require_relative "implementation_shared_examples" + +module ElasticGraph + class DatastoreCore + module IndexDefinition + RSpec.describe RolloverIndexTemplate do + include_examples "an IndexDefinition implementation (unit specs)" do + def define_datastore_core_with_index(index_name, schema_def: nil, config_overrides: {}, timestamp_path: "created_at", **index_options, &block) + build_datastore_core(schema_definition: lambda do |s| + s.object_type "NestedFields" do |t| + t.field "created_at", "DateTime" + t.field "created_on", "Date" + t.field "nested_id", "ID" + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "created_at", "DateTime!" + t.field "created_on", "Date" + t.field "nested_fields", "NestedFields" + t.index(index_name, **index_options) do |i| + i.rollover :monthly, "created_at" # timestamp_path + block&.call(i) + end + end + + schema_def&.call(s) + end, **config_overrides) + end + + it "inspects well" do + index = define_index("colors") + + expect(index.inspect).to eq "#" + end + + describe "#index_name_for_writes" do + shared_examples_for "index_name_for_writes" do |timestamp_field, timestamp_value| + let(:test_record) { {"id" => "1", timestamp_field => timestamp_value, "nested_fields" => {timestamp_field => timestamp_value}} } + + { + daily: "things_rollover__2020-04-23", + monthly: "things_rollover__2020-04", + yearly: "things_rollover__2020" + }.each do |frequency, expected_index_name| + it "returns the correct rollover index to write to when the frequency is `#{frequency.inspect}`" do + index = define_index("things") do |i| + i.rollover frequency, timestamp_field + end + + expect(index.index_name_for_writes(test_record)).to eq(expected_index_name) + end + end + + it "supports nested rollover timestamp fields" do + index = define_index "things" do |i| + i.rollover :monthly, "nested_fields.#{timestamp_field}" + end + + expect(index.index_name_for_writes(test_record)).to eq("things_rollover__2020-04") + end + + it "allows an alternate `timestamp_field_path` to be passed as an argument to support updates against a different type from the source event type" do + index1 = define_index "things" do |i| + i.rollover :monthly, "nested_fields.#{timestamp_field}" + end + + record_without_nested_fields = test_record.except("nested_fields") + expect(index1.index_name_for_writes(record_without_nested_fields, timestamp_field_path: timestamp_field)).to eq("things_rollover__2020-04") + + index2 = define_index "things" do |i| + i.rollover :monthly, timestamp_field + end + + record_without_timestamp_field = test_record.except(timestamp_field) + expect(index2.index_name_for_writes(record_without_timestamp_field, timestamp_field_path: "nested_fields.#{timestamp_field}")).to eq("things_rollover__2020-04") + end + + it "returns the name of a custom timestamp range index if the record's timestamp falls in a custom range" do + index = define_index("my_type", config_overrides: { + index_definitions: { + "my_type" => config_index_def_of(custom_timestamp_ranges: [ + { + "index_name_suffix" => "before_2015", + "lt" => "2015-01-01T00:00:00Z", + "setting_overrides" => {} + }, + { + "index_name_suffix" => "2016_and_2017", + "gte" => "2016-01-01T00:00:00Z", + "lt" => "2018-01-01T00:00:00Z", + "setting_overrides" => {} + } + ]) + } + }) do |i| + i.rollover :monthly, timestamp_field + end + + expect(index.index_name_for_writes({timestamp_field => normalize_timestamp_value("2014-01-01T00:00:00Z")})).to eq "my_type_rollover__before_2015" + expect(index.index_name_for_writes({timestamp_field => normalize_timestamp_value("2015-01-01T00:00:00Z")})).to eq "my_type_rollover__2015-01" + expect(index.index_name_for_writes({timestamp_field => normalize_timestamp_value("2016-01-01T00:00:00Z")})).to eq "my_type_rollover__2016_and_2017" + expect(index.index_name_for_writes({timestamp_field => normalize_timestamp_value("2017-01-01T00:00:00Z")})).to eq "my_type_rollover__2016_and_2017" + expect(index.index_name_for_writes({timestamp_field => normalize_timestamp_value("2018-01-01T00:00:00Z")})).to eq "my_type_rollover__2018-01" + end + + it "raises exception if rollover index configuration references missing field" do + index = define_index("things") do |i| + i.rollover :monthly, timestamp_field + end + + expect { + index.index_name_for_writes(test_record.except(timestamp_field)) + }.to raise_error(KeyError, a_string_including(timestamp_field)) + end + end + + context "when the rollover timestamp field is a `DateTime`" do + include_examples "index_name_for_writes", "created_at", "2020-04-23T18:25:43.511Z" do + def normalize_timestamp_value(value) + value + end + + it "returns the correct rollover index to write to when the frequency is :hourly" do + index = define_index("things") do |i| + i.rollover :hourly, "created_at" + end + + expect(index.index_name_for_writes(test_record)).to eq("things_rollover__2020-04-23-18") + end + end + + it "returns the correct rollover index from `related_rollover_index_for_timestamp`" do + index = define_index("things") do |i| + i.rollover :hourly, "created_at" + end + + rollover_index = index.related_rollover_index_for_timestamp("2018-01-01T08:07:03Z") + + expect(rollover_index).to be_a RolloverIndex + expect(rollover_index.name).to eq("things_rollover__2018-01-01-08") + end + end + + context "when the rollover timestamp field is a `Date`" do + include_examples "index_name_for_writes", "created_on", "2020-04-23" do + def normalize_timestamp_value(value) + value.split("T").first # pull out just the date part + end + + it "returns an index name based on the midnight hour if the frequency is :hourly" do + index = define_index("things") do |i| + i.rollover :hourly, "created_on" + end + + # `:hourly` doesn't really make sense to use with a `Date` field, but that's a schema definition + # mistake, and here it's reasonable to just use midnight as the hour for every date value. + expect(index.index_name_for_writes(test_record)).to eq("things_rollover__2020-04-23-00") + end + + it "returns the correct rollover index from `related_rollover_index_for_timestamp`" do + index = define_index("things") do |i| + i.rollover :daily, "created_on" + end + + rollover_index = index.related_rollover_index_for_timestamp("2018-01-01") + + expect(rollover_index).to be_a RolloverIndex + expect(rollover_index.name).to eq("things_rollover__2018-01-01") + end + end + end + end + end + + it "raises an exception when rollover frequency is unsupported" do + expect { + define_index do |i| + i.rollover :secondly, "created_at" + end + }.to raise_error(Errors::SchemaError, a_string_including("Rollover index config 'timestamp_field' or 'frequency' is invalid")) + end + + describe "its constants" do + specify "`ROLLOVER_SUFFIX_FORMATS_BY_FREQUENCY` and `TIME_UNIT_BY_FREQUENCY` have the same keys" do + expect(RolloverIndexTemplate::ROLLOVER_SUFFIX_FORMATS_BY_FREQUENCY.keys).to match_array(RolloverIndexTemplate::TIME_UNIT_BY_FREQUENCY.keys) + end + + specify "all `TIME_UNIT_BY_FREQUENCY` values are valid arguments to `Support::TimeUtil.advance_one_unit`" do + now = ::Time.now + + RolloverIndexTemplate::TIME_UNIT_BY_FREQUENCY.values.each do |unit| + expect(Support::TimeUtil.advance_one_unit(now, unit)).to be_a ::Time + end + end + end + end + end + end +end diff --git a/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core_spec.rb b/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core_spec.rb new file mode 100644 index 00000000..024670a9 --- /dev/null +++ b/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core_spec.rb @@ -0,0 +1,91 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core" +require "elastic_graph/datastore_core/config" +require "elastic_graph/elasticsearch/client" +require "logger" + +module ElasticGraph + # stub_datastore_client - the specs below depend on using a real datastore client (but don't send any + # requests to the datastore, so they are stll unit tests). + RSpec.describe DatastoreCore, stub_datastore_client: false do + it "returns non-nil values from each attribute" do + expect_to_return_non_nil_values_from_all_attributes(build_datastore_core( + # Give `client_customization_block` a non-nil value for this test... + client_customization_block: ->(client) { client } + )) + end + + describe ".from_parsed_yaml" do + it "can build an instance from a parsed settings YAML file" do + datastore_core = DatastoreCore.from_parsed_yaml(parsed_test_settings_yaml, for_context: :admin) + + expect(datastore_core).to be_an(DatastoreCore) + end + + it "allows the datastore clients to be customized via the passed block" do + customization_block = lambda { |conn| } + datastore_core = DatastoreCore.from_parsed_yaml(parsed_test_settings_yaml, for_context: :admin, &customization_block) + + expect(datastore_core.client_customization_block).to be(customization_block) + end + end + + context "when built for our tests" do + it "overrides settings on every index for optimal test speed (intentionally giving up data durability)" do + datastore_core = build_datastore_core + index_def_names = datastore_core.schema_artifacts.index_mappings_by_index_def_name.keys + + settings_for_each_index = index_def_names.map do |index_name| + datastore_core.config.index_definitions[index_name].setting_overrides + end + + expect(settings_for_each_index).to all include("translog.durability" => "async") + end + end + + it "builds the client with a logger when `config.log_traffic` is true" do + client_class = class_spy(Elasticsearch::Client).as_stubbed_const + build_datastore_core(log_traffic: true, datastore_backend: :elasticsearch).clients_by_name + + expect(client_class).to have_received(:new) + .with(an_instance_of(::String), hash_including(logger: an_object_responding_to(:info, :warn, :error))) + .at_least(:once) + end + + it "builds the client without a logger when `config.log_traffic` is false" do + client_class = class_spy(Elasticsearch::Client).as_stubbed_const + build_datastore_core(log_traffic: false, datastore_backend: :elasticsearch).clients_by_name + + expect(client_class).to have_received(:new) + .with(an_instance_of(::String), hash_including(logger: nil)) + .at_least(:once) + end + + it "allows `client_customization_block` to be injected, to support `elasticgraph-lambda` using an AWS signing client" do + expect { |probe| + build_datastore_core( + client_customization_block: probe + ) do |config| + # Ensure only one cluster is configured so `datastore_client_customization_block` is only used once. + config.with(clusters: config.clusters.first(1).to_h) + end.clients_by_name + }.to yield_with_args an_object_satisfying { |f| f.is_a?(::Faraday::Connection) } + end + + def build_datastore_core(client_faraday_adapter: nil, **options) + client_faraday_adapter &&= DatastoreCore::Configuration::ClientFaradayAdapter.new( + name: client_faraday_adapter.to_sym, + require: nil + ) + + super(client_faraday_adapter: client_faraday_adapter, **options) + end + end +end diff --git a/elasticgraph-elasticsearch/.rspec b/elasticgraph-elasticsearch/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-elasticsearch/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-elasticsearch/.yardopts b/elasticgraph-elasticsearch/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-elasticsearch/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-elasticsearch/Gemfile b/elasticgraph-elasticsearch/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-elasticsearch/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-elasticsearch/LICENSE.txt b/elasticgraph-elasticsearch/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-elasticsearch/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-elasticsearch/README.md b/elasticgraph-elasticsearch/README.md new file mode 100644 index 00000000..4d712287 --- /dev/null +++ b/elasticgraph-elasticsearch/README.md @@ -0,0 +1,3 @@ +# ElasticGraph::Elasticsearch + +Wraps the official Elasticsearch client for use by ElasticGraph. diff --git a/elasticgraph-elasticsearch/elasticgraph-elasticsearch.gemspec b/elasticgraph-elasticsearch/elasticgraph-elasticsearch.gemspec new file mode 100644 index 00000000..6d730f80 --- /dev/null +++ b/elasticgraph-elasticsearch/elasticgraph-elasticsearch.gemspec @@ -0,0 +1,20 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :datastore_adapter) do |spec, eg_version| + spec.summary = "Wraps the Elasticsearch client for use by ElasticGraph." + + spec.add_dependency "elasticgraph-support", eg_version + spec.add_dependency "elasticsearch", "~> 8.15" + spec.add_dependency "faraday", "~> 2.12" + spec.add_dependency "faraday-retry", "~> 2.2" + + spec.add_development_dependency "httpx", "~> 1.3" +end diff --git a/elasticgraph-elasticsearch/lib/elastic_graph/elasticsearch/client.rb b/elasticgraph-elasticsearch/lib/elastic_graph/elasticsearch/client.rb new file mode 100644 index 00000000..272cdea1 --- /dev/null +++ b/elasticgraph-elasticsearch/lib/elastic_graph/elasticsearch/client.rb @@ -0,0 +1,205 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/errors" +require "elastic_graph/support/faraday_middleware/msearch_using_get_instead_of_post" +require "elastic_graph/support/faraday_middleware/support_timeouts" +require "elasticsearch" +require "faraday/retry" + +module ElasticGraph + # @private + module Elasticsearch + # @private + class Client + # @dynamic cluster_name + attr_reader :cluster_name + + def initialize(cluster_name, url:, faraday_adapter: nil, retry_on_failure: 3, logger: nil) + @cluster_name = cluster_name + + @raw_client = ::Elasticsearch::Client.new( + adapter: faraday_adapter, + url: url, + retry_on_failure: retry_on_failure, + # We use `logger` for both the tracer and logger to log everything we can. While the trace and log output do overlap, one is + # not a strict superset of the other (for example, warnings go to `logger`, while full request bodies go to `tracer`). + logger: logger, + tracer: logger + ) do |faraday| + faraday.use Support::FaradayMiddleware::MSearchUsingGetInsteadOfPost + faraday.use Support::FaradayMiddleware::SupportTimeouts + + # Note: this overrides the default retry exceptions, which includes `Faraday::TimeoutError`. + # That's important because we do NOT want a retry on timeout -- a timeout indicates a slow, + # expensive query, and is transformed to a `Errors::RequestExceededDeadlineError` by `SupportTimeouts`, + # anyway. + # + # In addition, it's worth noting that the retry middleware ONLY retries known idempotent HTTP + # methods (e.g. get/put/delete/head/options). POST requests will not be retried. We could + # configure it to make it retry POSTs but we'd need to do an analysis of all ElasticGraph requests to + # make sure all POST requests are truly idempotent, and at least for now, it's sufficient to skip + # any POST requests we make. + faraday.request :retry, + exceptions: [::Faraday::ConnectionFailed, ::Faraday::RetriableResponse], + max: retry_on_failure, + retry_statuses: [500, 502, 503] # Internal Server Error, Bad Gateway, Service Unavailable + + yield faraday if block_given? + end + + # Here we call `app` on each Faraday connection as a way to force it to resolve + # all configured middlewares and adapters. If it cannot load a required dependency + # (e.g. `httpx`), it'll fail fast with a clear error. + # + # Without this, we would instead get an error when the client was used to make + # a request for the first time, which isn't as ideal. + @raw_client.transport.connections.each { |c| c.connection.app } + end + + # Cluster APIs + + def get_cluster_health + transform_errors { |c| c.cluster.health.body } + end + + def get_node_os_stats + transform_errors { |c| c.nodes.stats(metric: "os").body } + end + + def get_flat_cluster_settings + transform_errors { |c| c.cluster.get_settings(flat_settings: true).body } + end + + # We only support persistent settings here because Elasticsearch docs recommend against using transient settings: + # https://www.elastic.co/guide/en/elasticsearch/reference/8.13/cluster-update-settings.html + # + # > We no longer recommend using transient cluster settings. Use persistent cluster settings instead. If a cluster becomes unstable, + # > transient settings can clear unexpectedly, resulting in a potentially undesired cluster configuration. + def put_persistent_cluster_settings(settings) + transform_errors { |c| c.cluster.put_settings(body: {persistent: settings}).body } + end + + # Script APIs + + # Gets the script with the given ID. Returns `nil` if the script does not exist. + def get_script(id:) + transform_errors { |c| c.get_script(id: id).body } + rescue ::Elastic::Transport::Transport::Errors::NotFound + nil + end + + def put_script(id:, body:, context:) + transform_errors { |c| c.put_script(id: id, body: body, context: context).body } + end + + def delete_script(id:) + transform_errors { |c| c.delete_script(id: id).body } + rescue ::Elastic::Transport::Transport::Errors::NotFound + # it's ok if it's already not there. + end + + # Index Template APIs + + def get_index_template(index_template_name) + transform_errors do |client| + client.indices.get_index_template(name: index_template_name, flat_settings: true).fetch("index_templates").to_h do |entry| + [entry.fetch("name"), entry.fetch("index_template")] + end.dig(index_template_name) || {} + end + rescue ::Elastic::Transport::Transport::Errors::NotFound + {} + end + + def put_index_template(name:, body:) + transform_errors { |c| c.indices.put_index_template(name: name, body: body).body } + end + + def delete_index_template(index_template_name) + transform_errors { |c| c.indices.delete_index_template(name: [index_template_name], ignore: [404]).body } + end + + # Index APIs + + def get_index(index_name) + transform_errors do |client| + client.indices.get( + index: index_name, + ignore_unavailable: true, + flat_settings: true + )[index_name] || {} + end + end + + def list_indices_matching(index_expression) + transform_errors do |client| + client + .cat + .indices(index: index_expression, format: "json", h: ["index"]) + .map { |index_hash| index_hash.fetch("index") } + end + end + + def create_index(index:, body:) + transform_errors { |c| c.indices.create(index: index, body: body).body } + end + + def put_index_mapping(index:, body:) + transform_errors { |c| c.indices.put_mapping(index: index, body: body).body } + end + + def put_index_settings(index:, body:) + transform_errors { |c| c.indices.put_settings(index: index, body: body).body } + end + + def delete_indices(*index_names) + # `allow_no_indices: true` is needed when we attempt to delete a non-existing index to avoid errors. For rollover indices, + # when we delete the actual indices, we will always perform a wildcard deletion, and `allow_no_indices: true` is needed. + # + # Note that the Elasticsearch API documentation[^1] says that `allow_no_indices` defaults to `true` but a Elasticsearch Ruby + # client code comment[^2] says it defaults to `false`. Regardless, we don't want to rely on the default behavior that could change. + # + # [^1]: https://www.elastic.co/guide/en/elasticsearch/reference/8.12/indices-delete-index.html#delete-index-api-query-params + # [^2]: https://github.com/elastic/elasticsearch-ruby/blob/8.12/elasticsearch-api/lib/elasticsearch/api/actions/indices/delete.rb#L31 + transform_errors do |client| + client.indices.delete(index: index_names, ignore_unavailable: true, allow_no_indices: true).body + end + end + + # Document APIs + + def msearch(body:, headers: nil) + transform_errors { |c| c.msearch(body: body, headers: headers).body } + end + + def bulk(body:, refresh: false) + transform_errors { |c| c.bulk(body: body, filter_path: DATASTORE_BULK_FILTER_PATH, refresh: refresh).body } + end + + # Synchronously deletes all documents in the cluster. Intended for tests to give ourselves a clean slate. + # Supports an `index` argument so the caller can limit the deletion to a specific "scope" (e.g. a set of indices with a common prefix). + # + # Overrides `scroll` to `10s` to avoid getting a "Trying to create too many scroll contexts" error, as discussed here: + # https://discuss.elastic.co/t/too-many-scroll-contexts-with-update-by-query-and-or-delete-by-query/282325/1 + def delete_all_documents(index: "_all") + transform_errors do |client| + client.delete_by_query(index: index, body: {query: {match_all: _ = {}}}, refresh: true, scroll: "10s").body + end + end + + private + + def transform_errors + yield @raw_client + rescue ::Elastic::Transport::Transport::Errors::BadRequest => ex + raise Errors::BadDatastoreRequest, ex.message + end + end + end +end diff --git a/elasticgraph-elasticsearch/sig/elastic_graph/elasticsearch/client.rbs b/elasticgraph-elasticsearch/sig/elastic_graph/elasticsearch/client.rbs new file mode 100644 index 00000000..1b33577c --- /dev/null +++ b/elasticgraph-elasticsearch/sig/elastic_graph/elasticsearch/client.rbs @@ -0,0 +1,23 @@ +module ElasticGraph + module Elasticsearch + class Client + include DatastoreCore::_Client + extend DatastoreCore::_ClientClass + + def initialize: ( + ::String, + url: ::String, + ?faraday_adapter: ::Symbol?, + ?retry_on_failure: ::Integer, + ?logger: ::Logger? + ) ?{ (::Faraday::RackBuilder) -> void } -> void + + private + + @cluster_name: ::String + @raw_client: ::Elasticsearch::Client + + def transform_errors: [T] () { (::Elasticsearch::Client) -> T } -> T + end + end +end diff --git a/elasticgraph-elasticsearch/sig/elasticsearch.rbs b/elasticgraph-elasticsearch/sig/elasticsearch.rbs new file mode 100644 index 00000000..5e64a094 --- /dev/null +++ b/elasticgraph-elasticsearch/sig/elasticsearch.rbs @@ -0,0 +1,80 @@ +module Elasticsearch + type stringOrSymbolHash = ::Hash[(::String | ::Symbol), untyped] + + module API + module Cat + class CatClient + def indices: (?index: ::String, ?format: ::String, ?h: ::Array[::String]) -> ::Array[::Hash[::String, untyped]] + end + end + + module Cluster + class ClusterClient + def health: () -> Response + def get_settings: (?flat_settings: bool) -> Response + def put_settings: (body: stringOrSymbolHash) -> Response + end + end + + module Indices + class IndicesClient + def get_index_template: (name: ::String, ?flat_settings: bool) -> Response + def put_index_template: (name: ::String, body: stringOrSymbolHash) -> Response + def delete_index_template: (name: ::Array[::String], ?ignore: [::Integer]) -> Response + def get: (index: ::String, ?ignore_unavailable: bool, ?flat_settings: bool) -> Response + def create: (index: ::String, body: stringOrSymbolHash) -> Response + def put_mapping: (index: ::String, body: stringOrSymbolHash) -> Response + def put_settings: (index: ::String, body: stringOrSymbolHash) -> Response + def delete: (index: ::Array[::String], ?ignore_unavailable: bool, ?allow_no_indices: bool) -> Response + end + end + + module Nodes + class NodesClient + def stats: (metric: ::String) -> Response + end + end + + class Response + attr_reader body: ::Hash[::String, untyped] + def []: (::String) -> untyped + def fetch: (::String) -> untyped + end + end + + class Client + def initialize: ( + url: ::String, + ?retry_on_failure: ::Integer, + ?adapter: ::Symbol?, + ?logger: ::Logger?, + ?tracer: ::Logger? + ) { (::Faraday::RackBuilder) -> void } -> void + + def transport: () -> untyped + def get_script: (id: ::String) -> API::Response + def put_script: (id: ::String, body: stringOrSymbolHash, ?context: (::Symbol | ::String)) -> API::Response + def delete_script: (id: ::String) -> API::Response + def msearch: (body: ::Array[stringOrSymbolHash], ?headers: ::Hash[::String, untyped]?) -> API::Response + def bulk: (body: ::Array[stringOrSymbolHash], ?filter_path: ::String, ?refresh: bool) -> API::Response + def delete_by_query: (index: ::String, ?body: stringOrSymbolHash, ?refresh: bool, ?scroll: ::String) -> API::Response + def cat: () -> API::Cat::CatClient + def cluster: () -> API::Cluster::ClusterClient + def indices: () -> API::Indices::IndicesClient + def nodes: () -> API::Nodes::NodesClient + end +end + +module Elastic + module Transport + module Transport + module Errors + class BadRequest < StandardError + end + + class NotFound < StandardError + end + end + end + end +end diff --git a/elasticgraph-elasticsearch/sig/faraday.rbs b/elasticgraph-elasticsearch/sig/faraday.rbs new file mode 100644 index 00000000..45dd423f --- /dev/null +++ b/elasticgraph-elasticsearch/sig/faraday.rbs @@ -0,0 +1,4 @@ +module Faraday + class RetriableResponse < StandardError + end +end diff --git a/elasticgraph-elasticsearch/spec/spec_helper.rb b/elasticgraph-elasticsearch/spec/spec_helper.rb new file mode 100644 index 00000000..ed2c6b0b --- /dev/null +++ b/elasticgraph-elasticsearch/spec/spec_helper.rb @@ -0,0 +1,11 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-elasticsearch`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. diff --git a/elasticgraph-elasticsearch/spec/unit/elastic_graph/elasticsearch/client_spec.rb b/elasticgraph-elasticsearch/spec/unit/elastic_graph/elasticsearch/client_spec.rb new file mode 100644 index 00000000..5d4936fa --- /dev/null +++ b/elasticgraph-elasticsearch/spec/unit/elastic_graph/elasticsearch/client_spec.rb @@ -0,0 +1,95 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/elasticsearch/client" +require "elastic_graph/spec_support/datastore_client_shared_examples" + +module ElasticGraph + module Elasticsearch + RSpec.describe Client do + it_behaves_like "a datastore client" do + def define_stubs(stub, requested_stubs) + stub.get("/") { [200, {"X-Elastic-Product" => "Elasticsearch"}, ""] } + + requested_stubs.each do |stub_name, body| + case stub_name + + # Cluster APIs + in :get_cluster_health + stub.get("/_cluster/health") { |env| response_for(body, env) } + in :get_node_os_stats + stub.get("/_nodes/stats/os") { |env| response_for(body, env) } + in :get_flat_cluster_settings + stub.get("/_cluster/settings?flat_settings=true") { |env| response_for(body, env) } + in :put_persistent_cluster_settings + stub.put("/_cluster/settings") { |env| response_for(body, env) } + + # Script APIs + in :get_script_123 + stub.get("/_scripts/123") { |env| response_for(body, env) } + in :put_script_123 + stub.put("/_scripts/123/update") { |env| response_for(body, env) } + in :delete_script_123 + stub.delete("/_scripts/123") { |env| response_for(body, env) } + + # Index Template APIs + in :get_index_template_my_template + stub.get("/_index_template/my_template?flat_settings=true") { |env| response_for(body, env) } + in :put_index_template_my_template + stub.put("/_index_template/my_template") { |env| response_for(body, env) } + in :delete_index_template_my_template + stub.delete("/_index_template/my_template") { |env| response_for(body, env) } + + # Index APIs + in :get_index_my_index + stub.get("/my_index?flat_settings=true&ignore_unavailable=true") { |env| response_for(body, env) } + in :list_indices_matching_foo + stub.get("/_cat/indices/foo%2A?format=json&h=index") { |env| response_for(body, env) } + in :create_index_my_index + stub.put("/my_index") { |env| response_for(body, env) } + in :put_index_mapping_my_index + stub.put("/my_index/_mapping") { |env| response_for(body, env) } + in :put_index_settings_my_index + stub.put("/my_index/_settings") { |env| response_for(body, env) } + in :delete_indices_ind1_ind2 + stub.delete("/ind1,ind2?allow_no_indices=true&ignore_unavailable=true") { |env| response_for(body, env) } + + # Document APIs + in :get_msearch + stub.get("/_msearch") do |env| + env.request.timeout ? raise(::Faraday::TimeoutError) : response_for(body, env) + end + in :post_bulk + stub.post("/_bulk?filter_path=items.%2A.status%2Citems.%2A.result%2Citems.%2A.error&refresh=false") do |env| + response_for(body, env) + end + in :delete_all_documents + stub.post("/_all/_delete_by_query?refresh=true&scroll=10s") { |env| response_for(body, env) } + in :delete_test_env_7_documents + stub.post("/test_env_7_%2A/_delete_by_query?refresh=true&scroll=10s") { |env| response_for(body, env) } + + else + # :nocov: -- none of our current tests hit this case + raise "Unexpected stub name: #{stub_name.inspect}" + # :nocov: + end + end + end + + prepend Module.new { + def build_client(...) + super(...).tap do |client| + # Force the client to make a request to `/` so its "is the product Elasticsearch?" validation is satisfied. + client.instance_variable_get(:@raw_client).info + end + end + } + end + end + end +end diff --git a/elasticgraph-graphql/.rspec b/elasticgraph-graphql/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-graphql/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-graphql/.yardopts b/elasticgraph-graphql/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-graphql/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-graphql/Gemfile b/elasticgraph-graphql/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-graphql/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-graphql/LICENSE.txt b/elasticgraph-graphql/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-graphql/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-graphql/README.md b/elasticgraph-graphql/README.md new file mode 100644 index 00000000..141e3531 --- /dev/null +++ b/elasticgraph-graphql/README.md @@ -0,0 +1,3 @@ +# ElasticGraph::GraphQL + +Provides the ElasticGraph GraphQL query engine. diff --git a/elasticgraph-graphql/elasticgraph-graphql.gemspec b/elasticgraph-graphql/elasticgraph-graphql.gemspec new file mode 100644 index 00000000..a4893548 --- /dev/null +++ b/elasticgraph-graphql/elasticgraph-graphql.gemspec @@ -0,0 +1,23 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :core) do |spec, eg_version| + spec.summary = "The ElasticGraph GraphQL query engine." + + spec.add_dependency "elasticgraph-datastore_core", eg_version + spec.add_dependency "elasticgraph-schema_artifacts", eg_version + spec.add_dependency "graphql", "~> 2.3.19" + + spec.add_development_dependency "elasticgraph-admin", eg_version + spec.add_development_dependency "elasticgraph-elasticsearch", eg_version + spec.add_development_dependency "elasticgraph-opensearch", eg_version + spec.add_development_dependency "elasticgraph-indexer", eg_version + spec.add_development_dependency "elasticgraph-schema_definition", eg_version +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql.rb b/elasticgraph-graphql/lib/elastic_graph/graphql.rb new file mode 100644 index 00000000..42605acd --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql.rb @@ -0,0 +1,264 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core" +require "elastic_graph/graphql/config" +require "elastic_graph/support/from_yaml_file" + +module ElasticGraph + # The main entry point for ElasticGraph GraphQL handling. Instantiate this to get access to the + # different parts of this library. + class GraphQL + extend Support::FromYamlFile + + # @private + # @dynamic config, logger, runtime_metadata, graphql_schema_string, datastore_core, clock + attr_reader :config, :logger, :runtime_metadata, :graphql_schema_string, :datastore_core, :clock + + # @private + # A factory method that builds a GraphQL instance from the given parsed YAML config. + # `from_yaml_file(file_name, &block)` is also available (via `Support::FromYamlFile`). + def self.from_parsed_yaml(parsed_yaml, &datastore_client_customization_block) + new( + config: GraphQL::Config.from_parsed_yaml(parsed_yaml), + datastore_core: DatastoreCore.from_parsed_yaml(parsed_yaml, for_context: :graphql, &datastore_client_customization_block) + ) + end + + # @private + def initialize( + config:, + datastore_core:, + graphql_adapter: nil, + datastore_search_router: nil, + filter_interpreter: nil, + sub_aggregation_grouping_adapter: nil, + monotonic_clock: nil, + clock: ::Time + ) + @config = config + @datastore_core = datastore_core + @graphql_adapter = graphql_adapter + @datastore_search_router = datastore_search_router + @filter_interpreter = filter_interpreter + @sub_aggregation_grouping_adapter = sub_aggregation_grouping_adapter + @monotonic_clock = monotonic_clock + @clock = clock + @logger = @datastore_core.logger + @runtime_metadata = @datastore_core.schema_artifacts.runtime_metadata + @graphql_schema_string = @datastore_core.schema_artifacts.graphql_schema_string + + # Apply any extension modules that have been configured. + @config.extension_modules.each { |mod| extend mod } + @runtime_metadata.graphql_extension_modules.each { |ext_mod| extend ext_mod.extension_class } + end + + # @private + def graphql_http_endpoint + @graphql_http_endpoint ||= begin + require "elastic_graph/graphql/http_endpoint" + HTTPEndpoint.new( + query_executor: graphql_query_executor, + monotonic_clock: monotonic_clock, + client_resolver: config.client_resolver + ) + end + end + + # @private + def graphql_query_executor + @graphql_query_executor ||= begin + require "elastic_graph/graphql/query_executor" + QueryExecutor.new( + schema: schema, + monotonic_clock: monotonic_clock, + logger: logger, + slow_query_threshold_ms: @config.slow_query_latency_warning_threshold_in_ms, + datastore_search_router: datastore_search_router + ) + end + end + + # @private + def schema + @schema ||= begin + require "elastic_graph/graphql/schema" + + Schema.new( + graphql_schema_string: graphql_schema_string, + config: config, + runtime_metadata: runtime_metadata, + index_definitions_by_graphql_type: @datastore_core.index_definitions_by_graphql_type, + graphql_gem_plugins: graphql_gem_plugins + ) do |schema| + @graphql_adapter || begin + @schema = schema # assign this so that `#schema` returns the schema when `datastore_query_adapters` is called below + require "elastic_graph/graphql/resolvers/graphql_adapter" + Resolvers::GraphQLAdapter.new( + schema: schema, + datastore_query_builder: datastore_query_builder, + datastore_query_adapters: datastore_query_adapters, + runtime_metadata: runtime_metadata, + resolvers: graphql_resolvers + ) + end + end + end + end + + # @private + def datastore_search_router + @datastore_search_router ||= begin + require "elastic_graph/graphql/datastore_search_router" + DatastoreSearchRouter.new( + datastore_clients_by_name: @datastore_core.clients_by_name, + logger: logger, + monotonic_clock: monotonic_clock, + config: @config + ) + end + end + + # @private + def datastore_query_builder + @datastore_query_builder ||= begin + require "elastic_graph/graphql/datastore_query" + DatastoreQuery::Builder.with( + filter_interpreter: filter_interpreter, + runtime_metadata: runtime_metadata, + logger: logger, + default_page_size: @config.default_page_size, + max_page_size: @config.max_page_size + ) + end + end + + # @private + def graphql_gem_plugins + @graphql_gem_plugins ||= begin + require "graphql" + {::GraphQL::Dataloader => {}} + end + end + + # @private + def graphql_resolvers + @graphql_resolvers ||= begin + require "elastic_graph/graphql/resolvers/get_record_field_value" + require "elastic_graph/graphql/resolvers/list_records" + require "elastic_graph/graphql/resolvers/nested_relationships" + + nested_relationships = Resolvers::NestedRelationships.new( + schema_element_names: runtime_metadata.schema_element_names, + logger: logger + ) + + list_records = Resolvers::ListRecords.new + + get_record_field_value = Resolvers::GetRecordFieldValue.new( + schema_element_names: runtime_metadata.schema_element_names + ) + + [nested_relationships, list_records, get_record_field_value] + end + end + + # @private + def datastore_query_adapters + @datastore_query_adapters ||= begin + require "elastic_graph/graphql/aggregation/query_adapter" + require "elastic_graph/graphql/query_adapter/filters" + require "elastic_graph/graphql/query_adapter/pagination" + require "elastic_graph/graphql/query_adapter/sort" + require "elastic_graph/graphql/query_adapter/requested_fields" + + schema_element_names = runtime_metadata.schema_element_names + + [ + GraphQL::QueryAdapter::Pagination.new(schema_element_names: schema_element_names), + GraphQL::QueryAdapter::Filters.new( + schema_element_names: schema_element_names, + filter_args_translator: filter_args_translator, + filter_node_interpreter: filter_node_interpreter + ), + GraphQL::QueryAdapter::Sort.new(order_by_arg_name: schema_element_names.order_by), + Aggregation::QueryAdapter.new( + schema: schema, + config: config, + filter_args_translator: filter_args_translator, + runtime_metadata: runtime_metadata, + sub_aggregation_grouping_adapter: sub_aggregation_grouping_adapter + ), + GraphQL::QueryAdapter::RequestedFields.new(schema) + ] + end + end + + # @private + def filter_interpreter + @filter_interpreter ||= begin + require "elastic_graph/graphql/filtering/filter_interpreter" + Filtering::FilterInterpreter.new(filter_node_interpreter: filter_node_interpreter, logger: logger) + end + end + + # @private + def filter_node_interpreter + @filter_node_interpreter ||= begin + require "elastic_graph/graphql/filtering/filter_node_interpreter" + Filtering::FilterNodeInterpreter.new(runtime_metadata: runtime_metadata) + end + end + + # @private + def filter_args_translator + @filter_args_translator ||= begin + require "elastic_graph/graphql/filtering/filter_args_translator" + Filtering::FilterArgsTranslator.new(schema_element_names: runtime_metadata.schema_element_names) + end + end + + # @private + def sub_aggregation_grouping_adapter + @sub_aggregation_grouping_adapter ||= begin + require "elastic_graph/graphql/aggregation/non_composite_grouping_adapter" + Aggregation::NonCompositeGroupingAdapter + end + end + + # @private + def monotonic_clock + @monotonic_clock ||= begin + require "elastic_graph/support/monotonic_clock" + Support::MonotonicClock.new + end + end + + # @private + # Loads dependencies eagerly. In some environments (such as in an AWS Lambda) this is desirable as we to load all dependencies + # at boot time instead of deferring dependency loading until we handle the first query. In other environments (such as tests), + # it's nice to load dependencies when needed. + def load_dependencies_eagerly + # run a simple GraphQL query to force load any dependencies needed to handle GraphQL queries + graphql_query_executor.execute(EAGER_LOAD_QUERY, client: Client::ELASTICGRAPH_INTERNAL) + graphql_http_endpoint # force load this too. + end + + private + + EAGER_LOAD_QUERY = <<~EOS.strip + query ElasticGraphEagerLoadBootQuery { + __schema { + types { + kind + } + } + } + EOS + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/composite_grouping_adapter.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/composite_grouping_adapter.rb new file mode 100644 index 00000000..68bf0800 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/composite_grouping_adapter.rb @@ -0,0 +1,79 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + module Aggregation + # Grouping adapter that uses a `composite` aggregation. + # + # For now, only used for the outermost "root" aggregations but may be used for sub-aggregations in the future. + module CompositeGroupingAdapter + class << self + def meta_name + "comp" + end + + def grouping_detail_for(query) + sources = build_sources(query) + + inner_clauses = yield + inner_clauses = nil if inner_clauses.empty? + + return AggregationDetail.new(inner_clauses, {}) if sources.empty? + + clauses = { + query.name => { + "composite" => { + "size" => query.paginator.requested_page_size, + "sources" => sources, + "after" => composite_after(query) + }.compact, + "aggs" => inner_clauses + }.compact + } + + AggregationDetail.new(clauses, {"buckets_path" => [query.name]}) + end + + def prepare_response_buckets(sub_agg, buckets_path, meta) + sub_agg.dig(*buckets_path).fetch("buckets").map do |bucket| + bucket.merge({"doc_count_error_upper_bound" => 0}) + end + end + + private + + def composite_after(query) + return unless (cursor = query.paginator.search_after) + expected_keys = query.groupings.map(&:key) + + if cursor.sort_values.keys.sort == expected_keys.sort + cursor.sort_values + else + raise ::GraphQL::ExecutionError, "`#{cursor.encode}` is not a valid cursor for the current groupings." + end + end + + def build_sources(query) + # We don't want documents that have no value for a grouping field to be omitted, so we set `missing_bucket: true`. + # https://www.elastic.co/guide/en/elasticsearch/reference/8.11/search-aggregations-bucket-composite-aggregation.html#_missing_bucket + grouping_options = if query.paginator.search_in_reverse? + {"order" => "desc", "missing_bucket" => true} + else + {"missing_bucket" => true} + end + + query.groupings.map do |grouping| + {grouping.key => grouping.composite_clause(grouping_options: grouping_options)} + end + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/computation.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/computation.rb new file mode 100644 index 00000000..6f10442f --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/computation.rb @@ -0,0 +1,39 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/key" +require "elastic_graph/graphql/aggregation/field_path_encoder" + +module ElasticGraph + class GraphQL + module Aggregation + # Represents some sort of aggregation computation (min, max, avg, sum, etc) on a field. + # For the relevant Elasticsearch docs, see: + # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/search-aggregations-metrics-avg-aggregation.html + # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/search-aggregations-metrics-max-aggregation.html + # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/search-aggregations-metrics-min-aggregation.html + # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/search-aggregations-metrics-sum-aggregation.html + Computation = ::Data.define(:source_field_path, :computed_index_field_name, :detail) do + # @implements Computation + + def key(aggregation_name:) + Key::AggregatedValue.new( + aggregation_name: aggregation_name, + field_path: source_field_path.map(&:name_in_graphql_query), + function_name: computed_index_field_name + ).encode + end + + def clause + encoded_path = FieldPathEncoder.join(source_field_path.filter_map(&:name_in_index)) + {detail.function.to_s => {"field" => encoded_path}} + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/date_histogram_grouping.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/date_histogram_grouping.rb new file mode 100644 index 00000000..b7cfd1c3 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/date_histogram_grouping.rb @@ -0,0 +1,83 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/graphql/aggregation/field_path_encoder" +require "elastic_graph/support/memoizable_data" + +module ElasticGraph + class GraphQL + module Aggregation + # Represents a grouping of a timestamp field into a date histogram. + # For the relevant Elasticsearch docs, see: + # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/search-aggregations-bucket-datehistogram-aggregation.html + # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/search-aggregations-bucket-composite-aggregation.html#_date_histogram + class DateHistogramGrouping < Support::MemoizableData.define(:field_path, :interval, :time_zone, :offset) + def key + @key ||= FieldPathEncoder.encode(field_path.map(&:name_in_graphql_query)) + end + + def encoded_index_field_path + @encoded_index_field_path ||= FieldPathEncoder.join(field_path.filter_map(&:name_in_index)) + end + + def composite_clause(grouping_options: {}) + interval_options = INTERVAL_OPTIONS_BY_NAME.fetch(interval) do + raise ArgumentError, "#{interval.inspect} is an unsupported interval. Valid values: #{INTERVAL_OPTIONS_BY_NAME.keys.inspect}." + end + + inner_hash = interval_options.merge(grouping_options).merge({ + "field" => encoded_index_field_path, + "format" => DATASTORE_DATE_TIME_FORMAT, + "offset" => offset, + "time_zone" => time_zone + }.compact) + + {"date_histogram" => inner_hash} + end + + def non_composite_clause_for(query) + # `min_doc_count: 1` is important so we don't have excess buckets when there is a large gap + # between document dates. For example, if you group on a field at the year truncation unit, and + # a one-off rogue document has an incorrect timestamp for hundreds of years ago, you'll wind + # up with a bucket for each intervening year. `min_doc_count: 1` excludes those empty buckets. + composite_clause(grouping_options: {"min_doc_count" => 1}) + end + + def inner_meta + INNER_META + end + + INNER_META = { + # On a date histogram aggregation, the `key` is formatted as a number (milliseconds since epoch). We + # need it formatted as a string, which `key_as_string` provides. + "key_path" => ["key_as_string"], + # Date histogram aggregations do not have any doc count error. Our resolver is generic and expects + # there to always be a `doc_count_error_upper_bound`. So we want to tell it to merge an error of `0` + # into each bucket. + "merge_into_bucket" => {"doc_count_error_upper_bound" => 0} + } + + INTERVAL_OPTIONS_BY_NAME = { + # These intervals have only fixed intervals... + "millisecond" => {"fixed_interval" => "1ms"}, + "second" => {"fixed_interval" => "1s"}, + # ...but the rest have calendar intervals, which we prefer. + "minute" => {"calendar_interval" => "minute"}, + "hour" => {"calendar_interval" => "hour"}, + "day" => {"calendar_interval" => "day"}, + "week" => {"calendar_interval" => "week"}, + "month" => {"calendar_interval" => "month"}, + "quarter" => {"calendar_interval" => "quarter"}, + "year" => {"calendar_interval" => "year"} + } + private_constant :INTERVAL_OPTIONS_BY_NAME + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/field_path_encoder.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/field_path_encoder.rb new file mode 100644 index 00000000..7cc67445 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/field_path_encoder.rb @@ -0,0 +1,47 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" + +module ElasticGraph + class GraphQL + module Aggregation + module FieldPathEncoder + # Embedded fields need to be specified with dot separators. + DELIMITER = "." + + # Takes a list of field names (e.g., ["amountMoney", "amount"]) + # and returns a single field name path string (e.g., "amountMoney.amount"). + def self.encode(field_names) + field_names.each do |str| + verify_delimiters(str) + end + + join(field_names) + end + + # Joins together a list of encoded paths. + def self.join(encoded_paths) + encoded_paths.join(DELIMITER) + end + + # Takes a field path (e.g., "amountMoney.amount") and returns the field name parts + # (["amountMoney", "amount"]). + def self.decode(field_path) + field_path.split(DELIMITER) + end + + private_class_method def self.verify_delimiters(str) + if str.to_s.include?(DELIMITER) + raise Errors::InvalidArgumentValueError, %("#{str}" contains delimiter: "#{DELIMITER}") + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/field_term_grouping.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/field_term_grouping.rb new file mode 100644 index 00000000..9df291eb --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/field_term_grouping.rb @@ -0,0 +1,26 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/term_grouping" + +module ElasticGraph + class GraphQL + module Aggregation + class FieldTermGrouping < Support::MemoizableData.define(:field_path) + # @dynamic field_path + include TermGrouping + + private + + def terms_subclause + {"field" => encoded_index_field_path} + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/key.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/key.rb new file mode 100644 index 00000000..426cd3b6 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/key.rb @@ -0,0 +1,87 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/graphql/aggregation/field_path_encoder" + +module ElasticGraph + class GraphQL + module Aggregation + module Key + # The datastore only gives us an "aggregation key" (or name) to tie response values back to the part of + # request it came from. We use this delimiter to encode and decode aggregation keys. + DELIMITER = ":" + + # Aggregation key implementation used when we're dealing with `aggregated_values`. + class AggregatedValue < ::Data.define( + # The name of the aggregation encoded into this key. + :aggregation_name, + # The path to the field used by this aggregation (encoded as a string) + :encoded_field_path, + # The name of the aggregation function, such as "sum". + :function_name + ) + # We encode the field path as part of initialization to enforce an invariant that all `AggregatedValue` + # instances have valid values for all attributes. `FieldPathEncoder.encode` will raise an exception if + # the field path is invalid. + def initialize(aggregation_name:, function_name:, field_path: [], encoded_field_path: FieldPathEncoder.encode(field_path)) + Key.verify_no_delimiter_in(aggregation_name, function_name, *field_path) + + super( + aggregation_name: aggregation_name, + encoded_field_path: encoded_field_path, + function_name: function_name + ) + end + + def encode + Key.encode([aggregation_name, encoded_field_path, function_name]) + end + + def field_path + FieldPathEncoder.decode(encoded_field_path) + end + end + + # Encodes the key used for a `missing` aggregation used to provide a bucket for + # documents that are missing a value for the field being grouped on. + def self.missing_value_bucket_key(base_key) + Key.encode([base_key, "m"]) + end + + # Extracts an aggregation name from a string that could either already be an aggregation name, or could + # be an encoded key. We need this for dealing with the multiple forms that aggregation responses take: + # + # - When we use `grouped_by`, we run a composite aggregation that has the aggregation name, and + # that shows up as a key directly under `aggregations` in the datastore response. + # - For aggregations with no `grouped_by`, we encode the aggregation name in the key, and the keys + # directly under `aggregations` in the datastore response will take a from like: + # `[agg_name]:[field_path]:[function]`. + # + # It's also possible for these two forms to be mixed under `aggregations` on a datastore response, + # where some hash keys are in one form and some are in the other form. This can happen when we run + # multiple aggregations (some with `grouped_by`, some without) in the same query. + def self.extract_aggregation_name_from(agg_name_or_key) + agg_name_or_key.split(DELIMITER, 2).first || agg_name_or_key + end + + def self.encode(parts) + parts.join(DELIMITER) + end + + def self.verify_no_delimiter_in(*parts) + parts.each do |part| + if part.to_s.include?(DELIMITER) + raise Errors::InvalidArgumentValueError, %("#{part}" contains delimiter: "#{DELIMITER}") + end + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/nested_sub_aggregation.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/nested_sub_aggregation.rb new file mode 100644 index 00000000..a770824f --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/nested_sub_aggregation.rb @@ -0,0 +1,47 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/field_path_encoder" +require "elastic_graph/support/memoizable_data" + +module ElasticGraph + class GraphQL + module Aggregation + # Represents a sub-aggregation on a `nested` field. + # For the relevant Elasticsearch docs, see: + # https://www.elastic.co/guide/en/elasticsearch/reference/8.10/search-aggregations-bucket-nested-aggregation.html + class NestedSubAggregation < Support::MemoizableData.define(:nested_path, :query) + # The nested path in the GraphQL query from the parent aggregation to this-subaggregation, encoded + # for use as a hash key. + # + # This key will be unique in the scope of the parent aggregation query, and thus suitable as a key + # in a sub-aggregations hash. + def nested_path_key + @nested_path_key ||= FieldPathEncoder.encode(nested_path.map(&:name_in_graphql_query)) + end + + def build_agg_hash(filter_interpreter, parent_queries:) + detail = query.build_agg_detail(filter_interpreter, field_path: nested_path, parent_queries: parent_queries) + return {} if detail.nil? + + parent_query_names = parent_queries.map(&:name) + { + Key.encode(parent_query_names + [nested_path_key]) => { + "nested" => {"path" => FieldPathEncoder.encode(nested_path.filter_map(&:name_in_index))}, + "aggs" => detail.clauses, + "meta" => detail.meta.merge({ + "size" => query.paginator.desired_page_size, + "adapter" => query.grouping_adapter.meta_name + }) + }.compact + } + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/non_composite_grouping_adapter.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/non_composite_grouping_adapter.rb new file mode 100644 index 00000000..dc86570b --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/non_composite_grouping_adapter.rb @@ -0,0 +1,138 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + module Aggregation + # Grouping adapter that avoids using a `composite` aggregation, due to limitations with Elasticsearch/OpenSearch. + module NonCompositeGroupingAdapter + class << self + def meta_name + "non_comp" + end + + def grouping_detail_for(query) + date_groupings, term_groupings = query.groupings.partition do |grouping| + grouping.is_a?(DateHistogramGrouping) + end + + grouping_detail(date_groupings, query) do + # We want term groupings inside date groupings so that, when our bucket aggregations might produce + # inaccurate doc counts, the innermost grouping aggregation has `doc_count_error_upper_bound` on + # its buckets allowing us to expose information about the accuracy. + # + # Date histogram aggregations do not include `doc_count_error_upper_bound` because, on their own, they are + # always accurate, but they may not be accurate when used as a sub-aggregation of a `terms` aggregation. + # + # For more detail on the issue this ordering is designed to avoid, see: + # https://discuss.elastic.co/t/accuracy-of-date-histogram-sub-aggregation-doc-count-under-terms-aggregation/348685 + grouping_detail(term_groupings, query) do + inner_clauses = yield + inner_clauses = nil if inner_clauses.empty? + AggregationDetail.new(inner_clauses, {}) + end + end + end + + def prepare_response_buckets(sub_agg, buckets_path, meta) + sort_and_truncate_buckets(format_buckets(sub_agg, buckets_path), meta.fetch("size")) + end + + private + + def grouping_detail(groupings, query) + # Our `reduce` here builds the date grouping clauses from the inside out (since each reduction step + # wraps the prior step's result in an outer `aggs` hash). The natural result of that is a nested set of + # date grouping clauses that "feels" inside-out compared to what you would naturally expect. + # + # While that causes no concrete issue, it's nice to avoid. Here we use `reverse` to correct for that. + groupings.reverse.reduce(yield) do |inner_detail, grouping| + inner_detail.wrap_with_grouping(grouping, query: query) + end + end + + # Formats the result of a bucket aggregation into a format that we can easily resolve. There are two things + # this accomplishes: + # + # - Converts bucket keys into hashes that can be used to resolve `grouped_by` fields. + # - Recursively flattens multiple levels of aggregations (which happens when we need to mix multiple kinds of + # bucket aggregations to group in the way the client requested) into a single flat list. + def format_buckets(sub_agg, buckets_path, parent_key_fields: {}, parent_key_values: []) + agg_with_buckets = sub_agg.dig(*buckets_path) + + missing_bucket = { + # Doc counts in missing value buckets are always perfectly accurate. + "doc_count_error_upper_bound" => 0 + }.merge(sub_agg.dig(*missing_bucket_path_from(buckets_path))) # : ::Hash[::String, untyped] + + meta = agg_with_buckets.fetch("meta") + + grouping_field_names = meta.fetch("grouping_fields") # provides the names of the fields being grouped on + key_path = meta.fetch("key_path") # indicates whether we want to get the key values from `key` or `key_as_string`. + sub_buckets_path = meta["buckets_path"] # buckets_path is optional, so we don't use fetch. + merge_into_bucket = meta.fetch("merge_into_bucket") + + raw_buckets = agg_with_buckets.fetch("buckets") # : ::Array[::Hash[::String, untyped]] + + # If the missing bucket is non-empty, include it. This matches the behavior of composite aggregations + # when the `missing_bucket` option is used. + raw_buckets += [missing_bucket] if missing_bucket.fetch("doc_count") > 0 + + raw_buckets.flat_map do |raw_bucket| + # The key will either be a single value (e.g. `47`) if we used a `terms`/`date_histogram` aggregation, + # or a tuple of values (e.g. `[47, "abc"]`) if we used a `multi_terms` aggregation. Here we convert it + # to the form needed for resolving `grouped_by` fields: a hash like `{"size" => 47, "tag" => "abc"}`. + key_values = Array(raw_bucket.dig(*key_path)) + key_fields_hash = grouping_field_names.zip(key_values).to_h + + # If we have multiple levels of aggregations, we need to merge the key fields hash with the key fields from the parent levels. + key_fields = parent_key_fields.merge(key_fields_hash) + key_values = parent_key_values + key_values + + # If there's another level of aggregations, `buckets_path` will provide us with the path to that next level. + # We can use it to recurse as we build a flat list of buckets. + if sub_buckets_path + format_buckets(raw_bucket, sub_buckets_path, parent_key_fields: key_fields, parent_key_values: key_values) + else + [raw_bucket.merge(merge_into_bucket).merge({"key" => key_fields, "key_values" => key_values})] + end + end + end + + # A `terms` or `multi_terms` sub-aggregation is automatically sorted by `doc_count` and we pass + # `size` to the datastore to limit the number of returned buckets. + # + # A `date_histogram` sub-aggregation is sorted ascending by the date, and we don't limit the buckets + # in any way (there's no `size` parameter). + # + # To honor the requested page size and return buckets in a consistent order, we sort the buckets here + # (by doc count descending, then by the key values ascending), and then take only first `size`. + def sort_and_truncate_buckets(buckets, size) + buckets + .sort_by do |b| + # We convert key values to strings to ensure they are comparable. Otherwise, we can get an error like: + # + # > ArgumentError: comparison of Array with Array failed + # + # Note that this turns it into a lexicographical sort rather than a more type-aware sort + # (10 will sort before 2, for example), but that's fine. We only sort by `key_values` as + # a time breaker to ensure deterministic results, but don't particularly care which buckets + # come first. + [-b.fetch("doc_count"), b.fetch("key_values").map(&:to_s)] + end.first(size) + end + + def missing_bucket_path_from(buckets_path) + *all_but_last, last = buckets_path + all_but_last + [Key.missing_value_bucket_key(last.to_s)] + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/path_segment.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/path_segment.rb new file mode 100644 index 00000000..bb5192c5 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/path_segment.rb @@ -0,0 +1,31 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + module Aggregation + PathSegment = ::Data.define( + # The name of this segment's field in the GraphQL query. If it's an aliased field, this + # will be the alias name. + :name_in_graphql_query, + # The name of this segment's field in the datastore index. + :name_in_index + ) do + # Factory method that aids in building a `PathSegment` for a given `field` and `lookahead` node. + def self.for(lookahead:, field: nil) + ast_node = lookahead.ast_nodes.first # : ::GraphQL::Language::Nodes::Field + + new( + name_in_graphql_query: ast_node.alias || ast_node.name, + name_in_index: field&.name_in_index&.to_s + ) + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/query.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/query.rb new file mode 100644 index 00000000..4f3e2d83 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/query.rb @@ -0,0 +1,176 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/key" +require "elastic_graph/graphql/datastore_query" +require "elastic_graph/graphql/filtering/field_path" +require "elastic_graph/support/hash_util" + +module ElasticGraph + class GraphQL + module Aggregation + class Query < ::Data.define( + # Unique name for the aggregation + :name, + # Whether or not we need to get the document count for each bucket. + :needs_doc_count, + # Whether or not we need to get the error on the document count to satisfy the sub-aggregation query. + # https://www.elastic.co/guide/en/elasticsearch/reference/8.10/search-aggregations-bucket-terms-aggregation.html#_per_bucket_document_count_error + :needs_doc_count_error, + # Filter to apply to this sub-aggregation. + :filter, + # Paginator for handling size and other pagination concerns. + :paginator, + # A sub-aggregation query can have sub-aggregations of its own. + :sub_aggregations, + # Collection of `Computation` objects that specify numeric computations to perform. + :computations, + # Collection of `DateHistogramGrouping`, `FieldTermGrouping`, and `ScriptTermGrouping` objects that specify how this sub-aggregation should be grouped. + :groupings, + # Adapter to use for groupings. + :grouping_adapter + ) + def needs_total_doc_count? + # We only need a total document count when there are NO groupings and the doc count is requested. + # The datastore will return the number of hits in each grouping automatically, so we don't need + # a total doc count when there are groupings. And when the query isn't requesting the field, we + # don't need it, either. + needs_doc_count && groupings.empty? + end + + # Builds an aggregations hash. The returned value has a few different cases: + # + # - If `size` is 0, or `groupings` and `computations` are both empty, we return an empty hash, + # so that `to_datastore_body` is an empty hash. We do this so that we avoid sending + # the datastore any sort of aggregations query in these cases, as the client is not + # requesting any aggregation data. + # - If `SINGLETON_CURSOR` was provide for either `before` or `after`, we also return an empty hash, + # because we know there cannot be any results to return--the cursor is a reference to + # the one and only item in the list, and nothing can exist before or after it. + # - Otherwise, we return an aggregatinos hash based on the groupings, computations, and sub-aggregations. + def build_agg_hash(filter_interpreter) + build_agg_detail(filter_interpreter, field_path: [], parent_queries: [])&.clauses || {} + end + + def build_agg_detail(filter_interpreter, field_path:, parent_queries:) + return nil if paginator.desired_page_size.zero? || paginator.paginated_from_singleton_cursor? + queries = parent_queries + [self] # : ::Array[Query] + + filter_detail(filter_interpreter, field_path) do + grouping_adapter.grouping_detail_for(self) do + Support::HashUtil.disjoint_merge(computations_detail, sub_aggregation_detail(filter_interpreter, queries)) + end + end + end + + private + + def filter_detail(filter_interpreter, field_path) + filtering_field_path = Filtering::FieldPath.of(field_path.filter_map(&:name_in_index)) + filter_clause = filter_interpreter.build_query([filter].compact, from_field_path: filtering_field_path) + + inner_detail = yield + + return inner_detail if filter_clause.nil? + key = "#{name}:filtered" + + clause = { + key => { + "filter" => filter_clause, + "aggs" => inner_detail.clauses + }.compact + } + + inner_meta = inner_detail.meta + meta = + if (buckets_path = inner_detail.meta["buckets_path"]) + # In this case, we have some grouping aggregations applied, and the response will include a `buckets` array. + # Here we are prefixing the `buckets_path` with the `key` used for our filter aggregation to maintain its accuracy. + inner_meta.merge({"buckets_path" => [key] + buckets_path}) + else + # In this case, no grouping aggregations have been applied, and the response will _not_ have a `buckets` array. + # Instead, we'll need to treat the single unbucketed aggregation as a single bucket. To indicate that, we use + # `bucket_path` (singular) rather than `buckets_path` (plural). + inner_meta.merge({"bucket_path" => [key]}) + end + + AggregationDetail.new(clause, meta) + end + + def computations_detail + build_inner_aggregation_detail(computations) do |computation| + {computation.key(aggregation_name: name) => computation.clause} + end + end + + def sub_aggregation_detail(filter_interpreter, parent_queries) + build_inner_aggregation_detail(sub_aggregations.values) do |sub_agg| + sub_agg.build_agg_hash(filter_interpreter, parent_queries: parent_queries) + end + end + + def build_inner_aggregation_detail(collection, &block) + initial = {} # : ::Hash[::String, untyped] + collection.map(&block).reduce(initial) do |accum, hash| + Support::HashUtil.disjoint_merge(accum, hash) + end + end + end + + # The details of an aggregation level, including the `aggs` clauses themselves and `meta` + # that we want echoed back to us in the response for the aggregation level. + AggregationDetail = ::Data.define( + # Aggregation clauses that would go under `aggs. + :clauses, + # Custom metadata that will be echoed back to us in the response. + # https://www.elastic.co/guide/en/elasticsearch/reference/8.11/search-aggregations.html#add-metadata-to-an-agg + :meta + ) do + # @implements AggregationDetail + + # Wraps this aggregation detail in another aggregation layer for the given `grouping`, + # so that we can easily build up the necessary multi-level aggregation structure. + def wrap_with_grouping(grouping, query:) + agg_key = grouping.key + extra_inner_meta = grouping.inner_meta.merge({ + # The response just includes tuples of values for the key of each bucket. We need to know what fields those + # values come from, and this `meta` field indicates that. + "grouping_fields" => [agg_key] + }) + + inner_agg_hash = { + "aggs" => (clauses unless (clauses || {}).empty?), + "meta" => meta.merge(extra_inner_meta) + }.compact + + missing_bucket_inner_agg_hash = inner_agg_hash.key?("aggs") ? inner_agg_hash : {} # : ::Hash[::String, untyped] + + AggregationDetail.new( + { + agg_key => grouping.non_composite_clause_for(query).merge(inner_agg_hash), + + # Here we include a `missing` aggregation as a sibling to the main grouping aggregation. We do this + # so that we get a bucket of documents that have `null` values for the field we are grouping on, in + # order to provide the same behavior as the `CompositeGroupingAdapter` (which uses the built-in + # `missing_bucket` option). + # + # To work correctly, we need to include this `missing` aggregation as a sibling at _every_ level of + # the aggregation structure, and the `missing` aggregation needs the same child aggregations as the + # main grouping aggregation has. Given the recursive nature of how this is applied, this results in + # a fairly complex structure, even though conceptually the idea behind this isn't _too_ bad. + Key.missing_value_bucket_key(agg_key) => { + "missing" => {"field" => grouping.encoded_index_field_path} + }.merge(missing_bucket_inner_agg_hash) + }, + {"buckets_path" => [agg_key]} + ) + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/query_adapter.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/query_adapter.rb new file mode 100644 index 00000000..8e9e500c --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/query_adapter.rb @@ -0,0 +1,349 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/composite_grouping_adapter" +require "elastic_graph/graphql/aggregation/computation" +require "elastic_graph/graphql/aggregation/date_histogram_grouping" +require "elastic_graph/graphql/aggregation/field_term_grouping" +require "elastic_graph/graphql/aggregation/nested_sub_aggregation" +require "elastic_graph/graphql/aggregation/path_segment" +require "elastic_graph/graphql/aggregation/query" +require "elastic_graph/graphql/aggregation/script_term_grouping" +require "elastic_graph/graphql/schema/arguments" +require "elastic_graph/support/hash_util" +require "elastic_graph/support/memoizable_data" + +module ElasticGraph + class GraphQL + module Aggregation + # Responsible for taking in the incoming GraphQL request context, arguments, and the GraphQL + # schema and directives and populating the `aggregations` portion of `query`. + class QueryAdapter < Support::MemoizableData.define(:schema, :config, :filter_args_translator, :runtime_metadata, :sub_aggregation_grouping_adapter) + # @dynamic element_names + attr_reader :element_names + + def call(query:, lookahead:, args:, field:, context:) + return query unless field.type.unwrap_fully.indexed_aggregation? + + aggregations_node = extract_aggregation_node(lookahead, field, context.query) + return query unless aggregations_node + + aggregation_query = build_aggregation_query_for( + aggregations_node, + field: field, + grouping_adapter: CompositeGroupingAdapter, + # Filters on root aggregations applied to the search query body itself instead of + # using a filter aggregation, like sub-aggregations do, so we don't want a filter + # aggregation generated here. + unfiltered: true + ) + + query.merge_with(aggregations: {aggregation_query.name => aggregation_query}) + end + + private + + def after_initialize + @element_names = schema.element_names + end + + def extract_aggregation_node(lookahead, field, graphql_query) + return nil unless (ast_nodes = lookahead.ast_nodes) + + if ast_nodes.size > 1 + names = ast_nodes.map { |n| "`#{name_of(n)}`" } + raise_conflicting_grouping_requirement_selections("`#{lookahead.name}` selection with the same name", names) + end + + ::GraphQL::Execution::Lookahead.new( + query: graphql_query, + ast_nodes: ast_nodes, + field: lookahead.field, + owner_type: field.parent_type.graphql_type + ) + end + + def build_aggregation_query_for(aggregations_node, field:, grouping_adapter:, nested_path: [], unfiltered: false) + aggregation_name = name_of(_ = aggregations_node.ast_nodes.first) + + # Get the AST node for the `nodes` subfield (e.g. from `fooAggregations { nodes { ... } }`) + nodes_node = selection_above_grouping_fields(aggregations_node, element_names.nodes, aggregation_name) + + # Also get the AST node for `edges.node` (e.g. from `fooAggregations { edges { node { ... } } }`) + edges_node_node = [element_names.edges, element_names.node].reduce(aggregations_node) do |node, sub_selection| + selection_above_grouping_fields(node, sub_selection, aggregation_name) + end + + # ...and then determine which one is being used for nodes. + node_node = + if nodes_node.selected? && edges_node_node.selected? + raise_conflicting_grouping_requirement_selections("node selection", ["`#{element_names.nodes}`", "`#{element_names.edges}.#{element_names.node}`"]) + elsif !nodes_node.selected? + edges_node_node + else + nodes_node + end + + count_detail_node = node_node.selection(element_names.count_detail) + needs_doc_count_error = + # We need to know what the error is to determine if the approximate count is in fact the exact count. + count_detail_node.selects?(element_names.exact_value) || + # We need to know what the error is to determine the upper bound on the count. + count_detail_node.selects?(element_names.upper_bound) + + unless unfiltered + filter = filter_args_translator.translate_filter_args(field: field, args: field.args_to_schema_form(aggregations_node.arguments)) + end + + Query.new( + name: aggregation_name, + groupings: build_groupings_from(node_node, aggregation_name, from_field_path: nested_path), + computations: build_computations_from(node_node, from_field_path: nested_path), + sub_aggregations: build_sub_aggregations_from(node_node, parent_nested_path: nested_path), + needs_doc_count: count_detail_node.selected? || node_node.selects?(element_names.count), + needs_doc_count_error: needs_doc_count_error, + paginator: build_paginator_for(aggregations_node), + filter: filter, + grouping_adapter: grouping_adapter + ) + end + + # Helper method for dealing with lookahead selections above the grouping fields. If the caller selects + # such a field multiple times (e.g. with aliases) it leads to conflicting grouping requirements, so we + # do not allow it. + def selection_above_grouping_fields(node, sub_selection_name, aggregation_name) + node.selection(sub_selection_name).tap do |nested_node| + ast_nodes = nested_node.ast_nodes || [] + if ast_nodes.size > 1 + names = ast_nodes.map { |n| "`#{name_of(n)}`" } + raise_conflicting_grouping_requirement_selections("`#{sub_selection_name}` selection under `#{aggregation_name}`", names) + end + end + end + + def build_clauses_from(parent_node, &block) + get_children_nodes(parent_node).flat_map do |child_node| + transform_node_to_clauses(child_node, &block) + end.to_set + end + + # Takes a `GraphQL::Execution::Lookahead` node and returns an array of children + # lookahead nodes, excluding nodes for introspection fields. + def get_children_nodes(node) + node.selections.reject do |child| + child.field.introspection? + end + end + + # Takes a `GraphQL::Execution::Lookahead` node that conforms to our aggregate field + # conventions (`some_field: {Type}Metric`) and returns a Hash compatible with the `aggregations` + # argument to `DatastoreQuery.new`. + def transform_node_to_clauses(node, parent_path: [], &clause_builder) + field = field_from_node(node) + field_path = parent_path + [PathSegment.for(field: field, lookahead: node)] + + clause_builder.call(node, field, field_path) || get_children_nodes(node).flat_map do |embedded_field| + transform_node_to_clauses(embedded_field, parent_path: field_path, &clause_builder) + end + end + + def build_computations_from(node_node, from_field_path: []) + aggregated_values_node = node_node.selection(element_names.aggregated_values) + + build_clauses_from(aggregated_values_node) do |node, field, field_path| + if field.aggregated? + field_path = from_field_path + field_path + + get_children_nodes(node).map do |fn_node| + computed_field = field_from_node(fn_node) + computation_detail = field_from_node(fn_node).computation_detail # : SchemaArtifacts::RuntimeMetadata::ComputationDetail + + Aggregation::Computation.new( + source_field_path: field_path, + computed_index_field_name: computed_field.name_in_index.to_s, + detail: computation_detail + ) + end + end + end + end + + def build_groupings_from(node_node, aggregation_name, from_field_path: []) + grouped_by_node = selection_above_grouping_fields(node_node, element_names.grouped_by, aggregation_name) + + build_clauses_from(grouped_by_node) do |node, field, field_path| + field_path = from_field_path + field_path + + # New date/time grouping API (DateGroupedBy, DateTimeGroupedBy) + if field.type.elasticgraph_category == :date_grouped_by_object + date_time_groupings_from(field_path: field_path, node: node) + + elsif !field.type.object? + case field.type.name + # Legacy date grouping API + when :Date + legacy_date_histogram_groupings_from( + field_path: field_path, + node: node, + get_time_zone: ->(args) {}, + get_offset: ->(args) { args[element_names.offset_days]&.then { |days| "#{days}d" } } + ) + # Legacy datetime grouping API + when :DateTime + legacy_date_histogram_groupings_from( + field_path: field_path, + node: node, + get_time_zone: ->(args) { args.fetch(element_names.time_zone) }, + get_offset: ->(args) { datetime_offset_from(node, args) } + ) + # Non-date/time grouping + else + [FieldTermGrouping.new(field_path: field_path)] + end + end + end + end + + # Given a `GraphQL::Execution::Lookahead` node, returns the corresponding `Schema::Field` + def field_from_node(node) + schema.field_named(node.owner_type.graphql_name, node.field.name) + end + + # Returns an array of `...Grouping`, one for each child node (`as_date_time`, `as_date`, etc). + def date_time_groupings_from(field_path:, node:) + get_children_nodes(node).map do |child_node| + schema_args = Schema::Arguments.to_schema_form(child_node.arguments, child_node.field) + # Because `DateGroupedBy` doesn't have a `timeZone` argument, and we want to reuse the same + # script for both `Date` and `DateTime`, we fall back to "UTC" here. + time_zone = schema_args[element_names.time_zone] || "UTC" + child_field_path = field_path + [PathSegment.for(lookahead: child_node)] + + if child_node.field.name == element_names.as_day_of_week + ScriptTermGrouping.new( + field_path: child_field_path, + script_id: runtime_metadata.static_script_ids_by_scoped_name.fetch("field/as_day_of_week"), + params: { + "offset_ms" => datetime_offset_as_ms_from(child_node, schema_args), + "time_zone" => time_zone + } + ) + elsif child_node.field.name == element_names.as_time_of_day + ScriptTermGrouping.new( + field_path: child_field_path, + script_id: runtime_metadata.static_script_ids_by_scoped_name.fetch("field/as_time_of_day"), + params: { + "interval" => interval_from(child_node, schema_args, interval_unit_key: element_names.truncation_unit), + "offset_ms" => datetime_offset_as_ms_from(child_node, schema_args), + "time_zone" => time_zone + } + ) + else + DateHistogramGrouping.new( + field_path: child_field_path, + interval: interval_from(child_node, schema_args, interval_unit_key: element_names.truncation_unit), + offset: datetime_offset_from(child_node, schema_args), + time_zone: time_zone + ) + end + end + end + + def legacy_date_histogram_groupings_from(field_path:, node:, get_time_zone:, get_offset:) + schema_args = Schema::Arguments.to_schema_form(node.arguments, node.field) + + [DateHistogramGrouping.new( + field_path: field_path, + interval: interval_from(node, schema_args, interval_unit_key: element_names.granularity), + time_zone: get_time_zone.call(schema_args), + offset: get_offset.call(schema_args) + )] + end + + # Figure out the Date histogram grouping interval for the given node based on the `grouped_by` argument. + # Until `legacy_grouping_schema` is removed, we need to check both `granularity` and `truncation_unit`. + def interval_from(node, schema_args, interval_unit_key:) + enum_type_name = node.field.arguments.fetch(interval_unit_key).type.unwrap.graphql_name + enum_value_name = schema_args.fetch(interval_unit_key) + enum_value = schema.type_named(enum_type_name).enum_value_named(enum_value_name) + + _ = enum_value.runtime_metadata.datastore_value + end + + def datetime_offset_from(node, schema_args) + if (unit_name = schema_args.dig(element_names.offset, element_names.unit)) + enum_value = enum_value_from_offset(node, unit_name) + amount = schema_args.fetch(element_names.offset).fetch(element_names.amount) + "#{amount}#{enum_value.runtime_metadata.datastore_abbreviation}" + end + end + + # Convert from amount and unit to milliseconds, using runtime metadata `datastore_value` + def datetime_offset_as_ms_from(node, schema_args) + unit_name = schema_args.dig(element_names.offset, element_names.unit) + return 0 unless unit_name + + amount = schema_args.fetch(element_names.offset).fetch(element_names.amount) + enum_value = enum_value_from_offset(node, unit_name) + + amount * enum_value.runtime_metadata.datastore_value + end + + def enum_value_from_offset(node, unit_name) + offset_input_type = node.field.arguments.fetch(element_names.offset).type.unwrap # : ::GraphQL::Schema::InputObject + enum_type_name = offset_input_type.arguments.fetch(element_names.unit).type.unwrap.graphql_name + schema.type_named(enum_type_name).enum_value_named(unit_name) + end + + def name_of(ast_node) + ast_node.alias || ast_node.name + end + + def build_sub_aggregations_from(node_node, parent_nested_path: []) + key_sub_agg_pairs = + build_clauses_from(node_node.selection(element_names.sub_aggregations)) do |node, field, field_path| + if field.type.elasticgraph_category == :nested_sub_aggregation_connection + nested_path = parent_nested_path + field_path + nested_sub_agg = NestedSubAggregation.new( + nested_path: nested_path, + query: build_aggregation_query_for( + node, + field: field, + grouping_adapter: sub_aggregation_grouping_adapter, + nested_path: nested_path + ) + ) + + [[nested_sub_agg.nested_path_key, nested_sub_agg]] # : ::Array[[::String, NestedSubAggregation]] + end + end + + Support::HashUtil.strict_to_h(key_sub_agg_pairs) + end + + def build_paginator_for(node) + args = field_from_node(node).args_to_schema_form(node.arguments) + + DatastoreQuery::Paginator.new( + first: args[element_names.first], + after: args[element_names.after], + last: args[element_names.last], + before: args[element_names.before], + default_page_size: config.default_page_size, + max_page_size: config.max_page_size, + schema_element_names: schema.element_names + ) + end + + def raise_conflicting_grouping_requirement_selections(more_than_one_description, paths) + raise ::GraphQL::ExecutionError, "Cannot have more than one #{more_than_one_description} " \ + "(#{paths.join(", ")}), because that could lead to conflicting grouping requirements." + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/query_optimizer.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/query_optimizer.rb new file mode 100644 index 00000000..d05ab408 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/query_optimizer.rb @@ -0,0 +1,187 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/key" + +module ElasticGraph + class GraphQL + module Aggregation + # This class is used by `DatastoreQuery.perform` to optimize away an inefficiency that's present in + # our aggregations API. To explain what this does, it's useful to see an example: + # + # ``` + # query WigdetsBySizeAndColor($filter: WidgetFilterInput) { + # by_size: widgetAggregations(filter: $filter) { + # edges { node { + # size + # count + # } } + # } + # + # by_color: widgetAggregations(filter: $filter) { + # edges { node { + # color + # count + # } } + # } + # } + # ``` + # + # With this API, two separate datastore queries get built--one for `by_size`, and one + # for `by_color`. While we're able to send them to the datastore in a single `msearch` request, + # as it allows a single search to have multiple aggregations in it. The aggregations + # API we offered before April 2023 directly supported this, allowing for more efficient + # queries. (But it had other significant downsides). + # + # We found that sending 2 queries is significantly slower than sending one combined query + # (from benchmarks/aggregations_old_vs_new_api.rb): + # + # Benchmarks for old API (300 times): + # Average took value: 15 + # Median took value: 14 + # P99 took value: 45 + # + # Benchmarks for new API (300 times): + # Average took value: 28 + # Median took value: 25 + # P99 took value: 75 + # + # This class optimizes this case by merging `DatastoreQuery` objects together when we can safely do so, + # in order to execute fewer datastore queries. Notably, while this was designed for this specific + # aggregations case, the merging logic can also apply in non-aggregations case. + # + # Note that we want to err on the side of safety here. We only merge queries if their datastore + # payloads are byte-for-byte identical when aggregations are excluded. There are some cases where + # we _could_ merge slightly differing queries in clever ways (for example, if the only difference is + # `track_total_hits: false` vs `track_total_hits: true`, we could merge to a single query with + # `track_total_hits: true`), but that's significantly more complex and error prone, so we do not do it. + # We can always improve this further in the future to cover more cases. + # + # NOTE: the `QueryOptimizer` assumes that `Aggregation::Query` will always produce aggregation keys + # using `Aggregation::Query#name` such that `Aggregation::Key.extract_aggregation_name_from` is able + # to extract the original name from response keys. If that is violated, it will not work properly and + # subtle bugs can result. However, we have a test helper method which is hooked into our unit and + # integration tests for `DatastoreQuery` (`verify_aggregations_satisfy_optimizer_requirements`) which + # verifies that this requirement is satisfied. + class QueryOptimizer + def self.optimize_queries(queries) + return {} if queries.empty? + optimizer = new(queries, logger: (_ = queries.first).logger) + responses_by_query = yield optimizer.merged_queries + optimizer.unmerge_responses(responses_by_query) + end + + def initialize(original_queries, logger:) + @original_queries = original_queries + @logger = logger + last_id = 0 + @unique_prefix_by_query = ::Hash.new { |h, k| h[k] = "#{last_id += 1}_" } + end + + def merged_queries + original_queries_by_merged_query.keys + end + + def unmerge_responses(responses_by_merged_query) + original_queries_by_merged_query.flat_map do |merged, originals| + # When we only had a single query to start with, we didn't change the query at all, and don't need to unmerge the response. + needs_unmerging = originals.size > 1 + + originals.filter_map do |orig| + if (merged_response = responses_by_merged_query[merged]) + response = needs_unmerging ? unmerge_response(merged_response, orig) : merged_response + [orig, response] + end + end + end.to_h + end + + private + + def original_queries_by_merged_query + @original_queries_by_merged_query ||= queries_by_merge_key.values.to_h do |original_queries| + [merge_queries(original_queries), original_queries] + end + end + + NO_AGGREGATIONS = {} + + def queries_by_merge_key + @original_queries.group_by do |query| + # Here we group queries in the simplest, safest way possible: queries are safe to merge if + # their datastore payloads are byte-for-byte identical, excluding aggregations. + query.with(aggregations: NO_AGGREGATIONS) + end + end + + def merge_queries(queries) + # If we only have a single query, there's nothing to merge! + return (_ = queries.first) if queries.one? + + all_aggs_by_name = queries.flat_map do |query| + # It's possible for two queries to have aggregations with the same name but different parameters. + # In a merged query, each aggregation must have a different name. Here we guarantee that by adding + # a numeric prefix to the aggregations. For example, if both `query1` and `query2` have a `by_size` + # aggregation, on the merged query we'll have a `1_by_size` aggregation and a `2_by_size` aggregation. + prefix = @unique_prefix_by_query[query] + query.aggregations.values.map do |agg| + agg.with(name: "#{prefix}#{agg.name}") + end + end.to_h { |agg| [agg.name, agg] } + + @logger.info({ + "message_type" => "AggregationQueryOptimizerMergedQueries", + "query_count" => queries.size, + "aggregation_count" => all_aggs_by_name.size, + "aggregation_names" => all_aggs_by_name.keys.sort + }) + + (_ = queries.first).with(aggregations: all_aggs_by_name) + end + + # "Unmerges" a response to convert it to what it woulud have been if we hadn't merged queries. + # To do that, we need to do two things: + # + # - Filter down the aggregations to just the ones that are for the original query. + # - Remove the query-specific prefix (e.g. `1_`) from the parts of the response that + # contain the aggregation name. + def unmerge_response(response_from_merged_query, original_query) + # If there are no aggregations, there's nothing to unmerge--just return it as is. + return response_from_merged_query unless (aggs = response_from_merged_query["aggregations"]) + + prefix = @unique_prefix_by_query[original_query] + agg_names = original_query.aggregations.keys.map { |name| "#{prefix}#{name}" }.to_set + + filtered_aggs = aggs + .select { |key, agg_data| agg_names.include?(Key.extract_aggregation_name_from(key)) } + .to_h do |key, agg_data| + [key.delete_prefix(prefix), strip_prefix_from_agg_data(agg_data, prefix, key)] + end + + response_from_merged_query.merge("aggregations" => filtered_aggs) + end + + def strip_prefix_from_agg_data(agg_data, prefix, key) + case agg_data + when ::Hash + agg_data.to_h do |sub_key, sub_data| + sub_key = sub_key.delete_prefix(prefix) if sub_key.start_with?(key) + [sub_key, strip_prefix_from_agg_data(sub_data, prefix, key)] + end + when ::Array + agg_data.map do |element| + strip_prefix_from_agg_data(element, prefix, key) + end + else + agg_data + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/aggregated_values.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/aggregated_values.rb new file mode 100644 index 00000000..c45292f1 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/aggregated_values.rb @@ -0,0 +1,42 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/key" +require "elastic_graph/graphql/aggregation/path_segment" +require "elastic_graph/support/hash_util" + +module ElasticGraph + class GraphQL + module Aggregation + module Resolvers + class AggregatedValues < ::Data.define(:aggregation_name, :bucket, :field_path) + def can_resolve?(field:, object:) + true + end + + def resolve(field:, object:, args:, context:, lookahead:) + return with(field_path: field_path + [PathSegment.for(field: field, lookahead: lookahead)]) if field.type.object? + + key = Key::AggregatedValue.new( + aggregation_name: aggregation_name, + field_path: field_path.map(&:name_in_graphql_query), + function_name: field.name_in_index.to_s + ) + + result = Support::HashUtil.verbose_fetch(bucket, key.encode) + + # Aggregated value results always have a `value` key; in addition, for `date` field, they also have a `value_as_string`. + # In that case, `value` is a number (e.g. ms since epoch) whereas `value_as_string` is a formatted value. ElasticGraph + # works with date types as formatted strings, so we need to use `value_as_string` here if it is present. + result.fetch("value_as_string") { result.fetch("value") } + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/count_detail.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/count_detail.rb new file mode 100644 index 00000000..b0105e25 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/count_detail.rb @@ -0,0 +1,44 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/resolvers/resolvable_value" + +module ElasticGraph + class GraphQL + module Aggregation + module Resolvers + # Resolves the detailed `count` sub-fields of a sub-aggregation. It's an object because + # the count we get from the datastore may not be accurate and we have multiple + # fields we expose to give the client control over how much detail they want. + # + # Note: for now our resolver logic only uses the bucket fields returned to us by the datastore, + # but I believe we may have some opportunities to provide more accurate responses to these when custom shard + # routing and/or index rollover are in use. For example, when grouping on the custom shard routing field, + # we know that no term bucket will have data from more than one shard. The datastore isn't aware of our + # custom shard routing logic, though, and can't account for that in what it returns, so it may indicate + # a potential error upper bound where we can deduce there is none. + class CountDetail < GraphQL::Resolvers::ResolvableValue.new(:bucket) + # The (potentially approximate) `doc_count` returned by the datastore for a bucket. + def approximate_value + @approximate_value ||= bucket.fetch("doc_count") + end + + # The `doc_count`, if we know it was exact. (Otherwise, returns `nil`). + def exact_value + approximate_value if approximate_value == upper_bound + end + + # The upper bound on how large the doc count could be. + def upper_bound + @upper_bound ||= bucket.fetch("doc_count_error_upper_bound") + approximate_value + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/grouped_by.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/grouped_by.rb new file mode 100644 index 00000000..a175a743 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/grouped_by.rb @@ -0,0 +1,32 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/field_path_encoder" +require "elastic_graph/support/hash_util" + +module ElasticGraph + class GraphQL + module Aggregation + module Resolvers + class GroupedBy < ::Data.define(:bucket, :field_path) + def can_resolve?(field:, object:) + true + end + + def resolve(field:, object:, args:, context:, lookahead:) + new_field_path = field_path + [PathSegment.for(field: field, lookahead: lookahead)] + return with(field_path: new_field_path) if field.type.object? + + bucket_entry = Support::HashUtil.verbose_fetch(bucket, "key") + Support::HashUtil.verbose_fetch(bucket_entry, FieldPathEncoder.encode(new_field_path.map(&:name_in_graphql_query))) + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/node.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/node.rb new file mode 100644 index 00000000..3a1a1e9e --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/node.rb @@ -0,0 +1,64 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/resolvers/aggregated_values" +require "elastic_graph/graphql/aggregation/resolvers/grouped_by" +require "elastic_graph/graphql/decoded_cursor" +require "elastic_graph/graphql/resolvers/resolvable_value" + +module ElasticGraph + class GraphQL + module Aggregation + module Resolvers + class Node < GraphQL::Resolvers::ResolvableValue.new(:query, :parent_queries, :bucket, :field_path) + # This file defines a subclass of `Node` and can't be loaded until `Node` has been defined. + require "elastic_graph/graphql/aggregation/resolvers/sub_aggregations" + + def grouped_by + @grouped_by ||= GroupedBy.new(bucket, field_path) + end + + def aggregated_values + @aggregated_values ||= AggregatedValues.new(query.name, bucket, field_path) + end + + def sub_aggregations + @sub_aggregations ||= SubAggregations.new( + schema_element_names, + query.sub_aggregations, + parent_queries + [query], + bucket, + field_path + ) + end + + def count + bucket.fetch("doc_count") + end + + def count_detail + @count_detail ||= CountDetail.new(schema_element_names, bucket) + end + + def cursor + # If there's no `key`, then we aren't grouping by anything. We just have a single aggregation + # bucket containing computed values over the entire set of filtered documents. In that case, + # we still need a pagination cursor but we have no "key" to speak of that we can encode. Instead, + # we use the special SINGLETON cursor defined for this case. + @cursor ||= + if (key = bucket.fetch("key")).empty? + DecodedCursor::SINGLETON + else + DecodedCursor.new(key) + end + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/relay_connection_builder.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/relay_connection_builder.rb new file mode 100644 index 00000000..5bbba788 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/relay_connection_builder.rb @@ -0,0 +1,83 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/resolvers/node" +require "elastic_graph/graphql/datastore_query" +require "elastic_graph/graphql/resolvers/relay_connection/generic_adapter" + +module ElasticGraph + class GraphQL + module Aggregation + module Resolvers + module RelayConnectionBuilder + def self.build_from_search_response(query:, search_response:, schema_element_names:) + build_from_buckets(query: query, parent_queries: [], schema_element_names: schema_element_names) do + extract_buckets_from(search_response, for_query: query) + end + end + + def self.build_from_buckets(query:, parent_queries:, schema_element_names:, field_path: [], &build_buckets) + GraphQL::Resolvers::RelayConnection::GenericAdapter.new( + schema_element_names: schema_element_names, + raw_nodes: raw_nodes_for(query, parent_queries, schema_element_names, field_path, &build_buckets), + paginator: query.paginator, + get_total_edge_count: -> {}, + to_sort_value: ->(node, decoded_cursor) do + query.groupings.map do |grouping| + DatastoreQuery::Paginator::SortValue.new( + from_item: (_ = node).bucket.fetch("key").fetch(grouping.key), + from_cursor: decoded_cursor.sort_values.fetch(grouping.key), + sort_direction: :asc # we don't yet support any alternate sorting. + ) + end + end + ) + end + + private_class_method def self.raw_nodes_for(query, parent_queries, schema_element_names, field_path) + # The `DecodedCursor::SINGLETON` is a special case, so handle it here. + return [] if query.paginator.paginated_from_singleton_cursor? + + yield.map do |bucket| + Node.new( + schema_element_names: schema_element_names, + query: query, + parent_queries: parent_queries, + bucket: bucket, + field_path: field_path + ) + end + end + + private_class_method def self.extract_buckets_from(search_response, for_query:) + search_response.raw_data.dig( + "aggregations", + for_query.name, + "buckets" + ) || [build_bucket(for_query, search_response.raw_data)] + end + + private_class_method def self.build_bucket(query, response) + defaults = { + "key" => query.groupings.to_h { |g| [g.key, nil] }, + "doc_count" => response.dig("hits", "total", "value") || 0 + } + + empty_bucket_computations = query.computations.to_h do |computation| + [computation.key(aggregation_name: query.name), {"value" => computation.detail.empty_bucket_value}] + end + + defaults + .merge(empty_bucket_computations) + .merge(response["aggregations"] || {}) + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/sub_aggregations.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/sub_aggregations.rb new file mode 100644 index 00000000..1170aa3e --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/resolvers/sub_aggregations.rb @@ -0,0 +1,85 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/composite_grouping_adapter" +require "elastic_graph/graphql/aggregation/field_path_encoder" +require "elastic_graph/graphql/aggregation/key" +require "elastic_graph/graphql/aggregation/non_composite_grouping_adapter" +require "elastic_graph/graphql/aggregation/resolvers/count_detail" +require "elastic_graph/graphql/decoded_cursor" +require "elastic_graph/graphql/resolvers/resolvable_value" +require "elastic_graph/support/hash_util" + +module ElasticGraph + class GraphQL + module Aggregation + module Resolvers + class SubAggregations < ::Data.define(:schema_element_names, :sub_aggregations, :parent_queries, :sub_aggs_by_agg_key, :field_path) + def can_resolve?(field:, object:) + true + end + + def resolve(field:, object:, args:, context:, lookahead:) + path_segment = PathSegment.for(field: field, lookahead: lookahead) + new_field_path = field_path + [path_segment] + return with(field_path: new_field_path) unless field.type.elasticgraph_category == :nested_sub_aggregation_connection + + sub_agg_key = FieldPathEncoder.encode(new_field_path.map(&:name_in_graphql_query)) + sub_agg_query = Support::HashUtil.verbose_fetch(sub_aggregations, sub_agg_key).query + + RelayConnectionBuilder.build_from_buckets( + query: sub_agg_query, + parent_queries: parent_queries, + schema_element_names: schema_element_names, + field_path: new_field_path + ) { extract_buckets(sub_agg_key, args) } + end + + private + + def extract_buckets(aggregation_field_path, args) + # When the client passes `first: 0`, we omit the sub-aggregation from the query body entirely, + # and it wont' be in `sub_aggs_by_agg_key`. Instead, we can just return an empty list of buckets. + return [] if args[schema_element_names.first] == 0 + + sub_agg_key = Key.encode(parent_queries.map(&:name) + [aggregation_field_path]) + sub_agg = Support::HashUtil.verbose_fetch(sub_aggs_by_agg_key, sub_agg_key) + meta = sub_agg.fetch("meta") + + # When the sub-aggregation node of the GraphQL query has a `filter` argument, the direct sub-aggregation returned by + # the datastore will be the unfiltered sub-aggregation. To get the filtered sub-aggregation (the data our client + # actually cares about), we have a sub-aggregation under that. + # + # To indicate this case, our query includes a `meta` field which which tells us which sub-key # has the actual data we care about in it: + # - If grouping has been applied (leading to multiple buckets): `meta: {buckets_path: [path, to, bucket]}` + # - If no grouping has been applied (leading to a single bucket): `meta: {bucket_path: [path, to, bucket]}` + if (buckets_path = meta["buckets_path"]) + bucket_adapter = BUCKET_ADAPTERS.fetch(sub_agg.dig("meta", "adapter")) + bucket_adapter.prepare_response_buckets(sub_agg, buckets_path, meta) + else + singleton_bucket = + if (bucket_path = meta["bucket_path"]) + sub_agg.dig(*bucket_path) + else + sub_agg + end + + # When we have a single ungrouped bucket, we never have any error on the `doc_count`. + # Our resolver logic expects it to be present, though. + [singleton_bucket.merge({"doc_count_error_upper_bound" => 0})] + end + end + + BUCKET_ADAPTERS = [CompositeGroupingAdapter, NonCompositeGroupingAdapter].to_h do |adapter| + [adapter.meta_name, adapter] + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/script_term_grouping.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/script_term_grouping.rb new file mode 100644 index 00000000..7fb97454 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/script_term_grouping.rb @@ -0,0 +1,32 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/term_grouping" + +module ElasticGraph + class GraphQL + module Aggregation + # Used for term groupings that use a script instead of a field + class ScriptTermGrouping < Support::MemoizableData.define(:field_path, :script_id, :params) + # @dynamic field_path + include TermGrouping + + private + + def terms_subclause + { + "script" => { + "id" => script_id, + "params" => params.merge({"field" => encoded_index_field_path}) + } + } + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/term_grouping.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/term_grouping.rb new file mode 100644 index 00000000..e196221b --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/term_grouping.rb @@ -0,0 +1,118 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/field_path_encoder" +require "elastic_graph/support/memoizable_data" + +module ElasticGraph + class GraphQL + module Aggregation + # Represents a grouping on a term. + # For the relevant Elasticsearch docs, see: + # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/search-aggregations-bucket-terms-aggregation.html + # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/search-aggregations-bucket-composite-aggregation.html#_terms + module TermGrouping + def key + @key ||= FieldPathEncoder.encode(field_path.map(&:name_in_graphql_query)) + end + + def encoded_index_field_path + @encoded_index_field_path ||= FieldPathEncoder.join(field_path.filter_map(&:name_in_index)) + end + + def composite_clause(grouping_options: {}) + {"terms" => terms_subclause.merge(grouping_options)} + end + + def non_composite_clause_for(query) + clause_value = work_around_elasticsearch_bug(terms_subclause) + { + "terms" => clause_value.merge({ + "size" => query.paginator.desired_page_size, + "show_term_doc_count_error" => query.needs_doc_count_error + }) + } + end + + INNER_META = {"key_path" => ["key"], "merge_into_bucket" => {}} + + def inner_meta + INNER_META + end + + private + + # Here we force the `collect_mode` to `depth_first`. Without doing that, we've observed that some of our acceptance + # specs fail on CI when running against Elasticsearch 8.11 with an error like: + # + # ``` + # { + # "root_cause": [ + # { + # "type": "runtime_exception", + # "reason": "score for different docid, nesting an aggregation under a children aggregation and terms aggregation with collect mode breadth_first isn't possible" + # } + # ], + # "type": "search_phase_execution_exception", + # "reason": "all shards failed", + # "phase": "query", + # "grouped": true, + # "failed_shards": [ + # { + # "shard": 0, + # "index": "teams_camel", + # "node": "pDXJzLTsRJCRjKe83DqipA", + # "reason": { + # "type": "runtime_exception", + # "reason": "score for different docid, nesting an aggregation under a children aggregation and terms aggregation with collect mode breadth_first isn't possible" + # } + # } + # ], + # "caused_by": { + # "type": "runtime_exception", + # "reason": "score for different docid, nesting an aggregation under a children aggregation and terms aggregation with collect mode breadth_first isn't possible", + # "caused_by": { + # "type": "runtime_exception", + # "reason": "score for different docid, nesting an aggregation under a children aggregation and terms aggregation with collect mode breadth_first isn't possible" + # } + # } + # } + # ``` + # + # This specific exception message was introduced in https://github.com/elastic/elasticsearch/pull/89993, but that was done to provide + # a better error than a NullPointerException (which is what used to happen). This error also appears to be non-deterministic; I wasn't + # able to reproduce the CI failure locally until I forced `"collect_mode" => "breadth_first"`, at which point I did see the same error + # locally. The Elasticsearch docs[^1] mention that a heuristic (partially based on if a field's cardinality is known!) is used to pick + # whether `breadth_first` or `depth_first` is used when `collect_mode`is not specified: + # + # > The `breadth_first` is the default mode for fields with a cardinality bigger than the requested size or when the cardinality is unknown + # > (numeric fields or scripts for instance). + # + # In addition, the docs[^2] make it clear that `depth_first` is usually what you want: + # + # > The strategy we outlined previously—building the tree fully and then pruning—is called depth-first and it is the default. + # > Depth-first works well for the majority of aggregations, but can fall apart in situations like our actors and costars example. + # > + # > ... + # > + # > Breadth-first should be used only when you expect more buckets to be generated than documents landing in the buckets. + # + # So, for now we are forcing the collect mode to `depth_first`, as it avoids an issue with Elasticsearch and is a generally + # sane default. It may fall over in the case breadth-first is intended for, but we can cross that bridge when it comes. + # + # Long term, we're hoping to switch sub-aggregations to use a `composite` aggregation instead of `terms`, rendering this moot. + # + # [^1]: https://www.elastic.co/guide/en/elasticsearch/reference/8.11/search-aggregations-bucket-terms-aggregation.html#search-aggregations-bucket-terms-aggregation-collect + # [^2]: https://www.elastic.co/guide/en/elasticsearch/guide/current/_preventing_combinatorial_explosions.html#_depth_first_versus_breadth_first + def work_around_elasticsearch_bug(terms_clause) + terms_clause.merge({"collect_mode" => "depth_first"}) + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/client.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/client.rb new file mode 100644 index 00000000..6d4ff4da --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/client.rb @@ -0,0 +1,43 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + # Represents a client of an ElasticGraph GraphQL endpoint. + # `name` and `source_description` can really be any string, but `name` is + # meant to be a friendly/human readable string (such as a service name) + # where as `source_description` is meant to be an opaque string describing + # where `name` came from. + class Client < Data.define(:source_description, :name) + # `Data.define` provides the following methods: + # @dynamic initialize, name, source_description, with + + ANONYMOUS = new("(anonymous)", "(anonymous)") + ELASTICGRAPH_INTERNAL = new("(ElasticGraphInternal)", "(ElasticGraphInternal)") + + def description + if source_description == name + name + else + "#{name} (#{source_description})" + end + end + + # Default resolver used to determine the client for a given HTTP request. + # Also defines the interface of a client resolver. (This is why we define `initialize`). + class DefaultResolver + def initialize(config) + end + + def resolve(http_request) + Client::ANONYMOUS + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/config.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/config.rb new file mode 100644 index 00000000..8a3e4a55 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/config.rb @@ -0,0 +1,81 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/graphql/client" +require "elastic_graph/schema_artifacts/runtime_metadata/extension_loader" + +module ElasticGraph + class GraphQL + class Config < Data.define( + # Determines the size of our datastore search requests if the query does not specify. + :default_page_size, + # Determines the maximum size of a requested page. If the client requests a page larger + # than this value, `max_page_size` elements will be returned instead. + :max_page_size, + # Queries that take longer than this configured threshold will have a sanitized version logged. + :slow_query_latency_warning_threshold_in_ms, + # Object used to identify the client of a GraphQL query based on the HTTP request. + :client_resolver, + # Array of modules that will be extended onto the `GraphQL` instance to support extension libraries. + :extension_modules, + # Contains any additional settings that were in the settings file beyond settings that are expected as part of ElasticGraph + # itself. Extensions are free to use these extra settings. + :extension_settings + ) + def self.from_parsed_yaml(entire_parsed_yaml) + parsed_yaml = entire_parsed_yaml.fetch("graphql") + extra_keys = parsed_yaml.keys - EXPECTED_KEYS + + unless extra_keys.empty? + raise Errors::ConfigError, "Unknown `graphql` config settings: #{extra_keys.join(", ")}" + end + + extension_loader = SchemaArtifacts::RuntimeMetadata::ExtensionLoader.new(::Module.new) + extension_mods = parsed_yaml.fetch("extension_modules", []).map do |mod_hash| + extension_loader.load(mod_hash.fetch("extension_name"), from: mod_hash.fetch("require_path"), config: {}).extension_class.tap do |mod| + unless mod.instance_of?(::Module) + raise Errors::ConfigError, "`#{mod_hash.fetch("extension_name")}` is not a module, but all application extension modules must be modules." + end + end + end + + new( + default_page_size: parsed_yaml.fetch("default_page_size"), + max_page_size: parsed_yaml.fetch("max_page_size"), + slow_query_latency_warning_threshold_in_ms: parsed_yaml["slow_query_latency_warning_threshold_in_ms"] || 5000, + client_resolver: load_client_resolver(parsed_yaml), + extension_modules: extension_mods, + extension_settings: entire_parsed_yaml.except(*ELASTICGRAPH_CONFIG_KEYS) + ) + end + + # The keys we expect under `graphql`. + EXPECTED_KEYS = members.map(&:to_s) + + # The standard ElasticGraph root config setting keys; anything else is assumed to be extension settings. + ELASTICGRAPH_CONFIG_KEYS = %w[graphql indexer logger datastore schema_artifacts] + + private_class_method def self.load_client_resolver(parsed_yaml) + config = parsed_yaml.fetch("client_resolver") do + return Client::DefaultResolver.new({}) + end + + client_resolver_loader = SchemaArtifacts::RuntimeMetadata::ExtensionLoader.new(Client::DefaultResolver) + extension = client_resolver_loader.load( + config.fetch("extension_name"), + from: config.fetch("require_path"), + config: config.except("extension_name", "require_path") + ) + extension_class = extension.extension_class # : ::Class + + __skip__ = extension_class.new(extension.extension_config) + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query.rb new file mode 100644 index 00000000..9706ad22 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query.rb @@ -0,0 +1,372 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/graphql/aggregation/query" +require "elastic_graph/graphql/aggregation/query_optimizer" +require "elastic_graph/graphql/decoded_cursor" +require "elastic_graph/graphql/datastore_response/search_response" +require "elastic_graph/graphql/filtering/filter_interpreter" +require "elastic_graph/support/memoizable_data" + +module ElasticGraph + class GraphQL + # An immutable class that represents a datastore query. Since this represents + # a datastore query, and not a GraphQL query, all the data in it is modeled + # in datastore terms, not GraphQL terms. For example, any field names in a + # `Query` should be references to index fields, not GraphQL fields. + # + # Filters are modeled as a `Set` of filtering hashes. While we usually expect only + # a single `filter` hash, modeling it as a set makes it easy for us to support + # merging queries. The datastore knows how to apply multiple `must` clauses that + # apply to the same field, giving us the exact semantics we want in such a situation + # with minimal effort. + class DatastoreQuery < Support::MemoizableData.define( + :total_document_count_needed, :aggregations, :logger, :filter_interpreter, :routing_picker, + :index_expression_builder, :default_page_size, :search_index_definitions, :max_page_size, + :filters, :sort, :document_pagination, :requested_fields, :individual_docs_needed, + :monotonic_clock_deadline, :schema_element_names + ) { + def initialize( + filter: nil, + filters: nil, + sort: nil, + document_pagination: nil, + aggregations: nil, + requested_fields: nil, + individual_docs_needed: false, + total_document_count_needed: false, + monotonic_clock_deadline: nil, + **kwargs + ) + # Deal with `:filter` vs `:filters` input and normalize it to a single `filters` set. + filters = ::Set.new(filters || []) + filters << filter if filter && !filter.empty? + filters.freeze + + aggregations ||= {} + requested_fields ||= [] + + super( + filters: filters, + sort: sort || [], + document_pagination: document_pagination || {}, + aggregations: aggregations, + requested_fields: requested_fields.to_set, + individual_docs_needed: individual_docs_needed || !requested_fields.empty?, + total_document_count_needed: total_document_count_needed || aggregations.values.any?(&:needs_total_doc_count?), + monotonic_clock_deadline: monotonic_clock_deadline, + **kwargs + ) + + if search_index_definitions.empty? + raise Errors::SearchFailedError, "Query is invalid, since it contains no `search_index_definitions`." + end + end + } + # Load these files after the `Query` class has been defined, to avoid + # `TypeError: superclass mismatch for class Query` + require "elastic_graph/graphql/datastore_query/document_paginator" + require "elastic_graph/graphql/datastore_query/index_expression_builder" + require "elastic_graph/graphql/datastore_query/paginator" + require "elastic_graph/graphql/datastore_query/routing_picker" + + # Performs a list of queries by building a hash of datastore msearch header/body tuples (keyed + # by query), yielding them to the caller, and then post-processing the results. The caller is + # responsible for returning a hash of responses by query from its block. + # + # Note that some of the passed queries may not be yielded to the caller; when we can tell + # that a query does not have to be sent to the datastore we avoid yielding it from here. + # Therefore, the caller should not assume that all queries passed to this method will be + # yielded back. + # + # The return value is a hash of `DatastoreResponse::SearchResponse` objects by query. + # + # Note: this method uses `send` to work around ruby visibility rules. We do not want + # `#decoded_cursor_factory` to be public, as we only need it here, but we cannot access + # it from a class method without using `send`. + def self.perform(queries) + empty_queries, present_queries = queries.partition(&:empty?) + + responses_by_query = Aggregation::QueryOptimizer.optimize_queries(present_queries) do |optimized_queries| + header_body_tuples_by_query = optimized_queries.each_with_object({}) do |query, hash| + hash[query] = query.to_datastore_msearch_header_and_body + end + + yield(header_body_tuples_by_query) + end + + empty_responses = empty_queries.each_with_object({}) do |query, hash| + hash[query] = DatastoreResponse::SearchResponse::RAW_EMPTY + end + + empty_responses.merge(responses_by_query).each_with_object({}) do |(query, response), hash| + hash[query] = DatastoreResponse::SearchResponse.build(response, decoded_cursor_factory: query.send(:decoded_cursor_factory)) + end.tap do |responses_hash| + # Callers expect this `perform` method to provide an invariant: the returned hash MUST contain one entry + # for each of the `queries` passed in the args. In practice, violating this invariant primarily causes a + # problem when the caller uses the `GraphQL::Dataloader` (which happens for every GraphQL request in production...). + # However, our tests do not always run queries end-to-end, so this is an added check we want to do, so that + # anytime our logic here fails to include a query in the response in any test, we'll be notified of the + # problem. + expected_queries = queries.to_set + actual_queries = responses_hash.keys.to_set + + if expected_queries != actual_queries + missing_queries = expected_queries - actual_queries + extra_queries = actual_queries - expected_queries + + raise Errors::SearchFailedError, "The `responses_hash` does not have the expected set of queries as keys. " \ + "This can cause problems for the `GraphQL::Dataloader` and suggests a bug in the logic that should be fixed.\n\n" \ + "Missing queries (#{missing_queries.size}):\n#{missing_queries.map(&:inspect).join("\n")}.\n\n" \ + "Extra queries (#{extra_queries.size}): #{extra_queries.map(&:inspect).join("\n")}" + end + end + end + + # Merges the provided query, returning a new combined query object. + # Both query objects are left unchanged. + def merge(other_query) + if search_index_definitions != other_query.search_index_definitions + raise ElasticGraph::Errors::InvalidMergeError, "`search_index_definitions` conflict while merging between " \ + "#{search_index_definitions} and #{other_query.search_index_definitions}" + end + + with( + individual_docs_needed: individual_docs_needed || other_query.individual_docs_needed, + total_document_count_needed: total_document_count_needed || other_query.total_document_count_needed, + filters: filters + other_query.filters, + sort: merge_attribute(other_query, :sort), + requested_fields: requested_fields + other_query.requested_fields, + document_pagination: merge_attribute(other_query, :document_pagination), + monotonic_clock_deadline: [monotonic_clock_deadline, other_query.monotonic_clock_deadline].compact.min, + aggregations: aggregations.merge(other_query.aggregations) + ) + end + + # Convenience method for merging when you do not have access to an + # `DatastoreQuery::Builder`. Allows you to pass the query options you + # would like to merge. As with `#merge`, leaves the original query unchanged + # and returns a combined query object. + def merge_with(**query_options) + merge(with(**query_options)) + end + + # Pairs the multi-search headers and body into a tuple, as per the format required by the datastore: + # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-multi-search.html#search-multi-search-api-desc + def to_datastore_msearch_header_and_body + @to_datastore_msearch_header_and_body ||= [to_datastore_msearch_header, to_datastore_body] + end + + # Returns an index_definition expression string to use for searches. This string can specify + # multiple indices, use wildcards, etc. For info about what is supported, see: + # https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-index.html + def search_index_expression + @search_index_expression ||= index_expression_builder.determine_search_index_expression( + filters, + search_index_definitions, + # When we have aggregations, we must require indices to search. When we search no indices, the datastore does not return + # the standard aggregations response structure, which causes problems. + require_indices: !aggregations_datastore_body.empty? + ).to_s + end + + # Returns the name of the datastore cluster as a String where this query should be setn. + # Unless exactly 1 cluster name is found, this method raises a Errors::ConfigError. + def cluster_name + cluster_name = search_index_definitions.map(&:cluster_to_query).uniq + return cluster_name.first if cluster_name.size == 1 + raise Errors::ConfigError, "Found different datastore clusters (#{cluster_name}) to query " \ + "for query targeting indices: #{search_index_definitions}" + end + + # Returns a list of unique field paths that should be used for shard routing during searches. + # + # If a search is filtering on one of these fields, we can optimize the search by routing + # it to only the shards containing documents for that routing value. + # + # Note that this returns a list due to our support for type unions. A unioned type + # can be composed of subtypes that have use different shard routing; this will return + # the set union of them all. + def route_with_field_paths + search_index_definitions.map(&:route_with).uniq + end + + # The shard routing values used for this search. Can be `nil` if the query will hit all shards. + # `[]` means that we are routing to no shards. + def shard_routing_values + return @shard_routing_values if defined?(@shard_routing_values) + routing_values = routing_picker.extract_eligible_routing_values(filters, route_with_field_paths) + + @shard_routing_values ||= + if routing_values&.empty? && !aggregations_datastore_body.empty? + # If we return an empty array of routing values, no shards will get searched, which causes a problem for aggregations. + # When a query includes aggregations, there are normally aggregation structures on the respopnse (even when there are no + # search hits to aggregate over!) but if there are no routing values, those aggregation structures will be missing from + # the response. It's complex to handle that in our downstream response handling code, so we prefer to force a "fallback" + # routing value here to ensure that at least one shard gets searched. Which shard gets searched doesn't matter; the search + # filter that led to an empty set of routing values will match on documents on any shard. + ["fallback_shard_routing_value"] + elsif contains_ignored_values_for_routing?(routing_values) + nil + else + routing_values&.sort # order doesn't matter, but sorting it makes it easier to assert on in our tests. + end + end + + # Indicates if the query does not need any results from the datastore. As an optimization, + # we can reply with a default "empty" response for an empty query. + def empty? + # If we are searching no indices or routing to an empty set of shards, there is no need to query the datastore at all. + # This only happens when our filter processing has deduced that the query will match no results. + return true if search_index_expression.empty? || shard_routing_values&.empty? + + datastore_body = to_datastore_body + datastore_body.fetch(:size) == 0 && !datastore_body.fetch(:track_total_hits) && aggregations_datastore_body.empty? + end + + def inspect + description = to_datastore_msearch_header.merge(to_datastore_body).map do |key, value| + "#{key}=#{(key == :query) ? "" : value.inspect}" + end.join(" ") + + "#<#{self.class.name} #{description}>" + end + + def to_datastore_msearch_header + @to_datastore_msearch_header ||= {index: search_index_expression, routing: shard_routing_values&.join(",")}.compact + end + + # `DatastoreQuery` objects are used as keys in a hash. Computing `#hash` can be expensive (given how many fields + # an `DatastoreQuery` has) and it's safe to cache since `DatastoreQuery` instances are immutable, so we memoize it + # here. We've observed this making a very noticeable difference in our test suite runtime. + def hash + @hash ||= super + end + + def document_paginator + @document_paginator ||= DocumentPaginator.new( + sort_clauses: sort_with_tiebreaker, + individual_docs_needed: individual_docs_needed, + total_document_count_needed: total_document_count_needed, + decoded_cursor_factory: decoded_cursor_factory, + schema_element_names: schema_element_names, + paginator: Paginator.new( + default_page_size: default_page_size, + max_page_size: max_page_size, + first: document_pagination[:first], + after: document_pagination[:after], + last: document_pagination[:last], + before: document_pagination[:before], + schema_element_names: schema_element_names + ) + ) + end + + private + + def merge_attribute(other_query, attribute) + value = public_send(attribute) + other_value = other_query.public_send(attribute) + + if value.empty? + other_value + elsif other_value.empty? + value + elsif value == other_value + value + else + logger.warn("Tried to merge two queries that both define `#{attribute}`, using the value from the query being merged: #{value}, #{other_value}") + other_value + end + end + + TIEBREAKER_SORT_CLAUSES = [{"id" => {"order" => "asc"}}].freeze + + # We want to use `id` as a tiebreaker ONLY when `id` isn't explicitly specified as a sort field + def sort_with_tiebreaker + @sort_with_tiebreaker ||= remove_duplicate_sort_clauses(sort + TIEBREAKER_SORT_CLAUSES) + end + + def remove_duplicate_sort_clauses(sort_clauses) + seen_fields = Set.new + sort_clauses.select do |clause| + clause.keys.all? { |key| seen_fields.add?(key) } + end + end + + def decoded_cursor_factory + @decoded_cursor_factory ||= DecodedCursor::Factory.from_sort_list(sort_with_tiebreaker) + end + + def contains_ignored_values_for_routing?(routing_values) + ignored_values_for_routing.intersect?(routing_values.to_set) if routing_values + end + + def ignored_values_for_routing + @ignored_values_for_routing ||= search_index_definitions.flat_map { |i| i.ignored_values_for_routing.to_a }.to_set + end + + def to_datastore_body + @to_datastore_body ||= aggregations_datastore_body + .merge(document_paginator.to_datastore_body) + .merge({query: filter_interpreter.build_query(filters)}.compact) + .merge({_source: source}) + end + + def aggregations_datastore_body + @aggregations_datastore_body ||= begin + aggs = aggregations + .values + .map { |agg| agg.build_agg_hash(filter_interpreter) } + .reduce({}, :merge) + + aggs.empty? ? {} : {aggs: aggs} + end + end + + # Make our query as efficient as possible by limiting what parts of `_source` we fetch. + # For an id-only query (or a query that has no requested fields) we don't need to fetch `_source` + # at all--which means the datastore can avoid decompressing the _source field. Otherwise, + # we only ask for the fields we need to return. + def source + requested_source_fields = requested_fields - ["id"] + return false if requested_source_fields.empty? + # Merging in requested_fields as _source:{includes:} based on Elasticsearch documentation: + # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html#include-exclude + {includes: requested_source_fields.to_a} + end + + # Encapsulates dependencies of `Query`, giving us something we can expose off of `application` + # to build queries when desired. + class Builder < Support::MemoizableData.define(:runtime_metadata, :logger, :query_defaults) + def self.with(runtime_metadata:, logger:, **query_defaults) + new(runtime_metadata: runtime_metadata, logger: logger, query_defaults: query_defaults) + end + + def routing_picker + @routing_picker ||= RoutingPicker.new(schema_names: runtime_metadata.schema_element_names) + end + + def index_expression_builder + @index_expression_builder ||= IndexExpressionBuilder.new(schema_names: runtime_metadata.schema_element_names) + end + + def new_query(**options) + DatastoreQuery.new( + routing_picker: routing_picker, + index_expression_builder: index_expression_builder, + logger: logger, + schema_element_names: runtime_metadata.schema_element_names, + **query_defaults.merge(options) + ) + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query/document_paginator.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query/document_paginator.rb new file mode 100644 index 00000000..4435fc5b --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query/document_paginator.rb @@ -0,0 +1,100 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "graphql" + +module ElasticGraph + class GraphQL + class DatastoreQuery + # Contains query logic related to pagination. Mostly delegates to `Paginator`, which + # contains most of the logic. This merely adapts the `Paginator` to the needs of document + # pagination. (Paginator also supports aggregation bucket pagination.) + class DocumentPaginator < Support::MemoizableData.define( + :sort_clauses, :paginator, :decoded_cursor_factory, :schema_element_names, + # `individual_docs_needed`: when false, we request a `size` of 0. Set to `true` when the client is + # requesting any document fields, or if we need documents to compute any parts of the `PageInfo`. + :individual_docs_needed, + # `total_document_count_needed`: when false, `track_total_hits` will be 0 in our datastore query. + # This will prevent the datastore from doing extra work to get an accurate count + :total_document_count_needed + ) + # Builds a hash containing the portions of a datastore search body related to pagination. + def to_datastore_body + { + size: effective_size, + sort: effective_sort, + search_after: search_after, + track_total_hits: total_document_count_needed + }.reject { |key, value| Array(value).empty? } + end + + def sort + @sort ||= sort_clauses.map do |clause| + clause.transform_values do |options| + # As per the Elasticsearch docs[^1] missing/null values get sorted last by default, but we can control + # it here. We want to control it here to make our sorting behavior more consistent in a couple ways: + # + # 1. We want _document_ sorting and _aggregation_ sorting to behave the same. Aggregation sorting puts + # missing value buckets first when sorting ascending and last when sorting descending[^2]. Note that in + # Elasticsearch 7.16[^3] and above, you can control if missing buckets go first or last, but below that + # version you have no control. Here we match that behavior. + # 2. Clients are likely to expect that descending sorting will produce a list in reverse order from what + # ascending sorting produces, but with the default behavior (missing/null values get sorted last), this + # is not the case. We have to use the opposite `missing` option when the `order` is the opposite. + # + # [^1]: https://www.elastic.co/guide/en/elasticsearch/reference/7.10/sort-search-results.html#_missing_values + # [^2]: https://www.elastic.co/guide/en/elasticsearch/reference/7.10/search-aggregations-bucket-composite-aggregation.html#_missing_bucket + # [^3]: https://www.elastic.co/guide/en/elasticsearch/reference/7.16/search-aggregations-bucket-composite-aggregation.html#_missing_bucket + missing = (options.fetch("order") == "asc") ? "_first" : "_last" + options.merge({"missing" => missing}) + end + end + end + + private + + def effective_size + individual_docs_needed ? paginator.requested_page_size : 0 + end + + def effective_sort + return [] unless effective_size > 0 + paginator.search_in_reverse? ? reverse_sort : sort + end + + DIRECTION_OPPOSITES = {"asc" => "desc", "desc" => "asc"}.freeze + MISSING_OPPOSITES = {"_first" => "_last", "_last" => "_first"}.freeze + + def reverse_sort + @reverse_sort ||= sort.map do |sort_clause| + sort_clause.transform_values do |options| + { + "order" => DIRECTION_OPPOSITES.fetch(options.fetch("order")), + "missing" => MISSING_OPPOSITES.fetch(options.fetch("missing")) + } + end + end + end + + def search_after + paginator.search_after&.then do |cursor| + decoded_cursor_factory.sort_fields.map do |field| + cursor.sort_values.fetch(field) do + raise ::GraphQL::ExecutionError, "`#{cursor.encode}` is not a valid cursor for the current `#{schema_element_names.order_by}` argument." + end + end + end + end + end + + # `Query::DocumentPaginator` exists only for use by `Query` and is effectively private. + private_constant :DocumentPaginator + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query/index_expression_builder.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query/index_expression_builder.rb new file mode 100644 index 00000000..1c58026d --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query/index_expression_builder.rb @@ -0,0 +1,142 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/filtering/filter_value_set_extractor" +require "elastic_graph/support/time_set" + +module ElasticGraph + class GraphQL + class DatastoreQuery + # Responsible for building a search index expression for a specific query based on the filters. + class IndexExpressionBuilder + def initialize(schema_names:) + @filter_value_set_extractor = Filtering::FilterValueSetExtractor.new(schema_names, Support::TimeSet::ALL) do |operator, filter_value| + case operator + when :gt, :gte, :lt, :lte + if date_string?(filter_value) + # Here we translate into a range of time objects. When translating dates to times, + # we need to use an appropriate time suffix: + # + # - `> 2024-04-01` == `> 2024-04-01T23:59:59.999Z` + # - `≥ 2024-04-01` == `≥ 2024-04-01T00:00:00Z` + # - `< 2024-04-01` == `< 2024-04-01T00:00:00Z` + # - `≤ 2024-04-01` == `≤ 2024-04-01T23:59:59.999Z` + time_suffix = (operator == :gt || operator == :lte) ? "T23:59:59.999Z" : "T00:00:00Z" + Support::TimeSet.of_range(operator => ::Time.iso8601(filter_value + time_suffix)) + else + Support::TimeSet.of_range(operator => ::Time.iso8601(filter_value)) + end + when :equal_to_any_of + # This calls `.compact` to remove `nil` timestamp values. + ranges = filter_value.compact.map do |iso8601_string| + if date_string?(iso8601_string) + # When we have a date string, build a range for the entire day. + start_of_day = ::Time.iso8601("#{iso8601_string}T00:00:00Z") + end_of_day = ::Time.iso8601("#{iso8601_string}T23:59:59.999Z") + ::Range.new(start_of_day, end_of_day) + else + value = ::Time.iso8601(iso8601_string) + ::Range.new(value, value) + end + end + + Support::TimeSet.of_range_objects(ranges) + end + end + end + + # Returns an index_definition expression string to use for searches. This string can specify + # multiple indices, use wildcards, etc. For info about what is supported, see: + # https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-index.html + def determine_search_index_expression(filter_hashes, search_index_definitions, require_indices:) + # Here we sort the index expressions. It won't change the behavior in the datastore, but + # makes the return value here deterministic which makes it easier to assert on in tests. + search_index_definitions.sort_by(&:name).reduce(IndexExpression::EMPTY) do |index_expression, index_def| + index_expression + index_expression_for(filter_hashes, index_def, require_indices: require_indices) + end + end + + private + + def index_expression_for(filter_hashes, maybe_rollover_index_def, require_indices:) + unless maybe_rollover_index_def.rollover_index_template? + return IndexExpression.only(maybe_rollover_index_def.index_expression_for_search) + end + + # @type var index_def: DatastoreCore::IndexDefinition::RolloverIndexTemplate + index_def = _ = maybe_rollover_index_def + + time_set = @filter_value_set_extractor.extract_filter_value_set(filter_hashes, [index_def.timestamp_field_path]) + + if time_set.empty? + return require_indices ? + # Indices are required. Given the time set is empty, it's impossible for any documents to match our search. + # Therefore, which index we use here doesn't matter. We just pick the first one, alphabetically. + IndexExpression.only(index_def.known_related_query_rollover_indices.map(&:index_expression_for_search).min) : + # No indices are required, so we can return an empty index expression. + IndexExpression::EMPTY + end + + indices_to_exclude = index_def.known_related_query_rollover_indices.reject do |index| + time_set.intersect?(index.time_set) + end + + if require_indices && (index_def.known_related_query_rollover_indices - indices_to_exclude).empty? + # Indices are required, but all known indices have been excluded. We satisfy the requirement for an index by excluding one + # less index. This is preferable to the alternative ways to satisfy the requirement. + # + # - We could return an `IndexExpression` with no exclusions, but that would search across all indices, which is less efficient. + # - We could pick the first index to search (as we do for the `time_set.empty?` case), but that could cause matching documents + # to be be missed, because it's possible that matching documents exist in just-created index that is not in + # `known_related_query_rollover_indices`. Therefore, it's important that we still search the rollover wildcard expression, + # and we want to exclude all but one of the known indices. + indices_to_exclude = indices_to_exclude.drop(1) + end + + IndexExpression.new( + names_to_include: ::Set.new([index_def.index_expression_for_search]), + names_to_exclude: ::Set.new(indices_to_exclude.map(&:index_expression_for_search)) + ) + end + + def date_string?(string) + /\A\d{4}-\d{2}-\d{2}\z/.match?(string) + end + end + + class IndexExpression < ::Data.define(:names_to_include, :names_to_exclude) + EMPTY = new(names_to_include: ::Set.new, names_to_exclude: ::Set.new) + + def self.only(name) + IndexExpression.new(names_to_include: ::Set.new([name].compact), names_to_exclude: ::Set.new) + end + + def to_s + # Note: exclusions must come after inclusions. I can't find anything in the Elasticsearch or OpenSearch docs + # that mention this, but when exclusions come first I found that we got errors. + parts = names_to_include.sort + names_to_exclude.sort.map { |name| "-#{name}" } + parts.join(",") + end + + def +(other) + with( + names_to_include: names_to_include.union(other.names_to_include), + names_to_exclude: names_to_exclude.union(other.names_to_exclude) + ) + end + end + + # `Query::IndexExpressionBuilder` exists only for use by `Query` and is effectively private. + private_constant :IndexExpressionBuilder + + # Steep is complaining that it can't find some `Query` but they are not in this file... + # @dynamic aggregations, shard_routing_values, search_index_definitions, merge_with, search_index_expression + # @dynamic with, to_datastore_msearch_header_and_body, document_paginator + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query/paginator.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query/paginator.rb new file mode 100644 index 00000000..1d7f4105 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query/paginator.rb @@ -0,0 +1,199 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/support/memoizable_data" + +module ElasticGraph + class GraphQL + class DatastoreQuery + # A generic pagination implementation, designed to handle both document pagination and + # aggregation pagination. Not tested directly; tests drive the `Query` interface instead. + # + # Our pagination support is designed to support Facebook's Relay Cursor Connections Spec. + # The description of the pagination algorithm is directly implemented by this class: + # + # https://facebook.github.io/relay/graphql/connections.htm#sec-Pagination-algorithm + # + # As described by the spec, we support 4 pagination arguments, and apply them in this order: + # + # - `after`: items with a cursor value on or before this value are excluded + # - `before`: items with a cursor value on or after this value are excluded + # - `first`: after applying before/after, all but the first `N` items are excluded + # - `last`: after applying before/after/first, all but the last `N` items are excluded + # + # Note that `first` is applied before `last`, meaning that when both are provided (as in + # `first: 10, last: 4`) it is interpreted as "the last 4 of the first 10". However, the Relay + # spec itself discourages clients from passing both, but servers must still support it: + # + # > Including a value for both first and last is strongly discouraged, as it is likely to lead + # > to confusing queries and results. + # + # For document pagination, the relay semantics are implemented on top of Elasticsearch/OpenSearch's `search_after` feature: + # + # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-search-after.html + # + # For aggregation pagination, the relay semantics are implemented on top of the composite aggregation + # pagination feature: + # + # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/search-aggregations-bucket-composite-aggregation.html#_pagination + # + # In either case, the `search_after` (or `after`) argument is directly analogous to Relay's `after`. + # To support the full Relay spec, we have to do some additional clever things: + # + # - When necessary (such as for `last: 50, before: some_cursor`), we have to _reverse_ the + # sort, perform the query with a size of `last`, and then reverse the returned items + # to the originally requested order. + # - In some cases, we have to apply `after`, `before` or `last` as a post-processing step + # to the items returned by the datastore. + # + # Note, however, that the sort key data type used for these two cases is a bit different: + # + # - For document pagination, `search_after` is a list of scalar values, corresponding to the order + # of `sort` clauses. That is, if we are sorting on `amount` ascending and `createdAt` descending, + # then the `search_after` value (and the `sort` value of each document) will be an + # `[amount, createdAt]` tuple. + # - For aggregation pagination, `after` (and the `key` of each aggregation bucket is an unordered + # hash of sort values. The sort field order is instead implied by the composite aggregation + # `sources`. + class Paginator < Support::MemoizableData.define(:default_page_size, :max_page_size, :first, :after, :last, :before, :schema_element_names) + # These methods are provided by `Data.define`: + # @dynamic default_page_size, max_page_size, first, after, last, before, schema_element_names, initialize + + def requested_page_size + # `+ 1` so we can tell if there are more docs for `has_next_page`/`has_previous_page` + # ...but only if we need to get anything at all. + (desired_page_size == 0) ? 0 : desired_page_size + 1 + end + + # Indicates if we need to search in reverse or not in order to satisfy the Relay pagination args. + # If searching in reverse is necessary, `process_items_and_build_page_info` will take care of + # reversing the reversed results back to their original order. + def search_in_reverse? + # If `first` has been provided then we _must not_ search in reverse. + # The relay spec requires us to apply `first` before `last`, and searching + # in reverse would prevent us from being able to return the first `N`. + return false if first_n + + # If we do not have to return the first N results, we are free to search in + # reverse if needed. Either `last` or `before` requires it. + last_n || before + end + + # The cursor values to search after (if we need to search after one at all). + def search_after + search_in_reverse? ? before : after + end + + # In some cases, we're forced to search in reverse; in those caes, this is used to restore + # the ordering of the items to the intended order. + def restore_intended_item_order(items) + search_in_reverse? ? items.reverse : items + end + + # Used for post-processing a list of items from a search result, truncating the list as needed. Truncation + # may be necessary because we may request an extra item as part of our pagination implementation. + def truncate_items(items) + # Remove the extra doc we requested by doing `size: size + 1`, if an extra was returned. + # Removing the first or last doc (as this will do) will signal to `bulid_page_info` + # that there definitely is a previous or next page. + # Note: we use `to_a` to satisfy steep, since `Array#[]` can return `nil`--but with the arg + # we pass, never does when items is non-empty, which our conditional enforces here. + items = items[search_in_reverse? ? 1..-1 : 0...-1].to_a if items.size > desired_page_size + + # We can't always use `before` and `after` in the datastore query (such as when both are provided!), + # so here we drop items from the start that come on or before `after`, and items from the + # end that come on or after `before`. + if (after_cursor = after) + items = items.drop_while do |doc| + item_sort_values_satisfy?(yield(doc, after_cursor), :<=) + end + end + + if (before_cursor = before) + items = items.take_while do |doc| + item_sort_values_satisfy?(yield(doc, before_cursor), :<) + end + end + + # We are not always able to use `last` as the query `size` (such as when `first` is also provided) + # so here we apply `last`. If it has already been used this line will be a no-op. + items = (_ = items).last(last_n.to_i) if last_n + items + end + + def paginated_from_singleton_cursor? + before == DecodedCursor::SINGLETON || after == DecodedCursor::SINGLETON + end + + def desired_page_size + # The relay spec requires us to apply `first` before `last`, but if neither + # is provided we fall back to `default_page_size`. + @desired_page_size ||= [first_n || last_n || default_page_size, max_page_size].min.to_i + end + + private + + def first_n + @first_n ||= size_arg_value(:first, first) + end + + def last_n + @last_n ||= size_arg_value(:last, last) + end + + def size_arg_value(arg_name, value) + if value && value < 0 + raise ::GraphQL::ExecutionError, "`#{schema_element_names.public_send(arg_name)}` cannot be negative, but is #{value}." + else + value + end + end + + # A bit like `Array#<=>`, but understands ascending vs descending sorts. + # We can't simply use doc_sort_values <=> cursor_sort_values` because our + # sort might mix ascending and descending sorts. So, we have to go value-by-value + # and compare each. + def item_sort_values_satisfy?(sort_values, comparison_operator) + if (first_unequal_sort_value = sort_values.find(&:unequal?)) + # Since each subsequent sort field is a tie breaker that only gets used if two documents + # have the same values for all the prior sort fields, as soon as we find a sort value that + # is unequal we can just do the comparison based on it. + first_unequal_sort_value.item_satisfies_compared_to_cursor?(comparison_operator) + else + # The doc values and cursor values are all exactly equal. Return true or false on + # the basis of whether or not the comparison operator allows exact equality. + comparison_operator == :<= || comparison_operator == :>= + end + end + + SortValue = ::Data.define(:from_item, :from_cursor, :sort_direction) do + # @implements SortValue + def unequal? + from_item != from_cursor + end + + def item_satisfies_compared_to_cursor?(comparison_operator) + if from_item.nil? + # nil values sort first when sorting ascending, and last when sorting descending. + # (see `DocumentPaginator#sort` for a more thorough explanation). + sort_direction == :asc + elsif from_cursor.nil? + # nil values sort first when sorting ascending, and last when sorting descending. + # (see `DocumentPaginator#sort` for a more thorough explanation). + sort_direction == :desc + else # both `from_item` and `from_cursor` are non-nil, and can be compared. + result = from_item.public_send(comparison_operator, from_cursor) + (sort_direction == :asc) ? result : !result + end + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query/routing_picker.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query/routing_picker.rb new file mode 100644 index 00000000..67eb09c8 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query/routing_picker.rb @@ -0,0 +1,239 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/filtering/filter_value_set_extractor" + +module ElasticGraph + class GraphQL + class DatastoreQuery + # Responsible for picking routing values for a specific query based on the filters. + class RoutingPicker + def initialize(schema_names:) + # @type var all_values_set: _RoutingValueSet + all_values_set = RoutingValueSet::ALL + + @filter_value_set_extractor = Filtering::FilterValueSetExtractor.new(schema_names, all_values_set) do |operator, filter_value| + if operator == :equal_to_any_of + # This calls `.compact` to remove `nil` filter_value values + RoutingValueSet.of(filter_value.compact) + else # gt, lt, gte, lte, matches + # With one of these inexact/inequality operators, we don't have a way to precisely represent + # the set of values. Instead, we represent it with the special UnboundedWithExclusions + # implementation since when these operators are used the set is unbounded (there's an infinite + # number of values in the set) but it doesn't contain all values (it has some exclusions). + RoutingValueSet::UnboundedWithExclusions + end + end + end + + # Given a list of `filter_hashes` and a list of `routing_field_paths`, returns a list of + # routing values that can safely be used to limit what index shards we search + # without risking missing any matching documents that could exist on other shards. + # + # If an eligible list of routing values cannot be determined, returns `nil`. + # + # Importantly, we have to be careful to not return routing values unless we are 100% sure + # that the set of values will route to the full set of shards on which documents matching + # the filters could live. If a document matching the filters lived on a shard that our + # search does not route to, it will not be included in the search response. + # + # Essentially, this method guarantees that the following pseudo code is always satisfied: + # + # ``` ruby + # if (routing_values = extract_eligible_routing_values(filter_hashes, routing_field_paths)) + # Datastore.all_documents_matching(filter_hashes).each do |document| + # routing_field_paths.each do |field_path| + # expect(routing_values).to include(document.value_at(field_path)) + # end + # end + # end + # ``` + def extract_eligible_routing_values(filter_hashes, routing_field_paths) + @filter_value_set_extractor.extract_filter_value_set(filter_hashes, routing_field_paths).to_return_value + end + end + + class RoutingValueSet < Data.define(:type, :routing_values) + # @dynamic == + + def self.of(routing_values) + new(:inclusive, routing_values.to_set) + end + + def self.of_all_except(routing_values) + new(:exclusive, routing_values.to_set) + end + + ALL = of_all_except([]) + + def intersection(other_set) + # Here we return `self` to preserve the commutative property of `intersection`. Returning `self` + # here matches the behavior of `UnboundedWithExclusions.intersection`. See the comment there for + # rationale. + return self if other_set == UnboundedWithExclusions + + # @type var other: RoutingValueSet + other = _ = other_set + + if inclusive? && other.inclusive? + # Since both sets are inclusive, we can just delegate to `Set#intersection` here. + RoutingValueSet.of(routing_values.intersection(other.routing_values)) + elsif exclusive? && other.exclusive? + # Since both sets are exclusive, we need to return an exclusive set of the union of the + # excluded values. For example, when dealing with positive integers: + # + # s1 = RoutingValueSet.of_all_except([1, 2, 3]) # > 3 + # s2 = RoutingValueSet.of_all_except([3, 4, 5]) # 1, 2, > 5 + # + # s3 = s1.intersection(s2) + # + # Here s3 would be all values > 5 (the same as `RoutingValueSet.of_all_except([1, 2, 3, 4, 5])`) + RoutingValueSet.of_all_except(routing_values.union(other.routing_values)) + else + # Since one set is inclusive and one set is exclusive, we need to return an inclusive set of + # `included_values - excluded_values`. For example, when dealing with positive integers: + # + # s1 = RoutingValueSet.of([1, 2, 3]) # 1, 2, 3 + # s2 = RoutingValueSet.of_all_except([3, 4, 5]) # 1, 2, > 5 + # + # s3 = s1.intersection(s2) + # + # Here s3 would be just `1, 2`. + included_values, excluded_values = get_included_and_excluded_values(other) + RoutingValueSet.of(included_values - excluded_values) + end + end + + def union(other_set) + # Here we return `other` to preserve the commutative property of `union`. Returning `other` + # here matches the behavior of `UnboundedWithExclusions.union`. See the comment there for + # rationale. + return other_set if other_set == UnboundedWithExclusions + + # @type var other: RoutingValueSet + other = _ = other_set + + if inclusive? && other.inclusive? + # Since both sets are inclusive, we can just delegate to `Set#union` here. + RoutingValueSet.of(routing_values.union(other.routing_values)) + elsif exclusive? && other.exclusive? + # Since both sets are exclusive, we need to return an exclusive set of the intersection of the + # excluded values. For example, when dealing with positive integers: + # + # s1 = RoutingValueSet.of_all_except([1, 2, 3]) # > 3 + # s2 = RoutingValueSet.of_all_except([3, 4, 5]) # 1, 2, > 5 + # + # s3 = s1.union(s2) + # + # Here s3 would be all 1, 2, > 3 (the same as `RoutingValueSet.of_all_except([3])`) + RoutingValueSet.of_all_except(routing_values.intersection(other.routing_values)) + else + # Since one set is inclusive and one set is exclusive, we need to return an exclusive set of + # `excluded_values - included_values`. For example, when dealing with positive integers: + # + # s1 = RoutingValueSet.of([1, 2, 3]) # 1, 2, 3 + # s2 = RoutingValueSet.of_all_except([3, 4, 5]) # 1, 2, > 5 + # + # s3 = s1.union(s2) + # + # Here s3 would be 1, 2, 3, > 5 (the same as `RoutingValueSet.of_all_except([4, 5])`) + included_values, excluded_values = get_included_and_excluded_values(other) + RoutingValueSet.of_all_except(excluded_values - included_values) + end + end + + def negate + with(type: INVERTED_TYPES.fetch(type)) + end + + INVERTED_TYPES = {inclusive: :exclusive, exclusive: :inclusive} + + def to_return_value + # Elasticsearch/OpenSearch have no routing value syntax to tell it to avoid searching a specific shard + # (and the fact that we are excluding a routing value doesn't mean that other documents that + # live on the same shard with different routing values can't match!) so we return `nil` to + # force the datastore to search all shards. + return nil if exclusive? + + routing_values.to_a + end + + protected + + def inclusive? + type == :inclusive + end + + def exclusive? + type == :exclusive + end + + private + + def get_included_and_excluded_values(other) + inclusive? ? [routing_values, other.routing_values] : [other.routing_values, routing_values] + end + + # This `RoutingValueSet` implementation is used for otherwise unrepresentable cases. We use it when + # a filter on one of the `routing_field_paths` uses an inequality like: + # + # {routing_field: {gt: "abc"}} + # + # In a case like that, the set is unbounded (there's an infinite number of values that are greater + # than `"abc"`...), but it's not `RoutingValueSet::ALL`--since it's based on an inequality, there are + # _some_ values that are excluded from the set. But we can't use `RoutingValueSet.of_all_except(...)` + # because the set of exclusions is also unbounded! + # + # When our filter value extraction results in this set, we must search all shards of the index and + # cannot pass any `routing` value to the datastore at all. + module UnboundedWithExclusions + # @dynamic self.== + + def self.intersection(other) + # Technically, the "true" intersection would be `other - values_of(self)` but as we don't have + # any known values from this unbounded set, we just return `other`. It's OK to include extra values + # in the set (we'll search additional shards) but not OK to fail to include necessary values in + # the set (we'd avoid searching a shard that may have matching documents) so we err on the side of + # including more values. + other + end + + def self.union(other) + # Since our set here is unbounded, the resulting union is also unbounded. This errs on the side + # of safety since this set's `to_return_value` returns `nil` to cause the datastore to search + # all shards. + self + end + + def self.negate + # This here is the only difference in behavior of this set implementation vs `RoutingValueSet::ALL`. + # Where as `ALL.negate` returns an empty set, we treat `negate` as a no-op. We do that because the + # negation of an inexact unbounded set is still an inexact unbounded set. While it flips which values + # are in or out of the set, this object is still the representation in our datamodel for that case. + self + end + + def self.to_return_value + # Here we return `nil` to make sure that the datastore searches all shards, since we don't have + # any information we can use to safely limit what shards it searches. + nil + end + end + end + + # `Query::RoutingPicker` exists only for use by `Query` and is effectively private. + private_constant :RoutingPicker + # `RoutingValueSet` exists only for use here and is effectively private. + private_constant :RoutingValueSet + + # Steep is complaining that it can't find some `Query` but they are not in this file... + # @dynamic aggregations, shard_routing_values, search_index_definitions, merge_with, search_index_expression + # @dynamic with, to_datastore_msearch_header_and_body, document_paginator + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_response/document.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_response/document.rb new file mode 100644 index 00000000..38fa1799 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_response/document.rb @@ -0,0 +1,78 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/decoded_cursor" +require "elastic_graph/support/memoizable_data" +require "forwardable" + +module ElasticGraph + class GraphQL + module DatastoreResponse + # Represents a document fetched from the datastore. Exposes both the raw metadata + # provided by the datastore and the doc payload itself. In addition, you can treat + # it just like a document hash using `#[]` or `#fetch`. + Document = Support::MemoizableData.define(:raw_data, :payload, :decoded_cursor_factory) do + # @implements Document + extend Forwardable + def_delegators :payload, :[], :fetch + + def self.build(raw_data, decoded_cursor_factory: DecodedCursor::Factory::Null) + source = raw_data.fetch("_source") do + {} # : ::Hash[::String, untyped] + end + + new( + raw_data: raw_data, + # Since we no longer fetch _source for id only queries, merge id into _source to take care of that case + payload: source.merge("id" => raw_data["_id"]), + decoded_cursor_factory: decoded_cursor_factory + ) + end + + def self.with_payload(payload) + build({"_source" => payload}) + end + + def index_name + raw_data["_index"] + end + + def index_definition_name + index_name.split(ROLLOVER_INDEX_INFIX_MARKER).first # : ::String + end + + def id + raw_data["_id"] + end + + def sort + raw_data["sort"] + end + + def version + payload["version"] + end + + def cursor + @cursor ||= decoded_cursor_factory.build(raw_data.fetch("sort")) + end + + def datastore_path + # Path based on this API: + # https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html + "/#{index_name}/_doc/#{id}".squeeze("/") + end + + def to_s + "#<#{self.class.name} #{datastore_path}>" + end + alias_method :inspect, :to_s + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_response/search_response.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_response/search_response.rb new file mode 100644 index 00000000..f1322d64 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_response/search_response.rb @@ -0,0 +1,79 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/graphql/decoded_cursor" +require "elastic_graph/graphql/datastore_response/document" +require "forwardable" + +module ElasticGraph + class GraphQL + module DatastoreResponse + # Represents a search response from the datastore. Exposes both the raw metadata + # provided by the datastore and the collection of documents. Can be treated as a + # collection of documents when you don't care about the metadata. + class SearchResponse < ::Data.define(:raw_data, :metadata, :documents, :total_document_count) + include Enumerable + extend Forwardable + + def_delegators :documents, :each, :to_a, :size, :empty? + + EXCLUDED_METADATA_KEYS = %w[hits aggregations].freeze + + def self.build(raw_data, decoded_cursor_factory: DecodedCursor::Factory::Null) + documents = raw_data.fetch("hits").fetch("hits").map do |doc| + Document.build(doc, decoded_cursor_factory: decoded_cursor_factory) + end + + metadata = raw_data.except(*EXCLUDED_METADATA_KEYS) + metadata["hits"] = raw_data.fetch("hits").except("hits") + + # `hits.total` is exposed as an object like: + # + # { + # "value" => 200, + # "relation" => "eq", # or "gte" + # } + # + # This allows it to provide a lower bound on the number of hits, rather than having + # to give an exact count. We may want to handle the `gte` case differently at some + # point but for now we just use the value as-is. + # + # In the case where `track_total_hits` flag is set to `false`, `hits.total` field will be completely absent. + # This means the client intentionally chose not to query the total doc count, and `total_document_count` will be nil. + # In this case, we will throw an exception if the client later tries to access `total_document_count`. + total_document_count = metadata.dig("hits", "total", "value") + + new( + raw_data: raw_data, + metadata: metadata, + documents: documents, + total_document_count: total_document_count + ) + end + + # Benign empty response that can be used in place of datastore response errors as needed. + RAW_EMPTY = {"hits" => {"hits" => [], "total" => {"value" => 0}}}.freeze + EMPTY = build(RAW_EMPTY) + + def docs_description + (documents.size < 3) ? documents.inspect : "[#{documents.first}, ..., #{documents.last}]" + end + + def total_document_count + super || raise(Errors::CountUnavailableError, "#{__method__} is unavailable; set `query.total_document_count_needed = true` to make it available") + end + + def to_s + "#<#{self.class.name} size=#{documents.size} #{docs_description}>" + end + alias_method :inspect, :to_s + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_search_router.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_search_router.rb new file mode 100644 index 00000000..8d36b7ff --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/datastore_search_router.rb @@ -0,0 +1,151 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/errors" +require "elastic_graph/graphql/datastore_response/search_response" +require "elastic_graph/graphql/query_details_tracker" +require "elastic_graph/support/threading" + +module ElasticGraph + class GraphQL + # Responsible for routing datastore search requests to the appropriate cluster and index. + class DatastoreSearchRouter + def initialize( + datastore_clients_by_name:, + logger:, + monotonic_clock:, + config: + ) + @datastore_clients_by_name = datastore_clients_by_name + @logger = logger + @monotonic_clock = monotonic_clock + @config = config + end + + # Sends the datastore a multi-search request based on the given queries. + # Returns a hash of responses keyed by the query. + def msearch(queries, query_tracker: QueryDetailsTracker.empty) + DatastoreQuery.perform(queries) do |header_body_tuples_by_query| + # Here we set a client-side timeout, which causes the client to give up and close the connection. + # According to [1]--"We have a new way to cancel search requests efficiently from the client + # in 7.4 (by closing the underlying http channel)"--this should cause the server to stop + # executing the search, and more importantly, gives us a strictly enforced timeout. + # + # In addition, the datastore supports a `timeout` option on a search body, but this timeout is + # "best effort", applies to each shard (and not to the overall search request), and only interrupts + # certain kinds of operations. [2] and [3] below have more info. + # + # Note that I have not been able to observe this `timeout` on a search body ever working + # as documented. In our test suite, none of the slow queries I have tried (both via + # slow aggregation query and a slow script) have ever aborted early when that option is + # set. In Kibana in production, @bsorbo observed it aborting a `search` request early + # (but not necessarily an `msearch` request...), but even then, the response said `timed_out: false`! + # Other people ([4]) have reported observing timeout having no effect on msearch requests. + # + # So, the client-side timeout is the main one we want here, and for now we are not using the + # datastore search `timeout` option at all. + # + # For more info, see: + # + # [1] https://github.com/elastic/elasticsearch/issues/47716 + # [2] https://github.com/elastic/elasticsearch/pull/51858 + # [3] https://www.elastic.co/guide/en/elasticsearch/guide/current/_search_options.html#_timeout_2 + # [4] https://discuss.elastic.co/t/timeouts-ignored-in-multisearch/23673 + + # Unfortunately, the Elasticsearch/OpenSearch clients don't support setting a per-request client-side timeout, + # even though Faraday (the underlying HTTP client) does. To work around this, we pass our desired + # timeout in a specific header that the `SupportTimeouts` Faraday middleware will use. + headers = {TIMEOUT_MS_HEADER => msearch_request_timeout_from(queries)&.to_s}.compact + + queries_and_header_body_tuples_by_datastore_client = header_body_tuples_by_query.group_by do |(query, header_body_tuples)| + @datastore_clients_by_name.fetch(query.cluster_name) + end + + datastore_query_started_at = @monotonic_clock.now_in_ms + + server_took_and_results = Support::Threading.parallel_map(queries_and_header_body_tuples_by_datastore_client) do |datastore_client, query_and_header_body_tuples_for_cluster| + queries_for_cluster, header_body_tuples = query_and_header_body_tuples_for_cluster.transpose + msearch_body = header_body_tuples.flatten(1) + response = datastore_client.msearch(body: msearch_body, headers: headers) + debug_query(query: msearch_body, response: response) + ordered_responses = response.fetch("responses") + [response["took"], queries_for_cluster.zip(ordered_responses)] + end + + query_tracker.record_datastore_query_duration_ms( + client: @monotonic_clock.now_in_ms - datastore_query_started_at, + server: server_took_and_results.map(&:first).compact.max + ) + + server_took_and_results.flat_map(&:last).to_h.tap do |responses_by_query| + log_shard_failure_if_necessary(responses_by_query) + raise_search_failed_if_any_failures(responses_by_query) + end + end + end + + private + + # Prefix tests with `DEBUG_QUERY=1 ...` or run `export DEBUG_QUERY=1` to print the actual + # Elasticsearch/OpenSearch query and response. This is particularly useful for adding new specs. + def debug_query(**debug_messages) + return unless ::ENV["DEBUG_QUERY"] + + formatted_messages = debug_messages.map do |key, msg| + "#{key.to_s.upcase}:\n#{::JSON.pretty_generate(msg)}\n" + end.join("\n") + puts "\n#{formatted_messages}\n\n" + end + + def msearch_request_timeout_from(queries) + return nil unless (min_query_deadline = queries.map(&:monotonic_clock_deadline).compact.min) + + (min_query_deadline - @monotonic_clock.now_in_ms).tap do |timeout| + if timeout <= 0 + raise Errors::RequestExceededDeadlineError, "It is already #{timeout.abs} ms past the search deadline." + end + end + end + + def raise_search_failed_if_any_failures(responses_by_query) + failures = responses_by_query.each_with_index.select { |(_query, response), _index| response["error"] } + return if failures.empty? + + formatted_failures = failures.map do |(query, response), index| + # Note: we intentionally omit the body of the request here, because it could contain PII + # or other sensitive values that we don't want logged. + <<~ERROR + #{index + 1}) Header: #{::JSON.generate(query.to_datastore_msearch_header)} + #{response.fetch("error").inspect}" + On cluster: #{query.cluster_name} + ERROR + end.join("\n\n") + + raise Errors::SearchFailedError, "Got #{failures.size} search failure(s):\n\n#{formatted_failures}" + end + + # Examine successful query responses and log any shard failure they encounter + def log_shard_failure_if_necessary(responses_by_query) + shard_failures = responses_by_query.each_with_index.select do |(query, response), query_numeric_index| + (200..299).cover?(response["status"]) && response["_shards"]["failed"] != 0 + end + + unless shard_failures.empty? + formatted_failures = shard_failures.map do |(query, response), query_numeric_index| + "Query #{query_numeric_index + 1} against index `#{query.search_index_expression}` on cluster `#{query.cluster_name}`}: " + + JSON.pretty_generate(response["_shards"]) + end.join("\n\n") + + formatted_shard_failures = "The following queries have failed shards: \n\n#{formatted_failures}" + @logger.warn(formatted_shard_failures) + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/decoded_cursor.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/decoded_cursor.rb new file mode 100644 index 00000000..3f4344ee --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/decoded_cursor.rb @@ -0,0 +1,120 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "base64" +require "elastic_graph/constants" +require "elastic_graph/errors" +require "elastic_graph/support/memoizable_data" +require "json" + +module ElasticGraph + class GraphQL + # Provides the in-memory representation of a cursor after it has been decoded, as a simple hash of sort values. + # + # The datastore's `search_after` pagination uses an array of values (which represent values of the fields you are + # sorting by). A cursor returned when we applied one sort is generally not valid when we apply a completely + # different sort. To ensure we can detect this, the encoder encodes a hash of sort fields and values, ensuring + # each value in the cursor is properly labeled with what field it came from. This allows us + # to detect situations where the client uses a cursor with a completely different sort applied, while + # allowing some minor variation in the sort. The following are still allowed: + # + # - Changing the direction of the sort (from `asc` to `desc` or vice-versa) + # - Re-ordering the sort (e.g. changing from `[amount_money_DESC, created_at_ASC]` + # to `[created_at_ASC, amount_money_DESC]` + # - Removing fields from the sort (e.g. changing from `[amount_money_DESC, created_at_ASC]` + # to `[amount_money_DESC]`) -- but adding fields is not allowed + # + # While we don't necessarily recommend clients change these things between pagination requests (the + # behavior may be surprising to the user), there is no ambiguity in how to support them, and we do not + # feel like it makes sense to restrict it at this point. + class DecodedCursor < Support::MemoizableData.define(:sort_values) + # Methods provided by `MemoizableData.define`: + # @dynamic initialize, sort_values + + # Tries to decode the given string cursor, returning `nil` if it is invalid. + def self.try_decode(string) + decode!(string) + rescue Errors::InvalidCursorError + nil + end + + # Tries to decode the given string cursor, raising an `Errors::InvalidCursorError` if it's invalid. + def self.decode!(string) + return SINGLETON if string == SINGLETON_CURSOR + json = ::Base64.urlsafe_decode64(string) + new(::JSON.parse(json)) + rescue ::ArgumentError, ::JSON::ParserError + raise Errors::InvalidCursorError, "`#{string}` is an invalid cursor." + end + + # Encodes the cursor to a string using JSON and Base64 encoding. + def encode + @encode ||= begin + json = ::JSON.fast_generate(sort_values) + ::Base64.urlsafe_encode64(json, padding: false) + end + end + + # A special cursor instance for when we need a cursor but have only a static collection of a single + # element without any sort of key we can encode. + SINGLETON = new({}).tap do |sc| + # Ensure the special string value is returned even though our `sort_values` are empty. + def sc.encode + SINGLETON_CURSOR + end + end + + # Used to build decoded cursor values for the given `sort_fields`. + class Factory < Data.define(:sort_fields) + # Methods provided by `Data.define`: + # @dynamic initialize, sort_fields + + # Builds a factory from a list like: + # `[{ 'amount_money.amount' => 'asc' }, { 'created_at' => 'desc' }]`. + def self.from_sort_list(sort_list) + sort_fields = sort_list.map do |hash| + if hash.values.any? { |v| !v.is_a?(::Hash) } || hash.values.flat_map(&:keys) != ["order"] + raise Errors::InvalidSortFieldsError, + "Given `sort_list` contained an invalid entry. Each must be a flat hash with one entry. Got: #{sort_list.inspect}" + end + + # Steep thinks it could be `nil` because `hash.keys` could be empty, but we raise an error above in + # that case, so we know this will wind up being a `String`. `_` here silences Steep's type check error. + _ = hash.keys.first + end + + if sort_fields.uniq.size < sort_fields.size + raise Errors::InvalidSortFieldsError, + "Given `sort_list` contains a duplicate field, which the CursorEncoder cannot handler. " \ + "The caller is responsible for de-duplicating the sort list fist. Got: #{sort_list.inspect}" + end + + new(sort_fields) + end + + def build(sort_values) + unless sort_values.size == sort_fields.size + raise Errors::CursorEncodingError, + "size of sort values (#{sort_values.inspect}) does not match the " \ + "size of sort fields (#{sort_fields.inspect})" + end + + DecodedCursor.new(sort_fields.zip(sort_values).to_h) + end + + alias_method :to_s, :inspect + + module Null + def self.build(sort_values) + DecodedCursor.new(sort_values.map(&:to_s).zip(sort_values).to_h) + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/boolean_query.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/boolean_query.rb new file mode 100644 index 00000000..39bad74e --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/boolean_query.rb @@ -0,0 +1,45 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + module Filtering + # BooleanQuery is an internal class for composing a datastore query: + # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-bool-query.html + # + # It is composed of: + # 1) The occurrence type (:must, :filter, :should, or :must_not) + # 2) A list of query clauses evaluated by the given occurrence type + # 3) An optional flag indicating whether the occurrence should be negated + class BooleanQuery < ::Data.define(:occurrence, :clauses) + def self.must(*clauses) + new(:must, clauses) + end + + def self.filter(*clauses) + new(:filter, clauses) + end + + def self.should(*clauses) + new(:should, clauses) + end + + def merge_into(bool_node) + bool_node[occurrence].concat(clauses) + end + + # For `any_of: []` we need a way to force the datastore to match no documents, but + # I haven't found any sort of literal `false` we can pass in the compound expression + # or even a literal `1 = 0` as is sometimes used in SQL. Instead, we use this for that + # case. + empty_array = [] # : ::Array[untyped] + ALWAYS_FALSE_FILTER = filter({ids: {values: empty_array}}) + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/field_path.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/field_path.rb new file mode 100644 index 00000000..6fa84dfd --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/field_path.rb @@ -0,0 +1,81 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" + +module ElasticGraph + class GraphQL + module Filtering + # Tracks state related to field paths as we traverse our filtering data structure in order to translate + # it to its Elasticsearch/OpenSearch form. + # + # Instances of this class are immutable--callers must use the provided APIs (`+`, `counts_path`, `nested`) + # to get back new instances with state changes applied. + FieldPath = ::Data.define( + # The path from the overall document root. + :from_root, + # The path from the current parent document. Usually `from_parent` and `from_root` are the same, + # but they'll be different when we encounter a list field indexed using the `nested` mapping type. + # When we're traversing a subfield of a `nested` field, `from_root` will contain the full path from + # the original, overall document root, while `from_parent` will contain the path from the current + # nested document's root. + :from_parent + ) do + # @implements FieldPath + + # Builds an empty instance. + def self.empty + new([], []) + end + + def self.of(parts) + new(parts, parts) + end + + # Used when we encounter a `nested` field to restart the `from_parent` path (while preserving the `from_root` path). + def nested + FieldPath.new(from_root, []) + end + + # Creates a new instance with `sub_path` appended. + def +(other) + FieldPath.new(from_root + [other], from_parent + [other]) + end + + # Converts the current paths to what they need to be to be able to query our hidden `__counts` field (which + # is a map containing the counts of elements of every list field on the document). The `__counts` field + # sits a the root of every document (for both an overall root document and a `nested` document). Here's an + # example (which assumes `seasons` and `seasons.players` fields which are both `nested` and an `awards` field + # which is a list of strings). Given a filter like this: + # + # filter: {seasons: {any_satisfy: {players: {any_satisfy: {results: {awards: {count: {gt: 1}}}}}}}} + # + # ...after processing the `awards` key, our `FieldPath` will be: + # + # FieldPath.new(["seasons", "players", "results", "awards"], ["results", "awards"]) + # + # When we then reach the `count` sub field and `counts_path` is called on it, the following will be returned: + # + # FieldPath.new(["seasons", "players", LIST_COUNTS_FIELD, "results|awards"], [LIST_COUNTS_FIELD, "results|awards"]) + # + # This gives us what we want: + # - The path from the root is `seasons.players.__counts.results|awards`. + # - The path from the (nested) parent is `__counts.results|awards`. + # + # Note that our `__counts` field is a flat map which uses `|` (the `LIST_COUNTS_FIELD_PATH_KEY_SEPARATOR` character) + # to separate its parts (hence, it's `results|awards` instead of `results.awards`). + def counts_path + from_root_to_parent_of_counts_field = from_root[0...-from_parent.size] # : ::Array[::String] + counts_sub_field = [LIST_COUNTS_FIELD, from_parent.join(LIST_COUNTS_FIELD_PATH_KEY_SEPARATOR)] + + FieldPath.new(from_root_to_parent_of_counts_field + counts_sub_field, counts_sub_field) + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/filter_args_translator.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/filter_args_translator.rb new file mode 100644 index 00000000..9a483eae --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/filter_args_translator.rb @@ -0,0 +1,58 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + module Filtering + # Responsible for translating a `filter` expression from GraphQL field names to the internal + # `name_in_index` of each field. This is necessary so that when a field is defined with + # an alternate `name_in_index`, the query against the index uses that name even while + # the name in the GraphQL schema is different. + # + # In addition, we translate the enum value names to enum value objects, so that any runtime + # metadata associated with that enum value is available to our `FilterInterpreter`. + class FilterArgsTranslator < ::Data.define(:filter_arg_name) + def initialize(schema_element_names:) + super(filter_arg_name: schema_element_names.filter) + end + + # Translates the `filter` expression from the given `args` and `field` into their equivalent + # form using the `name_in_index` for any fields that are named differently in the index + # vs GraphQL. + def translate_filter_args(field:, args:) + return nil unless (filter_hash = args[filter_arg_name]) + filter_type = field.schema.type_from(field.graphql_field.arguments[filter_arg_name].type) + convert(filter_type, filter_hash) + end + + private + + def convert(parent_type, filter_object) + case filter_object + when ::Hash + filter_object.to_h do |key, value| + field = parent_type.field_named(key) + [field.name_in_index.to_s, convert(field.type.unwrap_fully, value)] + end + when ::Array + filter_object.map { |value| convert(parent_type, value) } + when nil + nil + else + if parent_type.enum? + # Replace the name of an enum value with the value itself. + parent_type.enum_value_named(filter_object) + else + filter_object + end + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/filter_interpreter.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/filter_interpreter.rb new file mode 100644 index 00000000..e8dbfa57 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/filter_interpreter.rb @@ -0,0 +1,376 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/filtering/field_path" +require "elastic_graph/graphql/filtering/filter_node_interpreter" +require "elastic_graph/support/graphql_formatter" +require "elastic_graph/support/memoizable_data" +require "graphql" + +module ElasticGraph + class GraphQL + module Filtering + # Responsible for interpreting a query's overall `filter`. Not tested directly; tests drive the `Query` interface instead. + # + # For more info on how this works, see: + # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-bool-query.html + # https://www.elastic.co/blog/lost-in-translation-boolean-operations-and-filters-in-the-bool-query + FilterInterpreter = Support::MemoizableData.define(:filter_node_interpreter, :schema_names, :logger) do + # @implements FilterInterpreter + + def initialize(filter_node_interpreter:, logger:) + super( + filter_node_interpreter: filter_node_interpreter, + schema_names: filter_node_interpreter.schema_names, + logger: logger + ) + end + + # Builds a datastore query from the given collection of filter hashes. + # + # Returns `nil` if there are no query clauses, to make it easy for a caller to `compact` out + # `query: {}` in a larger search request body. + # + # https://www.elastic.co/guide/en/elasticsearch/reference/8.11/query-dsl.html + def build_query(filter_hashes, from_field_path: FieldPath.empty) + build_bool_hash do |bool_node| + filter_hashes.each do |filter_hash| + process_filter_hash(bool_node, filter_hash, from_field_path) + end + end + end + + def to_s + # The inspect/to_s output of `filter_node_interpreter` and `logger` can be quite large and noisy. We generally don't care about + # those details but want to be able to tell at a glance if two `FilterInterpreter` instances are equal or not--and, if they + # aren't equal, which part is responsible for the inequality. + # + # Using the hash of the two initialize args provides us with that. + "#" + end + alias_method :inspect, :to_s + + private + + def process_filter_hash(bool_node, filter_hash, field_path) + filter_hash.each do |field_or_op, expression| + # `nil` filter predicates should be ignored, so we can safely `compact` them out. + # It also is simpler to handle them once here instead of the different branches + # below having to be aware of possible `nil` predicates. + expression = expression.compact if expression.is_a?(::Hash) + + case filter_node_interpreter.identify_node_type(field_or_op, expression) + when :empty + # This is an "empty" filter predicate and we can ignore it. + when :not + process_not_expression(bool_node, expression, field_path) + when :list_any_filter + process_list_any_filter_expression(bool_node, expression, field_path) + when :any_of + process_any_of_expression(bool_node, expression, field_path) + when :all_of + process_all_of_expression(bool_node, expression, field_path) + when :operator + process_operator_expression(bool_node, field_or_op, expression, field_path) + when :list_count + process_list_count_expression(bool_node, expression, field_path) + when :sub_field + process_sub_field_expression(bool_node, expression, field_path + field_or_op) + else + logger.warn("Ignoring unknown filtering operator (#{field_or_op}: #{expression.inspect}) on field `#{field_path.from_root.join(".")}`") + end + end + end + + # Indicates if the given `expression` applies filtering to subfields or just applies + # operators at the current field path. + def filters_on_sub_fields?(expression) + expression.any? do |field_or_op, sub_expression| + case filter_node_interpreter.identify_node_type(field_or_op, sub_expression) + when :sub_field + true + when :not, :list_any_filter + filters_on_sub_fields?(sub_expression) + when :any_of, :all_of + # These are the only two cases where the `sub_expression` is an array of filter sub expressions, + # so we use `.any?` on it here. (Even for `all_of`--the overall `expression` filters on sub fields so + # long as at least one of the sub expressions does, regardless of it being `any_of` vs `all_of`). + sub_expression.any? { |expr| filters_on_sub_fields?(expr) } + else # :empty, :operator, :unknown, :list_count + false + end + end + end + + def process_not_expression(bool_node, expression, field_path) + sub_filter = build_bool_hash do |inner_node| + process_filter_hash(inner_node, expression, field_path) + end + + return unless sub_filter + + # Prevent any negated filters from being unnecessarily double-negated by + # converting them to a positive filter (i.e., !!A == A). + if sub_filter[:bool].key?(:must_not) + # Pull clauses up to current bool_node to remove negation + sub_filter[:bool][:must_not].each do |negated_clause| + negated_clause[:bool].each { |k, v| bool_node[k].concat(v) } + end + end + + # Don't drop any other filters! Let's negate them now. + other_filters = sub_filter[:bool].except(:must_not) + bool_node[:must_not] << {bool: other_filters} unless other_filters.empty? + end + + # There are two cases for `any_satisfy`, each of which is handled differently: + # + # - List-of-scalars + # - List-of-nested-objects + # + # We can detect which it is by checking `filter` to see if it filters on any subfields. + # If so, we know the filter is being applied to a `nested` list field. We can count on + # this because we do not generate `any_satisfy` filters on `object` list fields (instead, + # they get generated on their leaf fields). + def process_list_any_filter_expression(bool_node, filter, field_path) + if filters_on_sub_fields?(filter) + process_any_satisfy_filter_expression_on_nested_object_list(bool_node, filter, field_path) + else + process_any_satisfy_filter_expression_on_scalar_list(bool_node, filter, field_path) + end + end + + def process_any_satisfy_filter_expression_on_nested_object_list(bool_node, filter, field_path) + sub_filter = build_bool_hash do |inner_node| + process_filter_hash(inner_node, filter, field_path.nested) + end + + if sub_filter + bool_node[:filter] << {nested: {path: field_path.from_root.join("."), query: sub_filter}} + end + end + + # On a list-of-leaf-values field, `any_satisfy` doesn't _do_ anything: it just expresses + # the fact that documents with any list element values matching the predicates will match + # the overall filter. + def process_any_satisfy_filter_expression_on_scalar_list(bool_node, filter, field_path) + return unless (processed = build_bool_hash { |node| process_filter_hash(node, filter, field_path) }) + + processed_bool_query = processed.fetch(:bool) + + # The semantics we want for `any_satisfy` are that it matches when a value exists in the list that + # satisfies all of the provided subfilter. That's the semantics the datastore provides when the bool + # query only requires one clause to match, but if multiple clauses are required to match there's a subtle + # issue. A document matches so long as each required clause matches *some* value, but it doesn't require + # that they all match the *same* value. The list field on a document could contain N values, where + # each value matches a different one of the required clauses, and the document will be a search hit. + # + # Rather than behaving in a surprising way here, we'd rather disallow a filter that has multiple required + # clauses, so we return an error in this case. + if required_matching_clause_count(processed_bool_query) > 1 + formatted_filter = Support::GraphQLFormatter.serialize( + {schema_names.any_satisfy => filter}, + wrap_hash_with_braces: false + ) + + raise ::GraphQL::ExecutionError, "`#{formatted_filter}` is not supported because it produces " \ + "multiple filtering clauses under `#{schema_names.any_satisfy}`, which doesn't work as expected. " \ + "Remove one or more of your `#{schema_names.any_satisfy}` predicates and try again." + else + bool_node.update(processed_bool_query) do |_, existing_clauses, any_satisfy_clauses| + existing_clauses + any_satisfy_clauses + end + end + end + + def process_any_of_expression(bool_node, expressions, field_path) + shoulds = expressions.filter_map do |expression| + build_bool_hash do |inner_bool_node| + process_filter_hash(inner_bool_node, expression, field_path) + end + end + + # When our `shoulds` array is empty, the filtering semantics we want is to match no documents. + # However, that's not the behavior the datastore will give us if we have an empty array in the + # query under `should`. To get the behavior we want, we need to pass the datastore some filter + # criteria that will evaluate to false for every document. + bool_query = shoulds.empty? ? BooleanQuery::ALWAYS_FALSE_FILTER : BooleanQuery.should(*shoulds) + bool_query.merge_into(bool_node) + end + + def process_all_of_expression(bool_node, expressions, field_path) + # `all_of` represents an AND. AND is the default way that `process_filter_hash` combines + # filters so we just have to call it for each sub-expression. + expressions.each do |sub_expression| + process_filter_hash(bool_node, sub_expression, field_path) + end + end + + def process_operator_expression(bool_node, operator, expression, field_path) + # `operator` is a filtering operator, and `expression` is the value the filtering + # operator should be applied to. The `op_applicator` lambda, when called, will + # return a Clause instance (defined in this module). + bool_query = filter_node_interpreter.filter_operators.fetch(operator).call(field_path.from_root.join("."), expression) + bool_query&.merge_into(bool_node) + end + + def process_sub_field_expression(bool_node, expression, field_path) + # `sub_field` is a field name, and `expression` is a hash of filters to apply to that field. + # We want to add the field name to the field path and recursively process the hash. + # + # However, if the hash has `any_of` in it, then we need to process the filter hash on + # a nested bool node instead of on the `bool_node` we are already operating on. + # + # To understand why, first consider a filter that has no `any_of` but does use field nesting: + # + # filter: { + # weight: {lt: 2000}, + # cost: { + # currency: {equal_to_any_of: ["USD"]} + # amount: {gt: 1000} + # } + # } + # + # While this `currency` and `amount` are expressed as sub-filters under `cost` in our GraphQL + # syntax, we do not actually need to create a nested bool node structure for the datastore + # query. We get a flat filter structure like this: + # + # {bool: {filter: [ + # {range: {"weight": {lt: 2000}}}, + # {terms: {"cost.currency": ["USD"]}}, + # {range: {"amount": {gt: 1000}}} + # ]}} + # + # The 3 filter conditions are ANDed together as a single list under `filter`. + # The nested field structure gets flattened using a dot-separated path. + # + # Now consider a filter that has multiple `any_of` sub-expressions: + # + # filter: { + # weight: {any_of: [ + # {gt: 9000}, + # {lt: 2000} + # ]}, + # cost: {any_of: [ + # currency: {equal_to_any_of: ["USD"]}, + # amount: {gt: 1000} + # ]} + # } + # + # If we did not make a nested structure, we would wind up with a single list of sub-expressions + # that are OR'd together: + # + # {bool: {filter: [{bool: {should: [ + # {range: {"weight": {gt: 9000}}}, + # {range: {"weight": {lt: 2000}}}, + # {terms: {"cost.currency": ["USD"]}}, + # {range: {"amount": {gt: 1000}}} + # ]}}]}} + # + # ...but that's clearly wrong. By creating a nested bool node based on the presence of `any_of`, + # we can instead produce a structure like this: + # + # {bool: {filter: [ + # {bool: {should: [ + # {range: {"weight": {gt: 9000}}}, + # {range: {"weight": {lt: 2000}}} + # ]}}, + # {bool: {should: [ + # {terms: {"cost.currency": ["USD"]}}, + # {range: {"amount": {gt: 1000}}} + # ]}} + # ]}} + # + # ...which will actually work correctly. + if expression.key?(schema_names.any_of) + sub_filter = build_bool_hash do |inner_node| + process_filter_hash(inner_node, expression, field_path) + end + + bool_node[:filter] << sub_filter if sub_filter + else + process_filter_hash(bool_node, expression, field_path) + end + end + + def process_list_count_expression(bool_node, expression, field_path) + # Normally, we don't have to do anything special for list count expressions. + # That's the case, for example, for an expression like: + # + # filter: {tags: {count: {gt: 2}}} + # + # However, if the count expression could match count of 0 (that is, if it doesn't + # exclude a count of zero), such as this: + # + # filter: {tags: {count: {lt: 1}}} + # + # ...then we need some special handling here. A count of 0 is equivalent to the list field not existing. + # While we index an explicit count of 0, the count field will be missing from documents indexed before + # the list field was defined on the ElasticGraph schema. To properly match those documents, we need to + # convert this into an OR (using `any_of`) to also match documents that lack the field entirely. + unless excludes_zero?(expression) + expression = {schema_names.any_of => [ + expression, + {schema_names.equal_to_any_of => [nil]} + ]} + end + + process_sub_field_expression(bool_node, expression, field_path.counts_path) + end + + def build_bool_hash(&block) + bool_node = Hash.new { |h, k| h[k] = [] }.tap(&block) + + # To ignore "empty" filter predicates we need to return `nil` here. + return nil if bool_node.empty? + + # According to https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-bool-query.html#bool-min-should-match, + # if the bool query includes at least one should clause and no must or filter clauses, the default value is 1. Otherwise, the default value is 0. + # However, we want should clauses to work with musts and filters, so we need to set it explicitly to 1 when we have should clauses. + bool_node[:minimum_should_match] = 1 if bool_node.key?(:should) + + {bool: bool_node} + end + + # Determines if the given filter expression excludes the value `0`. + def excludes_zero?(expression) + expression.any? do |operator, operand| + case operator + when schema_names.equal_to_any_of then !operand.include?(0) + when schema_names.lt then operand <= 0 + when schema_names.lte then operand < 0 + when schema_names.gt then operand >= 0 + when schema_names.gte then operand > 0 + else + # :nocov: -- all operators are covered above. But simplecov complains about an implicit `else` branch being uncovered, so here we've defined it to wrap it with `:nocov:`. + false + # :nocov: + end + end + end + + # Counts how many clauses in `bool_query` are required to match for a document to be a search hit. + def required_matching_clause_count(bool_query) + bool_query.reduce(0) do |count, (occurrence, clauses)| + case occurrence + when :should + # The number of required matching clauses imposed by `:should` depends on the `:minimum_should_match` value. + # https://www.elastic.co/guide/en/elasticsearch/reference/8.9/query-dsl-bool-query.html#bool-min-should-match + bool_query.fetch(:minimum_should_match) + when :minimum_should_match + 0 # doesn't have any clauses on its own, just controls how many `:should` clauses are required. + else + # For all other occurrences, each cluse must match. + clauses.size + end + count + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/filter_node_interpreter.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/filter_node_interpreter.rb new file mode 100644 index 00000000..2883f964 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/filter_node_interpreter.rb @@ -0,0 +1,181 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/graphql/filtering/boolean_query" +require "elastic_graph/graphql/filtering/range_query" +require "elastic_graph/graphql/schema/enum_value" +require "elastic_graph/support/memoizable_data" +require "elastic_graph/support/time_util" + +module ElasticGraph + class GraphQL + module Filtering + # Responsible for interpreting a single `node` in a `filter` expression. + FilterNodeInterpreter = Support::MemoizableData.define(:runtime_metadata, :schema_names) do + # @implements FilterNodeInterpreter + + def initialize(runtime_metadata:) + super(runtime_metadata: runtime_metadata, schema_names: runtime_metadata.schema_element_names) + end + + def identify_node_type(field_or_op, sub_expression) + return :empty if sub_expression.nil? || sub_expression == {} + return :not if field_or_op == schema_names.not + return :list_any_filter if field_or_op == schema_names.any_satisfy + return :all_of if field_or_op == schema_names.all_of + return :any_of if field_or_op == schema_names.any_of + return :operator if filter_operators.key?(field_or_op) + return :list_count if field_or_op == LIST_COUNTS_FIELD + return :sub_field if sub_expression.is_a?(::Hash) + :unknown + end + + def filter_operators + @filter_operators ||= build_filter_operators(runtime_metadata) + end + + private + + def build_filter_operators(runtime_metadata) + filter_by_time_of_day_script_id = runtime_metadata + .static_script_ids_by_scoped_name + .fetch("filter/by_time_of_day") + + { + schema_names.equal_to_any_of => ->(field_name, value) { + values = to_datastore_value(value.compact.uniq) # : ::Array[untyped] + + equality_sub_expression = + if field_name == "id" + # Use specialized "ids" query when querying on ID field. + # See: https://www.elastic.co/guide/en/elasticsearch/reference/7.15/query-dsl-ids-query.html + # + # We reject empty strings because we otherwise get an error from the datastore: + # "failed to create query: Ids can't be empty" + {ids: {values: values - [""]}} + else + {terms: {field_name => values}} + end + + exists_sub_expression = {exists: {"field" => field_name}} + + if !value.empty? && value.all?(&:nil?) + BooleanQuery.new(:must_not, [{bool: {filter: [exists_sub_expression]}}]) + elsif value.include?(nil) + BooleanQuery.filter({bool: { + minimum_should_match: 1, + should: [ + {bool: {filter: [equality_sub_expression]}}, + {bool: {must_not: [{bool: {filter: [exists_sub_expression]}}]}} + ] + }}) + else + BooleanQuery.filter(equality_sub_expression) + end + }, + schema_names.gt => ->(field_name, value) { RangeQuery.new(field_name, :gt, value) }, + schema_names.gte => ->(field_name, value) { RangeQuery.new(field_name, :gte, value) }, + schema_names.lt => ->(field_name, value) { RangeQuery.new(field_name, :lt, value) }, + schema_names.lte => ->(field_name, value) { RangeQuery.new(field_name, :lte, value) }, + schema_names.matches => ->(field_name, value) { BooleanQuery.must({match: {field_name => value}}) }, + schema_names.matches_query => ->(field_name, value) do + allowed_edits_per_term = value.fetch(schema_names.allowed_edits_per_term).runtime_metadata.datastore_abbreviation + + BooleanQuery.must( + { + match: { + field_name => { + query: value.fetch(schema_names.query), + # This is always a string field, even though the value is often an integer + fuzziness: allowed_edits_per_term.to_s, + operator: value[schema_names.require_all_terms] ? "AND" : "OR" + } + } + } + ) + end, + schema_names.matches_phrase => ->(field_name, value) { + BooleanQuery.must( + { + match_phrase_prefix: { + field_name => { + query: value.fetch(schema_names.phrase) + } + } + } + ) + }, + + # This filter operator wraps a geo distance query: + # https://www.elastic.co/guide/en/elasticsearch/reference/7.10/query-dsl-geo-distance-query.html + schema_names.near => ->(field_name, value) do + unit_abbreviation = value.fetch(schema_names.unit).runtime_metadata.datastore_abbreviation + + BooleanQuery.filter({geo_distance: { + "distance" => "#{value.fetch(schema_names.max_distance)}#{unit_abbreviation}", + field_name => { + "lat" => value.fetch(schema_names.latitude), + "lon" => value.fetch(schema_names.longitude) + } + }}) + end, + + schema_names.time_of_day => ->(field_name, value) do + # To filter on time of day, we use the `filter/by_time_of_day` script. We accomplish + # this with a script because Elasticsearch/OpenSearch do not support this natively, and it's + # incredibly hard to implement correctly with respect to time zones without using a + # script. We considered indexing the `time_of_day` as a separate index field + # that we could directly filter on, but since we need the time of day to be relative + # to a specific time zone, there's no way to make that work with the reality of + # daylight savings time. For example, the `America/Los_Angeles` time zone has a -07:00 + # UTC offset for part of the year and a `America/Los_Angeles` -08:00 UTC offset for + # part of the year. In a script we can use Java time zone APIs to handle this correctly. + params = { + field: field_name, + equal_to_any_of: list_of_nanos_of_day_from(value, schema_names.equal_to_any_of), + gt: nano_of_day_from(value, schema_names.gt), + gte: nano_of_day_from(value, schema_names.gte), + lt: nano_of_day_from(value, schema_names.lt), + lte: nano_of_day_from(value, schema_names.lte), + time_zone: value[schema_names.time_zone] + }.compact + + # If there are no comparison operators, return `nil` instead of a `Clause` so that we avoid + # invoking the script for no reason. Note that `field` and `time_zone` will always be in + # `params` so we can't just check for an empty hash here. + if (params.keys - [:field, :time_zone]).any? + BooleanQuery.filter({script: {script: {id: filter_by_time_of_day_script_id, params: params}}}) + end + end + }.freeze + end + + def to_datastore_value(value) + case value + when ::Array + value.map { |v| to_datastore_value(v) } + when Schema::EnumValue + value.name.to_s + else + value + end + end + + def nano_of_day_from(value, field) + local_time = value[field] + Support::TimeUtil.nano_of_day_from_local_time(local_time) if local_time + end + + def list_of_nanos_of_day_from(value, field) + value[field]&.map { |t| Support::TimeUtil.nano_of_day_from_local_time(t) } + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/filter_value_set_extractor.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/filter_value_set_extractor.rb new file mode 100644 index 00000000..649127b7 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/filter_value_set_extractor.rb @@ -0,0 +1,148 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + module Filtering + # Responsible for extracting a set of values from query filters, based on a using a custom + # set type that is able to efficiently model the "all values" case. + class FilterValueSetExtractor + def initialize(schema_names, all_values_set, &build_set_for_filter) + @schema_names = schema_names + @all_values_set = all_values_set + @build_set_for_filter = build_set_for_filter + end + + # Given a list of `filter_hashes` and a list of `target_field_paths`, returns a representation + # of a set that includes all values that could be matched by the given filters. + # + # Essentially, this method guarantees that the following pseudo code is always satisfied: + # + # ``` ruby + # filter_value_set = extract_filter_value_set(filter_hashes, target_field_paths) + # Datastore.all_documents_matching(filter_hashes).each do |document| + # target_field_paths.each do |field_path| + # expect(filter_value_set).to include(document.value_at(field_path)) + # end + # end + # ``` + def extract_filter_value_set(filter_hashes, target_field_paths) + # We union the filter values together in cases where we have multiple target field paths + # to make sure we cover all the values we need to. We generally do not have multiple + # `target_field_paths` except for specialized cases, such as when searching multiple + # indices in one query, where those indices are configured to use differing `routing_field_paths`. + # In such a situation we must use the set union of values. Remember: including additional + # routing values causes no adverse behavior (although it may introduce an inefficiency) + # but if we fail to route to a shard that contains a matching document, the search results + # will be incorrect. + map_reduce_sets(target_field_paths, :union, negate: false) do |target_field_path| + filter_value_set_for_target_field_path(target_field_path, filter_hashes) + end + end + + private + + # Determines a set of filter values for one of our `target_field_paths`, + # based on a list of `filter_hashes`. + def filter_value_set_for_target_field_path(target_field_path, filter_hashes) + # Pre-split the `target_field_path` to make it easy to compare as an array, + # since we build up the `traversed_field_path_parts` as an array as we recurse. We do this here + # outside the `map_reduce_sets` block below so we only do it once instead of N times. + target_field_path_parts = target_field_path.split(".") + + # Here we intersect the filter value setbecause when we have multiple `filter_hashes`, + # the filters are ANDed together. Only documents that match ALL the filters will be + # returned. Therefore, we want the intersection of filter value sets. + map_reduce_sets(filter_hashes, :intersection, negate: false) do |filter_hash| + filter_value_set_for_filter_hash(filter_hash, target_field_path_parts, negate: false) + end + end + + # Determines the set of filter values for one of our `target_field_paths` values and one + # `filter_hash` from a list of filter hashes. Note that this method is called recursively, + # with `traversed_field_path_parts` as an accumulator that accumulates that path to a nested + # field we are filtering on. + def filter_value_set_for_filter_hash(filter_hash, target_field_path_parts, traversed_field_path_parts = [], negate:) + # Here we intersect the filter value sets because when we have multiple entries in a filter hash, + # the filters are ANDed together. Only documents that match ALL the filters will be + # returned. Therefore, we want the intersection of filter value sets. + map_reduce_sets(filter_hash, :intersection, negate: negate) do |key, value| + filter_value_set_for_filter_hash_entry(key, value, target_field_path_parts, traversed_field_path_parts, negate: negate) + end + end + + # Determines the set of filter values for one of our `target_field_paths` and one + # entry from one `filter_hash`. The key/value pair from a single entry is passed as the + # first two arguments. Depending on where we are at in recursing through the nested structure, + # the key could identify either a field we are filtering on or a filtering operator to apply + # to a particular field. + def filter_value_set_for_filter_hash_entry(field_or_op, filter_value, target_field_path_parts, traversed_field_path_parts, negate:) + if filter_value.nil? + # Any filter with a `nil` value is effectively ignored by our filtering logic, so we need + # to return our `@all_values_set` to indicate this filter matches all documents. + @all_values_set + elsif field_or_op == @schema_names.not + filter_value_set_for_filter_hash(filter_value, target_field_path_parts, traversed_field_path_parts, negate: !negate) + elsif filter_value.is_a?(::Hash) + # the only time `value` is a hash is when `field_or_op` is a field name. + # In that case, `value` is a hash of filters that apply to that field. + filter_value_set_for_filter_hash(filter_value, target_field_path_parts, traversed_field_path_parts + [field_or_op], negate: negate) + elsif field_or_op == @schema_names.any_of + filter_value_set_for_any_of(filter_value, target_field_path_parts, traversed_field_path_parts, negate: negate) + elsif target_field_path_parts == traversed_field_path_parts + set = filter_value_set_for_field_filter(field_or_op, filter_value) + negate ? set.negate : set + else + # Otherwise, we have no information in this clause to limit our filter value set. + @all_values_set + end + end + + # Determines the set of filter values for an `any_of` clause, which is used for ORing multiple filters together. + def filter_value_set_for_any_of(filter_hashes, target_field_path_parts, traversed_field_path_parts, negate:) + # Here we union the filter value sets because `any_of` represents an OR. If we can determine specific + # filter values for all `any_of` clauses, we will OR them together. Alternately, if we cannot + # determine specific filter values for any clauses, we will union `@all_values_set`, + # which will result in a return value of `@all_values_set`. This is correct because if there + # is an `any_of` clause that does not match on the `target_field_path_parts` then the filter + # excludes no documents on the basis of the target filter. + map_reduce_sets(filter_hashes, :union, negate: negate) do |filter_hash| + filter_value_set_for_filter_hash(filter_hash, target_field_path_parts, traversed_field_path_parts, negate: negate) + end + end + + # Determines the set of filter values for a single filter on a single field. + def filter_value_set_for_field_filter(filter_op, filter_value) + operator_name = @schema_names.canonical_name_for(filter_op) + @build_set_for_filter.call(operator_name, filter_value) || @all_values_set + end + + # Maps over the provided `collection` by applying the given `map_transform` + # (which must transform a collection entry to an instance of our set representation), then reduces + # the resulting collection to a single set value. `reduction` will be either `:union` or `:intersection`. + # + # If the collection is empty, we return `@all_values_set` because it's the only "safe" value + # we can return. We don't have any information that would allow us to limit the set of filter + # values in any way. + def map_reduce_sets(collection, reduction, negate:, &map_transform) + return @all_values_set if collection.empty? + + # In the case where `negate` is true (`not` is present somewhere in the filtering expression), + # we negate the reduction operator. Utilizing De Morgan’s Law (¬(A ∪ B) <-> (¬A) ∩ (¬B)), + # the negation of the union of two sets is the intersection of the negation of each set (the negation + # of each set is the difference between @all_values_set and the given set)--and vice versa. + reduction = REDUCTION_INVERSIONS.fetch(reduction) if negate + + collection.map(&map_transform).reduce(reduction) + end + + REDUCTION_INVERSIONS = {union: :intersection, intersection: :union} + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/range_query.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/range_query.rb new file mode 100644 index 00000000..09a99cd6 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/filtering/range_query.rb @@ -0,0 +1,56 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/hash_util" + +module ElasticGraph + class GraphQL + module Filtering + # Alternate `BooleanQuery` implementation for range queries. When we get a filter like this: + # + # {some_field: {gt: 10, lt: 100}} + # + # ...we independently build a range query for each predicate. The datastore query structure would look like this: + # + # {filter: [ + # {range: {some_field: {gt: 10}}}, + # {range: {some_field: {lt: 100}}} + # ]} + # + # However, the `range` query allows these be combined, like so: + # + # {filter: [ + # {range: {some_field: {gt: 10, lt: 100}}} + # ]} + # + # While we haven't measured it, it's likely to be more efficient (certainly not _less_ efficient!), + # and it's essential that we combine them when we are using `any_satisfy`. Consider this filter: + # + # {some_field: {any_satisfy: {gt: 10, lt: 100}}} + # + # This should match a document with `some_field: [5, 45, 200]` (since 45 is between 10 and 100), + # and not match a document with `some_field: [5, 200]` (since `some_field` has no value between 10 and 100). + # However, if we keep the range clauses separate, this document would match, because `some_field` has + # a value > 10 and a value < 100 (even though no single value satisfies both parts!). When we combine + # the clauses into a single `range` query then the filtering works like we expect. + class RangeQuery < ::Data.define(:field_name, :operator, :value) + def merge_into(bool_node) + existing_range_index = bool_node[:filter].find_index { |clause| clause.dig(:range, field_name) } + new_range_clause = {range: {field_name => {operator => value}}} + + if existing_range_index + existing_range_clause = bool_node[:filter][existing_range_index] + bool_node[:filter][existing_range_index] = Support::HashUtil.deep_merge(existing_range_clause, new_range_clause) + else + bool_node[:filter] << new_range_clause + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/http_endpoint.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/http_endpoint.rb new file mode 100644 index 00000000..d423ece6 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/http_endpoint.rb @@ -0,0 +1,229 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/client" +require "elastic_graph/support/memoizable_data" +require "json" +require "uri" + +module ElasticGraph + class GraphQL + # Handles HTTP concerns for when ElasticGraph is served via HTTP. The logic here + # is based on the graphql.org recommendations: + # + # https://graphql.org/learn/serving-over-http/#http-methods-headers-and-body + # + # As that recommends, we support queries in 3 different HTTP forms: + # + # - A standard POST request as application/json with query/operationName/variables in the body. + # - A GET request with `query`, `operationName` and `variables` query params in the URL. + # - A POST as application/graphql with a query string in the body. + # + # Note that this is designed to be agnostic to what the calling HTTP context is (for example, + # AWS Lambda, or Rails, or Rack...). Instead, this uses simple Request/Response value objects + # that the calling context can easily translate to/from to use this in any HTTP context. + class HTTPEndpoint + APPLICATION_JSON = "application/json" + APPLICATION_GRAPHQL = "application/graphql" + + def initialize(query_executor:, monotonic_clock:, client_resolver:) + @query_executor = query_executor + @monotonic_clock = monotonic_clock + @client_resolver = client_resolver + end + + # Processes the given HTTP request, returning an HTTP response. + # + # `max_timeout_in_ms` is not a property of the HTTP request (the + # calling application will determine it instead!) so it is a separate argument. + # + # Note that this method does _not_ convert exceptions to 500 responses. It's up to + # the calling application to do that if it wants to (and to determine how much of the + # exception to return in the HTTP response...). + def process(request, max_timeout_in_ms: nil, start_time_in_ms: @monotonic_clock.now_in_ms) + client_or_response = @client_resolver.resolve(request) + return client_or_response if client_or_response.is_a?(HTTPResponse) + + with_parsed_request(request, max_timeout_in_ms: max_timeout_in_ms) do |parsed| + result = @query_executor.execute( + parsed.query_string, + variables: parsed.variables, + operation_name: parsed.operation_name, + client: client_or_response, + timeout_in_ms: parsed.timeout_in_ms, + context: parsed.context, + start_time_in_ms: start_time_in_ms + ) + + HTTPResponse.json(200, result.to_h) + end + rescue Errors::RequestExceededDeadlineError + HTTPResponse.error(504, "Search exceeded requested timeout.") + end + + private + + # Helper method that converts `HTTPRequest` to a parsed form we can work with. + # If the request can be successfully parsed, a `ParsedRequest` will be yielded; + # otherwise an `HTTPResponse` will be returned with an error. + def with_parsed_request(request, max_timeout_in_ms:) + with_request_params(request) do |params| + with_timeout(request, max_timeout_in_ms: max_timeout_in_ms) do |timeout_in_ms| + with_context(request) do |context| + yield ParsedRequest.new( + query_string: params["query"], + variables: params["variables"] || {}, + operation_name: params["operationName"], + timeout_in_ms: timeout_in_ms, + context: context + ) + end + end + end + end + + # Responsible for handling the 3 types of requests we need to handle: + # + # - A standard POST request as application/json with query/operationName/variables in the body. + # - A GET request with `query`, `operationName` and `variables` query params in the URL. + # - A POST as application/graphql with a query string in the body. + # + # This yields a hash containing the query/operationName/variables if successful; otherwise + # it returns an `HTTPResponse` with an error. + def with_request_params(request) + params = + # POST with application/json is the most common form requests take, so we have it as the first branch here. + if request.http_method == :post && request.content_type == APPLICATION_JSON + begin + ::JSON.parse(request.body.to_s) + rescue ::JSON::ParserError + # standard:disable Lint/NoReturnInBeginEndBlocks + return HTTPResponse.error(400, "Request body is invalid JSON.") + # standard:enable Lint/NoReturnInBeginEndBlocks + end + + elsif request.http_method == :post && request.content_type == APPLICATION_GRAPHQL + {"query" => request.body} + + elsif request.http_method == :post + return HTTPResponse.error(415, "`#{request.content_type}` is not a supported content type. Only `#{APPLICATION_JSON}` and `#{APPLICATION_GRAPHQL}` are supported.") + + elsif request.http_method == :get + ::URI.decode_www_form(::URI.parse(request.url).query.to_s).to_h.tap do |hash| + # Variables must come in as JSON, even if in the URL. express-graphql does it this way, + # which is a bit of a canonical implementation, as it is referenced from graphql.org: + # https://github.com/graphql/express-graphql/blob/v0.12.0/src/index.ts#L492-L497 + hash["variables"] &&= ::JSON.parse(hash["variables"]) + rescue ::JSON::ParserError + return HTTPResponse.error(400, "Variables are invalid JSON.") + end + + else + return HTTPResponse.error(405, "GraphQL only supports GET and POST requests.") + end + + # Ignore an empty string operationName. + params = params.merge("operationName" => nil) if params["operationName"] && params["operationName"].empty? + + yield params + end + + # Responsible for figuring out the timeout, based on a header and a provided max. + # If successful, yields the timeout value; otherwise will return an `HTTPResponse` with + # an error. + def with_timeout(request, max_timeout_in_ms:) + requested_timeout_in_ms = + if (timeout_in_ms_str = request.normalized_headers[HTTPRequest.normalize_header_name(TIMEOUT_MS_HEADER)]) + begin + Integer(timeout_in_ms_str) + rescue ::ArgumentError + # standard:disable Lint/NoReturnInBeginEndBlocks + return HTTPResponse.error(400, "`#{TIMEOUT_MS_HEADER}` header value of #{timeout_in_ms_str.inspect} is invalid") + # standard:enable Lint/NoReturnInBeginEndBlocks + end + end + + yield [max_timeout_in_ms, requested_timeout_in_ms].compact.min + end + + # Responsible for determining any `context` values to pass down into the `query_executor`, + # which in turn will make the values available to the GraphQL resolvers. + # + # By default, our only context value is the HTTP request. This method exists to provide an extension + # point so that ElasticGraph extensions can add `context` values based on the `request` as desired. + # + # Extensions can return an `HTTPResponse` with an error if the `request` is invalid according + # to their requirements. Otherwise, they must call `super` (to delegate to this and any other + # extensions) with a block. In the block, they must merge in their `context` values and then `yield`. + def with_context(request) + yield({http_request: request}) + end + + ParsedRequest = Data.define(:query_string, :variables, :operation_name, :timeout_in_ms, :context) + end + + # Represents an HTTP request, containing: + # + # - http_method: a symbol like :get or :post. + # - url: a string containing the full URL. + # - headers: a hash with string keys and values containing HTTP headers. The headers can + # be in any form like `Content-Type`, `content-type`, `CONTENT-TYPE`, `CONTENT_TYPE`, etc. + # - body: a string containing the request body, if there was one. + HTTPRequest = Support::MemoizableData.define(:http_method, :url, :headers, :body) do + # @implements HTTPRequest + + # HTTP headers are intended to be case-insensitive, and different Web frameworks treat them differently. + # For example, Rack uppercases them with `_` in place of `-`. With AWS Lambda proxy integrations API + # gateway HTTP APIs, header names are lowercased: + # https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html + # + # ...but for integration with API gateway REST APIs, header names are provided as-is: + # https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format + # + # To be maximally compatible here, this normalizes to uppercase form with dashes in place of underscores. + def normalized_headers + @normalized_headers ||= headers.transform_keys do |key| + HTTPRequest.normalize_header_name(key) + end + end + + def content_type + @content_type ||= normalized_headers["CONTENT-TYPE"] + end + + def self.normalize_header_name(header) + header.upcase.tr("_", "-") + end + end + + # Represents an HTTP response, containing: + # + # - status_code: an integer like 200. + # - headers: a hash with string keys and values containing HTTP response headers. + # - body: a string containing the response body. + HTTPResponse = Data.define(:status_code, :headers, :body) do + # @implements HTTPResponse + + # Helper method for building a JSON response. + def self.json(status_code, body) + new(status_code, {"Content-Type" => HTTPEndpoint::APPLICATION_JSON}, ::JSON.generate(body)) + end + + # Helper method for building an error response. + def self.error(status_code, message) + json(status_code, {"errors" => [{"message" => message}]}) + end + end + + # Steep weirdly expects them here... + # @dynamic initialize, config, logger, runtime_metadata, graphql_schema_string, datastore_core, clock + # @dynamic graphql_http_endpoint, graphql_query_executor, schema, datastore_search_router, filter_interpreter, filter_node_interpreter + # @dynamic datastore_query_builder, graphql_gem_plugins, graphql_resolvers, datastore_query_adapters, monotonic_clock + # @dynamic load_dependencies_eagerly, self.from_parsed_yaml, filter_args_translator, sub_aggregation_grouping_adapter + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/monkey_patches/schema_field.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/monkey_patches/schema_field.rb new file mode 100644 index 00000000..f835954c --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/monkey_patches/schema_field.rb @@ -0,0 +1,56 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "graphql" + +module ElasticGraph + class GraphQL + module MonkeyPatches + # This module is designed to monkey patch `GraphQL::Schema::Field`, but to do so in a + # conservative, safe way: + # + # - It defines no new methods. + # - It delegates to the original implementation with `super` unless we are sure that a type should be hidden. + # - It only changes the behavior for ElasticGraph schemas (as indicated by `:elastic_graph_schema` in the `context`). + module SchemaFieldVisibilityDecorator + def visible?(context) + # `DynamicFields` and `EntryPoints` are built-in introspection types that `field_named` below doesn't support: + # https://github.com/rmosolgo/graphql-ruby/blob/0df187995c971b399ed7cc1fbdcbd958af6c4ade/lib/graphql/introspection/entry_points.rb + # https://github.com/rmosolgo/graphql-ruby/blob/0df187995c971b399ed7cc1fbdcbd958af6c4ade/lib/graphql/introspection/dynamic_fields.rb + # + # ...so if the owner is one of those we just return `super` here. + return super if %w[DynamicFields EntryPoints].include?(owner.graphql_name) + + if context[:elastic_graph_schema]&.field_named(owner.graphql_name, graphql_name)&.hidden_from_queries? + return false + end + + super + end + end + end + end +end + +# As per https://graphql-ruby.org/authorization/visibility.html, the public API +# provided by the GraphQL gem to control visibility of object types is to define +# a `visible?` instance method on a custom subclass of `GraphQL::Schema::Field`. +# However, because we load our schema from an SDL definition rather than defining +# classes for each schema type, we don't have a way to register a custom subclass +# to be used for fields. +# +# So, here we solve this a slightly different way: we prepend a module onto +# the `GraphQL::Schema::Field class. This allows our module to act like a +# decorator and intercept calls to `visible?` so that it can hide types as needed. +module GraphQL + class Schema + class Field + prepend ::ElasticGraph::GraphQL::MonkeyPatches::SchemaFieldVisibilityDecorator + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/monkey_patches/schema_object.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/monkey_patches/schema_object.rb new file mode 100644 index 00000000..fb464a8d --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/monkey_patches/schema_object.rb @@ -0,0 +1,48 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "graphql" + +module ElasticGraph + class GraphQL + module MonkeyPatches + # This module is designed to monkey patch `GraphQL::Schema::Object`, but to do so in a + # conservative, safe way: + # + # - It defines no new methods. + # - It delegates to the original implementation with `super` unless we are sure that a type should be hidden. + # - It only changes the behavior for ElasticGraph schemas (as indicated by `:elastic_graph_schema` in the `context`). + module SchemaObjectVisibilityDecorator + def visible?(context) + if context[:elastic_graph_schema]&.type_named(graphql_name)&.hidden_from_queries? + context[:elastic_graph_query_tracker].record_hidden_type(graphql_name) + return false + end + + super + end + end + end + end +end + +# As per https://graphql-ruby.org/authorization/visibility.html, the public API +# provided by the GraphQL gem to control visibility of object types is to define +# a `visible?` class method on each of your type classes. However, because we load +# our schema from an SDL definition rather than defining classes for each schema +# type, we don't have a way to define the `visible?` on each of our type classes. +# +# So, here we solve this a slightly different way: we prepend a module onto +# the `GraphQL::Schema::Object` singleton class. This allows our module to +# act like a decorator and intercept calls to `visible?` so that it can hide +# types as needed. This works because all types must be defined as subclasses +# of `GraphQL::Schema::Object`, and in fact the GraphQL gem defined anonymous +# subclasses for each type in our SDL schema, as you can see here: +# +# https://github.com/rmosolgo/graphql-ruby/blob/v1.12.16/lib/graphql/schema/build_from_definition.rb#L312 +GraphQL::Schema::Object.singleton_class.prepend ElasticGraph::GraphQL::MonkeyPatches::SchemaObjectVisibilityDecorator diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/query_adapter/filters.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/query_adapter/filters.rb new file mode 100644 index 00000000..8e9e3452 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/query_adapter/filters.rb @@ -0,0 +1,158 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/graphql/filtering/filter_value_set_extractor" +require "elastic_graph/support/memoizable_data" + +module ElasticGraph + class GraphQL + class QueryAdapter + class Filters < Support::MemoizableData.define(:schema_element_names, :filter_args_translator, :filter_node_interpreter) + def call(field:, query:, args:, lookahead:, context:) + filter_from_args = filter_args_translator.translate_filter_args(field: field, args: args) + automatic_filter = build_automatic_filter(filter_from_args: filter_from_args, query: query) + filters = [filter_from_args, automatic_filter].compact + return query if filters.empty? + + query.merge_with(filters: filters) + end + + private + + def build_automatic_filter(filter_from_args:, query:) + # If an incomplete document could be hit by a search with our filters against any of the + # index definitions, we must add a filter that will exclude incomplete documents. + exclude_incomplete_docs_filter if query + .search_index_definitions + .any? { |index_def| search_could_hit_incomplete_docs?(index_def, filter_from_args || {}) } + end + + def exclude_incomplete_docs_filter + {"__sources" => {schema_element_names.equal_to_any_of => [SELF_RELATIONSHIP_NAME]}} + end + + # Indicates if a search against the given `index_def` using the given `filter_from_args` + # could hit an incomplete document. + def search_could_hit_incomplete_docs?(index_def, filter_from_args) + # If the index definition doesn't allow any searches to hit incomplete documents, we + # can immediately return `false` without checking the filters. + return false unless index_def.searches_could_hit_incomplete_docs? + + # ...otherwise, we have to look at how we are filtering. An incomplete document will have `null` + # values for all fields with a `SELF_RELATIONSHIP_NAME` source. Therefore, if we filter on a + # self-sourced field in a way that excludes documents with a `null` value, the search cannot + # hit incomplete documents. However, when in doubt we'd rather return `true` as that's the safer + # value to return (no bugs will result from returning `true` when we could have returned `false`, + # but the query may not be as efficient as we'd like). + # + # Here we determine what field paths we need to check (e.g. only those field paths that are against + # self-sourced fields). + paths_to_check = determine_paths_to_check(filter_from_args, index_def.fields_by_path) + + # If we have no paths to check, then our filters don't exclude incomplete documents and we must return `true`. + return true if paths_to_check.empty? + + # Finally, we look over each path. If all our filters allow the search to match documents that have `nil` + # at that path, then the search can hit incomplete documents. But if even one path excludes documents + # that have a `null` value for the field, we can safely return `false` for a more efficient query. + paths_to_check.all? { |path| can_match_nil_values_at?(path, filter_from_args) } + end + + # Figures out which field paths we need to check to see if a filter on it could match an incomplete document. + # This method returns the set intersection of: + # + # - The field paths we are filtering on. + # - The field paths that are sourced from `SELF_RELATIONSHIP_NAME`. + def determine_paths_to_check(expression, index_fields_by_path, parent_path: nil) + return [] unless expression.is_a?(::Hash) + + expression.compact.flat_map do |field_or_op, sub_expression| + if filter_node_interpreter.identify_node_type(field_or_op, sub_expression) == :sub_field + path = parent_path ? "#{parent_path}.#{field_or_op}" : field_or_op + if (index_field = index_fields_by_path[path]) + # We've recursed down to a leaf field path. We want that path to be returned if the + # field is sourced from SELF_RELATIONSHIP_NAME. + (index_field.source == SELF_RELATIONSHIP_NAME) ? [path] : [] + else + determine_paths_to_check(sub_expression, index_fields_by_path, parent_path: path) + end + elsif sub_expression.is_a?(::Array) + sub_expression.flat_map do |sub_filter| + determine_paths_to_check(sub_filter, index_fields_by_path, parent_path: parent_path) + end + else + determine_paths_to_check(sub_expression, index_fields_by_path, parent_path: parent_path) + end + end + end + + # Indicates if the given `filter` can match `nil` values at the given `path`. We rely + # on `filter_value_set_extractor` to determine it, since it understands the semantics + # of `any_of`, `not`, etc. + def can_match_nil_values_at?(path, filter) + filter_value_set_extractor.extract_filter_value_set([filter], [path]).includes_nil? + end + + def filter_value_set_extractor + @filter_value_set_extractor ||= + Filtering::FilterValueSetExtractor.new(schema_element_names, IncludesNilSet) do |operator, filter_value| + if operator == :equal_to_any_of && filter_value.include?(nil) + IncludesNilSet + else + ExcludesNilSet + end + end + end + + # Mixin for use with our set implementations that only care about if `nil` is an included value or not. + module NilFocusedSet + def union(other) + (includes_nil? || other.includes_nil?) ? IncludesNilSet : ExcludesNilSet + end + + def intersection(other) + (includes_nil? && other.includes_nil?) ? IncludesNilSet : ExcludesNilSet + end + end + + # A representation of a set that includes `nil`. + module IncludesNilSet + extend NilFocusedSet + + # Methods provided by `extend NilFocusedSet` + # @dynamic self.union, self.intersection + + def self.negate + ExcludesNilSet + end + + def self.includes_nil? + true + end + end + + # A representation of a set that excludes `nil`. + module ExcludesNilSet + extend NilFocusedSet + + # Methods provided by `extend NilFocusedSet` + # @dynamic self.union, self.intersection + + def self.negate + IncludesNilSet + end + + def self.includes_nil? + false + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/query_adapter/pagination.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/query_adapter/pagination.rb new file mode 100644 index 00000000..d5b93082 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/query_adapter/pagination.rb @@ -0,0 +1,27 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + class QueryAdapter + # Note: This class is not tested directly but indirectly through specs on `QueryAdapter` + Pagination = Data.define(:schema_element_names) do + # @implements Pagination + def call(query:, args:, lookahead:, field:, context:) + return query unless field.type.unwrap_fully.indexed_document? + + document_pagination = [:first, :before, :last, :after].to_h do |key| + [key, args[schema_element_names.public_send(key)]] + end + + query.merge_with(document_pagination: document_pagination) + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/query_adapter/requested_fields.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/query_adapter/requested_fields.rb new file mode 100644 index 00000000..c97ad890 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/query_adapter/requested_fields.rb @@ -0,0 +1,124 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + class QueryAdapter + # Query adapter that populates the `requested_fields` attribute of an `DatastoreQuery` in + # order to limit what fields we fetch from the datastore to only those that we actually + # need to satisfy the GraphQL query. This results in more efficient datastore queries, + # similar to doing `SELECT f1, f2, ...` instead of `SELECT *` for a SQL query. + class RequestedFields + def initialize(schema) + @schema = schema + end + + def call(field:, query:, lookahead:, args:, context:) + return query if field.type.unwrap_fully.indexed_aggregation? + + attributes = query_attributes_for(field: field, lookahead: lookahead) + query.merge_with(**attributes) + end + + def query_attributes_for(field:, lookahead:) + attributes = + if field.type.relay_connection? + { + individual_docs_needed: pagination_fields_need_individual_docs?(lookahead), + requested_fields: requested_fields_under(relay_connection_node_from(lookahead)) + } + else + { + requested_fields: requested_fields_under(lookahead) + } + end + + attributes.merge(total_document_count_needed: query_needs_total_document_count?(lookahead)) + end + + private + + # Identifies the fields we need to fetch from the datastore by looking for the fields + # under the given `node`. + # + # For nested relation fields, it is important that we start with this method, instead of + # `requested_fields_for`, because they need to be treated differently if we are building + # an `DatastoreQuery` for the nested relation field, or for a parent type. When we determine + # requested fields for a nested relation field, we need to look at its child fields, and we + # can ignore its foreign key; but when we are determining requested fields for a parent type, + # we need to identify the foreign key to request from the datastore, without recursing into + # its children. + def requested_fields_under(node, path_prefix: "") + fields = node.selections.flat_map do |child| + requested_fields_for(child, path_prefix: path_prefix) + end + + fields << "#{path_prefix}__typename" if field_for(node.field)&.type&.abstract? + fields + end + + # Identifies the fields we need to fetch from the datastore for the given node, + # and recursing into the fields under it as needed. + def requested_fields_for(node, path_prefix:) + return [] if graphql_dynamic_field?(node) + + # @type var field: Schema::Field + field = _ = field_for(node.field) + + if field.type.embedded_object? + requested_fields_under(node, path_prefix: "#{path_prefix}#{field.name_in_index}.") + else + field.index_field_names_for_resolution.map do |name| + "#{path_prefix}#{name}" + end + end + end + + def field_for(field) + return nil unless field + @schema.field_named(field.owner.graphql_name, field.name) + end + + def pagination_fields_need_individual_docs?(lookahead) + # If the client wants cursors, we need to request docs from the datastore so we get back the sort values + # for each node, which we can then encode into a cursor. + return true if lookahead.selection(@schema.element_names.edges).selects?(@schema.element_names.cursor) + + # Most subfields of `page_info` also require us to fetch documents from the datastore. For example, + # we cannot compute `has_next_page` or `has_previous_page` correctly if we do not fetch a full page + # of documents from the datastore. + lookahead.selection(@schema.element_names.page_info).selections.any? + end + + def relay_connection_node_from(lookahead) + node = lookahead.selection(@schema.element_names.nodes) + return node if node.selected? + + lookahead + .selection(@schema.element_names.edges) + .selection(@schema.element_names.node) + end + + # total_hits_count is needed when the connection explicitly specifies `total_edge_count` to + # be returned in the part of the GraphQL query we are processing. Note that the aggregation + # query adapter can also set it to true based on its needs. + def query_needs_total_document_count?(lookahead) + # If total edge count is explicitly specified in page_info, we have to return the total count + lookahead.selects?(@schema.element_names.total_edge_count) + end + + def graphql_dynamic_field?(node) + # As per https://spec.graphql.org/October2021/#sec-Objects, + # > All fields defined within an Object type must not have a name which begins with "__" + # > (two underscores), as this is used exclusively by GraphQL’s introspection system. + node.field.name.start_with?("__") + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/query_adapter/sort.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/query_adapter/sort.rb new file mode 100644 index 00000000..3175701b --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/query_adapter/sort.rb @@ -0,0 +1,32 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + class QueryAdapter + # Note: This class is not tested directly but indirectly through specs on `QueryAdapter` + Sort = Data.define(:order_by_arg_name) do + # @implements Sort + def call(query:, args:, field:, lookahead:, context:) + sort_clauses = field.sort_clauses_for(args[order_by_arg_name]) + + if sort_clauses.empty? + # When there are multiple search index definitions, we just need to pick one as the + # source of the default sort clauses. It doesn't really matter which (if the client + # really cared, they would have provided an `order_by` argument...) but we want our + # logic to be consistent and deterministic, so we just use the alphabetically first + # index here. + sort_clauses = (_ = query.search_index_definitions.min_by(&:name)).default_sort_clauses + end + + query.merge_with(sort: sort_clauses) + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/query_details_tracker.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/query_details_tracker.rb new file mode 100644 index 00000000..8c6aaa84 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/query_details_tracker.rb @@ -0,0 +1,60 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/client" +require "elastic_graph/support/hash_util" +require "graphql" + +module ElasticGraph + class GraphQL + # Class used to track details of what happens during a single GraphQL query for the purposes of logging. + # Here we use `Struct` instead of `Data` specifically because it is designed to be mutable. + class QueryDetailsTracker < Struct.new( + :hidden_types, + :shard_routing_values, + :search_index_expressions, + :query_counts_per_datastore_request, + :datastore_query_server_duration_ms, + :datastore_query_client_duration_ms, + :mutex + ) + def self.empty + new( + hidden_types: ::Set.new, + shard_routing_values: ::Set.new, + search_index_expressions: ::Set.new, + query_counts_per_datastore_request: [], + datastore_query_server_duration_ms: 0, + datastore_query_client_duration_ms: 0, + mutex: ::Thread::Mutex.new + ) + end + + def record_datastore_queries_for_single_request(queries) + mutex.synchronize do + shard_routing_values.merge(queries.flat_map { |q| q.shard_routing_values || [] }) + search_index_expressions.merge(queries.map(&:search_index_expression)) + query_counts_per_datastore_request << queries.size + end + end + + def record_hidden_type(type) + mutex.synchronize do + hidden_types << type + end + end + + def record_datastore_query_duration_ms(client:, server:) + mutex.synchronize do + self.datastore_query_client_duration_ms += client + self.datastore_query_server_duration_ms += server if server + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/query_executor.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/query_executor.rb new file mode 100644 index 00000000..b67cf6d2 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/query_executor.rb @@ -0,0 +1,200 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/client" +require "elastic_graph/graphql/query_details_tracker" +require "elastic_graph/support/hash_util" +require "graphql" + +module ElasticGraph + class GraphQL + # Responsible for executing queries. + class QueryExecutor + # @dynamic schema + attr_reader :schema + + def initialize(schema:, monotonic_clock:, logger:, slow_query_threshold_ms:, datastore_search_router:) + @schema = schema + @monotonic_clock = monotonic_clock + @logger = logger + @slow_query_threshold_ms = slow_query_threshold_ms + @datastore_search_router = datastore_search_router + end + + # Executes the given `query_string` using the provided `variables`. + # + # `timeout_in_ms` can be provided to limit how long the query runs for. If the timeout + # is exceeded, `Errors::RequestExceededDeadlineError` will be raised. Note that `timeout_in_ms` + # does not provide an absolute guarantee that the query will take no longer than the + # provided value; it is only used to halt datastore queries. In process computation + # can make the total query time exceeded the specified timeout. + # + # `context` is merged into the context hash passed to the resolvers. + def execute( + query_string, + client: Client::ANONYMOUS, + variables: {}, + timeout_in_ms: nil, + operation_name: nil, + context: {}, + start_time_in_ms: @monotonic_clock.now_in_ms + ) + # Before executing the query, prune any null-valued variable fields. This means we + # treat `foo: null` the same as if `foo` was unmentioned. With certain clients (e.g. + # code-gen'd clients in a statically typed language), it is non-trivial to avoid + # mentioning variable fields they aren't using. It makes it easier to evolve the + # schema if we ignore null-valued fields rather than potentially returning an error + # due to a null-valued field referencing an undefined schema element. + variables = ElasticGraph::Support::HashUtil.recursively_prune_nils_from(variables) + + query_tracker = QueryDetailsTracker.empty + + query, result = build_and_execute_query( + query_string: query_string, + variables: variables, + operation_name: operation_name, + client: client, + context: context.merge({ + monotonic_clock_deadline: timeout_in_ms&.+(start_time_in_ms), + elastic_graph_schema: @schema, + schema_element_names: @schema.element_names, + elastic_graph_query_tracker: query_tracker, + datastore_search_router: @datastore_search_router + }.compact) + ) + + unless result.to_h.fetch("errors", []).empty? + @logger.error <<~EOS + Query #{query.operation_name}[1] for client #{client.description} resulted in errors[2]. + + [1] #{full_description_of(query)} + + [2] #{::JSON.pretty_generate(result.to_h.fetch("errors"))} + EOS + end + + unless query_tracker.hidden_types.empty? + @logger.warn "#{query_tracker.hidden_types.size} GraphQL types were hidden from the schema due to their backing indices being inaccessible: #{query_tracker.hidden_types.sort.join(", ")}" + end + + duration = @monotonic_clock.now_in_ms - start_time_in_ms + + # Note: I also wanted to log the sanitized query if `result` has `errors`, but `GraphQL::Query#sanitized_query` + # returns `nil` on an invalid query, and I don't want to risk leaking PII by logging the raw query string, so + # we don't log any form of the query in that case. + if duration > @slow_query_threshold_ms + @logger.warn "Query #{query.operation_name} for client #{client.description} with shard routing values " \ + "#{query_tracker.shard_routing_values.sort.inspect} and search index expressions #{query_tracker.search_index_expressions.sort.inspect} took longer " \ + "(#{duration} ms) than the configured slow query threshold (#{@slow_query_threshold_ms} ms). " \ + "Sanitized query:\n\n#{query.sanitized_query_string}" + end + + unless client == Client::ELASTICGRAPH_INTERNAL + @logger.info({ + "message_type" => "ElasticGraphQueryExecutorQueryDuration", + "client" => client.name, + "query_fingerprint" => fingerprint_for(query), + "query_name" => query.operation_name, + "duration_ms" => duration, + # Here we log how long the datastore queries took according to what the datastore itself reported. + "datastore_server_duration_ms" => query_tracker.datastore_query_server_duration_ms, + # Here we log an estimate for how much overhead ElasticGraph added on top of how long the datastore took. + # This is based on the duration, excluding how long the datastore calls took from the client side + # (e.g. accounting for network latency, serialization time, etc) + "elasticgraph_overhead_ms" => duration - query_tracker.datastore_query_client_duration_ms, + # According to https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/FilterAndPatternSyntax.html#metric-filters-extract-json, + # > Value nodes can be strings or numbers...If a property selector points to an array or object, the metric filter won't match the log format. + # So, to allow flexibility to deal with cloud watch metric filters, we coerce these values to a string here. + "unique_shard_routing_values" => query_tracker.shard_routing_values.sort.join(", "), + # We also include the count of shard routing values, to make it easier to search logs + # for the case of no shard routing values. + "unique_shard_routing_value_count" => query_tracker.shard_routing_values.count, + "unique_search_index_expressions" => query_tracker.search_index_expressions.sort.join(", "), + # Indicates how many requests we sent to the datastore to satisfy the GraphQL query. + "datastore_request_count" => query_tracker.query_counts_per_datastore_request.size, + # Indicates how many individual datastore queries there were. One datastore request + # can contain many queries (since we use `msearch`), so these counts can be different. + "datastore_query_count" => query_tracker.query_counts_per_datastore_request.sum, + "over_slow_threshold" => (duration > @slow_query_threshold_ms).to_s, + "slo_result" => slo_result_for(query, duration) + }) + end + + result + end + + private + + # Note: this is designed so that `elasticgraph-query_registry` can hook into this method. It needs to be able + # to override how the query is built and executed. + def build_and_execute_query(query_string:, variables:, operation_name:, context:, client:) + query = ::GraphQL::Query.new( + @schema.graphql_schema, + query_string, + variables: variables, + operation_name: operation_name, + context: context + ) + + [query, execute_query(query, client: client)] + end + + # Executes the given query, providing some extra logging if an exception occurs. + def execute_query(query, client:) + # Log the query before starting to execute it, in case there's a lambda timeout, in which case + # we won't get any other logged messages for the query. + @logger.info "Starting to execute query #{fingerprint_for(query)} for client #{client.description}." + + query.result + rescue => ex + @logger.error <<~EOS + Query #{query.operation_name}[1] for client #{client.description} failed with an exception[2]. + + [1] #{full_description_of(query)} + + [2] #{ex.class}: #{ex.message} + EOS + + raise ex + end + + # Returns a string that describes the query as completely as we can. + # Note that `query.sanitized_query_string` is quite complete, but can be nil in + # certain situations (such as when the query string itself is invalid!); we include + # the fingerprint to make sure that we at least have some identification information + # about the query. + def full_description_of(query) + "#{fingerprint_for(query)} #{query.sanitized_query_string}" + end + + def fingerprint_for(query) + query.query_string ? query.fingerprint : "(no query string)" + end + + def slo_result_for(query, duration) + latency_slo = directives_from_query_operation(query) + .dig(schema.element_names.eg_latency_slo, schema.element_names.ms) + + if latency_slo.nil? + nil + elsif duration <= latency_slo + "good" + else + "bad" + end + end + + def directives_from_query_operation(query) + query.selected_operation&.directives&.to_h do |dir| + arguments = dir.arguments.to_h { |arg| [arg.name, arg.value] } + [dir.name, arguments] + end || {} + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/get_record_field_value.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/get_record_field_value.rb new file mode 100644 index 00000000..2f66194e --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/get_record_field_value.rb @@ -0,0 +1,49 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/datastore_response/document" +require "elastic_graph/graphql/resolvers/relay_connection/array_adapter" +require "elastic_graph/support/hash_util" + +module ElasticGraph + class GraphQL + module Resolvers + # Responsible for fetching a single field value from a document. + class GetRecordFieldValue + def initialize(schema_element_names:) + @schema_element_names = schema_element_names + end + + def can_resolve?(field:, object:) + object.is_a?(DatastoreResponse::Document) || object.is_a?(::Hash) + end + + def resolve(field:, object:, args:, context:, lookahead:) + field_name = field.name_in_index.to_s + data = + case object + when DatastoreResponse::Document + object.payload + else + object + end + + value = Support::HashUtil.fetch_value_at_path(data, field_name) do + field.type.list? ? [] : nil + end + + if field.type.relay_connection? + RelayConnection::ArrayAdapter.build(value, args, @schema_element_names, context) + else + value + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/graphql_adapter.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/graphql_adapter.rb new file mode 100644 index 00000000..f0a00b23 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/graphql_adapter.rb @@ -0,0 +1,114 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/resolvers/query_adapter" + +module ElasticGraph + class GraphQL + module Resolvers + # Adapts the GraphQL gem's resolver interface to the interface implemented by + # our resolvers. Responsible for routing a resolution request to the appropriate + # resolver. + class GraphQLAdapter + def initialize(schema:, datastore_query_builder:, datastore_query_adapters:, runtime_metadata:, resolvers:) + @schema = schema + @query_adapter = QueryAdapter.new( + datastore_query_builder: datastore_query_builder, + datastore_query_adapters: datastore_query_adapters + ) + + @resolvers = resolvers + + scalar_types_by_name = runtime_metadata.scalar_types_by_name + @coercion_adapters_by_scalar_type_name = ::Hash.new do |hash, name| + scalar_types_by_name.fetch(name).load_coercion_adapter.extension_class + end + end + + # To be a valid resolver, we must implement `call`, accepting the 5 arguments listed here. + # + # See https://graphql-ruby.org/api-doc/1.9.6/GraphQL/Schema.html#from_definition-class_method + # (specifically, the `default_resolve` argument) for the API documentation. + def call(parent_type, field, object, args, context) + schema_field = @schema.field_named(parent_type.graphql_name, field.name) + + # Extract the `:lookahead` extra that we have configured all fields to provide. + # See https://graphql-ruby.org/api-doc/1.10.8/GraphQL/Execution/Lookahead.html for more info. + # It is not a "real" arg in the schema and breaks `args_to_schema_form` when we call that + # so we need to peel it off here. + lookahead = args[:lookahead] + # Convert args to the form they were defined in the schema, undoing the normalization + # the GraphQL gem does to convert them to Ruby keyword args form. + args = schema_field.args_to_schema_form(args.except(:lookahead)) + + resolver = resolver_for(schema_field, object) do + raise <<~ERROR + No resolver yet implemented for this case. + + parent_type: #{schema_field.parent_type} + + field: #{schema_field} + + obj: #{object.inspect} + + args: #{args.inspect} + + ctx: #{context.inspect} + ERROR + end + + result = resolver.resolve(field: schema_field, object: object, args: args, context: context, lookahead: lookahead) do + @query_adapter.build_query_from(field: schema_field, args: args, lookahead: lookahead, context: context) + end + + # Give the field a chance to coerce the result before returning it. Initially, this is only used to deal with + # enum value overrides (e.g. so that if `DayOfWeek.MONDAY` has been overridden to `DayOfWeek.MON`, we can coerce + # a `MONDAY` value being returned by a painless script to `MON`), but this is designed to be general purpose + # and we may use it for other coercions in the future. + # + # Note that coercion of scalar values is handled by the `coerce_result` callback below. + schema_field.coerce_result(result) + end + + # In order to support unions and interfaces, we must implement `resolve_type`. + def resolve_type(supertype, object, context) + # If `__typename` is available, use that to resolve. It should be available on any embedded abstract types... + # (See `Inventor` in `config/schema.graphql` for an example of this kind of type union.) + if (typename = object["__typename"]) + @schema.graphql_schema.possible_types(supertype).find { |t| t.graphql_name == typename } + else + # ...otherwise infer the type based on what index the object came from. This is the case + # with unions/interfaces of individually indexed types. + # (See `Part` in `config/schema/widgets.rb` for an example of this kind of type union.) + @schema.document_type_stored_in(object.index_definition_name).graphql_type + end + end + + def coerce_input(type, value, ctx) + scalar_coercion_adapter_for(type).coerce_input(value, ctx) + end + + def coerce_result(type, value, ctx) + scalar_coercion_adapter_for(type).coerce_result(value, ctx) + end + + private + + def scalar_coercion_adapter_for(type) + @coercion_adapters_by_scalar_type_name[type.graphql_name] + end + + def resolver_for(field, object) + return object if object.respond_to?(:can_resolve?) && object.can_resolve?(field: field, object: object) + resolver = @resolvers.find { |r| r.can_resolve?(field: field, object: object) } + resolver || yield + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/list_records.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/list_records.rb new file mode 100644 index 00000000..3659cccb --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/list_records.rb @@ -0,0 +1,29 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/resolvers/query_source" +require "elastic_graph/graphql/resolvers/relay_connection" + +module ElasticGraph + class GraphQL + module Resolvers + # Responsible for fetching a a list of records of a particular type + class ListRecords + def can_resolve?(field:, object:) + field.parent_type.name == :Query && field.type.collection? + end + + def resolve(field:, context:, lookahead:, **) + query = yield + response = QuerySource.execute_one(query, for_context: context) + RelayConnection.maybe_wrap(response, field: field, context: context, lookahead: lookahead, query: query) + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/nested_relationships.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/nested_relationships.rb new file mode 100644 index 00000000..351ec01f --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/nested_relationships.rb @@ -0,0 +1,74 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/resolvers/relay_connection" + +module ElasticGraph + class GraphQL + module Resolvers + # Responsible for loading nested relationships that are stored as separate documents + # in the datastore. We use `QuerySource` for the datastore queries to avoid + # the N+1 query problem (giving us one datastore query per layer of our graph). + # + # Most of the logic for this lives in ElasticGraph::Schema::RelationJoin. + class NestedRelationships + def initialize(schema_element_names:, logger:) + @schema_element_names = schema_element_names + @logger = logger + end + + def can_resolve?(field:, object:) + !!field.relation_join + end + + def resolve(object:, field:, context:, lookahead:, **) + log_warning = ->(**options) { log_field_problem_warning(field: field, **options) } + join = field.relation_join + id_or_ids = join.extract_id_or_ids_from(object, log_warning) + filters = [ + build_filter(join.filter_id_field_name, nil, join.foreign_key_nested_paths, Array(id_or_ids)), + join.additional_filter + ].reject(&:empty?) + query = yield.merge_with(filters: filters) + + response = + case id_or_ids + when nil, [] + join.blank_value + else + initial_response = QuerySource.execute_one(query, for_context: context) + join.normalize_documents(initial_response) do |problem| + log_warning.call(document: {"id" => id_or_ids}, problem: "got #{problem} from the datastore search query") + end + end + + RelayConnection.maybe_wrap(response, field: field, context: context, lookahead: lookahead, query: query) + end + + private + + def log_field_problem_warning(field:, document:, problem:) + id = document.fetch("id", "") + @logger.warn "#{field.parent_type.name}(id: #{id}).#{field.name} had a problem: #{problem}" + end + + def build_filter(path, previous_nested_path, nested_paths, ids) + if nested_paths.empty? + path = path.delete_prefix("#{previous_nested_path}.") if previous_nested_path + {path => {@schema_element_names.equal_to_any_of => ids}} + else + next_nested_path, *rest_nested_paths = nested_paths + sub_filter = build_filter(path, next_nested_path, rest_nested_paths, ids) + next_nested_path = next_nested_path.delete_prefix("#{previous_nested_path}.") if previous_nested_path + {next_nested_path => {@schema_element_names.any_satisfy => sub_filter}} + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/query_adapter.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/query_adapter.rb new file mode 100644 index 00000000..154b7495 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/query_adapter.rb @@ -0,0 +1,85 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + module Resolvers + # Responsible for taking raw GraphQL query arguments and transforming + # them into a DatastoreQuery object. + class QueryAdapter + def initialize(datastore_query_builder:, datastore_query_adapters:) + @datastore_query_builder = datastore_query_builder + @datastore_query_adapters = datastore_query_adapters + end + + def build_query_from(field:, args:, lookahead:, context: {}) + monotonic_clock_deadline = context[:monotonic_clock_deadline] + + # Building an `DatastoreQuery` is not cheap; we do a lot of work to: + # + # 1) Convert the `args` to their schema form. + # 2) Reduce over our different query builders into a final `Query` object + # 3) ...and those individual query builders often do a lot of work (traversing lookaheads, etc). + # + # So it is beneficial to avoid re-creating the exact same `DatastoreQuery` object when + # we are resolving the same field in the context of a different object. For example, + # consider a query like: + # + # query { + # widgets { + # components { + # id + # parts { + # id + # } + # } + # } + # } + # + # Here `components` and `parts` are nested relation fields. If we load 50 of each collection, + # this `build_query_from` method will be called 50 times for the `Widget.components` field, + # and 2500 times (50 * 50) for the `Component.parts` field...but for a given field, the + # built `DatastoreQuery` will be exactly the same. + # + # Therefore, it is beneficial to memoize the `DatastoreQuery` to avoid re-doing the same work + # over and over again, provided we can do so safely. + # + # `context` is a hash-like `GraphQL::Query::Context` object. Each executed query gets its own + # instance, so we can safely cache things in it and trust that it will not "leak" to another + # query execution. We carefully build a cache key below to ensure that we only ever reuse + # the same `DatastoreQuery` in a situation that would produce the exact same `DatastoreQuery`. + context[:datastore_query_cache] ||= {} + context[:datastore_query_cache][cache_key_for(field, args, lookahead)] ||= + build_new_query_from(field, args, lookahead, context, monotonic_clock_deadline) + end + + private + + def build_new_query_from(field, args, lookahead, context, monotonic_clock_deadline) + unwrapped_type = field.type.unwrap_fully + + initial_query = @datastore_query_builder.new_query( + search_index_definitions: unwrapped_type.search_index_definitions, + monotonic_clock_deadline: monotonic_clock_deadline + ) + + @datastore_query_adapters.reduce(initial_query) do |query, adapter| + adapter.call(query: query, field: field, args: args, lookahead: lookahead, context: context) + end + end + + def cache_key_for(field, args, lookahead) + # Unfortunately, `Lookahead` does not define `==` according to its internal state, + # so `l1 == l2` with the same internal state returns false. So we have to pull + # out its individual state fields in the cache key for our caching to work here. + [field, args, lookahead.ast_nodes, lookahead.field, lookahead.owner_type] + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/query_source.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/query_source.rb new file mode 100644 index 00000000..415f58a2 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/query_source.rb @@ -0,0 +1,46 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "graphql/dataloader/source" + +module ElasticGraph + class GraphQL + module Resolvers + # Provides a way to avoid N+1 query problems by batching up multiple + # datastore queries into one `msearch` call. In general, it is recommended + # that you use this from any resolver that needs to query the datastore, to + # maximize our ability to combine multiple datastore requests. Importantly, + # this should never be instantiated directly; instead use the `execute` method from below. + class QuerySource < ::GraphQL::Dataloader::Source + def initialize(datastore_router, query_tracker) + @datastore_router = datastore_router + @query_tracker = query_tracker + end + + def fetch(queries) + responses_by_query = @datastore_router.msearch(queries, query_tracker: @query_tracker) + @query_tracker.record_datastore_queries_for_single_request(queries) + queries.map { |q| responses_by_query[q] } + end + + def self.execute_many(queries, for_context:) + datastore_router = for_context.fetch(:datastore_search_router) + query_tracker = for_context.fetch(:elastic_graph_query_tracker) + dataloader = for_context.dataloader + + responses = dataloader.with(self, datastore_router, query_tracker).load_all(queries) + queries.zip(responses).to_h + end + + def self.execute_one(query, for_context:) + execute_many([query], for_context: for_context).fetch(query) + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/relay_connection.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/relay_connection.rb new file mode 100644 index 00000000..cde1f8c0 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/relay_connection.rb @@ -0,0 +1,43 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/resolvers/relay_connection_builder" +require "elastic_graph/graphql/resolvers/relay_connection/search_response_adapter_builder" +require "elastic_graph/support/hash_util" + +module ElasticGraph + class GraphQL + module Resolvers + # Defines resolver logic related to relay connections. The relay connections spec is here: + # https://facebook.github.io/relay/graphql/connections.htm + module RelayConnection + # Conditionally wraps the given search response in the appropriate relay connection adapter, if needed. + def self.maybe_wrap(search_response, field:, context:, lookahead:, query:) + return search_response unless field.type.relay_connection? + + schema_element_names = context.fetch(:schema_element_names) + + unless field.type.unwrap_fully.indexed_aggregation? + return SearchResponseAdapterBuilder.build_from( + schema_element_names: schema_element_names, + search_response: search_response, + query: query + ) + end + + agg_name = lookahead.ast_nodes.first&.alias || lookahead.name + Aggregation::Resolvers::RelayConnectionBuilder.build_from_search_response( + schema_element_names: schema_element_names, + search_response: search_response, + query: Support::HashUtil.verbose_fetch(query.aggregations, agg_name) + ) + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/relay_connection/array_adapter.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/relay_connection/array_adapter.rb new file mode 100644 index 00000000..758e66e3 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/relay_connection/array_adapter.rb @@ -0,0 +1,71 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/resolvers/resolvable_value" +require "forwardable" + +module ElasticGraph + class GraphQL + module Resolvers + module RelayConnection + # Relay connection adapter for an array. Implemented primarily by `GraphQL::Relay::ArrayConnection`; + # here we just adapt it to the ElasticGraph internal resolver interface. + class ArrayAdapter < ResolvableValue.new(:graphql_impl) + # `ResolvableValue.new` provides the following methods: + # @dynamic initialize, graphql_impl, schema_element_names + + # `def_delegators` provides the following methods: + # @dynamic start_cursor, end_cursor, has_next_page, has_previous_page + extend Forwardable + def_delegators :graphql_impl, :start_cursor, :end_cursor, :has_next_page, :has_previous_page + + def self.build(nodes, args, schema_element_names, context) + # ElasticGraph supports any schema elements (like a `first` argument) being renamed, + # but `GraphQL::Relay::ArrayConnection` would not understand a renamed argument. + # Here we map the args back to the canonical relay args so `ArrayConnection` can + # understand them. + relay_args = [:first, :after, :last, :before].to_h do |arg_name| + [arg_name, args[schema_element_names.public_send(arg_name)]] + end.compact + + graphql_impl = ::GraphQL::Pagination::ArrayConnection.new(nodes || [], context: context, **relay_args) + new(schema_element_names, graphql_impl) + end + + def total_edge_count + graphql_impl.nodes.size + end + + def page_info + self + end + + def edges + @edges ||= graphql_impl.nodes.map do |node| + Edge.new(schema_element_names, graphql_impl, node) + end + end + + def nodes + @nodes ||= graphql_impl.nodes + end + + # Simple edge implementation for a node object. + class Edge < ResolvableValue.new(:graphql_impl, :node) + # `ResolvableValue.new` provides the following methods: + # @dynamic initialize, graphql_impl, schema_element_names, node + + def cursor + graphql_impl.cursor_for(node) + end + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/relay_connection/generic_adapter.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/relay_connection/generic_adapter.rb new file mode 100644 index 00000000..e2cf37a8 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/relay_connection/generic_adapter.rb @@ -0,0 +1,65 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/resolvers/relay_connection/page_info" +require "elastic_graph/graphql/resolvers/resolvable_value" + +module ElasticGraph + class GraphQL + module Resolvers + module RelayConnection + class GenericAdapter < ResolvableValue.new( + # Array of nodes for this page of data, before paginator truncation has been added. + :raw_nodes, + # The paginator that's being used. + :paginator, + # Lambda that is used to convert a node to a sort value during truncation. + :to_sort_value, + # Gets an optional count of total edges. + :get_total_edge_count + ) + # @dynamic initialize, with, schema_element_names, raw_nodes, paginator, to_sort_value, get_total_edge_count + + def page_info + @page_info ||= PageInfo.new( + schema_element_names: schema_element_names, + before_truncation_nodes: before_truncation_nodes, + edges: edges, + paginator: paginator + ) + end + + def total_edge_count + get_total_edge_count.call + end + + def edges + @edges ||= nodes.map { |node| Edge.new(schema_element_names, node) } + end + + def nodes + @nodes ||= paginator.truncate_items(before_truncation_nodes, &to_sort_value) + end + + private + + def before_truncation_nodes + @before_truncation_nodes ||= paginator.restore_intended_item_order(raw_nodes) + end + + # Implements an `Edge` as per the relay spec: + # https://facebook.github.io/relay/graphql/connections.htm#sec-Edge-Types + class Edge < ResolvableValue.new(:node) + # @dynamic initialize, node + def cursor = node.cursor.encode + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/relay_connection/page_info.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/relay_connection/page_info.rb new file mode 100644 index 00000000..b670e20e --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/relay_connection/page_info.rb @@ -0,0 +1,82 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/resolvers/resolvable_value" + +module ElasticGraph + class GraphQL + module Resolvers + module RelayConnection + # Provides the `PageInfo` field values required by the relay spec. + # + # The relay connections spec defines an algorithm behind `hasPreviousPage` and `hasNextPage`: + # https://facebook.github.io/relay/graphql/connections.htm#sec-undefined.PageInfo.Fields + # + # However, it has a couple bugs as currently written (https://github.com/facebook/relay/issues/2787), + # so we have implemented our own algorithm instead. It would be nice to calculate `hasPreviousPage` + # and `hasNextPage` on-demand in a resolver, so we do not spend any effort on it if the client has + # not requested those fields, but it is quite hard to calculate them after the fact: we need to know + # whether we removed any leading or trailing items while processing the list to accurately answer + # the question, "do we have a page before or after the one we are returning?". + # + # Note: it's not clear what values `hasPreviousPage` and `hasNextPage` should have when we are returning + # a blank page (the client isn't being returned any cursors to continue paginating from!). This logic, + # as written, will normally cause both fields to be `true` (our request of `size: size + 1` will get us + # a list of 1 document, which will then be removed, causing `items.first` and `items.last` to + # both change to `nil`). However, if the datastore returns an empty list to us than `false` will be returned + # for one or both fields, based on the presence or absence of the `before`/`after` cursors in the pagination + # arguments. Regardless, given that it's not clear what the correct value is, we are just doing the + # least-effort thing and not putting any special handling for this case in place. + class PageInfo < ResolvableValue.new( + # The array of nodes for this page before we applied necessary truncation. + :before_truncation_nodes, + # The array of edges for this page. + :edges, + # The paginator built from the field arguments. + :paginator + ) + # @dynamic initialize, with, before_truncation_nodes, edges, paginator + + def start_cursor + edges.first&.cursor + end + + def end_cursor + edges.last&.cursor + end + + def has_previous_page + # If we dropped the first node during truncation then it means we removed some leading docs, indicating a previous page. + return true if edges.first&.node != before_truncation_nodes.first + + # Nothing exists both before and after the same cursor, and there is therefore no page before that set of results. + return false if paginator.before == paginator.after + + # If an `after` cursor was passed then there is definitely at least one doc before the page we are + # returning (the one matching the cursor), assuming the client did not construct a cursor by hand + # (which we do not support). + !!paginator.after + end + + def has_next_page + # If we dropped the last node during truncation then it means we removed some trailing docs, indicating a next page. + return true if edges.last&.node != before_truncation_nodes.last + + # Nothing exists both before and after the same cursor, and there is therefore no page after that set of results. + return false if paginator.before == paginator.after + + # If a `before` cursor was passed then there is definitely at least one doc after the page we are + # returning (the one matching the cursor), assuming the client did not construct a cursor by hand + # (which we do not support). + !!paginator.before + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/relay_connection/search_response_adapter_builder.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/relay_connection/search_response_adapter_builder.rb new file mode 100644 index 00000000..9946296c --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/relay_connection/search_response_adapter_builder.rb @@ -0,0 +1,40 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/resolvers/relay_connection/generic_adapter" + +module ElasticGraph + class GraphQL + module Resolvers + module RelayConnection + # Adapts an `DatastoreResponse::SearchResponse` to what the graphql gem expects for a relay connection. + class SearchResponseAdapterBuilder + def self.build_from(schema_element_names:, search_response:, query:) + document_paginator = query.document_paginator + + GenericAdapter.new( + schema_element_names: schema_element_names, + raw_nodes: search_response.to_a, + paginator: document_paginator.paginator, + get_total_edge_count: -> { search_response.total_document_count }, + to_sort_value: ->(document, decoded_cursor) do + (_ = document).sort.zip(decoded_cursor.sort_values.values, document_paginator.sort).map do |from_document, from_cursor, sort_clause| + DatastoreQuery::Paginator::SortValue.new( + from_item: from_document, + from_cursor: from_cursor, + sort_direction: sort_clause.values.first.fetch("order").to_sym + ) + end + end + ) + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/resolvable_value.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/resolvable_value.rb new file mode 100644 index 00000000..45715153 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/resolvable_value.rb @@ -0,0 +1,55 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/support/memoizable_data" + +module ElasticGraph + class GraphQL + module Resolvers + # A class builder that is just like `Data` and also adapts itself to our + # resolver interface. Can resolve any field that is defined in `schema_element_names` + # and also has a corresponding method definition. + module ResolvableValue + # `MemoizableData.define` provides the following methods: + # @dynamic schema_element_names + + def self.new(*fields, &block) + Support::MemoizableData.define(:schema_element_names, *fields) do + # @implements ResolvableValueClass + include ResolvableValue + class_exec(&block) if block + end + end + + def resolve(field:, object:, context:, args:, lookahead:) + method_name = canonical_name_for(field.name, "Field") + public_send(method_name, **args_to_canonical_form(args)) + end + + def can_resolve?(field:, object:) + method_name = schema_element_names.canonical_name_for(field.name) + !!method_name && respond_to?(method_name) + end + + private + + def args_to_canonical_form(args) + args.to_h do |key, value| + [canonical_name_for(key, "Argument"), value] + end + end + + def canonical_name_for(name, element_type) + schema_element_names.canonical_name_for(name) || + raise(Errors::SchemaError, "#{element_type} `#{name}` is not a defined schema element") + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/cursor.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/cursor.rb new file mode 100644 index 00000000..1f67d755 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/cursor.rb @@ -0,0 +1,35 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/decoded_cursor" + +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + class Cursor + def self.coerce_input(value, ctx) + case value + when DecodedCursor + value + when ::String + DecodedCursor.try_decode(value) + end + end + + def self.coerce_result(value, ctx) + case value + when DecodedCursor + value.encode + when ::String + value if DecodedCursor.try_decode(value) + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/date.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/date.rb new file mode 100644 index 00000000..ccd49ab2 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/date.rb @@ -0,0 +1,64 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + class Date + def self.coerce_input(value, ctx) + return value if value.nil? + + # `::Date.iso8601` will happily parse a time ISO8601 string like `2021-11-10T12:30:00Z` + # but for simplicity we only want to support a Date string (like `2021-11-10`), + # so we detect that case here. + raise ::ArgumentError if value.is_a?(::String) && value.include?(":") + + date = ::Date.iso8601(value) + + # Verify we have a 4 digit year. The datastore `strict_date_time` format se use only supports 4 digit years: + # + # > Most of the below formats have a `strict` companion format, which means that year, month and day parts of the + # > week must use respectively 4, 2 and 2 digits exactly, potentially prepending zeros. + # + # https://www.elastic.co/guide/en/elasticsearch/reference/7.10/mapping-date-format.html#built-in-date-formats + raise_coercion_error(value) if date.year < 1000 || date.year > 9999 + + # We ultimately wind up passing input args to the datastore as our GraphQL engine receives + # them (it doesn't do any formatting of Date args to what the datastore needs) so we do + # that here instead. We have configured the datastore to expect Dates in `strict_date` + # format, so here we convert it to that format (which is just ISO8601 format). Ultimately, + # that means that this method just "roundtrips" the input string back to a string, but it + # validates the string is formatted correctly and returns a string in the exact format we + # need for the datastore. Also, we technically don't have to do this; ISO8601 format is + # the format that `Date` objects are serialized as in JSON, anyway. But we _have_ to do this + # for `DateTime` objects so we also do it here for parity/consistency. + date.iso8601 + rescue ArgumentError, ::TypeError + raise_coercion_error(value) + end + + def self.coerce_result(value, ctx) + case value + when ::Date + value.iso8601 + when ::String + ::Date.iso8601(value).iso8601 + end + rescue ::ArgumentError + nil + end + + private_class_method def self.raise_coercion_error(value) + raise ::GraphQL::CoercionError, + "Could not coerce value #{value.inspect} to Date: must be formatted " \ + "as an ISO8601 Date string (example: #{::Date.today.iso8601.inspect})." + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/date_time.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/date_time.rb new file mode 100644 index 00000000..5788a8a5 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/date_time.rb @@ -0,0 +1,60 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "time" + +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + class DateTime + PRECISION = 3 # millisecond precision + + def self.coerce_input(value, ctx) + return value if value.nil? + + time = ::Time.iso8601(value) + + # Verify we do not have more than 4 digits for the year. The datastore `strict_date_time` format we use only supports 4 digit years: + # + # > Most of the below formats have a `strict` companion format, which means that year, month and day parts of the + # > week must use respectively 4, 2 and 2 digits exactly, potentially prepending zeros. + # + # https://www.elastic.co/guide/en/elasticsearch/reference/8.12/mapping-date-format.html#built-in-date-formats + raise_coercion_error(value) if time.year > 9999 + + # We ultimately wind up passing input args to the datastore as our GraphQL engine receives + # them (it doesn't do any formatting of DateTime args to what the datastore needs) so we do + # that here instead. We have configured the datastore to expect DateTimes in `strict_date_time` + # format, so here we convert it to that format (which is just ISO8601 format). Ultimately, + # that means that this method just "roundtrips" the input string back to a string, but it validates + # the string is formatted correctly and returns a string in the exact format we need for the datastore. + time.iso8601(PRECISION) + rescue ::ArgumentError, ::TypeError + raise_coercion_error(value) + end + + def self.coerce_result(value, ctx) + case value + when ::Time + value.iso8601(PRECISION) + when ::String + ::Time.iso8601(value).iso8601(PRECISION) + end + rescue ::ArgumentError + nil + end + + private_class_method def self.raise_coercion_error(value) + raise ::GraphQL::CoercionError, + "Could not coerce value #{value.inspect} to DateTime: must be formatted as an ISO8601 " \ + "DateTime string with a 4 digit year (example: #{::Time.now.getutc.iso8601.inspect})." + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/local_time.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/local_time.rb new file mode 100644 index 00000000..4546f5e2 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/local_time.rb @@ -0,0 +1,30 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" + +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + class LocalTime + def self.coerce_input(value, ctx) + validated_value(value) || raise(::GraphQL::CoercionError, + "Could not coerce value #{value.inspect} to LocalTime: must be formatted as an RFC3339 partial time (such as `14:23:12` or `07:05:23.555`") + end + + def self.coerce_result(value, ctx) + validated_value(value) + end + + private_class_method def self.validated_value(value) + value if value.is_a?(::String) && VALID_LOCAL_TIME_REGEX.match?(value) + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/longs.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/longs.rb new file mode 100644 index 00000000..1ea819e0 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/longs.rb @@ -0,0 +1,47 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" + +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + class Longs + def self.to_ruby_int_in_range(value, min, max) + value = Integer(value, exception: false) + return nil if value.nil? || value > max || value < min + value + end + end + + class JsonSafeLong + def self.coerce_input(value, ctx) + Longs.to_ruby_int_in_range(value, JSON_SAFE_LONG_MIN, JSON_SAFE_LONG_MAX) + end + + def self.coerce_result(value, ctx) + Longs.to_ruby_int_in_range(value, JSON_SAFE_LONG_MIN, JSON_SAFE_LONG_MAX) + end + end + + class LongString + def self.coerce_input(value, ctx) + # Do not allow non-string input, to guard against the value potentially having been rounded off by + # the client before it got serialized into a JSON request. + return nil unless value.is_a?(::String) + + Longs.to_ruby_int_in_range(value, LONG_STRING_MIN, LONG_STRING_MAX) + end + + def self.coerce_result(value, ctx) + Longs.to_ruby_int_in_range(value, LONG_STRING_MIN, LONG_STRING_MAX)&.to_s + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/no_op.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/no_op.rb new file mode 100644 index 00000000..d67e76d7 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/no_op.rb @@ -0,0 +1,24 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + # No-op implementation of coercion interface. Used as the default adapter. + class NoOp + def self.coerce_input(value, ctx) + value + end + + def self.coerce_result(value, ctx) + value + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/time_zone.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/time_zone.rb new file mode 100644 index 00000000..44fc0328 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/time_zone.rb @@ -0,0 +1,44 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "did_you_mean" +require "elastic_graph/graphql/scalar_coercion_adapters/valid_time_zones" + +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + class TimeZone + SUGGESTER = ::DidYouMean::SpellChecker.new(dictionary: VALID_TIME_ZONES.to_a) + + def self.coerce_input(value, ctx) + return value if value.nil? || VALID_TIME_ZONES.include?(value) + + suggestions = SUGGESTER.correct(value).map(&:inspect) + suggestion_sentence = + if suggestions.size >= 3 + *initial, final = suggestions + " Possible alternatives: #{initial.join(", ")}, or #{final}." + elsif suggestions.size == 1 + " Possible alternative: #{suggestions.first}." + elsif suggestions.size > 0 + " Possible alternatives: #{suggestions.join(" or ")}." + end + + raise ::GraphQL::CoercionError, + "Could not coerce value #{value.inspect} to TimeZone: must be a valid IANA time zone identifier " \ + "(such as `America/Los_Angeles` or `UTC`).#{suggestion_sentence}" + end + + def self.coerce_result(value, ctx) + return value if value.nil? || VALID_TIME_ZONES.include?(value) + nil + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/untyped.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/untyped.rb new file mode 100644 index 00000000..5da289a6 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/untyped.rb @@ -0,0 +1,32 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/untyped_encoder" + +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + class Untyped + def self.coerce_input(value, ctx) + Support::UntypedEncoder.encode(value).tap do |encoded| + # Check to see if the encoded form, when parsed as JSON, gives us back the original value. If not, + # it's not a valid `Untyped` value! + if Support::UntypedEncoder.decode(encoded) != value + raise ::GraphQL::CoercionError, + "Could not coerce value #{value.inspect} to `Untyped`: not representable as JSON." + end + end + end + + def self.coerce_result(value, ctx) + Support::UntypedEncoder.decode(value) if value.is_a?(::String) + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/valid_time_zones.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/valid_time_zones.rb new file mode 100644 index 00000000..28303c0e --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/scalar_coercion_adapters/valid_time_zones.rb @@ -0,0 +1,634 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + # The set of all valid time zones. We expect this set to align with the official IANA + # list[^1][^2][^3], but ultimately we pass these time zones to the datastore and need + # to enumerate all time zones it supports. Since Elasticsearch and OpenSearch run on the JVM and the + # date format docs[^4] link to Java's `java.time.format.DateTimeFormatter` class, we + # can conclude that they support the time zones that the java class supports. This set + # is generated by `script/dump_time_zones`, which queries Java's `java.time.ZoneId`[^5] + # class to get the set of time zones supported on the JVM. + # + # DO NOT EDIT BY HAND. + # + # [^1]: https://www.iana.org/time-zones + # [^2]: https://en.wikipedia.org/wiki/Tz_database + # [^3]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List + # [^4]: https://www.elastic.co/guide/en/elasticsearch/reference/7.10/mapping-date-format.html#custom-date-formats + # [^5]: https://docs.oracle.com/javase/8/docs/api/java/time/ZoneId.html + VALID_TIME_ZONES = %w[ + Africa/Abidjan + Africa/Accra + Africa/Addis_Ababa + Africa/Algiers + Africa/Asmara + Africa/Asmera + Africa/Bamako + Africa/Bangui + Africa/Banjul + Africa/Bissau + Africa/Blantyre + Africa/Brazzaville + Africa/Bujumbura + Africa/Cairo + Africa/Casablanca + Africa/Ceuta + Africa/Conakry + Africa/Dakar + Africa/Dar_es_Salaam + Africa/Djibouti + Africa/Douala + Africa/El_Aaiun + Africa/Freetown + Africa/Gaborone + Africa/Harare + Africa/Johannesburg + Africa/Juba + Africa/Kampala + Africa/Khartoum + Africa/Kigali + Africa/Kinshasa + Africa/Lagos + Africa/Libreville + Africa/Lome + Africa/Luanda + Africa/Lubumbashi + Africa/Lusaka + Africa/Malabo + Africa/Maputo + Africa/Maseru + Africa/Mbabane + Africa/Mogadishu + Africa/Monrovia + Africa/Nairobi + Africa/Ndjamena + Africa/Niamey + Africa/Nouakchott + Africa/Ouagadougou + Africa/Porto-Novo + Africa/Sao_Tome + Africa/Timbuktu + Africa/Tripoli + Africa/Tunis + Africa/Windhoek + America/Adak + America/Anchorage + America/Anguilla + America/Antigua + America/Araguaina + America/Argentina/Buenos_Aires + America/Argentina/Catamarca + America/Argentina/ComodRivadavia + America/Argentina/Cordoba + America/Argentina/Jujuy + America/Argentina/La_Rioja + America/Argentina/Mendoza + America/Argentina/Rio_Gallegos + America/Argentina/Salta + America/Argentina/San_Juan + America/Argentina/San_Luis + America/Argentina/Tucuman + America/Argentina/Ushuaia + America/Aruba + America/Asuncion + America/Atikokan + America/Atka + America/Bahia + America/Bahia_Banderas + America/Barbados + America/Belem + America/Belize + America/Blanc-Sablon + America/Boa_Vista + America/Bogota + America/Boise + America/Buenos_Aires + America/Cambridge_Bay + America/Campo_Grande + America/Cancun + America/Caracas + America/Catamarca + America/Cayenne + America/Cayman + America/Chicago + America/Chihuahua + America/Ciudad_Juarez + America/Coral_Harbour + America/Cordoba + America/Costa_Rica + America/Creston + America/Cuiaba + America/Curacao + America/Danmarkshavn + America/Dawson + America/Dawson_Creek + America/Denver + America/Detroit + America/Dominica + America/Edmonton + America/Eirunepe + America/El_Salvador + America/Ensenada + America/Fort_Nelson + America/Fort_Wayne + America/Fortaleza + America/Glace_Bay + America/Godthab + America/Goose_Bay + America/Grand_Turk + America/Grenada + America/Guadeloupe + America/Guatemala + America/Guayaquil + America/Guyana + America/Halifax + America/Havana + America/Hermosillo + America/Indiana/Indianapolis + America/Indiana/Knox + America/Indiana/Marengo + America/Indiana/Petersburg + America/Indiana/Tell_City + America/Indiana/Vevay + America/Indiana/Vincennes + America/Indiana/Winamac + America/Indianapolis + America/Inuvik + America/Iqaluit + America/Jamaica + America/Jujuy + America/Juneau + America/Kentucky/Louisville + America/Kentucky/Monticello + America/Knox_IN + America/Kralendijk + America/La_Paz + America/Lima + America/Los_Angeles + America/Louisville + America/Lower_Princes + America/Maceio + America/Managua + America/Manaus + America/Marigot + America/Martinique + America/Matamoros + America/Mazatlan + America/Mendoza + America/Menominee + America/Merida + America/Metlakatla + America/Mexico_City + America/Miquelon + America/Moncton + America/Monterrey + America/Montevideo + America/Montreal + America/Montserrat + America/Nassau + America/New_York + America/Nipigon + America/Nome + America/Noronha + America/North_Dakota/Beulah + America/North_Dakota/Center + America/North_Dakota/New_Salem + America/Nuuk + America/Ojinaga + America/Panama + America/Pangnirtung + America/Paramaribo + America/Phoenix + America/Port-au-Prince + America/Port_of_Spain + America/Porto_Acre + America/Porto_Velho + America/Puerto_Rico + America/Punta_Arenas + America/Rainy_River + America/Rankin_Inlet + America/Recife + America/Regina + America/Resolute + America/Rio_Branco + America/Rosario + America/Santa_Isabel + America/Santarem + America/Santiago + America/Santo_Domingo + America/Sao_Paulo + America/Scoresbysund + America/Shiprock + America/Sitka + America/St_Barthelemy + America/St_Johns + America/St_Kitts + America/St_Lucia + America/St_Thomas + America/St_Vincent + America/Swift_Current + America/Tegucigalpa + America/Thule + America/Thunder_Bay + America/Tijuana + America/Toronto + America/Tortola + America/Vancouver + America/Virgin + America/Whitehorse + America/Winnipeg + America/Yakutat + America/Yellowknife + Antarctica/Casey + Antarctica/Davis + Antarctica/DumontDUrville + Antarctica/Macquarie + Antarctica/Mawson + Antarctica/McMurdo + Antarctica/Palmer + Antarctica/Rothera + Antarctica/South_Pole + Antarctica/Syowa + Antarctica/Troll + Antarctica/Vostok + Arctic/Longyearbyen + Asia/Aden + Asia/Almaty + Asia/Amman + Asia/Anadyr + Asia/Aqtau + Asia/Aqtobe + Asia/Ashgabat + Asia/Ashkhabad + Asia/Atyrau + Asia/Baghdad + Asia/Bahrain + Asia/Baku + Asia/Bangkok + Asia/Barnaul + Asia/Beirut + Asia/Bishkek + Asia/Brunei + Asia/Calcutta + Asia/Chita + Asia/Choibalsan + Asia/Chongqing + Asia/Chungking + Asia/Colombo + Asia/Dacca + Asia/Damascus + Asia/Dhaka + Asia/Dili + Asia/Dubai + Asia/Dushanbe + Asia/Famagusta + Asia/Gaza + Asia/Harbin + Asia/Hebron + Asia/Ho_Chi_Minh + Asia/Hong_Kong + Asia/Hovd + Asia/Irkutsk + Asia/Istanbul + Asia/Jakarta + Asia/Jayapura + Asia/Jerusalem + Asia/Kabul + Asia/Kamchatka + Asia/Karachi + Asia/Kashgar + Asia/Kathmandu + Asia/Katmandu + Asia/Khandyga + Asia/Kolkata + Asia/Krasnoyarsk + Asia/Kuala_Lumpur + Asia/Kuching + Asia/Kuwait + Asia/Macao + Asia/Macau + Asia/Magadan + Asia/Makassar + Asia/Manila + Asia/Muscat + Asia/Nicosia + Asia/Novokuznetsk + Asia/Novosibirsk + Asia/Omsk + Asia/Oral + Asia/Phnom_Penh + Asia/Pontianak + Asia/Pyongyang + Asia/Qatar + Asia/Qostanay + Asia/Qyzylorda + Asia/Rangoon + Asia/Riyadh + Asia/Saigon + Asia/Sakhalin + Asia/Samarkand + Asia/Seoul + Asia/Shanghai + Asia/Singapore + Asia/Srednekolymsk + Asia/Taipei + Asia/Tashkent + Asia/Tbilisi + Asia/Tehran + Asia/Tel_Aviv + Asia/Thimbu + Asia/Thimphu + Asia/Tokyo + Asia/Tomsk + Asia/Ujung_Pandang + Asia/Ulaanbaatar + Asia/Ulan_Bator + Asia/Urumqi + Asia/Ust-Nera + Asia/Vientiane + Asia/Vladivostok + Asia/Yakutsk + Asia/Yangon + Asia/Yekaterinburg + Asia/Yerevan + Atlantic/Azores + Atlantic/Bermuda + Atlantic/Canary + Atlantic/Cape_Verde + Atlantic/Faeroe + Atlantic/Faroe + Atlantic/Jan_Mayen + Atlantic/Madeira + Atlantic/Reykjavik + Atlantic/South_Georgia + Atlantic/St_Helena + Atlantic/Stanley + Australia/ACT + Australia/Adelaide + Australia/Brisbane + Australia/Broken_Hill + Australia/Canberra + Australia/Currie + Australia/Darwin + Australia/Eucla + Australia/Hobart + Australia/LHI + Australia/Lindeman + Australia/Lord_Howe + Australia/Melbourne + Australia/NSW + Australia/North + Australia/Perth + Australia/Queensland + Australia/South + Australia/Sydney + Australia/Tasmania + Australia/Victoria + Australia/West + Australia/Yancowinna + Brazil/Acre + Brazil/DeNoronha + Brazil/East + Brazil/West + CET + CST6CDT + Canada/Atlantic + Canada/Central + Canada/Eastern + Canada/Mountain + Canada/Newfoundland + Canada/Pacific + Canada/Saskatchewan + Canada/Yukon + Chile/Continental + Chile/EasterIsland + Cuba + EET + EST5EDT + Egypt + Eire + Etc/GMT + Etc/GMT+0 + Etc/GMT+1 + Etc/GMT+10 + Etc/GMT+11 + Etc/GMT+12 + Etc/GMT+2 + Etc/GMT+3 + Etc/GMT+4 + Etc/GMT+5 + Etc/GMT+6 + Etc/GMT+7 + Etc/GMT+8 + Etc/GMT+9 + Etc/GMT-0 + Etc/GMT-1 + Etc/GMT-10 + Etc/GMT-11 + Etc/GMT-12 + Etc/GMT-13 + Etc/GMT-14 + Etc/GMT-2 + Etc/GMT-3 + Etc/GMT-4 + Etc/GMT-5 + Etc/GMT-6 + Etc/GMT-7 + Etc/GMT-8 + Etc/GMT-9 + Etc/GMT0 + Etc/Greenwich + Etc/UCT + Etc/UTC + Etc/Universal + Etc/Zulu + Europe/Amsterdam + Europe/Andorra + Europe/Astrakhan + Europe/Athens + Europe/Belfast + Europe/Belgrade + Europe/Berlin + Europe/Bratislava + Europe/Brussels + Europe/Bucharest + Europe/Budapest + Europe/Busingen + Europe/Chisinau + Europe/Copenhagen + Europe/Dublin + Europe/Gibraltar + Europe/Guernsey + Europe/Helsinki + Europe/Isle_of_Man + Europe/Istanbul + Europe/Jersey + Europe/Kaliningrad + Europe/Kiev + Europe/Kirov + Europe/Kyiv + Europe/Lisbon + Europe/Ljubljana + Europe/London + Europe/Luxembourg + Europe/Madrid + Europe/Malta + Europe/Mariehamn + Europe/Minsk + Europe/Monaco + Europe/Moscow + Europe/Nicosia + Europe/Oslo + Europe/Paris + Europe/Podgorica + Europe/Prague + Europe/Riga + Europe/Rome + Europe/Samara + Europe/San_Marino + Europe/Sarajevo + Europe/Saratov + Europe/Simferopol + Europe/Skopje + Europe/Sofia + Europe/Stockholm + Europe/Tallinn + Europe/Tirane + Europe/Tiraspol + Europe/Ulyanovsk + Europe/Uzhgorod + Europe/Vaduz + Europe/Vatican + Europe/Vienna + Europe/Vilnius + Europe/Volgograd + Europe/Warsaw + Europe/Zagreb + Europe/Zaporozhye + Europe/Zurich + GB + GB-Eire + GMT + GMT0 + Greenwich + Hongkong + Iceland + Indian/Antananarivo + Indian/Chagos + Indian/Christmas + Indian/Cocos + Indian/Comoro + Indian/Kerguelen + Indian/Mahe + Indian/Maldives + Indian/Mauritius + Indian/Mayotte + Indian/Reunion + Iran + Israel + Jamaica + Japan + Kwajalein + Libya + MET + MST7MDT + Mexico/BajaNorte + Mexico/BajaSur + Mexico/General + NZ + NZ-CHAT + Navajo + PRC + PST8PDT + Pacific/Apia + Pacific/Auckland + Pacific/Bougainville + Pacific/Chatham + Pacific/Chuuk + Pacific/Easter + Pacific/Efate + Pacific/Enderbury + Pacific/Fakaofo + Pacific/Fiji + Pacific/Funafuti + Pacific/Galapagos + Pacific/Gambier + Pacific/Guadalcanal + Pacific/Guam + Pacific/Honolulu + Pacific/Johnston + Pacific/Kanton + Pacific/Kiritimati + Pacific/Kosrae + Pacific/Kwajalein + Pacific/Majuro + Pacific/Marquesas + Pacific/Midway + Pacific/Nauru + Pacific/Niue + Pacific/Norfolk + Pacific/Noumea + Pacific/Pago_Pago + Pacific/Palau + Pacific/Pitcairn + Pacific/Pohnpei + Pacific/Ponape + Pacific/Port_Moresby + Pacific/Rarotonga + Pacific/Saipan + Pacific/Samoa + Pacific/Tahiti + Pacific/Tarawa + Pacific/Tongatapu + Pacific/Truk + Pacific/Wake + Pacific/Wallis + Pacific/Yap + Poland + Portugal + ROK + Singapore + SystemV/AST4 + SystemV/AST4ADT + SystemV/CST6 + SystemV/CST6CDT + SystemV/EST5 + SystemV/EST5EDT + SystemV/HST10 + SystemV/MST7 + SystemV/MST7MDT + SystemV/PST8 + SystemV/PST8PDT + SystemV/YST9 + SystemV/YST9YDT + Turkey + UCT + US/Alaska + US/Aleutian + US/Arizona + US/Central + US/East-Indiana + US/Eastern + US/Hawaii + US/Indiana-Starke + US/Michigan + US/Mountain + US/Pacific + US/Samoa + UTC + Universal + W-SU + WET + Zulu + ].to_set + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/schema.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/schema.rb new file mode 100644 index 00000000..19956248 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/schema.rb @@ -0,0 +1,164 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "digest/md5" +require "forwardable" +require "graphql" +require "elastic_graph/constants" +require "elastic_graph/errors" +require "elastic_graph/graphql/monkey_patches/schema_field" +require "elastic_graph/graphql/monkey_patches/schema_object" +require "elastic_graph/graphql/schema/field" +require "elastic_graph/graphql/schema/type" +require "elastic_graph/support/hash_util" + +module ElasticGraph + # Wraps a GraphQL::Schema object in order to provide higher-level, more convenient APIs + # on top of that. The schema is assumed to be immutable, so this class memoizes many + # computations it does, ensuring we never need to traverse the schema graph multiple times. + class GraphQL + class Schema + BUILT_IN_TYPE_NAMES = ( + scalar_types = ::GraphQL::Schema::BUILT_IN_TYPES.keys # Int, ID, String, etc + introspection_types = ::GraphQL::Schema.types.keys # __Type, __Schema, etc + scalar_types.to_set.union(introspection_types) + ) + + attr_reader :element_names, :defined_types, :config, :graphql_schema, :runtime_metadata + + def initialize( + graphql_schema_string:, + config:, + runtime_metadata:, + index_definitions_by_graphql_type:, + graphql_gem_plugins:, + &build_resolver + ) + @element_names = runtime_metadata.schema_element_names + @config = config + @runtime_metadata = runtime_metadata + + @types_by_graphql_type = Hash.new do |hash, key| + hash[key] = Type.new( + self, + key, + index_definitions_by_graphql_type[key.graphql_name] || [], + runtime_metadata.object_types_by_name[key.graphql_name], + runtime_metadata.enum_types_by_name[key.graphql_name] + ) + end + + @types_by_name = Hash.new { |hash, key| hash[key] = lookup_type_by_name(key) } + @build_resolver = build_resolver + + # Note: as part of loading the schema, the GraphQL gem may use the resolver (such + # when a directive has a custom scalar) so we must wait to instantiate the schema + # as late as possible here. If we do this before initializing some of the instance + # variables above we'll get `NoMethodError` on `nil`. + @graphql_schema = ::GraphQL::Schema.from_definition( + graphql_schema_string, + default_resolve: LazyResolverAdapter.new(method(:resolver)), + using: graphql_gem_plugins + ) + + # Pre-load all defined types so that all field extras can get configured as part + # of loading the schema, before we execute the first query. + @defined_types = build_defined_types_array(@graphql_schema) + end + + def type_from(graphql_type) + @types_by_graphql_type[graphql_type] + end + + # Note: this does not support "wrapped" types (e.g. `Int!` or `[Int]` compared to `Int`), + # as the graphql schema object does not give us an index of those by name. You can still + # get type objects for wrapped types, but you need to get it from a field object of that + # type. + def type_named(type_name) + @types_by_name[type_name.to_s] + end + + def document_type_stored_in(index_definition_name) + indexed_document_types_by_index_definition_name.fetch(index_definition_name) do + if index_definition_name.include?(ROLLOVER_INDEX_INFIX_MARKER) + raise ArgumentError, "`#{index_definition_name}` is the name of a rollover index; pass the name of the parent index definition instead." + else + raise Errors::NotFoundError, "The index definition `#{index_definition_name}` does not appear to exist. Is it misspelled?" + end + end + end + + def field_named(type_name, field_name) + type_named(type_name).field_named(field_name) + end + + def enum_value_named(type_name, enum_value_name) + type_named(type_name).enum_value_named(enum_value_name) + end + + # The list of user-defined types that are indexed document types. (Indexed aggregation types will not be included in this.) + def indexed_document_types + @indexed_document_types ||= defined_types.select(&:indexed_document?) + end + + def to_s + "#<#{self.class.name} 0x#{__id__.to_s(16)} indexed_document_types=#{indexed_document_types.map(&:name).sort.to_s.delete(":")}>" + end + alias_method :inspect, :to_s + + private + + # Adapter class to allow us to lazily load the resolver instance. + # + # Necessary because the resolver must be provided to `GraphQL::Schema.from_definition`, + # but the resolver logic itself depends upon the loaded schema to know how to resolve. + # To work around the circular dependency, we build the schema with this lazy adapter, + # then build the resolver with the schema, and then the lazy resolver lazily loads the resolver. + LazyResolverAdapter = Struct.new(:builder) do + def resolver + @resolver ||= builder.call + end + + extend Forwardable + def_delegators :resolver, :call, :resolve_type, :coerce_input, :coerce_result + end + + def lookup_type_by_name(type_name) + type_from(@graphql_schema.types.fetch(type_name)) + rescue KeyError => e + msg = "No type named #{type_name} could be found" + msg += "; Possible alternatives: [#{e.corrections.join(", ").delete('"')}]." if e.corrections.any? + raise Errors::NotFoundError, msg + end + + def resolver + @resolver ||= @build_resolver.call(self) + end + + def build_defined_types_array(graphql_schema) + graphql_schema + .types + .values + .reject { |t| BUILT_IN_TYPE_NAMES.include?(t.graphql_name) } + .map { |t| type_named(t.graphql_name) } + end + + def indexed_document_types_by_index_definition_name + @indexed_document_types_by_index_definition_name ||= indexed_document_types.each_with_object({}) do |type, hash| + type.index_definitions.each do |index_def| + if hash.key?(index_def.name) + raise Errors::SchemaError, "DatastoreCore::IndexDefinition #{index_def.name} is used multiple times: #{type} vs #{hash[index_def.name]}" + end + + hash[index_def.name] = type + end + end.freeze + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/schema/arguments.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/schema/arguments.rb new file mode 100644 index 00000000..90e9d2e4 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/schema/arguments.rb @@ -0,0 +1,78 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + class Schema + # A utility module for working with GraphQL schema arguments. + module Arguments + # A utility method to convert the given `args_hash` to its schema form. + # The schema form is the casing of the arguments according to the GraphQL + # schema definition. For example, consider a case like: + # + # type Query { + # widgets(orderBy: [WidgetSort!]): [Widget!]! + # } + # + # The GraphQL gem converts all arguments to ruby keyword args style (symbolized, + # snake_case keys) before passing the args to us, but the `orderBy` argument in + # the schema definition uses camelCase. ElasticGraph was designed to flexibly + # support whatever casing the schema developer chooses to use but the GraphQL + # gem's conversion to keyword args style gets in the way. ElasticGraph needs to + # receive the arguments in the casing form defined in the schema so that, for example, + # when it creates a datastore query, it correctly filters on fields according + # to the casing of the fields in the index. + # + # This utility method converts an args hash back to its schema form (string keys, + # with the casing from the schema) by using the arg definitions themselves to get + # the arg names from the GraphQL schema. + # + # Example: + # + # to_schema_form({ order_by: ["size"] }, widgets_field) + # # => { "orderBy" => ["size"] } + # + # The implementation here was taken from a code snippet provided by the maintainer of + # the GraphQL gem: https://github.com/rmosolgo/graphql-ruby/issues/2869 + def self.to_schema_form(args_value, args_owner) + # For custom scalar types (such as `_Any` for apollo federation), `args_owner` won't + # response to `arguments`. + return args_value unless args_owner.respond_to?(:arguments) + + __skip__ = case args_value + when Hash, ::GraphQL::Schema::InputObject + arg_defns = args_owner.arguments.values + + {}.tap do |accumulator| + args_value.each do |key, value| + # Note: we could build `arg_defns` into a hash keyed by `keyword` + # outside of this loop, to give us an O(1) lookup here. However, + # usually there are a small number of args (e.g 1 or 2, maybe up + # to 6 in extreme cases) so it's probably likely to be ultimately + # slower to build the hash, particularly when you account for the + # extra memory allocation and GC for the hash. + arg_defn = arg_defns.find do |a| + a.keyword == key + end || raise(Errors::SchemaError, "Cannot find an argument definition for #{key.inspect} on `#{args_owner.name}`") + + next_owner = arg_defn.type.unwrap + accumulator[arg_defn.name] = to_schema_form(value, next_owner) + end + end + when Array + args_value.map { |arg_value| to_schema_form(arg_value, args_owner) } + else + # :nocov: -- not sure how to cover this but we want this default branch. + args_value + # :nocov: + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/schema/enum_value.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/schema/enum_value.rb new file mode 100644 index 00000000..41bd5c79 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/schema/enum_value.rb @@ -0,0 +1,30 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" + +module ElasticGraph + class GraphQL + class Schema + # Represents an enum value within a GraphQL schema. + class EnumValue < ::Data.define(:name, :type, :runtime_metadata) + def sort_clauses + sort_clause = runtime_metadata&.sort_field&.then { |sf| {sf.field_path => {"order" => sf.direction.to_s}} } || + raise(Errors::SchemaError, "Runtime metadata provides no `sort_field` for #{type.name}.#{name} enum value.") + + [sort_clause] + end + + def to_s + "#<#{self.class.name} #{type.name}.#{name}>" + end + alias_method :inspect, :to_s + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/schema/field.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/schema/field.rb new file mode 100644 index 00000000..2400bfa5 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/schema/field.rb @@ -0,0 +1,147 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/graphql/schema/relation_join" +require "elastic_graph/graphql/schema/arguments" + +module ElasticGraph + class GraphQL + class Schema + # Represents a field within a GraphQL type. + class Field + # The type in which the field resides. + attr_reader :parent_type + + attr_reader :schema, :schema_element_names, :graphql_field, :name_in_index, :relation, :computation_detail + + def initialize(schema, parent_type, graphql_field, runtime_metadata) + @schema = schema + @schema_element_names = schema.element_names + @parent_type = parent_type + @graphql_field = graphql_field + @relation = runtime_metadata&.relation + @computation_detail = runtime_metadata&.computation_detail + @name_in_index = runtime_metadata&.name_in_index&.to_sym || name + + # Adds the :extras required by ElasticGraph. For now, this blindly adds `:lookahead` + # to each field so that we have access to what the child selections are, as described here: + # + # https://graphql-ruby.org/queries/lookahead + # + # Currently we only need this when building an `DatastoreQuery` (which is not done for all + # fields) so a future optimization may only add this to fields where we actually need it. + # For now we add it to all fields because it's simplest and it's not clear if there is + # any performance benefit to not adding it when we do not use it. + # + # Note: input fields do not respond to `extras`, which is why we do it conditionally here. + # + # Note: on GraphQL gem introspection types (e.g. `__Field`), the fields respond to `:extras`, + # but that later causes a weird error (`ArgumentError: unknown keyword: :lookahead`) + # when those types are accessed in a Query. We don't really want to mutate the fields on the + # built-in types by adding `:lookahead` so it's best to avoid setting that extra on the built + # in types. + if @graphql_field.respond_to?(:extras) && !BUILT_IN_TYPE_NAMES.include?(parent_type.name.to_s) + @graphql_field.extras([:lookahead]) + end + end + + def type + @type ||= @schema.type_from(@graphql_field.type) + end + + def name + @name ||= @graphql_field.name.to_sym + end + + # Returns an object that knows how this field joins to its relation. + # Used by ElasticGraph::Resolvers::NestedRelationships. + def relation_join + # Not every field has a join relation, so it can be nil. But we do not want + # to re-compute that on every call, so we return @relation_join if it's already + # defined rather than if its truthy. + return @relation_join if defined?(@relation_join) + @relation_join = RelationJoin.from(self) + end + + # Given an array of sort enums, returns an array of datastore compatible sort clauses + def sort_clauses_for(sorts) + Array(sorts).flat_map { |sort| sort_argument_type.enum_value_named(sort).sort_clauses } + end + + # Indicates if this is an aggregated field (used inside an `Aggregation` type). + def aggregated? + type.unwrap_non_null.elasticgraph_category == :scalar_aggregated_values + end + + def args_to_schema_form(args) + Arguments.to_schema_form(args, @graphql_field) + end + + # Returns a list of field names that are required from the datastore in order + # to resolve this field at GraphQL query handling time. + def index_field_names_for_resolution + # For an embedded object, we do not require any fields because it is the nested fields + # that we will request from the datastore, which will be required to resolve them. But + # we do not need to request the embedded object field itself. + return [] if type.embedded_object? + return [] if parent_type.relay_connection? || parent_type.relay_edge? + return index_id_field_names_for_relation if relation_join + + [name_in_index.to_s] + end + + # Indicates this field should be hidden in the GraphQL schema so as to not be queryable. + # We only hide a field if resolving it would require using a datastore cluster that + # we can't access. For the most part, this just delegates to `Type#hidden_from_queries?` + # which does the index accessibility check. + def hidden_from_queries? + # The type has logic to check if the backing datastore index is accessible, so we just + # delegate to that logic here. + type.unwrap_fully.hidden_from_queries? + end + + def coerce_result(result) + return result unless parent_type.graphql_only_return_type + type.coerce_result(result) + end + + def description + "#{@parent_type.name}.#{name}" + end + + def to_s + "#<#{self.class.name} #{description}>" + end + alias_method :inspect, :to_s + + private + + # Returns the `order_by` arguments type field (unwrapped) + def sort_argument_type + @sort_argument_type ||= begin + graphql_argument = @graphql_field.arguments.fetch(schema_element_names.order_by) do + raise Errors::SchemaError, "`#{schema_element_names.order_by}` argument not defined for field `#{parent_type.name}.#{name}`." + end + @schema.type_from(graphql_argument.type.unwrap) + end + end + + def index_id_field_names_for_relation + if type.unwrap_fully == parent_type # means its a self-referential relation (e.g. child to parent of same type) + # Since it's self-referential, the `filter_id_field` (which lives on the "remote" type) also must + # exist as a field in our DatastoreCore::IndexDefinition. + [relation_join.document_id_field_name, relation_join.filter_id_field_name] + else + [relation_join.document_id_field_name] + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/schema/relation_join.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/schema/relation_join.rb new file mode 100644 index 00000000..4f8c9731 --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/schema/relation_join.rb @@ -0,0 +1,103 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/datastore_response/search_response" + +module ElasticGraph + class GraphQL + class Schema + # Represents the join between documents for a relation. + # + # Note that this class assumes a valid, well-formed schema definition, and makes no + # attempt to provide user-friendly errors when that is not the case. For example, + # we assume that a nested relationship field has at most one relationship directive. + # The (as yet unwritten) schema linter should validate such things eventually. + # When we do encounter errors at runtime (such as getting a scalar where we expect + # a list, or vice-versa), this class attempts to deal with as best as it can (sometimes + # simply picking one record or id from many!) and logs a warning. + # + # Note: this class isn't driven directly by tests. It exist purely to serve the needs + # of ElasticGraph::Resolvers::NestedRelationships, and is driven by that class's tests. + # It lives here because it's useful to expose it off of a `Field` since it's a property + # of the field and that lets us memoize it on the field itself. + class RelationJoin < ::Data.define(:field, :document_id_field_name, :filter_id_field_name, :id_cardinality, :doc_cardinality, :additional_filter, :foreign_key_nested_paths) + def self.from(field) + return nil if (relation = field.relation).nil? + + doc_cardinality = field.type.collection? ? Cardinality::Many : Cardinality::One + + if relation.direction == :in + # An inbound foreign key has some field (such as `foo_id`) on another document that points + # back to the `id` field on the document with the relation. + # + # The cardinality of the document id field on an inbound relation is always 1 since + # it is always the primary key `id` field. + new(field, "id", relation.foreign_key, Cardinality::One, doc_cardinality, relation.additional_filter, relation.foreign_key_nested_paths) + else + # An outbound foreign key has some field (such as `foo_id`) on the document with the relation + # that point out to the `id` field of another document. + new(field, relation.foreign_key, "id", doc_cardinality, doc_cardinality, relation.additional_filter, relation.foreign_key_nested_paths) + end + end + + def blank_value + doc_cardinality.blank_value + end + + # Extracts a single id or a list of ids from the given document, as required by the relation. + def extract_id_or_ids_from(document, log_warning) + id_or_ids = document.fetch(document_id_field_name) do + log_warning.call(document: document, problem: "#{document_id_field_name} is missing from the document") + blank_value + end + + normalize_ids(id_or_ids) do |problem| + log_warning.call(document: document, problem: "#{document_id_field_name}: #{problem}") + end + end + + # Normalizes the given documents, ensuring it has the expected cardinality. + def normalize_documents(response, &handle_warning) + doc_cardinality.normalize(response, handle_warning: handle_warning, &:id) + end + + private + + def normalize_ids(id_or_ids, &handle_warning) + id_cardinality.normalize(id_or_ids, handle_warning: handle_warning, &:itself) + end + + module Cardinality + module Many + def self.normalize(list_or_scalar, handle_warning:) + return list_or_scalar if list_or_scalar.is_a?(Enumerable) + handle_warning.call("scalar instead of a list") + Array(list_or_scalar) + end + + def self.blank_value + DatastoreResponse::SearchResponse::EMPTY + end + end + + module One + def self.normalize(list_or_scalar, handle_warning:, &deterministic_comparator) + return list_or_scalar unless list_or_scalar.is_a?(Enumerable) + handle_warning.call("list of more than one item instead of a scalar") if list_or_scalar.size > 1 + list_or_scalar.min_by(&deterministic_comparator) + end + + def self.blank_value + nil + end + end + end + end + end + end +end diff --git a/elasticgraph-graphql/lib/elastic_graph/graphql/schema/type.rb b/elasticgraph-graphql/lib/elastic_graph/graphql/schema/type.rb new file mode 100644 index 00000000..e6823c7b --- /dev/null +++ b/elasticgraph-graphql/lib/elastic_graph/graphql/schema/type.rb @@ -0,0 +1,263 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core/index_definition" +require "elastic_graph/errors" +require "elastic_graph/graphql/schema/field" +require "elastic_graph/graphql/schema/enum_value" +require "forwardable" + +module ElasticGraph + class GraphQL + class Schema + # Represents a GraphQL type. + class Type + attr_reader :graphql_type, :fields_by_name, :index_definitions, :elasticgraph_category, :graphql_only_return_type + + def initialize( + schema, + graphql_type, + index_definitions, + object_runtime_metadata, + enum_runtime_metadata + ) + @schema = schema + @graphql_type = graphql_type + @enum_values_by_name = Hash.new do |hash, key| + hash[key] = lookup_enum_value_by_name(key) + end + + @index_definitions = index_definitions + @object_runtime_metadata = object_runtime_metadata + @elasticgraph_category = object_runtime_metadata&.elasticgraph_category + @graphql_only_return_type = object_runtime_metadata&.graphql_only_return_type + @enum_runtime_metadata = enum_runtime_metadata + @enum_value_names_by_original_name = (enum_runtime_metadata&.values_by_name || {}).to_h do |name, value| + [value.alternate_original_name || name, name] + end + + @fields_by_name = build_fields_by_name_hash(schema, graphql_type).freeze + end + + def name + @name ||= @graphql_type.to_type_signature.to_sym + end + + # List of index definitions that should be searched for this type. + def search_index_definitions + @search_index_definitions ||= + if indexed_aggregation? + # For an indexed aggregation, we just delegate to its source type. This works better than + # dumping index definitions in the runtime metadata of the indexed aggregation type itself + # because of abstract (interface/union) types. The source document type handles that (since + # there is a supertype/subtype relationship on the document types) but that relationship + # does not exist on the indexed aggregation. + # + # For example, assume we have these indexed document types: + # - type Person {} + # - type Company {} + # - union Inventor = Person | Company + # + # We can go from `Inventor` to its subtypes to find the search indexes. However, `InventorAggregation` + # is NOT a union of `PersonAggregation` and `CompanyAggregation`, so we can't do the same thing on the + # indexed aggregation types. Delegating to the source type solves this case. + @schema.type_named(@object_runtime_metadata.source_type).search_index_definitions + else + @index_definitions.union(subtypes.flat_map(&:search_index_definitions)) + end + end + + # List of index definitions that should be indexed into for this type. + # For now this is just an alias for `search_index_definitions`, but + # in the future we expect to allow these to be different. We don't yet + # support defining multiple indices on one GraphQL type, though, which is + # where that would prove useful. Still, it's a useful abstraction to have + # this method available for callers now. + alias_method :indexing_index_definitions, :search_index_definitions + + # Unwraps the non-null type wrapping, if this type is non-null. If this type is nullable, + # returns it as-is. + def unwrap_non_null + return self if nullable? + @schema.type_from(@graphql_type.of_type) + end + + # Fully unwraps this type, in order to extracts the underlying type (an object or scalar) + # from its wrappings. As needed, this will unwrap any of these wrappings: + # + # - non-null + # - list + # - relay connection + def unwrap_fully + @unwrap_fully ||= begin + unwrapped = @schema.type_from(@graphql_type.unwrap) + + if unwrapped.relay_connection? + unwrapped + .field_named(@schema.element_names.edges).type.unwrap_fully + .field_named(@schema.element_names.node).type.unwrap_fully + else + unwrapped + end + end + end + + # Returns the subtypes of this type, if it has any. This is like `#possible_types` provided by the + # GraphQL gem, but that includes a type itself when you ask for the possible types of a non-abstract type. + def subtypes + @subtypes ||= @schema.graphql_schema.possible_types(graphql_type).map { |t| @schema.type_from(t) } - [self] + end + + def field_named(field_name) + @fields_by_name.fetch(field_name.to_s) + rescue KeyError => e + msg = "No field named #{field_name} (on type #{name}) could be found" + msg += "; Possible alternatives: [#{e.corrections.join(", ").delete('"')}]." if e.corrections.any? + raise Errors::NotFoundError, msg + end + + def enum_value_named(enum_value_name) + @enum_values_by_name[enum_value_name.to_s] + end + + def coerce_result(result) + @enum_value_names_by_original_name.fetch(result, result) + end + + def to_s + "#<#{self.class.name} #{name}>" + end + alias_method :inspect, :to_s + + # ******************************************************************************************** + # Predicates + # + # Below here are a bunch of predicates that can be used to ask questions of a type. GraphQL's + # "wrapping" type system (e.g. non-null wraps nullable; lists wrap objects or scalars) adds + # some complexity and nuance here. We have decided to implement these predicates to auto-unwrap + # non-null (e.g. SomeType! -> SomeType). For example, `object?` will return `true` from both a + # nullable and non-nullable object type, because both are fundamentally objects. Importantly, + # we do not ever auto-unwrap a type from its list or relay connection wrapping; if the caller + # wants that, they can manually unwrap before calling the predicate. + # + # Note also that `non_null?` and `nullable?` are an exception: since they check nullability, + # we do not auto-unwrap non-null on them, naturally. + # ******************************************************************************************** + + extend Forwardable + def_delegators :@graphql_type, :list?, :non_null? + + def nullable? + !non_null? + end + + def abstract? + return unwrap_non_null.abstract? if non_null? + @graphql_type.kind.abstract? + end + + def enum? + return unwrap_non_null.enum? if non_null? + @graphql_type.kind.enum? + end + + # Returns `true` if this type serializes as a JSON object, with sub-fields. + # Note this is slightly different from the GraphQL gem and GraphQL spec: it considers + # inputs to be distinct from objects, but for our purposes we consider inputs to be + # objects since they have sub-fields and serialize as JSON objects. + def object? + return unwrap_non_null.object? if non_null? + kind = @graphql_type.kind + kind.abstract? || kind.object? || kind.input_object? + end + + # Is the type a user-defined document type directly indexed in the index? + def indexed_document? + return unwrap_non_null.indexed_document? if non_null? + return false if indexed_aggregation? + return true if subtypes.any? && subtypes.all?(&:indexed_document?) + @index_definitions.any? + end + + def indexed_aggregation? + unwrapped_has_category?(:indexed_aggregation) + end + + # Indicates if this type is an object type that is embedded in another indexed type + # in the index mapping. Note: we have avoided the term `nested` here because it + # is a specific Elasticsearch/OpenSearch mapping type that we will not necessarily be using: + # https://www.elastic.co/guide/en/elasticsearch/reference/current/nested.html + def embedded_object? + return unwrap_non_null.embedded_object? if non_null? + return false if relay_edge? || relay_connection? || @graphql_type.kind.input_object? + object? && !indexed_document? && !indexed_aggregation? + end + + def collection? + list? || relay_connection? + end + + def relay_connection? + unwrapped_has_category?(:relay_connection) + end + + def relay_edge? + unwrapped_has_category?(:relay_edge) + end + + # Indicates this type should be hidden in the GraphQL schema so as to not be queryable. + # We only hide a type if both of the following are true: + # + # - It's backed by one or more search index definitions + # - None of the search index definitions are accessible from queries + def hidden_from_queries? + return false if search_index_definitions.empty? + search_index_definitions.none?(&:accessible_from_queries?) + end + + private + + def lookup_enum_value_by_name(enum_value_name) + graphql_enum_value = @graphql_type.values.fetch(enum_value_name) + + EnumValue.new( + name: graphql_enum_value.graphql_name.to_sym, + type: self, + runtime_metadata: @enum_runtime_metadata&.values_by_name&.dig(enum_value_name) + ) + rescue KeyError => e + msg = "No enum value named #{enum_value_name} (on type #{name}) could be found" + msg += "; Possible alternatives: [#{e.corrections.join(", ").delete('"')}]." if e.corrections.any? + raise Errors::NotFoundError, msg + end + + def build_fields_by_name_hash(schema, graphql_type) + fields_hash = + if graphql_type.respond_to?(:fields) + graphql_type.fields + elsif graphql_type.kind.input_object? + # Unfortunately, input objects do not have a `fields` method; instead it is called `arguments`. + graphql_type.arguments + else + {} + end + + # Eagerly fan out and instantiate all `Field` objects so that the :extras + # get added to each field as require before we execute the first query + fields_hash.each_with_object({}) do |(name, field), hash| + hash[name] = Field.new(schema, self, field, @object_runtime_metadata&.graphql_fields_by_name&.dig(name)) + end + end + + def unwrapped_has_category?(category) + unwrap_non_null.elasticgraph_category == category + end + end + end + end +end diff --git a/elasticgraph-graphql/script/dump_time_zones b/elasticgraph-graphql/script/dump_time_zones new file mode 100755 index 00000000..4fe5bd80 --- /dev/null +++ b/elasticgraph-graphql/script/dump_time_zones @@ -0,0 +1,81 @@ +#!/usr/bin/env ruby + +require "fileutils" + +updated_code_filename = "#{__dir__}/../../tmp/updated_valid_time_zones.rb" + +# Note: CI does not appear to have java 14 or 17 available, so we use java 11 here. +java_time_zones = `java --source 11 #{__dir__}/dump_time_zones.java`.split("\n") + +::File.write(updated_code_filename, <<~EOS) + # Copyright 2024 Block, Inc. + # + # Use of this source code is governed by an MIT-style + # license that can be found in the LICENSE file or at + # https://opensource.org/licenses/MIT. + # + # frozen_string_literal: true + + module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + # The set of all valid time zones. We expect this set to align with the official IANA + # list[^1][^2][^3], but ultimately we pass these time zones to the datastore and need + # to enumerate all time zones it supports. Since Elasticsearch and OpenSearch run on the JVM and the + # date format docs[^4] link to Java's `java.time.format.DateTimeFormatter` class, we + # can conclude that they support the time zones that the java class supports. This set + # is generated by `script/dump_time_zones`, which queries Java's `java.time.ZoneId`[^5] + # class to get the set of time zones supported on the JVM. + # + # DO NOT EDIT BY HAND. + # + # [^1]: https://www.iana.org/time-zones + # [^2]: https://en.wikipedia.org/wiki/Tz_database + # [^3]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List + # [^4]: https://www.elastic.co/guide/en/elasticsearch/reference/7.10/mapping-date-format.html#custom-date-formats + # [^5]: https://docs.oracle.com/javase/8/docs/api/java/time/ZoneId.html + VALID_TIME_ZONES = %w[ + #{java_time_zones.join("\n ")} + ].to_set + end + end + end +EOS + +verify_code_command = %(ruby -r#{updated_code_filename} -e "puts ElasticGraph::GraphQL::ScalarCoercionAdapters::VALID_TIME_ZONES.size") +verify_code_output = `#{verify_code_command} 2>&1` + +if verify_code_output.to_i < 600 # As of Nov 2022 there are 601 time zones. + abort <<~EOS.strip + It appears that the generated code is invalid. Check `#{updated_code_filename}` to see what was generated. + + Output from `#{verify_code_command}`: + + #{verify_code_output} + EOS +end + +if ARGV.include?("--print") + puts ::File.read(updated_code_filename) +else + filename = "#{__dir__}/../lib/elastic_graph/graphql/scalar_coercion_adapters/valid_time_zones.rb" + ::FileUtils.cp(updated_code_filename, filename) + puts "Timezones have been written to `#{filename}`" +end + +# if ARGV.include?("--verify") +# existing_contents = ::File.exist?(filename) ? ::File.read(filename) : "" +# +# if existing_contents == valid_timezones_ruby_code +# puts "`#{filename}` is up to date!" +# else +# diff = `git diff --no-index #{"--color" if $stdout.tty?} --binary #{updated_code_filename} #{filename}` +# +# abort <<~EOS.strip +# `#{filename}` is not up to date! Rerun `script/dump_time_zones` to correct. +# +# #{diff} +# EOS +# end +# else +# end diff --git a/elasticgraph-graphql/script/dump_time_zones.java b/elasticgraph-graphql/script/dump_time_zones.java new file mode 100755 index 00000000..fb0f0450 --- /dev/null +++ b/elasticgraph-graphql/script/dump_time_zones.java @@ -0,0 +1,17 @@ +import java.time.ZoneId; +import java.util.Set; + +/** + * Run this java file via `java --source 11 script/dump_time_zones.java`. + * `script/dump_time_zones` is a higher level wrapper that delegates to this + * and applies additional logic. + */ +public class DumpTimeZones { + public static void main(String[] args) { + Set availableZones = ZoneId.getAvailableZoneIds(); + + availableZones.stream() + .sorted() + .forEach(it -> System.out.println(it)); + } +} diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql.rbs new file mode 100644 index 00000000..0a3f7c99 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql.rbs @@ -0,0 +1,71 @@ +module ElasticGraph + class GraphQL + attr_reader config: Config + attr_reader logger: ::Logger + attr_reader runtime_metadata: SchemaArtifacts::RuntimeMetadata::Schema + attr_reader graphql_schema_string: ::String + attr_reader datastore_core: DatastoreCore + attr_reader clock: singleton(::Time) + + extend _BuildableFromParsedYaml[GraphQL] + extend Support::FromYamlFile[GraphQL] + + def initialize: ( + config: Config, + datastore_core: DatastoreCore, + ?graphql_adapter: Resolvers::GraphQLAdapter?, + ?datastore_search_router: DatastoreSearchRouter?, + ?filter_interpreter: Filtering::FilterInterpreter?, + ?sub_aggregation_grouping_adapter: Aggregation::groupingAdapter?, + ?monotonic_clock: Support::MonotonicClock?, + ?clock: singleton(::Time) + ) -> void + + @graphql_http_endpoint: HTTPEndpoint? + def graphql_http_endpoint: () -> HTTPEndpoint + + @graphql_query_executor: QueryExecutor? + def graphql_query_executor: () -> QueryExecutor + + @schema: Schema? + def schema: () -> Schema + + @datastore_search_router: DatastoreSearchRouter? + def datastore_search_router: () -> DatastoreSearchRouter + + @datastore_query_builder: DatastoreQuery::Builder? + def datastore_query_builder: () -> DatastoreQuery::Builder + + @graphql_gem_plugins: ::Hash[::Class, ::Hash[::Symbol, untyped]]? + def graphql_gem_plugins: () -> ::Hash[::Class, ::Hash[::Symbol, untyped]] + + @graphql_resolvers: ::Array[Resolvers::_Resolver]? + def graphql_resolvers: () -> ::Array[Resolvers::_Resolver] + + @datastore_query_adapters: ::Array[_QueryAdapter]? + def datastore_query_adapters: () -> ::Array[_QueryAdapter] + + @filter_interpreter: Filtering::FilterInterpreter? + def filter_interpreter: () -> Filtering::FilterInterpreter + + @filter_node_interpreter: Filtering::FilterNodeInterpreter? + def filter_node_interpreter: () -> Filtering::FilterNodeInterpreter + + @filter_args_translator: Filtering::FilterArgsTranslator? + def filter_args_translator: () -> Filtering::FilterArgsTranslator + + @sub_aggregation_grouping_adapter: Aggregation::groupingAdapter? + def sub_aggregation_grouping_adapter: () -> Aggregation::groupingAdapter + + @monotonic_clock: Support::MonotonicClock? + def monotonic_clock: () -> Support::MonotonicClock + + def load_dependencies_eagerly: () -> void + + private + + @datastore_core: DatastoreCore + @graphql_adapter: Resolvers::GraphQLAdapter? + EAGER_LOAD_QUERY: ::String + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/composite_grouping_adapter.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/composite_grouping_adapter.rbs new file mode 100644 index 00000000..c906c508 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/composite_grouping_adapter.rbs @@ -0,0 +1,22 @@ +module ElasticGraph + class GraphQL + module Aggregation + module CompositeGroupingAdapter + def self.meta_name: () -> ::String + + def self.grouping_detail_for: (Query) { () -> ::Hash[::String, untyped] } -> AggregationDetail + + def self.prepare_response_buckets: ( + ::Hash[::String, untyped], + ::Array[::String], + ::Hash[::String, untyped] + ) -> ::Array[::Hash[::String, untyped]] + + private + + def self.composite_after: (Query) -> ::Hash[::String, untyped]? + def self.build_sources: (Query) -> ::Array[::Hash[::String, ::Hash[::String, untyped]]] + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/computation.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/computation.rbs new file mode 100644 index 00000000..bf198c54 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/computation.rbs @@ -0,0 +1,26 @@ +module ElasticGraph + class GraphQL + module Aggregation + class Computation + attr_reader source_field_path: fieldPath + attr_reader computed_index_field_name: ::String + attr_reader detail: SchemaArtifacts::RuntimeMetadata::ComputationDetail + + def key: (aggregation_name: ::String) -> ::String + def clause: () -> ::Hash[::String, untyped] + + def initialize : ( + source_field_path: fieldPath, + computed_index_field_name: ::String, + detail: SchemaArtifacts::RuntimeMetadata::ComputationDetail + ) -> void + + def with: ( + ?source_field_path: fieldPath, + ?computed_index_field_name: ::String, + ?detail: SchemaArtifacts::RuntimeMetadata::ComputationDetail + ) -> instance + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/date_histogram_grouping.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/date_histogram_grouping.rbs new file mode 100644 index 00000000..59385590 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/date_histogram_grouping.rbs @@ -0,0 +1,40 @@ +module ElasticGraph + type timeGroupingInterval = "millisecond" | "second" | "minute" | "hour" | "day" | "week" | "month" | "quarter" | "year" + + class GraphQL + module Aggregation + class DateHistogramGroupingSuperclass + attr_reader field_path: fieldPath + attr_reader interval: timeGroupingInterval + attr_reader time_zone: ::String? + attr_reader offset: ::String? + + def initialize: ( + field_path: fieldPath, + interval: timeGroupingInterval, + time_zone: ::String?, + offset: ::String? + ) -> void + + def with: ( + ?field_path: fieldPath, + ?interval: timeGroupingInterval, + ?time_zone: ::String?, + ?offset: ::String? + ) -> DateHistogramGrouping + end + + class DateHistogramGrouping < DateHistogramGroupingSuperclass + attr_reader key: ::String + attr_reader encoded_index_field_path: ::String + + def composite_clause: (?grouping_options: ::Hash[::String, untyped]) -> ::Hash[::String, untyped] + def non_composite_clause_for: (Query) -> ::Hash[::String, untyped] + def inner_meta: () -> ::Hash[::String, untyped] + + INNER_META: ::Hash[::String, untyped] + INTERVAL_OPTIONS_BY_NAME: ::Hash[timeGroupingInterval, ::Hash[::String, ::String]] + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/field_path_encoder.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/field_path_encoder.rbs new file mode 100644 index 00000000..6d97d230 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/field_path_encoder.rbs @@ -0,0 +1,15 @@ +module ElasticGraph + class GraphQL + module Aggregation + module FieldPathEncoder + DELIMITER: ::String + + def self.encode: (::Array[::String]) -> ::String + def self.join: (::Array[::String]) -> ::String + def self.decode: (::String) -> ::Array[::String] + private + def self.verify_delimiters: (::String) -> void + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/field_term_grouping.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/field_term_grouping.rbs new file mode 100644 index 00000000..53cd7812 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/field_term_grouping.rbs @@ -0,0 +1,16 @@ +module ElasticGraph + class GraphQL + module Aggregation + class FieldTermGroupingSupertype + attr_reader field_path: fieldPath + def initialize: (field_path: fieldPath) -> void + def self.new: (field_path: fieldPath) -> instance | (fieldPath) -> instance + end + + class FieldTermGrouping < FieldTermGroupingSupertype + include TermGrouping + include _TermGroupingSubtype + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/interfaces.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/interfaces.rbs new file mode 100644 index 00000000..1e4c35cb --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/interfaces.rbs @@ -0,0 +1,9 @@ +module ElasticGraph + class GraphQL + module Aggregation + type fieldPath = ::Array[PathSegment] + type stringish = ::Symbol | ::String + type grouping = DateHistogramGrouping | FieldTermGrouping | ScriptTermGrouping + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/key.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/key.rbs new file mode 100644 index 00000000..2fedee45 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/key.rbs @@ -0,0 +1,38 @@ +module ElasticGraph + class GraphQL + module Aggregation + module Key + DELIMITER: ::String + + class AggregatedValueSupertype + attr_reader aggregation_name: ::String + attr_reader encoded_field_path: ::String + attr_reader function_name: ::String + + def initialize: ( + aggregation_name: ::String, + encoded_field_path: ::String, + function_name: ::String + ) -> void + end + + class AggregatedValue < AggregatedValueSupertype + def initialize: ( + aggregation_name: ::String, + ?encoded_field_path: ::String, + ?field_path: ::Array[::String], + function_name: ::String + ) -> void + + def encode: () -> ::String + def field_path: () -> ::Array[::String] + end + + def self.missing_value_bucket_key: (::String) -> ::String + def self.extract_aggregation_name_from: (::String) -> ::String + def self.encode: (::Array[::String]) -> ::String + def self.verify_no_delimiter_in: (*::String) -> void + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/nested_sub_aggregation.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/nested_sub_aggregation.rbs new file mode 100644 index 00000000..c7413453 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/nested_sub_aggregation.rbs @@ -0,0 +1,25 @@ +module ElasticGraph + class GraphQL + module Aggregation + class NestedSubAggregationSupertype + attr_reader nested_path: ::Array[PathSegment] + attr_reader query: Query + + def initialize: ( + nested_path: ::Array[PathSegment], + query: Query + ) -> void + end + + class NestedSubAggregation < NestedSubAggregationSupertype + @nested_path_key: ::String? + def nested_path_key: () -> ::String + + def build_agg_hash: ( + Filtering::FilterInterpreter, + parent_queries: ::Array[Query] + ) -> ::Hash[::String, untyped] + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/non_composite_grouping_adapter.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/non_composite_grouping_adapter.rbs new file mode 100644 index 00000000..082320ce --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/non_composite_grouping_adapter.rbs @@ -0,0 +1,38 @@ +module ElasticGraph + class GraphQL + module Aggregation + module NonCompositeGroupingAdapter + def self.meta_name: () -> ::String + + def self.grouping_detail_for: (Query) { () -> ::Hash[::String, untyped] } -> AggregationDetail + + def self.prepare_response_buckets: ( + ::Hash[::String, untyped], + ::Array[::String], + ::Hash[::String, untyped] + ) -> ::Array[::Hash[::String, untyped]] + + private + + def self.grouping_detail: ( + ::Array[grouping], + Query + ) { () -> AggregationDetail } -> AggregationDetail + + def self.format_buckets: ( + ::Hash[::String, untyped], + ::Array[::String], + ?parent_key_fields: ::Hash[::String, untyped], + ?parent_key_values: ::Array[untyped] + ) -> ::Array[::Hash[::String, untyped]] + + def self.sort_and_truncate_buckets: ( + ::Array[::Hash[::String, untyped]], + ::Integer + ) -> ::Array[::Hash[::String, untyped]] + + def self.missing_bucket_path_from: (::Array[::String]) -> ::Array[::String] + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/path_segment.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/path_segment.rbs new file mode 100644 index 00000000..2ef32a39 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/path_segment.rbs @@ -0,0 +1,20 @@ +module ElasticGraph + class GraphQL + module Aggregation + class PathSegment + attr_reader name_in_graphql_query: ::String + attr_reader name_in_index: ::String? + + def initialize: ( + name_in_graphql_query: ::String, + name_in_index: ::String? + ) -> void + + def self.for: ( + lookahead: ::GraphQL::Execution::Lookahead, + ?field: Schema::Field? + ) -> PathSegment + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/query.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/query.rbs new file mode 100644 index 00000000..e87f4606 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/query.rbs @@ -0,0 +1,89 @@ +module ElasticGraph + class GraphQL + module Aggregation + type groupingAdapter = singleton(NonCompositeGroupingAdapter) | singleton(CompositeGroupingAdapter) + + class QueryValueClass + attr_reader name: ::String + attr_reader needs_doc_count: bool + attr_reader needs_doc_count_error: bool + attr_reader filter: ::Hash[::String, untyped]? + attr_reader paginator: DatastoreQuery::Paginator[Resolvers::Node] + attr_reader sub_aggregations: ::Hash[::String, NestedSubAggregation] + attr_reader computations: ::Set[Computation] + attr_reader groupings: ::Set[grouping] + attr_reader grouping_adapter: groupingAdapter + + def initialize: ( + name: ::String, + needs_doc_count: bool, + needs_doc_count_error: bool, + filter: ::Hash[::String, untyped]?, + paginator: DatastoreQuery::Paginator[Resolvers::Node], + sub_aggregations: ::Hash[::String, NestedSubAggregation], + computations: ::Set[Computation], + groupings: ::Set[grouping], + grouping_adapter: groupingAdapter + ) -> void + + def with: ( + ?name: ::String, + ?needs_doc_count: bool, + ?needs_doc_count_error: bool, + ?filter: ::Hash[::String, untyped]?, + ?paginator: DatastoreQuery::Paginator[Resolvers::Node], + ?sub_aggregations: ::Hash[::String, NestedSubAggregation], + ?computations: ::Set[Computation], + ?groupings: ::Set[grouping], + ?grouping_adapter: groupingAdapter + ) -> Query + end + + class Query < QueryValueClass + def needs_total_doc_count?: () -> bool + def build_agg_hash: (Filtering::FilterInterpreter) -> ::Hash[::String, untyped] + + def build_agg_detail: ( + Filtering::FilterInterpreter, + field_path: ::Array[PathSegment], + parent_queries: ::Array[Query] + ) -> AggregationDetail? + + private + + def filter_detail: ( + Filtering::FilterInterpreter, + ::Array[PathSegment] + ) { () -> AggregationDetail } -> AggregationDetail + + def computations_detail: () -> ::Hash[::String, untyped] + + def sub_aggregation_detail: ( + Filtering::FilterInterpreter, + ::Array[Query] + ) -> ::Hash[::String, untyped] + + def build_inner_aggregation_detail: [E] ( + ::Enumerable[E] + ) { (E) -> ::Hash[::String, untyped] } -> ::Hash[::String, untyped] + end + + class AggregationDetail + attr_reader clauses: ::Hash[::String, untyped]? + attr_reader meta: ::Hash[::String, untyped] + + def initialize: ( + ::Hash[::String, untyped]?, + ::Hash[::String, untyped] + ) -> void + + def with: ( + ?clauses: ::Hash[::String, untyped]?, + ?meta: ::Hash[::String, untyped] + ) -> instance + + def wrap_with_grouping: (grouping, query: Query) -> AggregationDetail + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/query_adapter.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/query_adapter.rbs new file mode 100644 index 00000000..5a3faf69 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/query_adapter.rbs @@ -0,0 +1,139 @@ +module ElasticGraph + class GraphQL + module Aggregation + class QueryAdapterSupertype + attr_reader schema: Schema + attr_reader config: Config + attr_reader filter_args_translator: Filtering::FilterArgsTranslator + attr_reader runtime_metadata: SchemaArtifacts::RuntimeMetadata::Schema + attr_reader sub_aggregation_grouping_adapter: groupingAdapter + + def initialize: ( + schema: Schema, + config: Config, + filter_args_translator: Filtering::FilterArgsTranslator, + runtime_metadata: SchemaArtifacts::RuntimeMetadata::Schema, + sub_aggregation_grouping_adapter: groupingAdapter + ) -> void + + def with: ( + ?schema: Schema, + ?config: Config, + ?filter_args_translator: Filtering::FilterArgsTranslator, + ?runtime_metadata: SchemaArtifacts::RuntimeMetadata::Schema, + ?sub_aggregation_grouping_adapter: groupingAdapter + ) -> QueryAdapter + end + + class QueryAdapter < QueryAdapterSupertype + include _QueryAdapter + attr_reader element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + + private + + def extract_aggregation_node: ( + ::GraphQL::Execution::Lookahead, + Schema::Field, + ::GraphQL::Query + ) -> ::GraphQL::Execution::Lookahead? + + def build_aggregation_query_for: ( + ::GraphQL::Execution::Lookahead, + field: Schema::Field, + grouping_adapter: groupingAdapter, + ?nested_path: ::Array[PathSegment], + ?unfiltered: bool + ) -> Query + + def selection_above_grouping_fields: ( + ::GraphQL::Execution::Lookahead, + ::String, + ::String + ) -> ::GraphQL::Execution::Lookahead + + def build_clauses_from: [A] ( + ::GraphQL::Execution::Lookahead + ) { ( + ::GraphQL::Execution::Lookahead, + Schema::Field, + ::Array[PathSegment] + ) -> ::Array[A]? } -> ::Set[A] + + def get_children_nodes: ( + ::GraphQL::Execution::Lookahead + ) -> ::Array[::GraphQL::Execution::Lookahead] + + def transform_node_to_clauses: [A] ( + ::GraphQL::Execution::Lookahead, + ?parent_path: ::Array[PathSegment] + ) { ( + ::GraphQL::Execution::Lookahead, + Schema::Field, + ::Array[PathSegment] + ) -> ::Array[A]? } -> ::Array[A] + + def build_computations_from: ( + ::GraphQL::Execution::Lookahead, + ?from_field_path: ::Array[PathSegment] + ) -> ::Set[Computation] + + def build_groupings_from: ( + ::GraphQL::Execution::Lookahead, + ::String, + ?from_field_path: ::Array[PathSegment] + ) -> ::Set[grouping] + + def field_from_node: ( + ::GraphQL::Execution::Lookahead + ) -> Schema::Field + + def date_time_groupings_from: ( + field_path: ::Array[PathSegment], + node: ::GraphQL::Execution::Lookahead, + ) -> ::Array[DateHistogramGrouping | ScriptTermGrouping] + + def legacy_date_histogram_groupings_from: ( + field_path: ::Array[PathSegment], + node: ::GraphQL::Execution::Lookahead, + get_time_zone: ^(::Hash[::String, untyped]) -> ::String?, + get_offset: ^(::Hash[::String, untyped]) -> ::String? + ) -> ::Array[DateHistogramGrouping] + + def interval_from: ( + ::GraphQL::Execution::Lookahead, + ::Hash[::String, untyped], + interval_unit_key: ::String + ) -> timeGroupingInterval + + def datetime_offset_from: ( + ::GraphQL::Execution::Lookahead, + ::Hash[::String, untyped] + ) -> ::String? + + def datetime_offset_as_ms_from: ( + ::GraphQL::Execution::Lookahead, + ::Hash[::String, untyped] + ) -> ::Integer + + def enum_value_from_offset: ( + ::GraphQL::Execution::Lookahead, + ::String + ) -> Schema::EnumValue + + def name_of: (::GraphQL::Language::Nodes::Field) -> ::String + + def build_sub_aggregations_from: ( + ::GraphQL::Execution::Lookahead, + ?parent_nested_path: ::Array[PathSegment] + ) -> ::Hash[::String, NestedSubAggregation] + + def build_paginator_for: [I] (::GraphQL::Execution::Lookahead) -> DatastoreQuery::Paginator[I] + + def raise_conflicting_grouping_requirement_selections: ( + ::String, + ::Array[::String] + ) -> bot + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/query_optimizer.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/query_optimizer.rbs new file mode 100644 index 00000000..e4338da5 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/query_optimizer.rbs @@ -0,0 +1,32 @@ +module ElasticGraph + class GraphQL + module Aggregation + class QueryOptimizer + type response = ::Hash[::String, untyped] + + def self.optimize_queries: ( + ::Array[DatastoreQuery] + ) { (::Array[DatastoreQuery]) -> ::Hash[DatastoreQuery, response] } -> ::Hash[DatastoreQuery, response] + + def initialize: (::Array[DatastoreQuery], logger: ::Logger) -> void + @original_queries: ::Array[DatastoreQuery] + @logger: ::Logger + @unique_prefix_by_query: ::Hash[DatastoreQuery, ::String] + + def merged_queries: () -> ::Array[DatastoreQuery] + def unmerge_responses: (::Hash[DatastoreQuery, response]) -> ::Hash[DatastoreQuery, response] + + private + + attr_reader original_queries_by_merged_query: ::Hash[DatastoreQuery, ::Array[DatastoreQuery]] + + NO_AGGREGATIONS: ::Hash[::String, Aggregation::Query] + + def queries_by_merge_key: () -> ::Hash[untyped, ::Array[DatastoreQuery]] + def merge_queries: (::Array[DatastoreQuery]) -> DatastoreQuery + def unmerge_response: (response, DatastoreQuery) -> response + def strip_prefix_from_agg_data: (untyped, ::String, ::String) -> untyped + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/aggregated_values.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/aggregated_values.rbs new file mode 100644 index 00000000..0157f711 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/aggregated_values.rbs @@ -0,0 +1,39 @@ +module ElasticGraph + class GraphQL + module Aggregation + module Resolvers + class AggregatedValuesSupertype + attr_reader aggregation_name: ::String + attr_reader bucket: ::Hash[::String, untyped] + attr_reader field_path: ::Array[PathSegment] + + def initialize: ( + aggregation_name: ::String, + bucket: ::Hash[::String, untyped], + field_path: ::Array[PathSegment] + ) -> void + + def self.new: ( + aggregation_name: ::String, + bucket: ::Hash[::String, untyped], + field_path: ::Array[PathSegment] + ) -> instance | ( + ::String, + ::Hash[::String, untyped], + ::Array[PathSegment] + ) -> instance + + def with: ( + ?aggregation_name: ::String, + ?bucket: ::Hash[::String, untyped], + ?field_path: ::Array[PathSegment] + ) -> instance + end + + class AggregatedValues < AggregatedValuesSupertype + include GraphQL::Resolvers::_Resolver + end + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/count_detail.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/count_detail.rbs new file mode 100644 index 00000000..d361fbbe --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/count_detail.rbs @@ -0,0 +1,35 @@ +module ElasticGraph + class GraphQL + module Aggregation + module Resolvers + class CountDetailSupertype < GraphQL::Resolvers::ResolvableValueClass + attr_reader bucket: ::Hash[::String, untyped] + + def initialize: ( + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + bucket: ::Hash[::String, untyped] + ) -> void + + def self.new: ( + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + bucket: ::Hash[::String, untyped] + ) -> instance | ( + SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + ::Hash[::String, untyped] + ) -> instance + + def with: ( + ?schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + ?bucket: ::Hash[::String, untyped] + ) -> instance + end + + class CountDetail < CountDetailSupertype + attr_reader approximate_value: ::Integer + attr_reader exact_value: ::Integer? + attr_reader upper_bound: ::Integer + end + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/grouped_by.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/grouped_by.rbs new file mode 100644 index 00000000..270564ea --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/grouped_by.rbs @@ -0,0 +1,34 @@ +module ElasticGraph + class GraphQL + module Aggregation + module Resolvers + class GroupedBySupertype + attr_reader bucket: ::Hash[::String, untyped] + attr_reader field_path: ::Array[PathSegment] + + def initialize: ( + bucket: ::Hash[::String, untyped], + field_path: ::Array[PathSegment] + ) -> void + + def self.new: ( + bucket: ::Hash[::String, untyped], + field_path: ::Array[PathSegment] + ) -> instance | ( + ::Hash[::String, untyped], + ::Array[PathSegment] + ) -> instance + + def with: ( + ?bucket: ::Hash[::String, untyped], + ?field_path: ::Array[PathSegment] + ) -> instance + end + + class GroupedBy < GroupedBySupertype + include GraphQL::Resolvers::_Resolver + end + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/node.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/node.rbs new file mode 100644 index 00000000..78d0d548 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/node.rbs @@ -0,0 +1,45 @@ +module ElasticGraph + class GraphQL + module Aggregation + module Resolvers + class NodeSupertype < GraphQL::Resolvers::ResolvableValueClass + attr_reader query: Query + attr_reader parent_queries: ::Array[Query] + attr_reader bucket: ::Hash[::String, untyped] + attr_reader field_path: ::Array[PathSegment] + + def self.new: ( + SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + Query, + ::Array[Query], + ::Hash[::String, untyped], + ::Array[PathSegment] + ) -> instance | ( + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + query: Query, + parent_queries: ::Array[Query], + bucket: ::Hash[::String, untyped], + field_path: ::Array[PathSegment] + ) -> instance + + def with: ( + ?schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + ?query: Query, + ?parent_queries: ::Array[Query], + ?bucket: ::Hash[::String, untyped], + ?field_path: ::Array[PathSegment] + ) -> instance + end + + class Node < NodeSupertype + attr_reader grouped_by: GroupedBy + attr_reader aggregated_values: AggregatedValues + attr_reader sub_aggregations: SubAggregations + def count: () -> ::Integer + attr_reader count_detail: CountDetail + attr_reader cursor: DecodedCursor + end + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/relay_connection_builder.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/relay_connection_builder.rbs new file mode 100644 index 00000000..24e3a2c8 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/relay_connection_builder.rbs @@ -0,0 +1,38 @@ +module ElasticGraph + class GraphQL + module Aggregation + module Resolvers + module RelayConnectionBuilder + def self.build_from_search_response: ( + query: Query, + search_response: DatastoreResponse::SearchResponse, + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + ) -> GraphQL::Resolvers::RelayConnection::GenericAdapter[Node] + + def self.build_from_buckets: ( + query: Query, + parent_queries: ::Array[Query], + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + ?field_path: ::Array[PathSegment] + ) { () -> ::Array[::Hash[::String, untyped]] } -> GraphQL::Resolvers::RelayConnection::GenericAdapter[Node] + + private + + def self.raw_nodes_for: ( + Query, + ::Array[Query], + SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + ::Array[PathSegment] + ) { () -> ::Array[::Hash[::String, untyped]] } -> ::Array[Node] + + def self.extract_buckets_from: ( + DatastoreResponse::SearchResponse, + for_query: Query + ) -> ::Array[::Hash[::String, untyped]] + + def self.build_bucket: (Query, ::Hash[::String, untyped]) -> ::Hash[::String, untyped] + end + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/sub_aggregations.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/sub_aggregations.rbs new file mode 100644 index 00000000..a281db2a --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/resolvers/sub_aggregations.rbs @@ -0,0 +1,47 @@ +module ElasticGraph + class GraphQL + module Aggregation + module Resolvers + class SubAggregationsSupertype + attr_reader schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + attr_reader sub_aggregations: ::Hash[::String, NestedSubAggregation] + attr_reader parent_queries: ::Array[Query] + attr_reader sub_aggs_by_agg_key: ::Hash[::String, ::Hash[::String, untyped]] + attr_reader field_path: ::Array[PathSegment] + + def self.new: ( + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + sub_aggregations: ::Hash[::String, NestedSubAggregation], + parent_queries: ::Array[Query], + sub_aggs_by_agg_key: ::Hash[::String, ::Hash[::String, untyped]], + field_path: ::Array[PathSegment] + ) -> instance | ( + SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + ::Hash[::String, NestedSubAggregation], + ::Array[Query], + ::Hash[::String, ::Hash[::String, untyped]], + ::Array[PathSegment] + ) -> instance + + def with: ( + ?schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + ?sub_aggregations: ::Hash[::String, NestedSubAggregation], + ?parent_queries: ::Array[Query], + ?sub_aggs_by_agg_key: ::Hash[::String, ::Hash[::String, untyped]], + ?field_path: ::Array[PathSegment] + ) -> instance + end + + class SubAggregations < SubAggregationsSupertype + include GraphQL::Resolvers::_Resolver + + private + + def extract_buckets: (::String, ::Hash[::String, untyped]) -> ::Array[::Hash[::String, untyped]] + + BUCKET_ADAPTERS: ::Hash[::String, groupingAdapter] + end + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/script_term_grouping.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/script_term_grouping.rbs new file mode 100644 index 00000000..ac9c5370 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/script_term_grouping.rbs @@ -0,0 +1,28 @@ +module ElasticGraph + class GraphQL + module Aggregation + class ScriptTermGroupingSuperclass + attr_reader field_path: fieldPath + attr_reader script_id: ::String + attr_reader params: ::Hash[::String, untyped] + + def initialize: ( + field_path: fieldPath, + script_id: ::String, + params: ::Hash[::String, untyped] + ) -> void + + def with: ( + ?field_path: fieldPath, + ?script_id: ::String, + ?params: ::Hash[::String, untyped], + ) -> ScriptTermGrouping + end + + class ScriptTermGrouping < ScriptTermGroupingSuperclass + include TermGrouping + include _TermGroupingSubtype + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/term_grouping.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/term_grouping.rbs new file mode 100644 index 00000000..59b1b161 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/aggregation/term_grouping.rbs @@ -0,0 +1,25 @@ +module ElasticGraph + class GraphQL + module Aggregation + module TermGrouping: _TermGroupingSubtype + attr_reader key: ::String + attr_reader encoded_index_field_path: ::String + + def composite_clause: (?grouping_options: ::Hash[::String, untyped]) -> ::Hash[::String, untyped] + def non_composite_clause_for: (Query) -> ::Hash[::String, untyped] + def inner_meta: () -> ::Hash[::String, untyped] + + INNER_META: ::Hash[::String, untyped] + + private + + def work_around_elasticsearch_bug: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + end + + interface _TermGroupingSubtype + def field_path: () -> fieldPath + def terms_subclause: () -> ::Hash[::String, untyped] + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/client.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/client.rbs new file mode 100644 index 00000000..aee7df94 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/client.rbs @@ -0,0 +1,28 @@ +module ElasticGraph + class GraphQL + class Client + attr_reader name: ::String + attr_reader source_description: ::String + def initialize: (name: ::String, source_description: ::String) -> void + def with: (?name: ::String, ?source_description: ::String) -> Client + + def self.new: + (name: ::String, source_description: ::String) -> instance + | (::String, ::String) -> instance + + ANONYMOUS: Client + ELASTICGRAPH_INTERNAL: Client + + def description: () -> ::String + + interface _Resolver + def initialize: (::Hash[::String, untyped]) -> void + def resolve: (HTTPRequest) -> (Client | HTTPResponse) + end + + class DefaultResolver + include _Resolver + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/config.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/config.rbs new file mode 100644 index 00000000..9608cf2d --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/config.rbs @@ -0,0 +1,42 @@ +module ElasticGraph + class GraphQL + class ConfigSupertype + attr_reader default_page_size: ::Integer + attr_reader max_page_size: ::Integer + attr_reader slow_query_latency_warning_threshold_in_ms: ::Integer + attr_reader client_resolver: Client::_Resolver + attr_reader extension_modules: ::Array[::Module] + attr_reader extension_settings: parsedYamlSettings + + def initialize: ( + default_page_size: ::Integer, + max_page_size: ::Integer, + slow_query_latency_warning_threshold_in_ms: ::Integer, + client_resolver: Client::_Resolver, + extension_modules: ::Array[::Module], + extension_settings: parsedYamlSettings + ) -> void + + def with: ( + ?default_page_size: ::Integer, + ?max_page_size: ::Integer, + ?slow_query_latency_warning_threshold_in_ms: ::Integer, + ?client_resolver: Client::_Resolver, + ?extension_modules: ::Array[::Module], + ?extension_settings: parsedYamlSettings + ) -> Config + + def self.members: () -> ::Array[::Symbol] + + private + + def self.load_client_resolver: (::Hash[::String, untyped]) -> Client::_Resolver + end + + class Config < ConfigSupertype + extend _BuildableFromParsedYaml[Config] + EXPECTED_KEYS: ::Array[::String] + ELASTICGRAPH_CONFIG_KEYS: ::Array[::String] + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_query.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_query.rbs new file mode 100644 index 00000000..daeed11d --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_query.rbs @@ -0,0 +1,27 @@ +module ElasticGraph + class GraphQL + # Note: this is a partial signature definition (`query.rb` is ignored in `Steepfile`) + class DatastoreQuery + attr_reader search_index_definitions: ::Array[DatastoreCore::_IndexDefinition] + attr_reader aggregations: ::Hash[::String, Aggregation::Query] + attr_reader document_paginator: DocumentPaginator + + def shard_routing_values: () -> ::Array[::String]? + def merge_with: (**untyped) -> DatastoreQuery + def search_index_expression: () -> ::String + def with: (**untyped) -> DatastoreQuery + + def to_datastore_msearch_header_and_body: () -> [::Hash[::String, untyped], ::Hash[::String, untyped]] + + class Builder + def self.with: ( + runtime_metadata: SchemaArtifacts::RuntimeMetadata::Schema, + logger: ::Logger, + **untyped + ) -> Builder + + def new_query: (**untyped) -> DatastoreQuery + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_query/document_paginator.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_query/document_paginator.rbs new file mode 100644 index 00000000..78b19eee --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_query/document_paginator.rbs @@ -0,0 +1,12 @@ +module ElasticGraph + class GraphQL + class DatastoreQuery + # Note: this class is not fully type checked yet. + class DocumentPaginator + attr_reader paginator: Paginator[DatastoreResponse::Document] + + def sort: () -> ::Array[::Hash[::String, untyped]] + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_query/index_expression_builder.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_query/index_expression_builder.rbs new file mode 100644 index 00000000..b58ee21a --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_query/index_expression_builder.rbs @@ -0,0 +1,46 @@ +module ElasticGraph + class GraphQL + class DatastoreQuery + class IndexExpressionBuilder + def initialize: (schema_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames) -> void + def determine_search_index_expression: ( + ::Array[::Hash[::String, untyped]], + ::Array[DatastoreCore::_IndexDefinition], + require_indices: bool + ) -> IndexExpression + + private + + @filter_value_set_extractor: Filtering::FilterValueSetExtractor[Support::TimeSet] + def index_expression_for: ( + ::Array[::Hash[::String, untyped]], + DatastoreCore::_IndexDefinition, + require_indices: bool + ) -> IndexExpression + + def date_string?: (::String) -> bool + end + + class IndexExpressionSupertype + attr_reader names_to_include: ::Set[::String] + attr_reader names_to_exclude: ::Set[::String] + + def initialize: ( + names_to_include: ::Set[::String], + names_to_exclude: ::Set[::String] + ) -> void + + def with: ( + ?names_to_include: ::Set[::String], + ?names_to_exclude: ::Set[::String] + ) -> instance + end + + class IndexExpression < IndexExpressionSupertype + EMPTY: IndexExpression + def self.only: (::String?) -> IndexExpression + def +: (IndexExpression) -> IndexExpression + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_query/paginator.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_query/paginator.rbs new file mode 100644 index 00000000..c42566aa --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_query/paginator.rbs @@ -0,0 +1,64 @@ +module ElasticGraph + class GraphQL + class DatastoreQuery + # I is the type of item (document or aggregation bucket) + class Paginator[I] + type comparisonOperator = :< | :<= | :> | :>= + type sortDirection = :asc | :desc + type scalarValue = untyped + + attr_reader default_page_size: ::Integer + attr_reader max_page_size: ::Integer + attr_reader first: ::Integer? + attr_reader after: DecodedCursor? + attr_reader last: ::Integer? + attr_reader before: DecodedCursor? + attr_reader schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + + def initialize: ( + default_page_size: ::Integer, + max_page_size: ::Integer, + first: ::Integer?, + after: DecodedCursor?, + last: ::Integer?, + before: DecodedCursor?, + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames) -> void + + def requested_page_size: () -> ::Integer + def search_in_reverse?: () -> boolish + def search_after: () -> DecodedCursor? + def restore_intended_item_order: [E] (::Array[E]) -> ::Array[E] + def truncate_items: [E] (::Array[E]) { (E, DecodedCursor) -> ::Array[SortValue] } -> ::Array[E] + def paginated_from_singleton_cursor?: () -> bool + + @desired_page_size: ::Integer? + def desired_page_size: () -> ::Integer + + private + + @first_n: ::Integer? + def first_n: () -> ::Integer? + + @last_n: ::Integer? + def last_n: () -> ::Integer? + + def size_arg_value: (::Symbol, ::Integer?) -> ::Integer? + def item_sort_values_satisfy?: (::Array[SortValue], comparisonOperator) -> bool + + class SortValue + attr_reader from_item: scalarValue + attr_reader from_cursor: scalarValue + attr_reader sort_direction: sortDirection + + def initialize: ( + from_item: scalarValue, + from_cursor: scalarValue, + sort_direction: sortDirection) -> void + + def unequal?: () -> bool + def item_satisfies_compared_to_cursor?: (comparisonOperator) -> bool + end + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_query/routing_picker.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_query/routing_picker.rbs new file mode 100644 index 00000000..c2bc34b8 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_query/routing_picker.rbs @@ -0,0 +1,65 @@ +module ElasticGraph + class GraphQL + class DatastoreQuery + class RoutingPicker + def initialize: (schema_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames) -> void + def extract_eligible_routing_values: ( + ::Array[::Hash[::String, untyped]], + ::Array[::String] + ) -> ::Array[routingValue]? + + private + + @filter_value_set_extractor: Filtering::FilterValueSetExtractor[_RoutingValueSet] + end + + type routingValue = untyped + + interface _RoutingValueSet + include Support::_NegatableSet[_RoutingValueSet] + def to_return_value: () -> ::Array[routingValue]? + def ==: (untyped) -> bool + end + + type routingValueSetType = :inclusive | :exclusive + + class RoutingValueSetSupertype + attr_reader type: routingValueSetType + attr_reader routing_values: ::Set[routingValue] + + def initialize: (routingValueSetType, ::Set[routingValue]) -> void + def self.with: ( + type: routingValueSetType, + routing_values: ::Set[routingValue] + ) -> RoutingValueSet + + def with: ( + ?type: routingValueSetType, + ?routing_values: ::Set[routingValue] + ) -> RoutingValueSet + end + + class RoutingValueSet < RoutingValueSetSupertype + include _RoutingValueSet + def self.of: (::Enumerable[routingValue]) -> RoutingValueSet + def self.of_all_except: (::Enumerable[routingValue]) -> RoutingValueSet + + ALL: RoutingValueSet + INVERTED_TYPES: ::Hash[routingValueSetType, routingValueSetType] + + def inclusive?: () -> bool + def exclusive?: () -> bool + + private + + def get_included_and_excluded_values: ( + RoutingValueSet + ) -> [::Set[routingValue], ::Set[routingValue]] + + module UnboundedWithExclusions + extend _RoutingValueSet + end + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_response/document.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_response/document.rbs new file mode 100644 index 00000000..b298722d --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_response/document.rbs @@ -0,0 +1,43 @@ +module ElasticGraph + class GraphQL + module DatastoreResponse + class Document + extend Forwardable + + attr_reader raw_data: ::Hash[::String, untyped] + attr_reader payload: ::Hash[::String, untyped] + attr_reader decoded_cursor_factory: DecodedCursor::_Factory + + def initialize: ( + raw_data: ::Hash[::String, untyped], + payload: ::Hash[::String, untyped], + decoded_cursor_factory: DecodedCursor::_Factory + ) -> void + + def with: ( + ?raw_data: ::Hash[::String, untyped], + ?payload: ::Hash[::String, untyped], + ?decoded_cursor_factory: DecodedCursor::_Factory + ) -> Document + + def self.build: ( + ::Hash[::String, untyped], + ?decoded_cursor_factory: DecodedCursor::_Factory + ) -> Document + + def []: (::String) -> untyped + def fetch: (::String) -> untyped + def index_name: () -> ::String + def index_definition_name: () -> ::String + def id: () -> ::String + def sort: () -> ::Array[untyped] + def version: () -> ::Integer + + @cursor: DecodedCursor? + def cursor: () -> DecodedCursor + + def datastore_path: () -> ::String + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_response/search_response.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_response/search_response.rbs new file mode 100644 index 00000000..f32560ef --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_response/search_response.rbs @@ -0,0 +1,18 @@ +module ElasticGraph + class GraphQL + module DatastoreResponse + # Note: this is a partial signature definition (`search_response.rb` is ignored in `Steepfile`) + class SearchResponse + include ::Enumerable[Document] + attr_reader raw_data: ::Hash[::String, untyped] + attr_reader total_document_count: ::Integer + attr_reader documents: Array[Document] + attr_reader docs_description: ::String + attr_reader size: ::Integer + + def each: () { (Document) -> void } -> void | () -> Enumerator[Document, void] + end + end + end +end + diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_search_router.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_search_router.rbs new file mode 100644 index 00000000..38ffa438 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/datastore_search_router.rbs @@ -0,0 +1,14 @@ +module ElasticGraph + class GraphQL + class DatastoreSearchRouter + def initialize: ( + datastore_clients_by_name: ::Hash[::String, DatastoreCore::_Client], + logger: ::Logger, + monotonic_clock: Support::MonotonicClock, + config: Config + ) -> void + + def msearch: (::Array[DatastoreQuery], ?query_tracker: QueryDetailsTracker) -> ::Hash[DatastoreQuery, DatastoreResponse::SearchResponse] + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/decoded_cursor.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/decoded_cursor.rbs new file mode 100644 index 00000000..244320ec --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/decoded_cursor.rbs @@ -0,0 +1,31 @@ +module ElasticGraph + class GraphQL + type scalarUntyped = ::String | ::Numeric | bool + + class DecodedCursor + def initialize: (::Hash[::String, scalarUntyped]) -> void + def sort_values: () -> ::Hash[::String, scalarUntyped] + def encode: () -> ::String + def self.try_decode: (::String) -> DecodedCursor? + def self.decode!: (::String) -> DecodedCursor + @encode: ::String? + + SINGLETON: DecodedCursor + + interface _Factory + def build: (::Array[scalarUntyped]) -> DecodedCursor + end + + class Factory + include _Factory + attr_reader sort_fields: ::Array[::String] + def initialize: (::Array[::String]) -> Factory + def self.from_sort_list: (::Array[::Hash[::String, {"order" => "asc" | "desc"}]]) -> Factory + + module Null + extend _Factory + end + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/boolean_query.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/boolean_query.rbs new file mode 100644 index 00000000..dbcc7d36 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/boolean_query.rbs @@ -0,0 +1,33 @@ +module ElasticGraph + class GraphQL + module Filtering + type queryClause = RangeQuery | BooleanQuery + type stringOrSymbolHash = ::Hash[(::String | ::Symbol), untyped] + type occurrence = :must | :must_not | :filter | :should + + class BooleanQuerySupertype + attr_reader occurrence: occurrence + attr_reader clauses: ::Array[stringOrSymbolHash] + + def initialize: ( + occurrence, + ::Array[stringOrSymbolHash] + ) -> void + + def with: ( + ?occurrence: occurrence, + ?clauses: ::Array[stringOrSymbolHash] + ) -> BooleanQuery + end + + class BooleanQuery < BooleanQuerySupertype + def self.must: (*stringOrSymbolHash) -> BooleanQuery + def self.filter: (*stringOrSymbolHash) -> BooleanQuery + def self.should: (*stringOrSymbolHash) -> BooleanQuery + def merge_into: (stringOrSymbolHash) -> void + + ALWAYS_FALSE_FILTER: BooleanQuery + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/field_path.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/field_path.rbs new file mode 100644 index 00000000..c437501b --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/field_path.rbs @@ -0,0 +1,21 @@ +module ElasticGraph + class GraphQL + module Filtering + class FieldPath + attr_reader from_root: ::Array[::String] + attr_reader from_parent: ::Array[::String] + + def initialize: ( + ::Array[::String], + ::Array[::String] + ) -> void + + def self.empty: () -> FieldPath + def self.of: (::Array[::String]) -> FieldPath + def nested: () -> FieldPath + def +: (::String) -> FieldPath + def counts_path: () -> FieldPath + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/filter_args_translator.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/filter_args_translator.rbs new file mode 100644 index 00000000..16ad592c --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/filter_args_translator.rbs @@ -0,0 +1,22 @@ +module ElasticGraph + class GraphQL + module Filtering + class FilterArgsTranslatorSupertype + attr_reader filter_arg_name: ::String + def initialize: (filter_arg_name: ::String) -> void + end + + class FilterArgsTranslator < FilterArgsTranslatorSupertype + def initialize: (schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames) -> void + def translate_filter_args: (field: Schema::Field, args: ::Hash[::String, untyped]) -> ::Hash[::String, untyped]? + + private + + def convert: + (Schema::Type, ::Hash[::String, untyped]) -> ::Hash[::String, untyped]? + | [E] (Schema::Type, ::Array[E]) -> ::Array[E] + | (Schema::Type, untyped) -> untyped + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/filter_interpreter.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/filter_interpreter.rbs new file mode 100644 index 00000000..b087f433 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/filter_interpreter.rbs @@ -0,0 +1,49 @@ +module ElasticGraph + class GraphQL + module Filtering + class FilterInterpreterSupertype + attr_reader filter_node_interpreter: FilterNodeInterpreter + attr_reader schema_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + attr_reader logger: ::Logger + + def initialize: ( + filter_node_interpreter: FilterNodeInterpreter, + schema_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + logger: ::Logger + ) -> void + end + + class FilterInterpreter < FilterInterpreterSupertype + type stringHash = ::Hash[::String, untyped] + attr_reader logger: ::Logger + + def initialize: ( + filter_node_interpreter: FilterNodeInterpreter, + logger: ::Logger + ) -> void + + def build_query: ( + ::Enumerable[stringHash], + ?from_field_path: FieldPath + ) -> stringOrSymbolHash? + + private + + def process_filter_hash: (stringOrSymbolHash, stringHash, FieldPath) -> void + def filters_on_sub_fields?: (stringHash) -> bool + def process_not_expression: (stringOrSymbolHash, stringHash, FieldPath) -> void + def process_list_any_filter_expression: (stringOrSymbolHash, stringHash, FieldPath) -> void + def process_any_satisfy_filter_expression_on_nested_object_list: (stringOrSymbolHash, stringHash, FieldPath) -> void + def process_any_satisfy_filter_expression_on_scalar_list: (stringOrSymbolHash, stringHash, FieldPath) -> void + def process_any_of_expression: (stringOrSymbolHash, ::Array[stringHash], FieldPath) -> void + def process_all_of_expression: (stringOrSymbolHash, ::Array[stringHash], FieldPath) -> void + def process_operator_expression: (stringOrSymbolHash, ::String, stringHash, FieldPath) -> void + def process_sub_field_expression: (stringOrSymbolHash, stringHash, FieldPath) -> void + def process_list_count_expression: (stringOrSymbolHash, stringHash, FieldPath) -> void + def build_bool_hash: () { (stringOrSymbolHash) -> void } -> stringOrSymbolHash? + def excludes_zero?: (stringHash) -> bool + def required_matching_clause_count: (stringOrSymbolHash) -> ::Integer + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/filter_node_interpreter.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/filter_node_interpreter.rbs new file mode 100644 index 00000000..cc3d3d02 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/filter_node_interpreter.rbs @@ -0,0 +1,45 @@ +module ElasticGraph + class GraphQL + module Filtering + class FilterNodeInterpreterSupertype + attr_reader runtime_metadata: SchemaArtifacts::RuntimeMetadata::Schema + attr_reader schema_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + + def initialize: ( + runtime_metadata: SchemaArtifacts::RuntimeMetadata::Schema, + schema_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + ) -> void + end + + class FilterNodeInterpreter < FilterNodeInterpreterSupertype + type stringHash = ::Hash[::String, untyped] + + type nodeType = :empty | + :not | + :list_any_filter | + :all_of | + :any_of | + :operator | + :list_count | + :sub_field | + :unknown + + def initialize: ( + runtime_metadata: SchemaArtifacts::RuntimeMetadata::Schema + ) -> void + + def identify_node_type: (::String, stringHash) -> nodeType + + attr_reader filter_operators: ::Hash[::String, ^(::String, untyped) -> queryClause?] + + def build_filter_operators: ( + SchemaArtifacts::RuntimeMetadata::Schema + ) -> ::Hash[::String, ^(::String, untyped) -> queryClause?] + + def to_datastore_value: (untyped) -> untyped + def nano_of_day_from: (stringHash, ::String) -> ::Integer? + def list_of_nanos_of_day_from: (stringHash, ::String) -> ::Array[::Integer]? + end + end + end +end \ No newline at end of file diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/filter_value_set_extractor.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/filter_value_set_extractor.rbs new file mode 100644 index 00000000..7ca44788 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/filter_value_set_extractor.rbs @@ -0,0 +1,30 @@ +module ElasticGraph + class GraphQL + module Filtering + class FilterValueSetExtractor[S < Support::_NegatableSet[S]] + def initialize: (SchemaArtifacts::RuntimeMetadata::SchemaElementNames, S) { (::Symbol, untyped) -> S? } -> void + def extract_filter_value_set: (::Array[::Hash[::String, untyped]], ::Array[::String]) -> S + + private + + @schema_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + @all_values_set: S + @build_set_for_filter: ^(::Symbol, untyped) -> S? + + def filter_value_set_for_target_field_path: (::String, ::Array[::Hash[::String, untyped]]) -> S + def filter_value_set_for_filter_hash: (::Hash[::String, untyped], ::Array[::String], ?::Array[::String], negate: bool) -> S + def filter_value_set_for_filter_hash_entry: (::String, untyped, ::Array[::String], ::Array[::String], negate: bool) -> S + def filter_value_set_for_any_of: (::Array[::Hash[::String, untyped]], ::Array[::String], ::Array[::String], negate: bool) -> S + def filter_value_set_for_field_filter: (::String, untyped) -> S + + type setReductionOperator = :union | :intersection + + def map_reduce_sets: + [E] (::Array[E], setReductionOperator, negate: bool) { (E) -> S } -> S | + [K, V] (::Hash[K, V], setReductionOperator, negate: bool) { ([K, V]) -> S } -> S + + REDUCTION_INVERSIONS: ::Hash[setReductionOperator, setReductionOperator] + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/range_query.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/range_query.rbs new file mode 100644 index 00000000..9b4511e5 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/filtering/range_query.rbs @@ -0,0 +1,23 @@ +module ElasticGraph + class GraphQL + module Filtering + class RangeQuerySupertype + attr_reader field_name: ::String + attr_reader operator: ::Symbol + attr_reader value: untyped + + def initialize: (::String, ::Symbol, untyped) -> void + + def with: ( + ?field_name: ::String, + ?operator: ::Symbol, + ?value: untyped + ) -> RangeQuery + end + + class RangeQuery < RangeQuerySupertype + def merge_into: (stringOrSymbolHash) -> void + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/http_endpoint.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/http_endpoint.rbs new file mode 100644 index 00000000..ca5d905a --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/http_endpoint.rbs @@ -0,0 +1,116 @@ +module ElasticGraph + class GraphQL + class HTTPEndpoint + APPLICATION_JSON: ::String + APPLICATION_GRAPHQL: ::String + + def initialize: ( + query_executor: QueryExecutor, + monotonic_clock: Support::MonotonicClock, + client_resolver: Client::_Resolver + ) -> void + + def process: ( + HTTPRequest, + ?max_timeout_in_ms: ::Integer?, + ?start_time_in_ms: ::Integer + ) -> HTTPResponse + + private + + @query_executor: QueryExecutor + @monotonic_clock: Support::MonotonicClock + @client_resolver: Client::_Resolver + + def with_parsed_request: ( + HTTPRequest, + max_timeout_in_ms: ::Integer? + ) { (ParsedRequest) -> HTTPResponse } -> HTTPResponse + + def with_request_params: ( + HTTPRequest + ) { (::Hash[::String, untyped]) -> HTTPResponse } -> HTTPResponse + + def with_timeout: ( + HTTPRequest, + max_timeout_in_ms: ::Integer? + ) { (::Integer?) -> HTTPResponse } -> HTTPResponse + + def with_context: ( + HTTPRequest + ) { (::Hash[::Symbol, untyped]) -> HTTPResponse } -> HTTPResponse + + class ParsedRequest + attr_reader query_string: ::String + attr_reader variables: ::Hash[::String, untyped] + attr_reader operation_name: ::String? + attr_reader timeout_in_ms: ::Integer? + attr_reader context: ::Hash[::Symbol, untyped] + + def initialize: ( + query_string: ::String, + variables: ::Hash[::String, untyped], + operation_name: ::String?, + timeout_in_ms: ::Integer?, + context: ::Hash[::Symbol, untyped]) -> void + + def with: ( + ?query_string: ::String, + ?variables: ::Hash[::String, untyped], + ?operation_name: ::String?, + ?timeout_in_ms: ::Integer?, + ?context: ::Hash[::Symbol, untyped]) -> ParsedRequest + end + end + + class HTTPRequest + attr_reader http_method: ::Symbol + attr_reader url: ::String + attr_reader headers: ::Hash[::String, ::String] + attr_reader body: ::String? + + def initialize: ( + http_method: ::Symbol, + url: ::String, + headers: ::Hash[::String, ::String], + body: ::String?) -> void + + def with: ( + ?http_method: ::Symbol, + ?url: ::String, + ?headers: ::Hash[::String, ::String], + ?body: ::String) -> HTTPRequest + + @normalized_headers: ::Hash[::String, ::String]? + def normalized_headers: () -> ::Hash[::String, ::String] + + @content_type: ::String? + def content_type: () -> ::String + + def self.normalize_header_name: (::String) -> ::String + end + + class HTTPResponse + attr_reader status_code: ::Integer + attr_reader headers: ::Hash[::String, ::String] + attr_reader body: ::String + + def initialize: ( + status_code: ::Integer, + headers: ::Hash[::String, ::String], + body: ::String) -> void + + def self.new: + (status_code: ::Integer, headers: ::Hash[::String, ::String], body: ::String) -> instance + | (::Integer, ::Hash[::String, ::String], ::String) -> instance + + def with: ( + ?status_code: ::Integer, + ?headers: ::Hash[::String, ::String], + ?body: ::String) -> HTTPResponse + + def self.json: (::Integer, ::Hash[::String, untyped]) -> HTTPResponse + def self.error: (::Integer, ::String) -> HTTPResponse + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/query_adapter/filters.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/query_adapter/filters.rbs new file mode 100644 index 00000000..94076668 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/query_adapter/filters.rbs @@ -0,0 +1,58 @@ +module ElasticGraph + class GraphQL + class QueryAdapter + class FiltersSupertype + attr_reader schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + attr_reader filter_args_translator: Filtering::FilterArgsTranslator + attr_reader filter_node_interpreter: Filtering::FilterNodeInterpreter + + def initialize: ( + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + filter_args_translator: Filtering::FilterArgsTranslator, + filter_node_interpreter: Filtering::FilterNodeInterpreter + ) -> void + end + + class Filters < FiltersSupertype + include _QueryAdapter + + private + + def build_automatic_filter: (filter_from_args: ::Hash[::String, untyped]?, query: DatastoreQuery) -> ::Hash[::String, untyped]? + def exclude_incomplete_docs_filter: () -> ::Hash[::String, untyped] + def search_could_hit_incomplete_docs?: (DatastoreCore::_IndexDefinition, ::Hash[::String, untyped]) -> bool + + def determine_paths_to_check: ( + ::Object, + ::Hash[::String, SchemaArtifacts::RuntimeMetadata::IndexField], + ?parent_path: ::String? + ) -> ::Array[::String] + + def can_match_nil_values_at?: (::String, ::Hash[::String, untyped]) -> bool + + @filter_value_set_extractor: Filtering::FilterValueSetExtractor[_NilFocusedSet]? + def filter_value_set_extractor: () -> Filtering::FilterValueSetExtractor[_NilFocusedSet] + + interface _NilFocusedSet + include Support::_NegatableSet[_NilFocusedSet] + def includes_nil?: () -> bool + end + + module NilFocusedSet: _NilFocusedSet + def union: (_NilFocusedSet) -> _NilFocusedSet + def intersection: (_NilFocusedSet) -> _NilFocusedSet + end + + module IncludesNilSet + extend _NilFocusedSet + extend NilFocusedSet + end + + module ExcludesNilSet + extend _NilFocusedSet + extend NilFocusedSet + end + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/query_adapter/interface.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/query_adapter/interface.rbs new file mode 100644 index 00000000..2255487c --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/query_adapter/interface.rbs @@ -0,0 +1,13 @@ +module ElasticGraph + class GraphQL + interface _QueryAdapter + def call: ( + query: DatastoreQuery, + field: Schema::Field, + args: ::Hash[::String, untyped], + lookahead: ::GraphQL::Execution::Lookahead, + context: ::GraphQL::Query::Context + ) -> DatastoreQuery + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/query_adapter/pagination.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/query_adapter/pagination.rbs new file mode 100644 index 00000000..8d2cb2b4 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/query_adapter/pagination.rbs @@ -0,0 +1,15 @@ +module ElasticGraph + class GraphQL + class QueryAdapter + class Pagination + include _QueryAdapter + + attr_reader schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + + def initialize: ( + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + ) -> void + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/query_adapter/requested_fields.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/query_adapter/requested_fields.rbs new file mode 100644 index 00000000..b4ee3b8a --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/query_adapter/requested_fields.rbs @@ -0,0 +1,46 @@ +module ElasticGraph + class GraphQL + class QueryAdapter + class RequestedFields + include _QueryAdapter + def initialize: (Schema) -> void + + def query_attributes_for: ( + field: Schema::Field, + lookahead: ::GraphQL::Execution::Lookahead) -> ::Hash[::Symbol, untyped] + + private + + @schema: Schema + + def requested_fields_under: ( + ::GraphQL::Execution::Lookahead, + ?path_prefix: ::String + ) -> ::Array[::String] + + def requested_fields_for: ( + ::GraphQL::Execution::Lookahead, + path_prefix: ::String + ) -> ::Array[::String] + + def field_for: (::GraphQL::Schema::Field?) -> Schema::Field? + + def pagination_fields_need_individual_docs?: ( + ::GraphQL::Execution::Lookahead + ) -> bool + + def relay_connection_node_from: ( + ::GraphQL::Execution::Lookahead + ) -> ::GraphQL::Execution::Lookahead + + def query_needs_total_document_count?: ( + ::GraphQL::Execution::Lookahead + ) -> bool + + def graphql_dynamic_field?: ( + ::GraphQL::Execution::Lookahead + ) -> bool + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/query_adapter/sort.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/query_adapter/sort.rbs new file mode 100644 index 00000000..36f52ecd --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/query_adapter/sort.rbs @@ -0,0 +1,15 @@ +module ElasticGraph + class GraphQL + class QueryAdapter + class Sort + include _QueryAdapter + + attr_reader order_by_arg_name: ::String + + def initialize: ( + order_by_arg_name: ::String + ) -> void + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/query_details_tracker.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/query_details_tracker.rbs new file mode 100644 index 00000000..c3989fc3 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/query_details_tracker.rbs @@ -0,0 +1,30 @@ +module ElasticGraph + class GraphQL + class QueryDetailsTrackerSupertype + attr_accessor hidden_types: ::Set[::String] + attr_accessor shard_routing_values: ::Set[::String] + attr_accessor search_index_expressions: ::Set[::String] + attr_accessor query_counts_per_datastore_request: ::Array[::Integer] + attr_accessor datastore_query_server_duration_ms: ::Integer + attr_accessor datastore_query_client_duration_ms: ::Integer + attr_accessor mutex: ::Thread::Mutex + + def initialize: ( + hidden_types: ::Set[::String], + shard_routing_values: ::Set[::String], + search_index_expressions: ::Set[::String], + query_counts_per_datastore_request: ::Array[::Integer], + datastore_query_server_duration_ms: ::Integer, + datastore_query_client_duration_ms: ::Integer, + mutex: ::Thread::Mutex + ) -> void + end + + class QueryDetailsTracker < QueryDetailsTrackerSupertype + def self.empty: () -> QueryDetailsTracker + def record_datastore_queries_for_single_request: (::Array[DatastoreQuery]) -> void + def record_hidden_type: (::String) -> void + def record_datastore_query_duration_ms: (client: ::Integer, server: ::Integer?) -> void + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/query_executor.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/query_executor.rbs new file mode 100644 index 00000000..766ad418 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/query_executor.rbs @@ -0,0 +1,45 @@ +module ElasticGraph + class GraphQL + class QueryExecutor + def initialize: ( + schema: Schema, + monotonic_clock: Support::MonotonicClock, + logger: ::Logger, + slow_query_threshold_ms: ::Integer, + datastore_search_router: DatastoreSearchRouter + ) -> void + + def execute: ( + ::String?, + ?client: Client, + ?variables: ::Hash[::String, untyped], + ?timeout_in_ms: ::Integer?, + ?operation_name: ::String?, + ?context: ::Hash[::Symbol, untyped], + ?start_time_in_ms: ::Integer + ) -> ::GraphQL::Query::Result + + private + + attr_reader schema: Schema + @monotonic_clock: Support::MonotonicClock + @logger: ::Logger + @slow_query_threshold_ms: ::Integer + @datastore_search_router: DatastoreSearchRouter + + def build_and_execute_query: ( + query_string: ::String?, + variables: ::Hash[::String, untyped], + operation_name: ::String?, + context: ::Hash[::Symbol, untyped], + client: Client + ) -> [::GraphQL::Query, ::GraphQL::Query::Result] + + def execute_query: (::GraphQL::Query, client: Client) -> ::GraphQL::Query::Result + def full_description_of: (::GraphQL::Query) -> ::String + def fingerprint_for: (::GraphQL::Query) -> ::String + def directives_from_query_operation: (::GraphQL::Query) -> ::Hash[::String, ::Hash[::String, untyped]] + def slo_result_for: (::GraphQL::Query, ::Integer) -> ::String? + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/get_record_field_value.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/get_record_field_value.rbs new file mode 100644 index 00000000..cc34194e --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/get_record_field_value.rbs @@ -0,0 +1,13 @@ +module ElasticGraph + class GraphQL + module Resolvers + class GetRecordFieldValue + include _Resolver + + def initialize: ( + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + ) -> void + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/graphql_adapter.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/graphql_adapter.rbs new file mode 100644 index 00000000..5a274740 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/graphql_adapter.rbs @@ -0,0 +1,15 @@ +module ElasticGraph + class GraphQL + module Resolvers + class GraphQLAdapter + def initialize: ( + schema: Schema, + datastore_query_builder: DatastoreQuery::Builder, + datastore_query_adapters: ::Array[_QueryAdapter], + runtime_metadata: SchemaArtifacts::RuntimeMetadata::Schema, + resolvers: ::Array[Resolvers::_Resolver] + ) -> void + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/interfaces.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/interfaces.rbs new file mode 100644 index 00000000..5aba2388 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/interfaces.rbs @@ -0,0 +1,17 @@ +module ElasticGraph + class GraphQL + module Resolvers + type fieldArgs = ::Hash[::String, untyped] + + interface _Resolver + def can_resolve?: (field: Schema::Field, object: untyped) -> bool + def resolve: ( + field: Schema::Field, + object: untyped, + context: ::GraphQL::Query::Context, + args: fieldArgs, + lookahead: ::GraphQL::Execution::Lookahead) -> untyped + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/list_records.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/list_records.rbs new file mode 100644 index 00000000..b9e57c27 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/list_records.rbs @@ -0,0 +1,9 @@ +module ElasticGraph + class GraphQL + module Resolvers + class ListRecords + include _Resolver + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/nested_relationships.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/nested_relationships.rbs new file mode 100644 index 00000000..f3e1a914 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/nested_relationships.rbs @@ -0,0 +1,14 @@ +module ElasticGraph + class GraphQL + module Resolvers + class NestedRelationships + include _Resolver + + def initialize: ( + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + logger: ::Logger + ) -> void + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/query_source.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/query_source.rbs new file mode 100644 index 00000000..a790f16f --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/query_source.rbs @@ -0,0 +1,22 @@ +module ElasticGraph + class GraphQL + module Resolvers + class QuerySource < ::GraphQL::Dataloader::Source + def initialize: (DatastoreSearchRouter, QueryDetailsTracker) -> void + @datastore_router: DatastoreSearchRouter + @query_tracker: QueryDetailsTracker + def fetch: (::Array[DatastoreQuery]) -> ::Array[DatastoreResponse::SearchResponse] + + def self.execute_many: ( + ::Array[DatastoreQuery], + for_context: ::GraphQL::Query::Context + ) -> ::Hash[DatastoreQuery, DatastoreResponse::SearchResponse] + + def self.execute_one: ( + DatastoreQuery, + for_context: ::GraphQL::Query::Context + ) -> DatastoreResponse::SearchResponse + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection.rbs new file mode 100644 index 00000000..fde98e0d --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection.rbs @@ -0,0 +1,15 @@ +module ElasticGraph + class GraphQL + module Resolvers + module RelayConnection + def self.maybe_wrap: ( + DatastoreResponse::SearchResponse, + field: Schema::Field, + context: ::GraphQL::Query::Context, + lookahead: ::GraphQL::Execution::Lookahead, + query: DatastoreQuery + ) -> (DatastoreResponse::SearchResponse | GenericAdapter[DatastoreResponse::Document] | GenericAdapter[Aggregation::Resolvers::Node]) + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection/array_adapter.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection/array_adapter.rbs new file mode 100644 index 00000000..d070dc60 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection/array_adapter.rbs @@ -0,0 +1,40 @@ +module ElasticGraph + class GraphQL + module Resolvers + module RelayConnection + class ArrayAdapter[N] + extend Forwardable + include _RelayConnection[N] + include _RelayPageInfo + attr_reader schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + attr_reader graphql_impl: ::GraphQL::Pagination::ArrayConnection[N] + + def initialize: (SchemaArtifacts::RuntimeMetadata::SchemaElementNames, ::GraphQL::Pagination::ArrayConnection[N]) -> void + + def self.build: [N] (::Array[N], ::Hash[::String, untyped], SchemaArtifacts::RuntimeMetadata::SchemaElementNames, ::GraphQL::Query::Context) -> ArrayAdapter[N] + def total_edge_count: () -> ::Integer + + @edges: ::Array[_RelayEdge[N]]? + @nodes: ::Array[N]? + + class Edge[N] + include _RelayEdge[N] + attr_reader schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + attr_reader graphql_impl: ::GraphQL::Pagination::ArrayConnection[N] + def initialize: (SchemaArtifacts::RuntimeMetadata::SchemaElementNames, ::GraphQL::Pagination::ArrayConnection[N], N) -> void + end + end + end + end + end +end + +module GraphQL + module Pagination + class ArrayConnection[N] + def initialize: (::Array[N], context: ::GraphQL::Query::Context, **untyped) -> void + attr_reader nodes: ::Array[N] + def cursor_for: (N) -> ::String + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection/generic_adapter.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection/generic_adapter.rbs new file mode 100644 index 00000000..3e7ef7d1 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection/generic_adapter.rbs @@ -0,0 +1,53 @@ +module ElasticGraph + class GraphQL + module Resolvers + module RelayConnection + interface _NodeWithCursor + def cursor: () -> DecodedCursor + def ==: (untyped) -> bool + def !=: (untyped) -> bool + end + + class GenericAdapter[N < _NodeWithCursor] < ResolvableValueClass + include _RelayConnection[N] + + attr_reader schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + attr_reader raw_nodes: ::Array[N] + attr_reader paginator: DatastoreQuery::Paginator[N] + attr_reader to_sort_value: ^(N, DecodedCursor) -> ::Array[DatastoreQuery::Paginator::SortValue] + attr_reader get_total_edge_count: ^() -> ::Integer? + + def initialize: ( + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + raw_nodes: ::Array[N], + paginator: DatastoreQuery::Paginator[N], + to_sort_value: ^(N, DecodedCursor) -> ::Array[DatastoreQuery::Paginator::SortValue], + get_total_edge_count: ^() -> ::Integer? + ) -> void + + def with: ( + ?schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + ?raw_nodes: ::Array[N], + ?paginator: DatastoreQuery::Paginator[N], + ?to_sort_value: ^(N, DecodedCursor) -> ::Array[DatastoreQuery::Paginator::SortValue], + ?get_total_edge_count: ^() -> ::Integer? + ) -> GenericAdapter[N] + + private + + @page_info: PageInfo[N]? + @edges: ::Array[_RelayEdge[N]]? + @nodes: ::Array[N]? + + @before_truncation_nodes: ::Array[N]? + def before_truncation_nodes: () -> ::Array[N] + + class Edge[N < _NodeWithCursor] < ResolvableValueClass + include _RelayEdge[N] + def initialize: (SchemaArtifacts::RuntimeMetadata::SchemaElementNames, N) -> void + end + end + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection/interfaces.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection/interfaces.rbs new file mode 100644 index 00000000..f78ecbda --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection/interfaces.rbs @@ -0,0 +1,25 @@ +module ElasticGraph + class GraphQL + module Resolvers + module RelayConnection + interface _RelayConnection[N] + def page_info: () -> _RelayPageInfo + def edges: () -> ::Array[_RelayEdge[N]] + def nodes: () -> ::Array[N] + end + + interface _RelayPageInfo + def start_cursor: () -> String? + def end_cursor: () -> String? + def has_previous_page: () -> bool + def has_next_page: () -> bool + end + + interface _RelayEdge[N] + def node: () -> N + def cursor: () -> String + end + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection/page_info.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection/page_info.rbs new file mode 100644 index 00000000..37e35bb8 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection/page_info.rbs @@ -0,0 +1,29 @@ +module ElasticGraph + class GraphQL + module Resolvers + module RelayConnection + class PageInfo[N < _NodeWithCursor] < ResolvableValueClass + include _RelayPageInfo + + attr_reader before_truncation_nodes: ::Array[N] + attr_reader edges: ::Array[_RelayEdge[N]] + attr_reader paginator: DatastoreQuery::Paginator[N] + + def initialize: [N] ( + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + before_truncation_nodes: ::Array[N], + edges: ::Array[_RelayEdge[N]], + paginator: DatastoreQuery::Paginator[N] + ) -> void + + def with: ( + ?schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + ?before_truncation_nodes: ::Array[N], + ?edges: ::Array[_RelayEdge[N]], + ?paginator: DatastoreQuery::Paginator[N] + ) -> PageInfo[N] + end + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection/search_response_adapter_builder.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection/search_response_adapter_builder.rbs new file mode 100644 index 00000000..1f049204 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/relay_connection/search_response_adapter_builder.rbs @@ -0,0 +1,15 @@ +module ElasticGraph + class GraphQL + module Resolvers + module RelayConnection + class SearchResponseAdapterBuilder + def self.build_from: ( + query: DatastoreQuery, + search_response: DatastoreResponse::SearchResponse, + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + ) -> GraphQL::Resolvers::RelayConnection::GenericAdapter[DatastoreResponse::Document] + end + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/resolvable_value.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/resolvable_value.rbs new file mode 100644 index 00000000..7d6ac2f7 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/resolvers/resolvable_value.rbs @@ -0,0 +1,21 @@ +module ElasticGraph + class GraphQL + module Resolvers + module ResolvableValue + def self.new: (*Symbol) ?{ () [self: self] -> void } -> ResolvableValueClass + + include _Resolver + attr_reader schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + + private + + def args_to_canonical_form: (fieldArgs) -> ::Hash[::Symbol, untyped] + def canonical_name_for: (::String | ::Symbol, ::String) -> ::Symbol + end + + class ResolvableValueClass + include ResolvableValue + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/cursor.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/cursor.rbs new file mode 100644 index 00000000..3b613567 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/cursor.rbs @@ -0,0 +1,9 @@ +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + class Cursor + extend SchemaArtifacts::_ScalarCoercionAdapter[DecodedCursor, ::String] + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/date.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/date.rbs new file mode 100644 index 00000000..b89a1e24 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/date.rbs @@ -0,0 +1,13 @@ +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + class Date + extend SchemaArtifacts::_ScalarCoercionAdapter[::String, ::String] + + private + + def self.raise_coercion_error: (::String) -> bot + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/date_time.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/date_time.rbs new file mode 100644 index 00000000..d92dbe57 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/date_time.rbs @@ -0,0 +1,14 @@ +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + class DateTime + extend SchemaArtifacts::_ScalarCoercionAdapter[::String, ::String] + PRECISION: ::Integer + + private + + def self.raise_coercion_error: (::String) -> bot + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/local_time.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/local_time.rbs new file mode 100644 index 00000000..611ee9ce --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/local_time.rbs @@ -0,0 +1,11 @@ +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + class LocalTime + extend SchemaArtifacts::_ScalarCoercionAdapter[::String, ::String] + + def self.validated_value: (untyped) -> ::String? + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/longs.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/longs.rbs new file mode 100644 index 00000000..bdefd993 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/longs.rbs @@ -0,0 +1,17 @@ +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + class Longs + def self.to_ruby_int_in_range: (untyped, ::Integer, ::Integer) -> ::Integer? + end + + class JsonSafeLong + extend SchemaArtifacts::_ScalarCoercionAdapter[::Integer, ::Integer] + end + + class LongString + extend SchemaArtifacts::_ScalarCoercionAdapter[::Integer, ::String] + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/no_op.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/no_op.rbs new file mode 100644 index 00000000..f1b4ecd2 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/no_op.rbs @@ -0,0 +1,9 @@ +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + class NoOp + extend SchemaArtifacts::_ScalarCoercionAdapter[untyped, untyped] + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/time_zone.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/time_zone.rbs new file mode 100644 index 00000000..829943cc --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/time_zone.rbs @@ -0,0 +1,10 @@ +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + class TimeZone + SUGGESTER: ::DidYouMean::SpellChecker + extend SchemaArtifacts::_ScalarCoercionAdapter[::String, ::String] + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/untyped.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/untyped.rbs new file mode 100644 index 00000000..952aebc3 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/untyped.rbs @@ -0,0 +1,9 @@ +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + class Untyped + extend SchemaArtifacts::_ScalarCoercionAdapter[::String, untyped] + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/valid_time_zones.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/valid_time_zones.rbs new file mode 100644 index 00000000..a0da4a8d --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/scalar_coercion_adapters/valid_time_zones.rbs @@ -0,0 +1,7 @@ +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + VALID_TIME_ZONES: ::Set[::String] + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/schema.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/schema.rbs new file mode 100644 index 00000000..72b84cb9 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/schema.rbs @@ -0,0 +1,22 @@ +module ElasticGraph + class GraphQL + # Note: this is a partial signature definition (`schema.rb` is ignored in `Steepfile`) + class Schema + attr_reader graphql_schema: ::GraphQL::Schema + attr_reader element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + + def initialize: ( + graphql_schema_string: ::String, + config: Config, + runtime_metadata: SchemaArtifacts::RuntimeMetadata::Schema, + index_definitions_by_graphql_type: ::Hash[::String, ::Array[DatastoreCore::_IndexDefinition]], + graphql_gem_plugins: ::Hash[::Class, ::Hash[::Symbol, untyped]] + ) { (Schema) -> Resolvers::GraphQLAdapter } -> void + + def type_from: (::GraphQL::Schema::_Type) -> Type + def type_named: (::String | ::Symbol) -> Type + def field_named: (::String | ::Symbol, ::String | ::Symbol) -> Field + def indexed_document_types: () -> ::Array[Type] + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/schema/arguments.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/schema/arguments.rbs new file mode 100644 index 00000000..9bfcce9c --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/schema/arguments.rbs @@ -0,0 +1,12 @@ +module ElasticGraph + class GraphQL + class Schema + module Arguments + def self.to_schema_form: ( + ::Hash[::String, untyped], + untyped + ) -> ::Hash[::String, untyped] + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/schema/enum_value.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/schema/enum_value.rbs new file mode 100644 index 00000000..ad8f8b3b --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/schema/enum_value.rbs @@ -0,0 +1,33 @@ +module ElasticGraph + class GraphQL + class Schema + class EnumValueSupertype + attr_reader name: ::Symbol + attr_reader type: Type + attr_reader runtime_metadata: SchemaArtifacts::RuntimeMetadata::Enum::Value + + def initialize: ( + ::Symbol, + Type, + SchemaArtifacts::RuntimeMetadata::Enum::Value + ) -> void + + def self.with: ( + name: ::Symbol, + type: Type, + runtime_metadata: SchemaArtifacts::RuntimeMetadata::Enum::Value + ) -> EnumValue + + def with: ( + ?name: ::Symbol, + ?type: Type, + ?runtime_metadata: SchemaArtifacts::RuntimeMetadata::Enum::Value + ) -> EnumValue + end + + class EnumValue < EnumValueSupertype + def sort_clauses: () -> ::Array[::Hash[::String, ::Hash[::String, ::String]]] + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/schema/field.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/schema/field.rbs new file mode 100644 index 00000000..151c21ce --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/schema/field.rbs @@ -0,0 +1,22 @@ +module ElasticGraph + class GraphQL + class Schema + # Note: this is a partial signature definition (`field.rb` is ignored in `Steepfile`) + class Field + attr_reader description: ::String + attr_reader schema: Schema + attr_reader name: ::Symbol + attr_reader name_in_index: ::Symbol + attr_reader parent_type: Type + attr_reader type: Type + attr_reader graphql_field: ::GraphQL::Schema::Field + attr_reader computation_detail: SchemaArtifacts::RuntimeMetadata::ComputationDetail? + + def aggregated?: () -> bool + def sort_clauses_for: (::Array[::String]) -> ::Array[::Hash[::String, ::Hash[::String, ::String]]] + def args_to_schema_form: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + def index_field_names_for_resolution: () -> ::Array[::String] + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/schema/field_name.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/schema/field_name.rbs new file mode 100644 index 00000000..9f6794fe --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/schema/field_name.rbs @@ -0,0 +1,12 @@ +module ElasticGraph + class GraphQL + class Schema + class FieldName + attr_reader name_in_graphql: ::String + attr_reader name_in_index: ::String + + def initialize: (::String, ::String) -> void + end + end + end +end diff --git a/elasticgraph-graphql/sig/elastic_graph/graphql/schema/type.rbs b/elasticgraph-graphql/sig/elastic_graph/graphql/schema/type.rbs new file mode 100644 index 00000000..4e781f18 --- /dev/null +++ b/elasticgraph-graphql/sig/elastic_graph/graphql/schema/type.rbs @@ -0,0 +1,24 @@ +module ElasticGraph + class GraphQL + class Schema + # Note: this is a partial signature definition (`type.rb` is ignored in `Steepfile`) + class Type + attr_reader name: ::Symbol + attr_reader fields_by_name: ::Hash[::String, Field] + attr_reader elasticgraph_category: SchemaArtifacts::RuntimeMetadata::elasticGraphCategory? + attr_reader graphql_type: ::GraphQL::Schema::_Type + def search_index_definitions: () -> ::Array[DatastoreCore::_IndexDefinition] + def unwrap_fully: () -> Type + def field_named: (::String | ::Symbol) -> Field + def object?: () -> bool + def relay_connection?: () -> bool + def abstract?: () -> bool + def enum?: () -> bool + def embedded_object?: () -> bool + def indexed_document?: () -> bool + def indexed_aggregation?: () -> bool + def enum_value_named: (::String | ::Symbol) -> EnumValue + end + end + end +end diff --git a/elasticgraph-graphql/sig/graphql_gem.rbs b/elasticgraph-graphql/sig/graphql_gem.rbs new file mode 100644 index 00000000..802b6409 --- /dev/null +++ b/elasticgraph-graphql/sig/graphql_gem.rbs @@ -0,0 +1,191 @@ +module GraphQL + class CoercionError < StandardError + end + + class Dataloader + class Source + def load_all: [Req, Res] (::Array[Req]) -> ::Array[Res] + end + + def with: (Class, *untyped, **untyped) -> Source + end + + module Execution + class Lookahead + attr_reader ast_nodes: ::Array[Language::Nodes::Field] + attr_reader name: ::String + attr_reader field: Schema::Field + attr_reader owner_type: Schema::_Type + attr_reader arguments: ::Hash[::String, untyped] + + def initialize: ( + query: Query, + ast_nodes: ::Array[Language::Nodes::AbstractNode], + field: Schema::Field, + owner_type: Schema::_Type + ) -> void + + def selects?: (::String) -> bool + def selection: (::String) -> Lookahead + def selections: () -> ::Array[Lookahead] + def selected?: () -> bool + end + end + + class ExecutionError < StandardError + end + + module Language + module Nodes + class AbstractNode + def to_query_string: () -> ::String + def merge: (::Hash[::Symbol, untyped]) -> self + end + + class Argument < AbstractNode + attr_reader name: ::String + attr_reader value: untyped + end + + class Directive < AbstractNode + attr_reader name: ::String + attr_reader arguments: ::Array[Argument] + end + + class Document < AbstractNode + attr_reader definitions: ::Array[FragmentDefinition | OperationDefinition] + end + + class Field < AbstractNode + attr_reader alias: ::String? + attr_reader name: ::String + end + + class FragmentDefinition < AbstractNode + attr_reader directives: ::Array[Directive] + end + + class OperationDefinition < AbstractNode + attr_reader name: ::String? + attr_reader variables: ::Array[VariableDefinition] + attr_reader directives: ::Array[Directive] + end + + type typeReference = TypeName | WrapperType + + class TypeName < AbstractNode + attr_reader name: ::String + end + + class VariableDefinition < AbstractNode + attr_reader line: ::Integer + attr_reader col: ::Integer + attr_reader name: ::String + attr_reader type: typeReference + end + + class WrapperType < AbstractNode + def of_type: () -> typeReference + end + end + end + + class Query + def initialize: ( + ::GraphQL::Schema, + ::String?, + ?document: Language::Nodes::Document?, + ?validate: bool, + ?variables: ::Hash[::String, untyped]?, + ?operation_name: ::String?, + ?context: ::Hash[::Symbol, untyped]? + ) -> void + + attr_reader document: Language::Nodes::Document + attr_reader operation_name: ::String? + attr_reader operations: ::Hash[::String, Language::Nodes::OperationDefinition] + attr_reader fingerprint: ::String + attr_reader sanitized_query_string: ::String? + attr_reader result: Result + attr_reader query_string: ::String? + + def selected_operation: () -> Language::Nodes::OperationDefinition? + def static_errors: () -> ::Array[_ValidationError] + + class Context + attr_reader query: Query + def []: (::Symbol | ::String) -> untyped + def fetch: (untyped) -> untyped + def add_error: (ExecutionError) -> void + def dataloader: () -> Dataloader + end + + class Result + def initialize: (query: Query?, values: ::Hash[::String, untyped]) -> void + attr_reader to_h: ::Hash[::String, untyped] + end + end + + class Schema + attr_reader types: ::Hash[::String, _Type] + attr_reader static_validator: StaticValidation::Validator + + def execute: (::String, **untyped) -> ::Hash[::String, untyped] + def self.from_definition: (::String) -> Schema + def to_definition: () -> ::String + def type_from_ast: (Language::Nodes::AbstractNode) -> _Type? + + interface _Type + def kind: () -> TypeKinds::TypeKind + def to_type_signature: () -> ::String + def unwrap: () -> _Type + def ==: (untyped) -> boolish + def graphql_name: () -> ::String + end + + class InputObject + include _Type + attr_reader arguments: ::Hash[::String, Argument] + end + + class Argument + attr_reader name: ::String + attr_reader type: _Type + end + + class Field + include _Member + attr_reader name: ::String + def arguments: () -> ::Hash[::String, Argument] + def owner: () -> _Member + def introspection?: () -> bool + end + + interface _Member + def graphql_name: () -> ::String + end + + class Printer + def self.print_schema: (Schema, **untyped) -> ::String + end + end + + module StaticValidation + class Validator + def validate: (Query) -> {errors: ::Array[_ValidationError]} + end + end + + module TypeKinds + class TypeKind + def input_object?: () -> bool + def enum?: () -> bool + end + end + + type validationErrorHash = ::Hash[::String, untyped] + + interface _ValidationError + def to_h: () -> validationErrorHash + end +end diff --git a/elasticgraph-graphql/spec/acceptance/aggregations_spec.rb b/elasticgraph-graphql/spec/acceptance/aggregations_spec.rb new file mode 100644 index 00000000..309d870f --- /dev/null +++ b/elasticgraph-graphql/spec/acceptance/aggregations_spec.rb @@ -0,0 +1,2488 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "elasticgraph_graphql_acceptance_support" +require "elastic_graph/graphql/scalar_coercion_adapters/valid_time_zones" +require "elastic_graph/graphql/datastore_query" + +module ElasticGraph + RSpec.describe "ElasticGraph::GraphQL--aggregations" do + include_context "ElasticGraph GraphQL acceptance aggregation support" + + with_both_casing_forms do + let(:amount_cents) { case_correctly("amount_cents") } + let(:aggregated_values) { case_correctly("aggregated_values") } + let(:grouped_by) { case_correctly("grouped_by") } + let(:approximate_distinct_value_count) { case_correctly("approximate_distinct_value_count") } + + it "returns empty aggregation results when querying a rollover index that does not yet have any concrete index (e.g. before the first document is indexed)" do + allow(GraphQL::DatastoreQuery).to receive(:perform).and_wrap_original do |original, queries, &block| + queries.each do |query| + original_to_datastore_msearch_header = query.to_datastore_msearch_header + to_datastore_msearch_header = + if original_to_datastore_msearch_header[:index].start_with?("widgets") + original_to_datastore_msearch_header.merge(index: "rollover_index_with_no_concrete_indexes__*") + else + original_to_datastore_msearch_header + end + + allow(query).to receive(:to_datastore_msearch_header).and_return(to_datastore_msearch_header) + end + + original.call(queries, &block) + end + + # Test grouped_by + aggregations = group_widgets_by_tag + expect(aggregations).to eq [{ + grouped_by => {"tag" => nil}, + "count" => 0, + aggregated_values => { + "#{amount_cents}1" => {"sum" => 0}, + "#{amount_cents}2" => {"sum" => 0} + } + }] + + # Test aggregated_values + aggregations = list_widgets_with_aggregations(all_amount_aggregations) + expect(aggregations).to contain_exactly({ + aggregated_values => { + amount_cents => expected_aggregated_amounts_of([]), + "cost" => {amount_cents => expected_aggregated_amounts_of([])} + }, + "count" => 0 + }) + end + + it "returns aggregates (terms, date histogram) and nested aggregates" do + # Test grouped_by before anything is indexed + aggregations = group_widgets_by_tag + expect(aggregations).to eq [] + + # Test aggregated_values before anything is indexed. + aggregations = list_widgets_with_aggregations(all_amount_aggregations) + expect(aggregations).to contain_exactly({ + aggregated_values => { + amount_cents => expected_aggregated_amounts_of([]), + "cost" => {amount_cents => expected_aggregated_amounts_of([])} + }, + "count" => 0 + }) + + ten_usd_and_cad = [{currency: "USD", amount_cents: 1000}, {currency: "CAD", amount_cents: 1000}] + index_records( + # we build some components in order to have some data for our `components` query that exercises an edge case, + # and also to support grouping on a numeric field (position x and y) + component = build(:component, position: {x: 10, y: 20}), + build(:component, position: {x: 10, y: 30}), + widget1 = build(:widget, name: "w100", amount_cents: 100, options: build(:widget_options, size: "SMALL", color: "BLUE"), cost_currency: "USD", created_at: "2019-06-01T12:02:20Z", release_timestamps: ["2019-06-01T12:02:20Z", "2019-06-04T19:19:19Z"], components: [component], tags: [], fees: []), + build(:widget, name: "w200", amount_cents: 200, options: build(:widget_options, size: "SMALL", color: "RED"), cost_currency: "GBP", created_at: "2019-06-02T12:02:20Z", release_timestamps: ["2019-06-02T12:02:20Z"], tags: ["small", "red"], fees: ten_usd_and_cad), + build(:widget, name: "w300", amount_cents: 300, options: build(:widget_options, size: "MEDIUM", color: "RED"), cost_currency: "USD", created_at: "2019-06-01T13:03:30Z", release_timestamps: ["2019-06-01T13:03:30Z"], tags: ["medium", "red", "red"], fees: []) + ) + + # Verify that we can group on a list field (which groups by the individual values of that field) + aggregations = group_widgets_by_tag + expect(aggregations).to eq [ + { + grouped_by => {"tag" => nil}, + "count" => 1, + aggregated_values => { + "#{amount_cents}1" => {"sum" => 100}, + "#{amount_cents}2" => {"sum" => 100} + } + }, + { + grouped_by => {"tag" => "medium"}, + "count" => 1, + aggregated_values => { + "#{amount_cents}1" => {"sum" => 300}, + "#{amount_cents}2" => {"sum" => 300} + } + }, + { + grouped_by => {"tag" => "red"}, + "count" => 2, # demonstrates that a document isn't included in a grouping twice even if its list has that value twice. + aggregated_values => { + "#{amount_cents}1" => {"sum" => 500}, + "#{amount_cents}2" => {"sum" => 500} + } + }, + { + grouped_by => {"tag" => "small"}, + "count" => 1, + aggregated_values => { + "#{amount_cents}1" => {"sum" => 200}, + "#{amount_cents}2" => {"sum" => 200} + } + } + ] + + # Verify that we can group on a subfield of list of objects field (which groups by the individual values of that field) + aggregations = group_widgets_by_fees_currency_with_approximate_distinct_value_counts + expect(aggregations).to eq [ + { + grouped_by => {"fees" => {"currency" => nil}}, + "count" => 2, + aggregated_values => {amount_cents => {"sum" => 400}, "id" => {approximate_distinct_value_count => 2}, "tags" => {approximate_distinct_value_count => 2}} + }, + { + grouped_by => {"fees" => {"currency" => "CAD"}}, + "count" => 1, + aggregated_values => {amount_cents => {"sum" => 200}, "id" => {approximate_distinct_value_count => 1}, "tags" => {approximate_distinct_value_count => 2}} + }, + { + grouped_by => {"fees" => {"currency" => "USD"}}, + "count" => 1, + aggregated_values => {amount_cents => {"sum" => 200}, "id" => {approximate_distinct_value_count => 1}, "tags" => {approximate_distinct_value_count => 2}} + } + ] + + # Verify that we can group on a graphql-only field which is an alias for a child field. + aggregations = list_widgets_with_aggregations(amount_aggregation("size")) + expect(aggregations).to match [ + { + grouped_by => {"size" => enum_value("MEDIUM")}, + "count" => 1, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(300), + "cost" => {amount_cents => expected_aggregated_amounts_of(300)} + } + }, + { + grouped_by => {"size" => enum_value("SMALL")}, + "count" => 2, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100, 200), + "cost" => {amount_cents => expected_aggregated_amounts_of(100, 200)} + } + } + ] + + # Verify that `aggregated_values` supports a graphql-only field which is an alias for a child field + size_uniq_count = widget_ungrouped_aggregated_values_for("size { approximate_distinct_value_count }") + expect(size_uniq_count).to eq({"size" => {case_correctly("approximate_distinct_value_count") => 2}}) + + aggregations = group_widget_currencies_by_widget_name + expect(aggregations).to eq [ + {"count" => 1, case_correctly("grouped_by") => {case_correctly("widget_name") => "w100"}}, + {"count" => 1, case_correctly("grouped_by") => {case_correctly("widget_name") => "w200"}}, + {"count" => 1, case_correctly("grouped_by") => {case_correctly("widget_name") => "w300"}} + ] + + # Verify non-group aggregations + aggregations = list_widgets_with_aggregations(amount_aggregation) + + expect(aggregations).to contain_exactly({ + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100, 200, 300), + "cost" => {amount_cents => expected_aggregated_amounts_of(100, 200, 300)} + }, + "count" => 3 + }) + + # Verify single grouping aggregations + expected_aggs = [ + { + grouped_by => { + "options" => {"size" => enum_value("MEDIUM")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(300), + "cost" => {amount_cents => expected_aggregated_amounts_of(300)} + }, + "count" => 1 + }, + { + grouped_by => { + "options" => {"size" => enum_value("SMALL")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100, 200), + "cost" => {amount_cents => expected_aggregated_amounts_of(100, 200)} + }, + "count" => 2 + } + ] + aggregations = list_widgets_with_aggregations(amount_aggregation("options.size", first: 1)) + expect(aggregations).to match([expected_aggs.first]).or match([expected_aggs.last]) + + aggregations = list_widgets_with_aggregations(amount_aggregation("options.size", first: 0)) + expect(aggregations).to eq([]) + + aggregations = list_widgets_with_aggregations(amount_aggregation("options.size")) + expect(aggregations).to match_array(expected_aggs) + + # Verify that the same query, when aggregating on a numeric field with divergent index/graphql field names, works properly. + aggregations_with_divergent_agg_field_names = list_widgets_with_aggregations( + amount_aggregation("options.size").sub("amount_cents", "amount_cents2") + ) + expect(aggregations_with_divergent_agg_field_names).to eq(aggregations.map do |agg| + agg.merge(aggregated_values => agg.fetch(aggregated_values).transform_keys do |k| + (k == case_correctly("amount_cents")) ? case_correctly("amount_cents2") : k + end) + end) + + # Verify that the same query, when grouping a leaf field that has a divergent index/graphql field name, works properly. + aggregations_with_divergent_field_names = list_widgets_with_aggregations(amount_aggregation("options.the_size")) + + expected_result_with_divergent_field_names = aggregations.map do |agg| + agg_grouped_by = agg.fetch(grouped_by) + + updated_options = agg_grouped_by.fetch("options") + .merge(case_correctly("the_size") => agg_grouped_by.dig("options", "size")) + .except("size") + + agg.merge(grouped_by => agg_grouped_by.merge("options" => updated_options)) + end + expect(aggregations_with_divergent_field_names).to eq(expected_result_with_divergent_field_names) + + # Verify single grouping aggregation on an nested field + aggregations = list_widgets_with_aggregations(amount_aggregation("cost.currency")) + + expect(aggregations).to contain_exactly( + { + grouped_by => { + "cost" => {"currency" => "USD"} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100, 300), + "cost" => {amount_cents => expected_aggregated_amounts_of(100, 300)} + }, + "count" => 2 + }, + { + grouped_by => { + "cost" => {"currency" => "GBP"} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200), + "cost" => {amount_cents => expected_aggregated_amounts_of(200)} + }, + "count" => 1 + } + ) + + # Verify single grouping on a numeric field + x_and_y_groupings = group_components_by_position_x_and_y + expect(x_and_y_groupings).to eq({ + case_correctly("by_x") => {"edges" => [ + {"node" => {"count" => 2, grouped_by => {"position" => {"x" => 10}}}} + ]}, + case_correctly("by_y") => {"edges" => [ + {"node" => {"count" => 1, grouped_by => {"position" => {"y" => 20}}}}, + {"node" => {"count" => 1, grouped_by => {"position" => {"y" => 30}}}} + ]} + }) + + # Verify multiple grouping aggregations + aggregations = list_widgets_with_aggregations(amount_aggregation("options.size", "options.color")) + + expect(aggregations).to contain_exactly( + { + grouped_by => { + "options" => {"size" => enum_value("SMALL"), "color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200), + "cost" => {amount_cents => expected_aggregated_amounts_of(200)} + }, + "count" => 1 + }, + { + grouped_by => { + "options" => {"size" => enum_value("SMALL"), "color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + }, + { + grouped_by => { + "options" => {"size" => enum_value("MEDIUM"), "color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(300), + "cost" => {amount_cents => expected_aggregated_amounts_of(300)} + }, + "count" => 1 + } + ) + + # Verify that the same query, when grouping on subfields of a parent field that has a divergent index/graphql field name, works properly. + aggregations_with_divergent_field_names = list_widgets_with_aggregations(amount_aggregation("the_options.the_size", "the_options.color")) + + expect(aggregations_with_divergent_field_names).to contain_exactly( + { + grouped_by => { + case_correctly("the_options") => {case_correctly("the_size") => enum_value("SMALL"), "color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200), + "cost" => {amount_cents => expected_aggregated_amounts_of(200)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("the_options") => {case_correctly("the_size") => enum_value("SMALL"), "color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("the_options") => {case_correctly("the_size") => enum_value("MEDIUM"), "color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(300), + "cost" => {amount_cents => expected_aggregated_amounts_of(300)} + }, + "count" => 1 + } + ) + + # Verify date/time aggregations + # DateTime: as_date_time() + aggregations = list_widgets_with_aggregations(amount_aggregation("created_at {as_date_time(truncation_unit: DAY)}", "options.color")) + expect(aggregations).to include( + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_date_time") => "2019-06-01T00:00:00.000Z" + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(300), + "cost" => {amount_cents => expected_aggregated_amounts_of(300)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_date_time") => "2019-06-01T00:00:00.000Z" + }, "options" => {"color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_date_time") => "2019-06-02T00:00:00.000Z" + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200), + "cost" => {amount_cents => expected_aggregated_amounts_of(200)} + }, + "count" => 1 + } + ) + + # DateTime: as_date() + aggregations = list_widgets_with_aggregations(amount_aggregation("created_at {as_date(truncation_unit: DAY)}", "options.color")) + expect(aggregations).to include( + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_date") => "2019-06-01" + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(300), + "cost" => {amount_cents => expected_aggregated_amounts_of(300)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_date") => "2019-06-01" + }, "options" => {"color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_date") => "2019-06-02" + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200), + "cost" => {amount_cents => expected_aggregated_amounts_of(200)} + }, + "count" => 1 + } + ) + + # DateTime: as_day_of_week() for a scalar field (`DateTime!`) + aggregations = list_widgets_with_aggregations(amount_aggregation("created_at {as_day_of_week}", "options.color")) + expect(aggregations).to include( + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_day_of_week") => enum_value("SATURDAY") + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(300), + "cost" => {amount_cents => expected_aggregated_amounts_of(300)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_day_of_week") => enum_value("SATURDAY") + }, "options" => {"color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_day_of_week") => enum_value("SUNDAY") + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200), + "cost" => {amount_cents => expected_aggregated_amounts_of(200)} + }, + "count" => 1 + } + ) + + # DateTime: as_day_of_week() for a list of scalar fields (`[DateTime!]!`) + aggregations = list_widgets_with_aggregations(amount_aggregation("release_timestamp {as_day_of_week(offset: {amount: -1, unit: DAY})}", "options.color")) + expect(aggregations).to include( + { + grouped_by => { + case_correctly("release_timestamp") => { + case_correctly("as_day_of_week") => enum_value("FRIDAY") + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(300), + "cost" => {amount_cents => expected_aggregated_amounts_of(300)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("release_timestamp") => { + case_correctly("as_day_of_week") => enum_value("FRIDAY") + }, "options" => {"color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("release_timestamp") => { + case_correctly("as_day_of_week") => enum_value("SATURDAY") + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200), + "cost" => {amount_cents => expected_aggregated_amounts_of(200)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("release_timestamp") => { + case_correctly("as_day_of_week") => enum_value("MONDAY") # 2nd DateTime value in list for widget1 + }, "options" => {"color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + } + ) + + # DateTime: as_time_of_day() truncated to SECOND + aggregations = list_widgets_with_aggregations(amount_aggregation("created_at {as_time_of_day(truncation_unit: SECOND)}", "options.color")) + expect(aggregations).to include( + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_time_of_day") => "12:02:20" + }, "options" => {"color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_time_of_day") => "12:02:20" + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200), + "cost" => {amount_cents => expected_aggregated_amounts_of(200)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_time_of_day") => "13:03:30" + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(300), + "cost" => {amount_cents => expected_aggregated_amounts_of(300)} + }, + "count" => 1 + } + ) + + # DateTime: as_time_of_day() truncated to MINUTE + aggregations = list_widgets_with_aggregations(amount_aggregation("created_at {as_time_of_day(truncation_unit: MINUTE)}", "options.color")) + expect(aggregations).to include( + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_time_of_day") => "12:02:00" + }, "options" => {"color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_time_of_day") => "12:02:00" + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200), + "cost" => {amount_cents => expected_aggregated_amounts_of(200)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_time_of_day") => "13:03:00" + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(300), + "cost" => {amount_cents => expected_aggregated_amounts_of(300)} + }, + "count" => 1 + } + ) + + # DateTime: as_time_of_day() truncated to HOUR + aggregations = list_widgets_with_aggregations(amount_aggregation("created_at {as_time_of_day(truncation_unit: HOUR)}", "options.color")) + expect(aggregations).to include( + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_time_of_day") => "12:00:00" + }, "options" => {"color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_time_of_day") => "12:00:00" + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200), + "cost" => {amount_cents => expected_aggregated_amounts_of(200)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_at") => { + case_correctly("as_time_of_day") => "13:00:00" + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(300), + "cost" => {amount_cents => expected_aggregated_amounts_of(300)} + }, + "count" => 1 + } + ) + + # DateTime: as_time_of_day() for a list of scalar fields (`[DateTime!]!`) + aggregations = list_widgets_with_aggregations(amount_aggregation("release_timestamp {as_time_of_day(truncation_unit: MINUTE, offset: {amount: -3, unit: HOUR})}", "options.color")) + expect(aggregations).to include( + { + grouped_by => { + case_correctly("release_timestamp") => { + case_correctly("as_time_of_day") => "09:02:00" + }, "options" => {"color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("release_timestamp") => { + case_correctly("as_time_of_day") => "09:02:00" + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200), + "cost" => {amount_cents => expected_aggregated_amounts_of(200)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("release_timestamp") => { + case_correctly("as_time_of_day") => "10:03:00" + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(300), + "cost" => {amount_cents => expected_aggregated_amounts_of(300)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("release_timestamp") => { + case_correctly("as_time_of_day") => "16:19:00" + }, "options" => {"color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + } + ) + + # Date: as_date() + aggregations = list_widgets_with_aggregations(amount_aggregation("created_on {as_date(truncation_unit: DAY)}", "options.color")) + expect(aggregations).to include( + { + grouped_by => { + case_correctly("created_on") => { + case_correctly("as_date") => "2019-06-01" + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(300), + "cost" => {amount_cents => expected_aggregated_amounts_of(300)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_on") => { + case_correctly("as_date") => "2019-06-01" + }, "options" => {"color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_on") => { + case_correctly("as_date") => "2019-06-02" + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200), + "cost" => {amount_cents => expected_aggregated_amounts_of(200)} + }, + "count" => 1 + } + ) + + # Date: as_day_of_week() + aggregations = list_widgets_with_aggregations(amount_aggregation("created_on {as_day_of_week}", "options.color")) + expect(aggregations).to include( + { + grouped_by => { + case_correctly("created_on") => { + case_correctly("as_day_of_week") => enum_value("SATURDAY") + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(300), + "cost" => {amount_cents => expected_aggregated_amounts_of(300)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_on") => { + case_correctly("as_day_of_week") => enum_value("SATURDAY") + }, "options" => {"color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_on") => { + case_correctly("as_day_of_week") => enum_value("SUNDAY") + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200), + "cost" => {amount_cents => expected_aggregated_amounts_of(200)} + }, + "count" => 1 + } + ) + + # Date: as_day_of_week() for a list of scalar fields (`[Date!]!`) + aggregations = list_widgets_with_aggregations(amount_aggregation("release_date {as_day_of_week(offset: {amount: 2, unit: DAY})}", "options.color")) + expect(aggregations).to include( + { + grouped_by => { + case_correctly("release_date") => { + case_correctly("as_day_of_week") => enum_value("MONDAY") + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(300), + "cost" => {amount_cents => expected_aggregated_amounts_of(300)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("release_date") => { + case_correctly("as_day_of_week") => enum_value("MONDAY") + }, "options" => {"color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("release_date") => { + case_correctly("as_day_of_week") => enum_value("TUESDAY") + }, "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200), + "cost" => {amount_cents => expected_aggregated_amounts_of(200)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("release_date") => { + case_correctly("as_day_of_week") => enum_value("THURSDAY") # 2nd Date value in list for widget1 + }, "options" => {"color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + } + ) + + verify_all_timestamp_groupings_valid(widget1.fetch(:id), truncation_unit_type: "DateTimeGroupingTruncationUnitInput", field: "created_at") + verify_all_datetime_offset_units_valid(widget1.fetch(:id)) + + # Legacy date/time grouping API + aggregations = list_widgets_with_aggregations(amount_aggregation("created_at_legacy(granularity: DAY)", "options.color")) + + expect(aggregations).to include( + { + grouped_by => { + case_correctly("created_at_legacy") => "2019-06-01T00:00:00.000Z", + "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(300), + "cost" => {amount_cents => expected_aggregated_amounts_of(300)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_at_legacy") => "2019-06-01T00:00:00.000Z", + "options" => {"color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_at_legacy") => "2019-06-02T00:00:00.000Z", + "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200), + "cost" => {amount_cents => expected_aggregated_amounts_of(200)} + }, + "count" => 1 + } + ) + + # Verify date histogram groupings when the datetime field has a different name in the index + aggs_with_divergent_date_field_names = list_widgets_with_aggregations(amount_aggregation("created_at2_legacy(granularity: DAY)", "options.color")) + expect(aggs_with_divergent_date_field_names).to eq(aggregations.map do |agg| + agg.merge(grouped_by => agg.fetch(grouped_by).transform_keys do |k| + (k == case_correctly("created_at_legacy")) ? case_correctly("created_at2_legacy") : k + end) + end) + + # Verify that grouping on the same timestamp field at different granularities succeeds (even though it's not really useful...). + aggregations = list_widgets_with_aggregations(amount_aggregation("by_day: created_at_legacy(granularity: DAY)", "by_month: created_at_legacy(granularity: MONTH)")) + expect(aggregations).to include( + { + grouped_by => { + case_correctly("by_day") => "2019-06-01T00:00:00.000Z", + case_correctly("by_month") => "2019-06-01T00:00:00.000Z" + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100, 300), + "cost" => {amount_cents => expected_aggregated_amounts_of(100, 300)} + }, + "count" => 2 + }, + { + grouped_by => { + case_correctly("by_day") => "2019-06-02T00:00:00.000Z", + case_correctly("by_month") => "2019-06-01T00:00:00.000Z" + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200), + "cost" => {amount_cents => expected_aggregated_amounts_of(200)} + }, + "count" => 1 + } + ) + + legacy_verify_all_timestamp_groupings_valid(widget1.fetch(:id), granularity_type: "DateTimeGroupingGranularityInput", field: "created_at_legacy") + legacy_verify_all_datetime_offset_units_valid(widget1.fetch(:id)) + + # Verify date histogram groupings + aggregations = list_widgets_with_aggregations(amount_aggregation("created_on_legacy(granularity: DAY)", "options.color")) + + expect(aggregations).to include( + { + grouped_by => { + case_correctly("created_on_legacy") => "2019-06-01", + "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(300), + "cost" => {amount_cents => expected_aggregated_amounts_of(300)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_on_legacy") => "2019-06-01", + "options" => {"color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + }, + { + grouped_by => { + case_correctly("created_on_legacy") => "2019-06-02", + "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200), + "cost" => {amount_cents => expected_aggregated_amounts_of(200)} + }, + "count" => 1 + } + ) + + aggregations = list_widgets_with_aggregations(amount_aggregation("created_on_legacy(granularity: MONTH, offset_days: 4)", "options.color")) + + expect(aggregations).to include( + { + grouped_by => { + case_correctly("created_on_legacy") => "2019-05-05", # When we shift the date boundaries 4 days, 2019-06-01 falls into the month bucket starting 2019-05-05 + "options" => {"color" => enum_value("RED")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(200, 300), + "cost" => {amount_cents => expected_aggregated_amounts_of(200, 300)} + }, + "count" => 2 + }, + { + grouped_by => { + case_correctly("created_on_legacy") => "2019-05-05", + "options" => {"color" => enum_value("BLUE")} + }, + aggregated_values => { + amount_cents => expected_aggregated_amounts_of(100), + "cost" => {amount_cents => expected_aggregated_amounts_of(100)} + }, + "count" => 1 + } + ) + + legacy_verify_all_timestamp_groupings_valid(widget1.fetch(:id), granularity_type: "DateGroupingGranularityInput", field: "created_on_legacy") + end + + def expected_aggregated_amounts_of(*raw_values) + raw_values = raw_values.flatten # to allow an empty array to be passed for explicitness + min = raw_values.min + max = raw_values.max + sum = raw_values.sum + avg = raw_values.empty? ? nil : sum.to_f / raw_values.size + + { + case_correctly("approximate_sum") => float_of(sum), + case_correctly("exact_sum") => int_of(sum), + case_correctly("approximate_avg") => float_of(avg), + case_correctly("exact_min") => int_of(min), + case_correctly("exact_max") => int_of(max) + } + end + + def verify_all_timestamp_groupings_valid(widget_id, truncation_unit_type:, field:) + truncation_unit_type = apply_derived_type_customizations(truncation_unit_type) + datetime_granularities = graphql.schema.type_named(truncation_unit_type).graphql_type.values.keys + + datetime_granularities.each do |truncation_unit| + aggregations = list_widgets_with_aggregations(amount_aggregation("#{field} { as_date_time(truncation_unit: #{truncation_unit})}")) + + expect(aggregations).not_to be_empty, "Expected valid results for grouping truncation_unit #{truncation_unit}, got: #{aggregations.inspect}" + + if truncation_unit.to_s == "WEEK" + # Elasticsearch/OpenSearch do not document which day they treat as the first day of week for WEEK groupings. + # We've observed that they treat Monday as the first day, and have documented it as such. Here + # we verify that that still holds true so that we know our documentation is accurate. + # + # We rely on this in the documentation generated in this file: + # elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb + timestamps_are_mondays = aggregations.map { |a| ::Date.iso8601(a.fetch(case_correctly("grouped_by")).fetch(case_correctly(field)).fetch(case_correctly("as_date_time"))).monday? }.uniq + expect(timestamps_are_mondays).to eq [true] + end + end + end + + def legacy_verify_all_timestamp_groupings_valid(widget_id, granularity_type:, field:) + granularity_type = apply_derived_type_customizations(granularity_type) + datetime_granularities = graphql.schema.type_named(granularity_type).graphql_type.values.keys + + datetime_granularities.each do |granularity| + aggregations = list_widgets_with_aggregations(amount_aggregation("#{field}(granularity: #{granularity})")) + + expect(aggregations).not_to be_empty, "Expected valid results for grouping granularity #{granularity}, got: #{aggregations.inspect}" + + if granularity.to_s == "WEEK" + # Elasticsearch/OpenSearch do not document which day they treat as the first day of week for WEEK groupings. + # We've observed that they treat Monday as the first day, and have documented it as such. Here + # we verify that that still holds true so that we know our documentation is accurate. + # + # We rely on this in the documentation generated in this file: + # elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb + timestamps_are_mondays = aggregations.map { |a| ::Date.iso8601(a.fetch(case_correctly("grouped_by")).fetch(case_correctly(field))).monday? }.uniq + expect(timestamps_are_mondays).to eq [true] + end + end + end + + def verify_all_datetime_offset_units_valid(widget_id) + date_time_unit = apply_derived_type_customizations("DateTimeUnitInput") + agg_queries = graphql.schema.type_named(date_time_unit).graphql_type.values.keys.map do |unit| + <<~EOS + #{unit}: widget_aggregations { + edges { + node { + grouped_by { + created_at { + as_date_time(truncation_unit: MONTH, offset: {amount: 4, unit: #{unit}}) + } + } + count + } + } + } + EOS + end + + results = call_graphql_query(<<~QUERY) + query { + DAY_FOR_HOUR: widget_aggregations { + edges { + node { + grouped_by { + # Verify that when the offset is much larger than the grouping truncation_unit, it does something reasonable even though this isn't useful. + created_at { + as_date_time(truncation_unit: HOUR, offset: {amount: 4, unit: DAY}) + } + } + count + } + } + } + + #{agg_queries.join("\n")} + } + QUERY + + timestamps_by_unit = results.dig("data").to_h do |k, v| + [k, v.dig("edges", 0, "node", case_correctly("grouped_by"), case_correctly("created_at"))] + end + + expect(timestamps_by_unit).to eq({ + "DAY_FOR_HOUR" => {case_correctly("as_date_time") => "2019-06-01T12:00:00.000Z"}, # When grouping by hour but offset by 4 days, the offset is effectively ignored. + "DAY" => {case_correctly("as_date_time") => "2019-05-05T00:00:00.000Z"}, # 2019-06-01 falls into the month starting 2019-05-05 after the shift + "HOUR" => {case_correctly("as_date_time") => "2019-06-01T04:00:00.000Z"}, + "MINUTE" => {case_correctly("as_date_time") => "2019-06-01T00:04:00.000Z"}, + "SECOND" => {case_correctly("as_date_time") => "2019-06-01T00:00:04.000Z"}, + "MILLISECOND" => {case_correctly("as_date_time") => "2019-06-01T00:00:00.004Z"} + }) + end + + def legacy_verify_all_datetime_offset_units_valid(widget_id) + date_time_unit = apply_derived_type_customizations("DateTimeUnitInput") + agg_queries = graphql.schema.type_named(date_time_unit).graphql_type.values.keys.map do |unit| + <<~EOS + #{unit}: widget_aggregations { + edges { + node { + grouped_by { + created_at_legacy(granularity: MONTH, offset: {amount: 4, unit: #{unit}}) + } + count + } + } + } + EOS + end + + results = call_graphql_query(<<~QUERY) + query { + DAY_FOR_HOUR: widget_aggregations { + edges { + node { + grouped_by { + # Verify that when the offset is much larger than the grouping granularity, it does something reasonable even though this isn't useful. + created_at_legacy(granularity: HOUR, offset: {amount: 4, unit: DAY}) + } + count + } + } + } + + #{agg_queries.join("\n")} + } + QUERY + + timestamps_by_unit = results.dig("data").to_h do |k, v| + [k, v.dig("edges", 0, "node", case_correctly("grouped_by"), case_correctly("created_at_legacy"))] + end + + expect(timestamps_by_unit).to eq({ + "DAY_FOR_HOUR" => "2019-06-01T12:00:00.000Z", # When grouping by hour but offset by 4 days, the offset is effectively ignored. + "DAY" => "2019-05-05T00:00:00.000Z", # 2019-06-01 falls into the month starting 2019-05-05 after the shift + "HOUR" => "2019-06-01T04:00:00.000Z", + "MINUTE" => "2019-06-01T00:04:00.000Z", + "SECOND" => "2019-06-01T00:00:04.000Z", + "MILLISECOND" => "2019-06-01T00:00:00.004Z" + }) + end + + it "supports all IANA timezone ids for timestamp grouping" do + index_records(build(:widget, created_at: "2022-11-23T03:00:00Z")) + + agg_queries = GraphQL::ScalarCoercionAdapters::VALID_TIME_ZONES.map.with_index do |time_zone, index| + # Use 2 different args as part of our assertions on the impact of `Aggregation::QueryOptimizer`. + args = + if time_zone.start_with?("America/") + {filter: {"id" => {"not" => {"equal_to_any_of" => [nil]}}}} + else + {} + end + + <<~EOS + tz#{index}: widget_aggregations#{graphql_args(args)} { + edges { + node { + grouped_by { + created_at { + as_date_time(truncation_unit: DAY, time_zone: "#{time_zone}") + } + } + count + } + } + } + EOS + end + + # If the datastore does not understand any of our timestamp values, we'll get an exception and this will fail here. + results = call_graphql_query(<<~QUERY) + query { + #{agg_queries.join("\n")} + } + QUERY + + uniq_timestamps = results.dig("data").map { |k, v| v.dig("edges", 0, "node", case_correctly("grouped_by"), case_correctly("created_at"), case_correctly("as_date_time")) }.uniq + + # Here we verify that the `created_at` values we got back run the gamut from -12:00 to +12:00, + # demonstrating that `created_at` values are correctly converted to values in the specified time zone. + # There are some additional values we're not asserting on here (such as `+03:30...) but we don't need + # to be exhaustive here. + expect(uniq_timestamps).to include( + "2022-11-22T00:00:00.000-12:00", + "2022-11-22T00:00:00.000-11:00", + "2022-11-22T00:00:00.000-10:00", + "2022-11-22T00:00:00.000-09:00", + "2022-11-22T00:00:00.000-08:00", + "2022-11-22T00:00:00.000-07:00", + "2022-11-22T00:00:00.000-06:00", + "2022-11-22T00:00:00.000-05:00", + "2022-11-22T00:00:00.000-04:00", + # Since the `created_at` timestamp is at `03:00:00Z`, the ones above here are on 2022-11-22, and the ones below on 2022-11-23. + "2022-11-23T00:00:00.000-03:00", + "2022-11-23T00:00:00.000-02:00", + "2022-11-23T00:00:00.000-01:00", + "2022-11-23T00:00:00.000Z", + "2022-11-23T00:00:00.000+01:00", + "2022-11-23T00:00:00.000+02:00", + "2022-11-23T00:00:00.000+03:00", + "2022-11-23T00:00:00.000+04:00", + "2022-11-23T00:00:00.000+05:00", + "2022-11-23T00:00:00.000+06:00", + "2022-11-23T00:00:00.000+07:00", + "2022-11-23T00:00:00.000+08:00", + "2022-11-23T00:00:00.000+09:00", + "2022-11-23T00:00:00.000+10:00", + "2022-11-23T00:00:00.000+11:00", + "2022-11-23T00:00:00.000+12:00" + ) + + expect(datastore_msearch_requests("main").size).to eq 1 + + # Due to our `Aggregation::QueryOptimizer`, we're able to serve this with 2 searches in a single msearch request. + # It would be 1, but we intentionally are using `filter` for some aggregations and omitting it for others in order + # to demonstrate that separate searches are used when filters differ. + expect(count_of_searches_in(datastore_msearch_requests("main").first)).to eq 2 + + # Verify that we logged positive values for `elasticgraph_overhead_ms` and `datastore_server_duration_ms` + # to ensure our detailed duration tracking is working end-to-end. We check it in this test because this + # query is one of the slowest ones in our test suite. Some of the others are fast enough that we can't + # always count on positive values for both numbers here even when things are working correctly. + expect(logged_jsons_of_type("ElasticGraphQueryExecutorQueryDuration").last).to include( + "elasticgraph_overhead_ms" => a_value > 0, + "datastore_server_duration_ms" => a_value > 0 + ) + end + + it "supports all IANA timezone ids for legacy timestamp grouping" do + index_records(build(:widget, created_at: "2022-11-23T03:00:00Z")) + + agg_queries = GraphQL::ScalarCoercionAdapters::VALID_TIME_ZONES.map.with_index do |time_zone, index| + # Use 2 different args as part of our assertions on the impact of `Aggregation::QueryOptimizer`. + args = + if time_zone.start_with?("America/") + {filter: {"id" => {"not" => {"equal_to_any_of" => [nil]}}}} + else + {} + end + + <<~EOS + tz#{index}: widget_aggregations#{graphql_args(args)} { + edges { + node { + grouped_by { + created_at_legacy(granularity: DAY, time_zone: "#{time_zone}") + } + count + } + } + } + EOS + end + + # If the datastore does not understand any of our timestamp values, we'll get an exception and this will fail here. + results = call_graphql_query(<<~QUERY) + query { + #{agg_queries.join("\n")} + } + QUERY + + uniq_timestamps = results.dig("data").map { |k, v| v.dig("edges", 0, "node", case_correctly("grouped_by"), case_correctly("created_at_legacy")) }.uniq + + # Here we verify that the `created_at_legacy` values we got back run the gamut from -12:00 to +12:00, + # demonstrating that `created_at_legacy` values are correctly converted to values in the specified time zone. + # There are some additional values we're not asserting on here (such as `+03:30...) but we don't need + # to be exhaustive here. + expect(uniq_timestamps).to include( + "2022-11-22T00:00:00.000-12:00", + "2022-11-22T00:00:00.000-11:00", + "2022-11-22T00:00:00.000-10:00", + "2022-11-22T00:00:00.000-09:00", + "2022-11-22T00:00:00.000-08:00", + "2022-11-22T00:00:00.000-07:00", + "2022-11-22T00:00:00.000-06:00", + "2022-11-22T00:00:00.000-05:00", + "2022-11-22T00:00:00.000-04:00", + # Since the `created_at_legacy` timestamp is at `03:00:00Z`, the ones above here are on 2022-11-22, and the ones below on 2022-11-23. + "2022-11-23T00:00:00.000-03:00", + "2022-11-23T00:00:00.000-02:00", + "2022-11-23T00:00:00.000-01:00", + "2022-11-23T00:00:00.000Z", + "2022-11-23T00:00:00.000+01:00", + "2022-11-23T00:00:00.000+02:00", + "2022-11-23T00:00:00.000+03:00", + "2022-11-23T00:00:00.000+04:00", + "2022-11-23T00:00:00.000+05:00", + "2022-11-23T00:00:00.000+06:00", + "2022-11-23T00:00:00.000+07:00", + "2022-11-23T00:00:00.000+08:00", + "2022-11-23T00:00:00.000+09:00", + "2022-11-23T00:00:00.000+10:00", + "2022-11-23T00:00:00.000+11:00", + "2022-11-23T00:00:00.000+12:00" + ) + + expect(datastore_msearch_requests("main").size).to eq 1 + + # Due to our `Aggregation::QueryOptimizer`, we're able to serve this with 2 searches in a single msearch request. + # It would be 1, but we intentionally are using `filter` for some aggregations and omitting it for others in order + # to demonstrate that separate searches are used when filters differ. + expect(count_of_searches_in(datastore_msearch_requests("main").first)).to eq 2 + + # Verify that we logged positive values for `elasticgraph_overhead_ms` and `datastore_server_duration_ms` + # to ensure our detailed duration tracking is working end-to-end. We check it in this test because this + # query is one of the slowest ones in our test suite. Some of the others are fast enough that we can't + # always count on positive values for both numbers here even when things are working correctly. + expect(logged_jsons_of_type("ElasticGraphQueryExecutorQueryDuration").last).to include( + "elasticgraph_overhead_ms" => a_value > 0, + "datastore_server_duration_ms" => a_value > 0 + ) + end + + it "supports grouping and counting on a type union or an interface" do + index_into( + graphql, + build(:electrical_part, name: "p1"), + build(:mechanical_part, name: "p1"), + build(:mechanical_part, name: "p2") + ) + + # Run on aggregation on a type union (`parts`) + results = call_graphql_query(<<~EOS).dig("data", case_correctly("part_aggregations"), "edges").map { |e| e["node"] } + query { + part_aggregations { + edges { + node { + grouped_by { name } + count + } + } + } + } + EOS + + expect(results).to contain_exactly( + {case_correctly("grouped_by") => {"name" => "p1"}, "count" => 2}, + {case_correctly("grouped_by") => {"name" => "p2"}, "count" => 1} + ) + + # Run on aggregation on an interface (`named_entities`) + results = call_graphql_query(<<~EOS).dig("data", case_correctly("named_entity_aggregations"), "edges").map { |e| e["node"] } + query { + named_entity_aggregations { + edges { + node { + grouped_by { name } + count + } + } + } + } + EOS + + expect(results).to contain_exactly( + {case_correctly("grouped_by") => {"name" => "p1"}, "count" => 2}, + {case_correctly("grouped_by") => {"name" => "p2"}, "count" => 1} + ) + end + + it "supports using aggregations only for counts" do + index_records( + build(:widget, amount_cents: 100, options: build(:widget_options, size: "SMALL", color: "BLUE"), created_at: "2019-06-01T12:00:00Z"), + build(:widget, amount_cents: 200, options: build(:widget_options, size: "SMALL", color: "RED"), created_at: "2019-06-02T12:00:00Z"), + build(:widget, amount_cents: 300, options: build(:widget_options, size: "MEDIUM", color: "RED"), created_at: "2019-06-01T12:00:00Z") + ) + + # Verify non-group aggregations + aggregations = list_widgets_with_aggregations(count_aggregation) + + expect(aggregations).to contain_exactly( + {"count" => 3} + ) + + # Verify single grouping aggregations + aggregations = list_widgets_with_aggregations(count_aggregation("options.size")) + + expect(aggregations).to contain_exactly( + {grouped_by => {"options" => {"size" => enum_value("MEDIUM")}}, "count" => 1}, + {grouped_by => {"options" => {"size" => enum_value("SMALL")}}, "count" => 2} + ) + + # Verify multiple grouping aggregations + aggregations = list_widgets_with_aggregations(count_aggregation("options.size", "options.color")) + + expect(aggregations).to contain_exactly( + {grouped_by => {"options" => {"size" => enum_value("SMALL"), "color" => enum_value("RED")}}, "count" => 1}, + {grouped_by => {"options" => {"size" => enum_value("SMALL"), "color" => enum_value("BLUE")}}, "count" => 1}, + {grouped_by => {"options" => {"size" => enum_value("MEDIUM"), "color" => enum_value("RED")}}, "count" => 1} + ) + + # Verify date histogram groupings + aggregations = list_widgets_with_aggregations(count_aggregation("created_at { as_date_time(truncation_unit: DAY) }", "options.color")) + + expect(aggregations).to include( + {grouped_by => {case_correctly("created_at") => {case_correctly("as_date_time") => "2019-06-01T00:00:00.000Z"}, "options" => {"color" => enum_value("RED")}}, "count" => 1}, + {grouped_by => {case_correctly("created_at") => {case_correctly("as_date_time") => "2019-06-01T00:00:00.000Z"}, "options" => {"color" => enum_value("BLUE")}}, "count" => 1}, + {grouped_by => {case_correctly("created_at") => {case_correctly("as_date_time") => "2019-06-02T00:00:00.000Z"}, "options" => {"color" => enum_value("RED")}}, "count" => 1} + ) + end + + it "supports using legacy aggregations only for counts" do + index_records( + build(:widget, amount_cents: 100, options: build(:widget_options, size: "SMALL", color: "BLUE"), created_at: "2019-06-01T12:00:00Z"), + build(:widget, amount_cents: 200, options: build(:widget_options, size: "SMALL", color: "RED"), created_at: "2019-06-02T12:00:00Z"), + build(:widget, amount_cents: 300, options: build(:widget_options, size: "MEDIUM", color: "RED"), created_at: "2019-06-01T12:00:00Z") + ) + + # Verify non-group aggregations + aggregations = list_widgets_with_aggregations(count_aggregation) + + expect(aggregations).to contain_exactly( + {"count" => 3} + ) + + # Verify single grouping aggregations + aggregations = list_widgets_with_aggregations(count_aggregation("options.size")) + + expect(aggregations).to contain_exactly( + {grouped_by => {"options" => {"size" => enum_value("MEDIUM")}}, "count" => 1}, + {grouped_by => {"options" => {"size" => enum_value("SMALL")}}, "count" => 2} + ) + + # Verify multiple grouping aggregations + aggregations = list_widgets_with_aggregations(count_aggregation("options.size", "options.color")) + + expect(aggregations).to contain_exactly( + {grouped_by => {"options" => {"size" => enum_value("SMALL"), "color" => enum_value("RED")}}, "count" => 1}, + {grouped_by => {"options" => {"size" => enum_value("SMALL"), "color" => enum_value("BLUE")}}, "count" => 1}, + {grouped_by => {"options" => {"size" => enum_value("MEDIUM"), "color" => enum_value("RED")}}, "count" => 1} + ) + + # Verify date histogram groupings + aggregations = list_widgets_with_aggregations(count_aggregation("created_at_legacy(granularity: DAY)", "options.color")) + + expect(aggregations).to include( + {grouped_by => {case_correctly("created_at_legacy") => "2019-06-01T00:00:00.000Z", "options" => {"color" => enum_value("RED")}}, "count" => 1}, + {grouped_by => {case_correctly("created_at_legacy") => "2019-06-01T00:00:00.000Z", "options" => {"color" => enum_value("BLUE")}}, "count" => 1}, + {grouped_by => {case_correctly("created_at_legacy") => "2019-06-02T00:00:00.000Z", "options" => {"color" => enum_value("RED")}}, "count" => 1} + ) + end + + it "supports using aggregation on `list of objects` fields" do + index_records( + build(:widget, fees: [{amount_cents: 500, currency: "USD"}, {amount_cents: 1500, currency: "USD"}]), + build(:widget, fees: [{amount_cents: 10, currency: "USD"}]), + build(:widget, fees: [{amount_cents: 100, currency: "USD"}, {amount_cents: 90, currency: "USD"}]) + ) + + result = call_graphql_query(<<~QUERY) + query { + widget_aggregations { + nodes { + aggregated_values { + fees { + amount_cents { + approximate_sum + approximate_avg + exact_min + exact_max + } + } + } + } + } + } + QUERY + + expect(result.dig("data", case_correctly("widget_aggregations"), "nodes")).to contain_exactly({ + case_correctly("aggregated_values") => { + "fees" => { + case_correctly("amount_cents") => { + case_correctly("approximate_sum") => float_of(2200.0), + case_correctly("approximate_avg") => float_of(440.0), + case_correctly("exact_min") => int_of(10.0), + case_correctly("exact_max") => int_of(1500.0) + } + } + } + }) + end + + it "supports using aggregations for a list of numbers" do + index_records( + build(:widget, amounts: [5, 10]), + build(:widget, amounts: [15]), + build(:widget, amounts: [20, 25]) + ) + + result = call_graphql_query(<<~QUERY) + query { + widget_aggregations { + edges { + node { + aggregated_values { + amounts { + approximate_sum + approximate_avg + exact_min + exact_max + } + } + } + } + } + } + QUERY + + expect(result.dig("data", case_correctly("widget_aggregations"), "edges")).to contain_exactly({ + "node" => { + case_correctly("aggregated_values") => { + "amounts" => { + case_correctly("approximate_sum") => float_of(75.0), + case_correctly("approximate_avg") => float_of(15.0), + case_correctly("exact_min") => int_of(5), + case_correctly("exact_max") => int_of(25) + } + } + } + }) + end + + it "supports using aggregations for calendar types (Date, DateTime, LocalTime)" do + index_records( + build(:widget, created_at: "2023-10-09T12:30:12.345Z"), + build(:widget, created_at: "2024-01-09T09:30:12.456Z"), + build(:widget, created_at: "2023-12-01T22:30:12.789Z") + ) + + result = call_graphql_query(<<~QUERY) + query { + widget_aggregations { + nodes { + aggregated_values { + created_at { exact_min, exact_max, approximate_avg, approximate_distinct_value_count } + created_on { exact_min, exact_max, approximate_avg, approximate_distinct_value_count } + created_at_time_of_day { exact_min, exact_max, approximate_avg, approximate_distinct_value_count } + } + } + } + } + QUERY + + expect(result.dig("data", case_correctly("widget_aggregations"), "nodes")).to contain_exactly({ + case_correctly("aggregated_values") => { + case_correctly("created_at") => { + case_correctly("exact_min") => "2023-10-09T12:30:12.345Z", + case_correctly("exact_max") => "2024-01-09T09:30:12.456Z", + case_correctly("approximate_avg") => "2023-11-26T22:50:12.530Z", + case_correctly("approximate_distinct_value_count") => 3 + }, + case_correctly("created_on") => { + case_correctly("exact_min") => "2023-10-09", + case_correctly("exact_max") => "2024-01-09", + case_correctly("approximate_avg") => "2023-11-26", + case_correctly("approximate_distinct_value_count") => 3 + }, + case_correctly("created_at_time_of_day") => { + case_correctly("exact_min") => "09:30:12", + case_correctly("exact_max") => "22:30:12", + case_correctly("approximate_avg") => "14:50:12", + case_correctly("approximate_distinct_value_count") => 3 + } + } + }) + end + + it "supports using legacy aggregations for calendar types (Date, DateTime, LocalTime)" do + index_records( + build(:widget, created_at: "2023-10-09T12:30:12.345Z"), + build(:widget, created_at: "2024-01-09T09:30:12.456Z"), + build(:widget, created_at: "2023-12-01T22:30:12.789Z") + ) + + result = call_graphql_query(<<~QUERY) + query { + widget_aggregations { + nodes { + aggregated_values { + created_at_legacy { exact_min, exact_max, approximate_avg, approximate_distinct_value_count } + created_on_legacy { exact_min, exact_max, approximate_avg, approximate_distinct_value_count } + created_at_time_of_day { exact_min, exact_max, approximate_avg, approximate_distinct_value_count } + } + } + } + } + QUERY + + expect(result.dig("data", case_correctly("widget_aggregations"), "nodes")).to contain_exactly({ + case_correctly("aggregated_values") => { + case_correctly("created_at_legacy") => { + case_correctly("exact_min") => "2023-10-09T12:30:12.345Z", + case_correctly("exact_max") => "2024-01-09T09:30:12.456Z", + case_correctly("approximate_avg") => "2023-11-26T22:50:12.530Z", + case_correctly("approximate_distinct_value_count") => 3 + }, + case_correctly("created_on_legacy") => { + case_correctly("exact_min") => "2023-10-09", + case_correctly("exact_max") => "2024-01-09", + case_correctly("approximate_avg") => "2023-11-26", + case_correctly("approximate_distinct_value_count") => 3 + }, + case_correctly("created_at_time_of_day") => { + case_correctly("exact_min") => "09:30:12", + case_correctly("exact_max") => "22:30:12", + case_correctly("approximate_avg") => "14:50:12", + case_correctly("approximate_distinct_value_count") => 3 + } + } + }) + end + + it "supports aggregating at different dimensions/granularities in a single GraphQL query using aliases" do + index_records( + build(:widget, amount_cents: 100, options: build(:widget_options, size: "SMALL", color: "BLUE"), created_at: "2019-06-01T12:00:00Z"), + build(:widget, amount_cents: 200, options: build(:widget_options, size: "SMALL", color: "RED"), created_at: "2019-07-02T12:00:00Z"), + build(:widget, amount_cents: 400, options: build(:widget_options, size: "MEDIUM", color: "RED"), created_at: "2020-06-01T12:00:00Z") + ) + + result = call_graphql_query(<<~QUERY).dig("data") + query { + size: widget_aggregations { + edges { + node { + grouped_by { options { size } } + count + aggregated_values { amount_cents { exact_sum } } + } + } + } + + color: widget_aggregations { + edges { + node { + grouped_by { options { color } } + count + aggregated_values { amount_cents { exact_sum } } + } + } + } + + month: widget_aggregations { + edges { + node { + grouped_by { created_at { as_date_time(truncation_unit: MONTH) }} + count + aggregated_values { amount_cents { exact_sum } } + } + } + } + + year: widget_aggregations { + edges { + node { + grouped_by { created_at { as_date_time(truncation_unit: YEAR) }} + count + aggregated_values { amount_cents { exact_sum } } + } + } + } + + avg: widget_aggregations { + edges { + node { + aggregated_values { amount_cents { approximate_avg } } + count + } + } + } + + minmax: widget_aggregations { + edges { + node { + aggregated_values { amount_cents { exact_min, exact_max } } + } + } + } + + count: widget_aggregations { + edges { + node { + count + } + } + } + } + QUERY + + expect(result.keys).to contain_exactly("size", "color", "month", "year", "avg", "minmax", "count") + + expect(result.fetch("size").fetch("edges").map { |e| e["node"] }).to eq [ + {grouped_by => {"options" => {"size" => enum_value("MEDIUM")}}, "count" => 1, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 400}}}, + {grouped_by => {"options" => {"size" => enum_value("SMALL")}}, "count" => 2, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 300}}} + ] + + expect(result.fetch("color").fetch("edges").map { |e| e["node"] }).to eq [ + {grouped_by => {"options" => {"color" => enum_value("BLUE")}}, "count" => 1, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 100}}}, + {grouped_by => {"options" => {"color" => enum_value("RED")}}, "count" => 2, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 600}}} + ] + + expect(result.fetch("month").fetch("edges").map { |e| e["node"] }).to eq [ + {grouped_by => {case_correctly("created_at") => {case_correctly("as_date_time") => "2019-06-01T00:00:00.000Z"}}, "count" => 1, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 100}}}, + {grouped_by => {case_correctly("created_at") => {case_correctly("as_date_time") => "2019-07-01T00:00:00.000Z"}}, "count" => 1, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 200}}}, + {grouped_by => {case_correctly("created_at") => {case_correctly("as_date_time") => "2020-06-01T00:00:00.000Z"}}, "count" => 1, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 400}}} + ] + + expect(result.fetch("year").fetch("edges").map { |e| e["node"] }).to eq [ + {grouped_by => {case_correctly("created_at") => {case_correctly("as_date_time") => "2019-01-01T00:00:00.000Z"}}, "count" => 2, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 300}}}, + {grouped_by => {case_correctly("created_at") => {case_correctly("as_date_time") => "2020-01-01T00:00:00.000Z"}}, "count" => 1, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 400}}} + ] + + expect(result.fetch("avg").fetch("edges").map { |e| e["node"] }).to match [ + {"count" => 3, aggregated_values => {case_correctly("amount_cents") => {case_correctly("approximate_avg") => a_value_within(0.1).of(233.3)}}} + ] + + expect(result.fetch("minmax").fetch("edges").map { |e| e["node"] }).to eq [ + {aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_min") => 100, case_correctly("exact_max") => 400}}} + ] + + expect(datastore_msearch_requests("main").size).to eq 1 + # We expect only 2 searches in our single msearch request: + # + # * 1 for size/color/month/year/minmax + # * 1 for avg/count + # + # avg and count cannot be combined with the others because they request the document count WITHOUT a `grouped_by`. + # The datastore aggregations API does not provide a way to get a count without grouping; instead you have to request the + # document count from the main search body. As a result, the `DatastoreQuery` for avg/count has differences from + # the others beyond just the aggregations. With out current `Aggregation::QueryOptimizer` implementation, we don't + # combine these. + expect(count_of_searches_in(datastore_msearch_requests("main").first)).to eq 2 + expect(logged_jsons_of_type("AggregationQueryOptimizerMergedQueries").size).to eq 2 + expect(logged_jsons_of_type("AggregationQueryOptimizerMergedQueries").first).to include( + "aggregation_names" => ["1_size", "2_color", "3_month", "4_year", "5_minmax"], + "aggregation_count" => 5 + ) + expect(logged_jsons_of_type("AggregationQueryOptimizerMergedQueries").last).to include( + "aggregation_names" => ["6_avg", "7_count"], + "aggregation_count" => 2, + "query_count" => 2 + ) + end + + it "supports legacy aggregating at different dimensions/granularities in a single GraphQL query using aliases" do + index_records( + build(:widget, amount_cents: 100, options: build(:widget_options, size: "SMALL", color: "BLUE"), created_at: "2019-06-01T12:00:00Z"), + build(:widget, amount_cents: 200, options: build(:widget_options, size: "SMALL", color: "RED"), created_at: "2019-07-02T12:00:00Z"), + build(:widget, amount_cents: 400, options: build(:widget_options, size: "MEDIUM", color: "RED"), created_at: "2020-06-01T12:00:00Z") + ) + + result = call_graphql_query(<<~QUERY).dig("data") + query { + size: widget_aggregations { + edges { + node { + grouped_by { options { size } } + count + aggregated_values { amount_cents { exact_sum } } + } + } + } + + color: widget_aggregations { + edges { + node { + grouped_by { options { color } } + count + aggregated_values { amount_cents { exact_sum } } + } + } + } + + month: widget_aggregations { + edges { + node { + grouped_by { created_at_legacy(granularity: MONTH) } + count + aggregated_values { amount_cents { exact_sum } } + } + } + } + + year: widget_aggregations { + edges { + node { + grouped_by { created_at_legacy(granularity: YEAR) } + count + aggregated_values { amount_cents { exact_sum } } + } + } + } + + avg: widget_aggregations { + edges { + node { + aggregated_values { amount_cents { approximate_avg } } + count + } + } + } + + minmax: widget_aggregations { + edges { + node { + aggregated_values { amount_cents { exact_min, exact_max } } + } + } + } + + count: widget_aggregations { + edges { + node { + count + } + } + } + } + QUERY + + expect(result.keys).to contain_exactly("size", "color", "month", "year", "avg", "minmax", "count") + + expect(result.fetch("size").fetch("edges").map { |e| e["node"] }).to eq [ + {grouped_by => {"options" => {"size" => enum_value("MEDIUM")}}, "count" => 1, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 400}}}, + {grouped_by => {"options" => {"size" => enum_value("SMALL")}}, "count" => 2, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 300}}} + ] + + expect(result.fetch("color").fetch("edges").map { |e| e["node"] }).to eq [ + {grouped_by => {"options" => {"color" => enum_value("BLUE")}}, "count" => 1, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 100}}}, + {grouped_by => {"options" => {"color" => enum_value("RED")}}, "count" => 2, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 600}}} + ] + + expect(result.fetch("month").fetch("edges").map { |e| e["node"] }).to eq [ + {grouped_by => {case_correctly("created_at_legacy") => "2019-06-01T00:00:00.000Z"}, "count" => 1, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 100}}}, + {grouped_by => {case_correctly("created_at_legacy") => "2019-07-01T00:00:00.000Z"}, "count" => 1, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 200}}}, + {grouped_by => {case_correctly("created_at_legacy") => "2020-06-01T00:00:00.000Z"}, "count" => 1, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 400}}} + ] + + expect(result.fetch("year").fetch("edges").map { |e| e["node"] }).to eq [ + {grouped_by => {case_correctly("created_at_legacy") => "2019-01-01T00:00:00.000Z"}, "count" => 2, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 300}}}, + {grouped_by => {case_correctly("created_at_legacy") => "2020-01-01T00:00:00.000Z"}, "count" => 1, aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_sum") => 400}}} + ] + + expect(result.fetch("avg").fetch("edges").map { |e| e["node"] }).to match [ + {"count" => 3, aggregated_values => {case_correctly("amount_cents") => {case_correctly("approximate_avg") => a_value_within(0.1).of(233.3)}}} + ] + + expect(result.fetch("minmax").fetch("edges").map { |e| e["node"] }).to eq [ + {aggregated_values => {case_correctly("amount_cents") => {case_correctly("exact_min") => 100, case_correctly("exact_max") => 400}}} + ] + + expect(datastore_msearch_requests("main").size).to eq 1 + # We expect only 2 searches in our single msearch request: + # + # * 1 for size/color/month/year/minmax + # * 1 for avg/count + # + # avg and count cannot be combined with the others because they request the document count WITHOUT a `grouped_by`. + # The datastore aggregations API does not provide a way to get a count without grouping; instead you have to request the + # document count from the main search body. As a result, the `DatastoreQuery` for avg/count has differences from + # the others beyond just the aggregations. With out current `Aggregation::QueryOptimizer` implementation, we don't + # combine these. + expect(count_of_searches_in(datastore_msearch_requests("main").first)).to eq 2 + expect(logged_jsons_of_type("AggregationQueryOptimizerMergedQueries").size).to eq 2 + expect(logged_jsons_of_type("AggregationQueryOptimizerMergedQueries").first).to include( + "aggregation_names" => ["1_size", "2_color", "3_month", "4_year", "5_minmax"], + "aggregation_count" => 5 + ) + expect(logged_jsons_of_type("AggregationQueryOptimizerMergedQueries").last).to include( + "aggregation_names" => ["6_avg", "7_count"], + "aggregation_count" => 2, + "query_count" => 2 + ) + end + + it "supports pagination on an aggregation connection when grouping by something" do + index_records( + build(:widget, workspace_id: "w1"), + build(:widget, workspace_id: "w2"), + build(:widget, workspace_id: "w3"), + build(:widget, workspace_id: "w2"), + build(:widget, workspace_id: "w4"), + build(:widget, workspace_id: "w5") + ) + + forward_paginate_through_workspace_id_groupings + backward_paginate_through_workspace_id_groupings + end + + def forward_paginate_through_workspace_id_groupings + page_info, workspace_nodes = list_widget_workspace_id_groupings(first: 2) + + expect(page_info).to match( + case_correctly("end_cursor") => /\w+/, + case_correctly("start_cursor") => /\w+/, + case_correctly("has_next_page") => true, + case_correctly("has_previous_page") => false + ) + + expect(workspace_nodes).to eq [ + {"count" => 1, grouped_by => {case_correctly("workspace_id") => "w1"}}, + {"count" => 2, grouped_by => {case_correctly("workspace_id") => "w2"}} + ] + + expect { + response = list_widget_workspace_id_groupings(first: 2, after: [1, 2, 3], expect_errors: true) + expect(response["errors"]).to contain_exactly(a_hash_including("message" => "Argument 'after' on Field '#{case_correctly("widget_aggregations")}' has an invalid value ([1, 2, 3]). Expected type 'Cursor'.")) + }.to log_warning a_string_including("Argument 'after' on Field '#{case_correctly("widget_aggregations")}' has an invalid value", "[1, 2, 3]") + + broken_cursor = page_info.fetch(case_correctly("end_cursor")) + "-broken" + expect { + response = list_widget_workspace_id_groupings(first: 2, after: broken_cursor, expect_errors: true) + expect(response["errors"]).to contain_exactly(a_hash_including("message" => "Argument 'after' on Field '#{case_correctly("widget_aggregations")}' has an invalid value (#{broken_cursor.inspect}). Expected type 'Cursor'.")) + }.to log_warning a_string_including("Argument 'after' on Field '#{case_correctly("widget_aggregations")}' has an invalid value", broken_cursor) + + page_info, workspace_nodes = list_widget_workspace_id_groupings(first: 2, after: page_info.fetch(case_correctly("end_cursor"))) + + expect(page_info).to match( + case_correctly("end_cursor") => /\w+/, + case_correctly("start_cursor") => /\w+/, + case_correctly("has_next_page") => true, + case_correctly("has_previous_page") => true + ) + + expect(workspace_nodes).to eq [ + {"count" => 1, grouped_by => {case_correctly("workspace_id") => "w3"}}, + {"count" => 1, grouped_by => {case_correctly("workspace_id") => "w4"}} + ] + + page_info, workspace_nodes = list_widget_workspace_id_groupings(first: 2, after: page_info.fetch(case_correctly("end_cursor"))) + + expect(page_info).to match( + case_correctly("end_cursor") => /\w+/, + case_correctly("start_cursor") => /\w+/, + case_correctly("has_next_page") => false, + case_correctly("has_previous_page") => true + ) + + expect(workspace_nodes).to eq [ + {"count" => 1, grouped_by => {case_correctly("workspace_id") => "w5"}} + ] + + page_info, workspace_nodes = list_widget_workspace_id_groupings(first: 2, after: page_info.fetch(case_correctly("end_cursor"))) + + expect(page_info).to match( + case_correctly("end_cursor") => nil, + case_correctly("start_cursor") => nil, + case_correctly("has_next_page") => false, + case_correctly("has_previous_page") => true + ) + + expect(workspace_nodes).to eq [] + end + + def backward_paginate_through_workspace_id_groupings + page_info, workspace_nodes = list_widget_workspace_id_groupings(last: 2) + + expect(page_info).to match( + case_correctly("end_cursor") => /\w+/, + case_correctly("start_cursor") => /\w+/, + case_correctly("has_next_page") => false, + case_correctly("has_previous_page") => true + ) + + expect(workspace_nodes).to eq [ + {"count" => 1, grouped_by => {case_correctly("workspace_id") => "w4"}}, + {"count" => 1, grouped_by => {case_correctly("workspace_id") => "w5"}} + ] + + page_info, workspace_nodes = list_widget_workspace_id_groupings(last: 2, before: page_info.fetch(case_correctly("start_cursor"))) + + expect(page_info).to match( + case_correctly("end_cursor") => /\w+/, + case_correctly("start_cursor") => /\w+/, + case_correctly("has_next_page") => true, + case_correctly("has_previous_page") => true + ) + + expect(workspace_nodes).to eq [ + {"count" => 2, grouped_by => {case_correctly("workspace_id") => "w2"}}, + {"count" => 1, grouped_by => {case_correctly("workspace_id") => "w3"}} + ] + + page_info, workspace_nodes = list_widget_workspace_id_groupings(last: 2, before: page_info.fetch(case_correctly("start_cursor"))) + + expect(page_info).to match( + case_correctly("end_cursor") => /\w+/, + case_correctly("start_cursor") => /\w+/, + case_correctly("has_next_page") => true, + case_correctly("has_previous_page") => false + ) + + expect(workspace_nodes).to eq [ + {"count" => 1, grouped_by => {case_correctly("workspace_id") => "w1"}} + ] + + page_info, workspace_nodes = list_widget_workspace_id_groupings(last: 2, before: page_info.fetch(case_correctly("start_cursor"))) + + expect(page_info).to match( + case_correctly("end_cursor") => nil, + case_correctly("start_cursor") => nil, + case_correctly("has_next_page") => true, + case_correctly("has_previous_page") => false + ) + + expect(workspace_nodes).to eq [] + end + + def list_widget_workspace_id_groupings(expect_errors: false, **pagination_args) + results = call_graphql_query(<<~QUERY, allow_errors: expect_errors) + query { + widget_aggregations#{graphql_args(pagination_args)} { + page_info { + end_cursor + start_cursor + has_next_page + has_previous_page + } + + edges { + node { + grouped_by { + workspace_id + } + count + } + } + } + } + QUERY + + return results if expect_errors + + results = results.dig("data", case_correctly("widget_aggregations")) + nodes = results.fetch("edges").map { |e| e.fetch("node") } + [results.fetch(case_correctly("page_info")), nodes] + end + + it "supports pagination on an ungrouped aggregation connection" do + index_records( + build(:widget, amount_cents: 100), + build(:widget, amount_cents: 200) + ) + + forward_paginate_through_ungrouped_aggregations( + "count", + {"count" => 2} + ) + backward_paginate_through_ungrouped_aggregations( + "count", + {"count" => 2} + ) + + forward_paginate_through_ungrouped_aggregations( + "aggregated_values { amount_cents { exact_sum } }", + { + case_correctly("aggregated_values") => { + case_correctly("amount_cents") => {case_correctly("exact_sum") => 300} + } + } + ) + backward_paginate_through_ungrouped_aggregations( + "aggregated_values { amount_cents { exact_sum } }", + { + case_correctly("aggregated_values") => { + case_correctly("amount_cents") => {case_correctly("exact_sum") => 300} + } + } + ) + + # Verify that we an query just `page_info` (no groupings or aggregated values) + results = call_graphql_query(<<~QUERY).dig("data", case_correctly("widget_aggregations"), case_correctly("page_info")) + query { + widget_aggregations { + page_info { + end_cursor + start_cursor + has_next_page + has_previous_page + } + } + } + QUERY + + expect(results).to match( + case_correctly("end_cursor") => /\w+/, + case_correctly("start_cursor") => /\w+/, + case_correctly("has_next_page") => false, + case_correctly("has_previous_page") => false + ) + end + + it "supports aggregations on just nodes" do + index_records( + build(:widget, name: "a", amount_cents: 100), + build(:widget, name: "a", amount_cents: 50), + build(:widget, name: "b", amount_cents: 200) + ) + + results = call_graphql_query(<<~QUERY).dig("data") + query { + widget_aggregations { + nodes { + grouped_by { + name + } + aggregated_values { + amount_cents { + exact_sum + } + } + } + } + } + QUERY + + expect(results).to eq(case_correctly("widget_aggregations") => { + "nodes" => [ + { + case_correctly("grouped_by") => { + "name" => "a" + }, + case_correctly("aggregated_values") => { + case_correctly("amount_cents") => { + case_correctly("exact_sum") => 150 + } + } + }, + { + case_correctly("grouped_by") => { + "name" => "b" + }, + case_correctly("aggregated_values") => { + case_correctly("amount_cents") => { + case_correctly("exact_sum") => 200 + } + } + } + ] + }) + end + + def forward_paginate_through_ungrouped_aggregations(select, expected_value) + page_info, nodes = list_widget_ungrouped_aggregations(select, first: 2) + + expect(page_info).to match( + case_correctly("end_cursor") => /\w+/, + case_correctly("start_cursor") => /\w+/, + case_correctly("has_next_page") => false, + case_correctly("has_previous_page") => false + ) + + expect(nodes).to eq [expected_value] + + page_info, nodes = list_widget_ungrouped_aggregations(select, first: 2, after: page_info.fetch(case_correctly("end_cursor"))) + + expect(page_info).to match( + case_correctly("end_cursor") => nil, + case_correctly("start_cursor") => nil, + case_correctly("has_next_page") => false, + case_correctly("has_previous_page") => true + ) + + expect(nodes).to eq [] + end + + def backward_paginate_through_ungrouped_aggregations(select, expected_value) + page_info, nodes = list_widget_ungrouped_aggregations(select, last: 2) + + expect(page_info).to match( + case_correctly("end_cursor") => /\w+/, + case_correctly("start_cursor") => /\w+/, + case_correctly("has_next_page") => false, + case_correctly("has_previous_page") => false + ) + + expect(nodes).to eq [expected_value] + + page_info, nodes = list_widget_ungrouped_aggregations(select, last: 2, before: page_info.fetch(case_correctly("start_cursor"))) + + expect(page_info).to match( + case_correctly("end_cursor") => nil, + case_correctly("start_cursor") => nil, + case_correctly("has_next_page") => true, + case_correctly("has_previous_page") => false + ) + + expect(nodes).to eq [] + end + + def list_widget_ungrouped_aggregations(select, **pagination_args) + results = call_graphql_query(<<~QUERY).dig("data", case_correctly("widget_aggregations")) + query { + widget_aggregations#{graphql_args(pagination_args)} { + page_info { + end_cursor + start_cursor + has_next_page + has_previous_page + } + + edges { + node { + #{select} + } + } + } + } + QUERY + + nodes = results.fetch("edges").map { |e| e.fetch("node") } + [results.fetch(case_correctly("page_info")), nodes] + end + end + + def float_of(value) + return nil if value.nil? + a_value_within(0.1).percent_of(value).and a_kind_of(::Float) + end + + def int_of(value) + return nil if value.nil? + (a_value == value).and a_kind_of(::Integer) + end + + def amount_aggregation(*fields, **agg_args) + unless fields.empty? + grouped_by = <<~EOS + grouped_by { + #{fields.reject { |f| f.include?(".") }.join("\n")} + #{sub_fields("options", fields)} + #{sub_fields("the_options", fields)} + #{"cost { currency }" if fields.include?("cost.currency")} + } + EOS + end + + <<~AGG + widget_aggregations#{graphql_args(agg_args)} { + edges { + node { + #{grouped_by} + + count + + aggregated_values { + amount_cents { + approximate_sum + exact_sum + approximate_avg + exact_min + exact_max + } + + cost { + amount_cents { + approximate_sum + exact_sum + approximate_avg + exact_min + exact_max + } + } + } + } + } + } + AGG + end + + def all_amount_aggregations + <<~AGG + widget_aggregations { + edges { + node { + count + + aggregated_values { + amount_cents { + approximate_sum + exact_sum + approximate_avg + exact_min + exact_max + } + + cost { + amount_cents { + approximate_sum + exact_sum + approximate_avg + exact_min + exact_max + } + } + } + } + } + } + AGG + end + + def count_aggregation(*fields, **agg_args) + unless fields.empty? + grouped_by = <<~EOS + grouped_by { + #{fields.reject { |f| f.include?(".") }.join("\n")} + #{sub_fields("options", fields)} + } + EOS + end + + <<~AGG + widget_aggregations#{graphql_args(agg_args)} { + edges { + node { + #{grouped_by} + count + } + } + } + AGG + end + + def sub_fields(parent, fields) + sub_fields = fields.filter_map { |f| f.sub("#{parent}.", "") if f.start_with?("#{parent}.") } + return "" if sub_fields.empty? + + <<~OPTS + #{parent} { + #{sub_fields.join("\n")} + } + OPTS + end + + def group_widgets_by_tag + call_graphql_query(<<~QUERY).dig("data", case_correctly("widget_aggregations"), "nodes") + query { + widget_aggregations { + nodes { + grouped_by { tag } + count + aggregated_values { + amount_cents1: amount_cents { sum: exact_sum } + amount_cents2: amount_cents { sum: exact_sum } + } + } + } + } + QUERY + end + + def widget_ungrouped_aggregated_values_for(field_selections) + call_graphql_query(<<~QUERY).dig("data", case_correctly("widget_aggregations"), "nodes", 0, case_correctly("aggregated_values")) + query { + widget_aggregations { + nodes { + aggregated_values { + #{field_selections} + } + } + } + } + QUERY + end + + def group_widgets_by_fees_currency_with_approximate_distinct_value_counts + call_graphql_query(<<~QUERY).dig("data", case_correctly("widget_aggregations"), "nodes") + query { + widget_aggregations { + nodes { + grouped_by { fees { currency } } + count + aggregated_values { + id { + approximate_distinct_value_count + } + tags { + approximate_distinct_value_count + } + amount_cents { + sum: exact_sum + } + } + } + } + } + QUERY + end + + def group_widget_currencies_by_widget_name + call_graphql_query(<<~QUERY).dig("data", case_correctly("widget_currency_aggregations"), "nodes") + query { + widget_currency_aggregations { + nodes { + grouped_by { widget_name } + count + } + } + } + QUERY + end + + def list_widgets_with_aggregations(widget_aggregation, **query_args) + call_graphql_query(<<~QUERY).dig("data", case_correctly("widget_aggregations"), "edges").map { |edge| edge["node"] } + query { + widgets#{graphql_args(query_args)} { + edges { + node { + id + name + amount_cents + options { + size + color + } + + inventor { + ... on Person { + name + nationality + } + + ... on Company { + name + stock_ticker + } + } + } + } + } + #{widget_aggregation} + + # Also query `components.widgets` (even though we do not do anything with the returned data) + # to exercise an odd aggregations edge case where a nested relationship field exists with the + # same name as an earlier field that had aggregations. + components { + edges { + node { + widgets { + edges { + node { + id + } + } + } + } + } + } + } + QUERY + end + + def group_components_by_position_x_and_y + call_graphql_query(<<~QUERY).dig("data") + query { + by_x: component_aggregations { + edges { + node { + grouped_by { position { x } } + count + } + } + } + + by_y: component_aggregations { + edges { + node { + grouped_by { position { y } } + count + } + } + } + } + QUERY + end + end +end diff --git a/elasticgraph-graphql/spec/acceptance/datastore_spec.rb b/elasticgraph-graphql/spec/acceptance/datastore_spec.rb new file mode 100644 index 00000000..0ff0a027 --- /dev/null +++ b/elasticgraph-graphql/spec/acceptance/datastore_spec.rb @@ -0,0 +1,189 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "elasticgraph_graphql_acceptance_support" + +module ElasticGraph + RSpec.describe "ElasticGraph::GraphQL--datastore tests" do + include_context "ElasticGraph GraphQL acceptance support" + + with_both_casing_forms do + it "avoids querying the datastore when it does not need to" do + datastore_requests("main").clear + + # If we are just inspecting the schema, we shouldn't need to query the datastore... + expect { + results = call_graphql_query(<<~EOS).dig("data", "widgets") + query { + widgets { + __typename + } + } + EOS + + expect(results).to eq({"__typename" => apply_derived_type_customizations("WidgetConnection")}) + }.to make_no_datastore_calls("main") + + # ... or if we are asking for 0 results, we shouldn't have to... + expect { + results = call_graphql_query(<<~EOS).dig("data", "widgets") + query { + widgets(first: 0) { + edges { + node { + id + } + } + } + } + EOS + + expect(results).to eq({"edges" => []}) + }.to make_no_datastore_calls("main") + + # ...but if we are just asking for page info, we still need to; there may still be a next page. + expect { + results = call_graphql_query(<<~EOS).dig("data", "widgets") + query { + widgets { + page_info { + has_next_page + } + } + } + EOS + + expect(results).to eq({case_correctly("page_info") => {case_correctly("has_next_page") => false}}) + }.to make_datastore_calls("main", "GET /_msearch") + end + + it "batches datastore queries when requesting indexed types in parallel" do + expect { + results = query_all_indexed_type_counts + + expect(results).to eq( + "addresses" => {case_correctly("total_edge_count") => 0}, + "components" => {case_correctly("total_edge_count") => 0}, + case_correctly("electrical_parts") => {case_correctly("total_edge_count") => 0}, + "manufacturers" => {case_correctly("total_edge_count") => 0}, + case_correctly("mechanical_parts") => {case_correctly("total_edge_count") => 0}, + "parts" => {case_correctly("total_edge_count") => 0}, + "widgets" => {case_correctly("total_edge_count") => 0}, + case_correctly("widget_currencies") => {case_correctly("total_edge_count") => 0} + ) + }.to query_datastore("main", 1).time + end + + it "handles queries against non-existing fields in the datastore gracefully--such as when a new field is added to a rollover index template after the template has been used" do + index_records( + address1 = build(:address, created_at: "2019-09-10T12:00:00.000Z", full_address: "123"), + address2 = build(:address, created_at: "2019-09-11T12:00:00.000Z", full_address: "456"), + address3 = build(:address, created_at: "2019-09-12T12:00:00.000Z", full_address: "789") + ) + + schema_def_string = raw_schema_def_string.sub('schema.object_type "Address" do |t|', <<~EOS) + schema.object_type "AddressLines" do |t| + t.field "line1", "String" + t.field "line2", "String" + end + + schema.object_type "Address" do |t| + t.field "lines", "AddressLines" + t.field "postal_code", "String" + EOS + + graphql_with_new_schema = build_graphql(schema_definition: ->(schema) do + # standard:disable Security/Eval -- it's ok here in a test. + schema.as_active_instance { eval(case_schema_def_correctly(schema_def_string)) } + # standard:enable Security/Eval + end) + + addresses = list_addresses(gql: graphql_with_new_schema, fields: <<~EOS, order_by: [:timestamps_created_at_ASC]) + full_address + timestamps { + created_at + } + postal_code + lines { + line1 + line2 + } + EOS + + expect(addresses).to match([ + string_hash_of(address1, :timestamps, :full_address, "postal_code" => nil, "lines" => nil), + string_hash_of(address2, :timestamps, :full_address, "postal_code" => nil, "lines" => nil), + string_hash_of(address3, :timestamps, :full_address, "postal_code" => nil, "lines" => nil) + ]) + end + + describe "timeout behavior" do + it "raises `Errors::RequestExceededDeadlineError` if the specified timeout is exceeded by a datastore query" do + expect { + call_graphql_query(<<~QUERY, timeout_in_ms: 0) + query { widgets { edges { node { id } } } } + QUERY + }.to raise_error(Errors::RequestExceededDeadlineError) + .and log(a_string_including("failed with an exception", "Errors::RequestExceededDeadlineError")) + end + + it "applies shorter timeouts to each subsequent datastore query as the monotonic clock passes so that the passed timeout applies to the entire GraphQL query", :expect_search_routing do + component = build(:component) + widget = build(:widget, components: [component]) + index_records(widget, component) + + # use a long (10 minute) timeout so that we definitely won't hit it, even if a debugger is used. + long_timeout_seconds = 600 + + call_graphql_query(<<~QUERY, timeout_in_ms: long_timeout_seconds * 1000) + query { + widgets { edges { node { + components { edges { node { id } } } + } } } + } + QUERY + + timeouts = datastore_msearch_requests("main").map(&:timeout) + + expect(timeouts.size).to eq 2 + expect(timeouts.first).to be < long_timeout_seconds + expect(timeouts.last).to be < timeouts.first + end + end + + def list_addresses(fields:, gql: graphql, **query_args) + call_graphql_query(<<~QUERY, gql: gql).dig("data", "addresses", "edges").map { |we| we.fetch("node") } + query { + addresses#{graphql_args(query_args)} { + edges { + node { + #{fields} + } + } + } + } + QUERY + end + + def query_all_indexed_type_counts + call_graphql_query(<<~QUERY).fetch("data") + query { + addresses { total_edge_count } + components { total_edge_count } + electrical_parts { total_edge_count } + manufacturers { total_edge_count } + mechanical_parts { total_edge_count } + parts { total_edge_count } + widgets { total_edge_count } + widget_currencies { total_edge_count } + } + QUERY + end + end + end +end diff --git a/elasticgraph-graphql/spec/acceptance/elasticgraph_graphql_acceptance_support.rb b/elasticgraph-graphql/spec/acceptance/elasticgraph_graphql_acceptance_support.rb new file mode 100644 index 00000000..1046feb3 --- /dev/null +++ b/elasticgraph-graphql/spec/acceptance/elasticgraph_graphql_acceptance_support.rb @@ -0,0 +1,417 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_definition/schema_elements/type_namer" +require "elastic_graph/spec_support/builds_admin" +require "graphql" +require "support/graphql" + +module ElasticGraph + RSpec.shared_context "ElasticGraph GraphQL acceptance support", :factories, :uses_datastore, :capture_logs, :builds_indexer, :builds_admin do + include GraphQLSupport + include PreventSearchesFromUsingWriteRequests + + let(:graphql) { build_graphql } + let(:indexer) { build_indexer(datastore_core: graphql.datastore_core) } + let(:admin) { build_admin(datastore_core: graphql.datastore_core) } + + # Need to use a local variable instead of an instance variable for the context state, + # to avoid issues related to VCR re-recording that is implemented using `rspec-retry` + # (which clears instance variables between attempts). + raw_schema_def_string = nil + define_method(:raw_schema_def_string) { raw_schema_def_string } + + before(:context) do + raw_schema_def_string = %w[schema/teams.rb schema/widgets.rb].map do |file| + File.read(File.join(CommonSpecHelpers::REPO_ROOT, "config", file)) + end.join("\n\n") + end + + before do + # Perform any cached calls to the datastore to happen before our `query_datastore` + # matcher below which tries to assert which specific requests get made, since index definitions + # have caching behavior that can make the presence or absence of that request slightly non-deterministic. + pre_cache_index_state(graphql) + end + + def self.with_both_casing_forms(&block) + context "with a snake_case schema" do + include SnakeCaseGraphQLAcceptanceAdapter + module_exec(&block) + end + + context "with a camelCase schema, alternate derived type naming, and enum value overrides" do + include CamelCaseGraphQLAcceptanceAdapter + + # Need to use a local variable instead of an instance variable for the context state, + # to avoid issues related to VCR re-recording that is implemented using `rspec-retry` + # (which clears instance variables between attempts). + extra_build_options = nil + + before(:context) do + enum_types = ::GraphQL::Schema.from_definition( + stock_schema_artifacts(for_context: :graphql).graphql_schema_string + ).types.values.select { |t| t.kind.enum? } + + # For each enum type, we want to override the values to be different, as a forcing function to make + # sure our GraphQL implementation works with any overrides. + enum_value_overrides_by_type = enum_types.to_h do |enum_type| + value_overrides = enum_type.values.keys.to_h { |v| [v, "#{v}2"] } + [enum_type.graphql_name, value_overrides] + end + + derived_type_name_formats = SchemaDefinition::SchemaElements::TypeNamer::DEFAULT_FORMATS.transform_values do |format| + apply_derived_type_customizations(format) + end + + # replace `snake_case` schema field names with `camelCase` names. + camel_case_schema_def = case_schema_def_correctly(raw_schema_def_string) + extra_build_options = { + datastore_backend: datastore_backend, + schema_element_name_form: :camelCase, + derived_type_name_formats: derived_type_name_formats, + enum_value_overrides_by_type: enum_value_overrides_by_type, + schema_definition: ->(schema) do + # standard:disable Security/Eval -- it's ok here in a test. + schema.as_active_instance { eval(camel_case_schema_def) } + # standard:enable Security/Eval + end + } + + admin = BuildsAdmin.build_admin(**extra_build_options) do |config| + configure_for_camel_case(config) + end + + manage_cluster_for(admin: admin, state_file_name: "camelCase_indices.yaml") + end + + define_method :build_graphql do |**options, &method_block| + super(**extra_build_options.merge(options)) do |config| + config = configure_for_camel_case(config) + config = method_block.call(config) if method_block + config + end + end + + module_exec(&block) + end + end + end + + # Our IAM policies in AWS grant GET/HEAD permissions but not POST permissions to + # IAM users/roles that are intended to only read but not write to the datastore. + # In elasticsearch-ruby 7.9.0, they changed it to use POST instead of GET with + # msearch due to the request body. This broke our lambdas when we deployed with + # the gem upgraded. Since our GraphQL lambda lacks POST/PUT/DELETE permissions we + # want to enforce that those HTTP verbs are not used while handling GraphQL queries, + # so we enforce that here. + module PreventSearchesFromUsingWriteRequests + def call_graphql_query(query, gql: graphql, **options) + response = nil + + expect { + response = super(query, gql: gql, **options) + }.to make_no_datastore_write_calls("main") + + response + end + end + + module SnakeCaseGraphQLAcceptanceAdapter + def enum_value(value) + value + end + + def case_correctly(word) + ::ElasticGraph::SchemaArtifacts::RuntimeMetadata::SchemaElementNamesDefinition::SnakeCaseConverter.normalize_case(word) + end + + def index_definition_name_for(snake_case_name) + snake_case_name + end + + def case_schema_def_correctly(snake_case_schema_def) + snake_case_schema_def + end + + def apply_derived_type_customizations(type_name) + # In the snake_case context we don't want to customize derived types, so return `type_name` as-is. + type_name + end + + # For parity with our `camelCase` context, also roundtrip factory-built records through JSON. + # Otherwise we can have subtle, surprising differences between the two casing contexts. For + # example, if the factory puts a `Date` object in a record, the JSON roundtripping will convert + # it to an ISO8601 date string, but it would be left as a `Date` object if we did not roundtrip + # it here. + def build(*args, **opts) + JSON.parse(JSON.generate(super), symbolize_names: true) + end + end + + module CamelCaseGraphQLAcceptanceAdapter + def enum_value(value) + case value + when ::Symbol + :"#{value}2" + else + "#{value}2" + end + end + + def configure_for_camel_case(config) + # Provide the same index definition settings, but for the `_camel` indices. + original_index_defs = config.index_definitions + config.with(index_definitions: Hash.new do |hash, index_def_name| + hash[index_def_name] = original_index_defs[index_def_name.delete_suffix("_camel")] + end) + end + + def case_schema_def_correctly(snake_case_schema_def) + # replace `snake_case` schema field names with `camelCase` names. + camel_case_schema_def = to_camel_case(snake_case_schema_def, only_quoted_strings: true) + # However, the datastore does not support `camelCase` index names (using one + # yields an "Invalid index name, must be lowercased" error). In addition, we + # want to use a different index for this context than for the `snake_case` + # context, so that the datastore does not use the same mapping for both. So, + # here we replace a string like `index: "mechanicalParts"` with `index: "mechanical_parts_camel"`. + camel_case_schema_def.gsub(/(\.index\s+")(\w+)(")/) do + "#{$1}#{index_definition_name_for(word_to_snake_case($2))}#{$3}" + end + # `geo_shape` needs to stay in snake_case because it's a datastore mapping type, and cannot be in camelCase + .gsub("geoShape", "geo_shape") + end + + DERIVED_NAME_PARTS = [ + "Connection", + "Edge", + "Filter", + "Sort", # for SortOrder + "Aggregated", # for AggregatedValues + "Grouped", # for GroupedBy + "Aggregation", # for Aggregation and SubAggregation + "Aggregations" # for SubAggregations + ].to_set + + def apply_derived_type_customizations(type_name) + # `AggregationCountDetail` has "Aggregation" in it, but is _not_ a derived type, + # so we need to return it as-is. + return type_name if type_name == "AggregationCountDetail" + + # These have the `Input` suffix, but not due to a naming format (they're not derived types), so we leave them alone. + return type_name if type_name == "DateTimeGroupingOffsetInput" + return type_name if type_name == "DateGroupingOffsetInput" + return type_name if type_name == "DayOfWeekGroupingOffsetInput" + return type_name if type_name == "LocalTimeGroupingOffsetInput" + + # We want to test _not_ using separate input and output enum types, so we remove the `Input` suffix used for input enums by default. + type_name = type_name.delete_suffix("Input") + + # Here we split on capital letters (without "consuming" them in the regex) to convert + # a type like `WidgetSubAggregations` to ["Widget", "Sub", "Aggregations"] + name_parts = type_name.split(/(?=[A-Z])/).to_set + + # Some derived types are "compound" types. For example, type like `Widget` gets + # a derived type like `WidgetAggregation`, and that then gets a derived type like + # `WidgetAggregationConnection`. + # + # For each derived name part, we want to apply our customizations: + # - Prefix it with `Pre` + # - Suffix it with `Post` + # - Reverse it (e.g. `Filter` -> `Retlif`) + # + # We want all 3 of these customizations as a forcing function to make sure that we don't take any incorrect + # short cuts when working with derived types: + # + # - The prefix ensures that `start_with?` can't be used to detect a derived type + # - The suffix ensures that `ends_with?` can't be used to detect a derived type + # - The reversing ensures that `include?` can't be used to detect a derived type + parts_needing_adjustment = name_parts.intersection(DERIVED_NAME_PARTS) + parts_needing_adjustment.reduce(type_name) do |adjusted_type_name, part| + "Pre#{adjusted_type_name.sub(/#{part}(?=\z|[A-Z])/) { |p| "#{p.reverse.downcase.capitalize}Post" }}" + end + end + + def index_definition_name_for(snake_case_name) + if snake_case_name.include?("_rollover__") + template_name, suffix = snake_case_name.split("_rollover__") + "#{template_name}_camel_rollover__#{suffix}" + else + "#{snake_case_name}_camel" + end + end + + # Override `build` to have it build hashes with camelCase key names, and to update enum values + # according to our overrides. + def build(*args, **opts) + raw_data = super + + schema_artifacts = stock_schema_artifacts(for_context: :graphql) + json_schema_defs = schema_artifacts.json_schemas_for(schema_artifacts.latest_json_schema_version).fetch("$defs") + + if (typename = raw_data[:__typename]) + raw_data = update_enum_values_in(raw_data, json_schema_defs, typename) + end + + JSON.parse(to_camel_case(JSON.generate(raw_data)), symbolize_names: true) + end + + def update_enum_values_in(data, json_schema_defs, type_name) + # There's nothing to do with the `Untyped` type, but it's non-standard and leads to errors if we try to + # handle it with the standard logic below. + return data if type_name == "Untyped" + + case data + when Hash + json_schema_def = json_schema_defs.fetch(type_name) + if json_schema_def["required"] == ["__typename"] && (data_type_name = data[:__typename]) + # `type_name` contains an abstract type. Lookup the concrete type and use that instead. + update_enum_values_in(data, json_schema_defs, data_type_name) + else + props = json_schema_def.fetch("properties") + data.to_h do |field_name, field_value| + unless [:__version, :__typename, :__json_schema_version].include?(field_name) + field_type = props.fetch(word_to_snake_case(field_name.to_s)).fetch("ElasticGraph").fetch("type")[/\w+/] + field_value = update_enum_values_in(field_value, json_schema_defs, field_type) + end + + [field_name, field_value] + end + end + when Array + data.map { |v| update_enum_values_in(v, json_schema_defs, type_name) } + else + if (enum = json_schema_defs.fetch(type_name)["enum"]) + # We append a `2` conditionally because it's possible it's already been appended. In particular, + # in a case like `build(:widget, options: build(:widget_options, ...))`, the update will have already + # been applied to the `widget_options` hash, and we don't want to apply it a second time when processing + # the `widget` hash here. + data = "#{data}2" if enum.include?(data) + end + + data + end + end + + # Override `string_hash_of` to have it convert key names to camelCase as + # it plucks values out of the source hash. + def string_hash_of(source_hash, *direct_fields, **fields_with_values) + direct_fields = direct_fields.map { |f| word_to_camel_case(f.to_s).to_sym } + fields_with_values = fields_with_values.map { |k, v| [word_to_camel_case(k.to_s).to_sym, v] }.to_h + super(source_hash, *direct_fields, **fields_with_values) + end + + # Override `call_graphql_query` so that the query is converted to camelCase + # before we send it to ElasticGraph. + def call_graphql_query(query, gql: graphql, allow_errors: false, **options) + # Here we convert the query to its camelCase form before executing it. + # However, orderBy options require special handling: + # + # - orderBy enum values have an `_ASC` or `_DESC` suffix which prevents the `to_camel_case` + # translation from working automatically for it. + # - Both orderBy and groupBy enum values use `_` as a separator between parent and child + # field names when referencing a nested field--e.g. `cost.amountCents` has an enum + # option of `cost_amountCents` for a camelCase schema and `cost_amount_cents` in a snake_case + # schema. There are no uniform translation rules that would allow us to translate from + # `cost_amount_cents` to `cost_amountCents` here, so we just special case it with the below + # hash. + special_cases_source = "#{__FILE__}:#{__LINE__ + 1}" + special_cases = { + # orderBy enum values + "amount_cents" => "amountCents", + "amount_cents2" => "amountCents2", + "cost_amount_cents" => "cost_amountCents", + "created_at_time_of_day" => "createdAtTimeOfDay", + "created_at_legacy" => "createdAtLegacy", + "created_at" => "createdAt", + "created_on_legacy" => "createdOnLegacy", + "created_on" => "createdOn", + "full_address" => "fullAddress", + "weight_in_ng_str" => "weightInNgStr", + "weight_in_ng" => "weightInNg" + } + + query = to_camel_case(query).gsub(/#{Regexp.union(special_cases.keys)}/, special_cases) + + super(query, gql: gql, allow_errors: allow_errors, **options).tap do |response| + unless allow_errors + expect(response["errors"]).to eq([]).or(eq(nil)), <<~EOS + #{"=" * 80} + camelCase query[1] failed with errors[2]: + + [1] + #{query}" + + [2] + #{::JSON.pretty_generate(response["errors"])} + + Note that camelCase queries need some special case translation to deal with orderBy + enum options (since they use `_` to denote parent-child field nesting). If you've uncovered + the need for an additional special case, register in the `special_cases` hash at + #{special_cases_source}. + #{"=" * 80} + EOS + end + end + end + + # Helper method to convert `snake_case` identifiers in a string + # to `camelCase` identifiers. + def to_camel_case(string, only_quoted_strings: false) + # https://rubular.com/r/9R8CG8wHD08mM8 + inner_regex = "[a-z][a-z.]*+_[a-z_0-9.]+" + regex = only_quoted_strings ? /"#{inner_regex}"/ : /\b#{inner_regex}\b/ + + string.gsub(regex) do |snake_case_word| + word_to_camel_case(snake_case_word) + end + end + + def word_to_camel_case(word) + ::ElasticGraph::SchemaArtifacts::RuntimeMetadata::SchemaElementNamesDefinition::CamelCaseConverter.normalize_case(word) + end + alias_method :case_correctly, :word_to_camel_case + + def word_to_snake_case(word) + ::ElasticGraph::SchemaArtifacts::RuntimeMetadata::SchemaElementNamesDefinition::SnakeCaseConverter.normalize_case(word) + end + end + + RSpec.shared_context "ElasticGraph GraphQL acceptance aggregation support" do + include_context "ElasticGraph GraphQL acceptance support" + + # To ensure that aggregation queries are as efficient as possible, we want these hold true for all aggregation queries: + # + # - `size: 0` should be passed to avoid requesting individual documents. + # - No `sort` option should be passed since we're not requesting individual documents. + # - `_source: false` should be passed to disable the fetching of any `_source` fields. + # + # Here we check all `_msearch` requests executed as part of the example to verify that these things held true. + after(:example) do + datastore_msearch_requests("main").each do |req| + req.body.split("\n").each_slice(2) do |(_header_line, body_line)| + search_body = ::JSON.parse(body_line) + if search_body.key?("aggs") + problems = [] + # :nocov: -- the branches below are only fully covered when we have a regression. + problems << "Search body has `size: #{search_body["size"]}` but it should be `size: 0`." unless search_body["size"] == 0 + problems << "Search body has `sort` but it is not needed." if search_body.key?("sort") + problems << "Search body has `_source: #{search_body["_source"]}` but it should be `_source: false`." unless search_body["_source"] == false + + unless problems.empty? + fail "Aggregation query[1] included some inefficiencies in its search body parameters: \n\n" \ + "#{problems.map { |prob| " - #{prob} " }.join("\n")}\n\n" \ + "[1] #{::JSON.pretty_generate(search_body)}" + end + # :nocov: + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/acceptance/graphql_types_spec.rb b/elasticgraph-graphql/spec/acceptance/graphql_types_spec.rb new file mode 100644 index 00000000..957eda10 --- /dev/null +++ b/elasticgraph-graphql/spec/acceptance/graphql_types_spec.rb @@ -0,0 +1,489 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "elasticgraph_graphql_acceptance_support" + +module ElasticGraph + RSpec.describe "ElasticGraph::GraphQL--GraphQL types" do + include_context "ElasticGraph GraphQL acceptance support" + + with_both_casing_forms do + let(:page_info) { case_correctly("page_info") } + let(:grouped_by) { case_correctly("grouped_by") } + let(:aggregated_values) { case_correctly("aggregated_values") } + let(:amount_cents) { case_correctly("amount_cents") } + + it "accepts and returns arbitrary JSON for `Untyped` scalar fields" do + index_records( + _widget1 = build(:widget, metadata: 9), + _widget2 = build(:widget, metadata: 3.75), + _widget3 = build(:widget, metadata: true), + _widget4 = build(:widget, metadata: "abc"), + _widget5 = build(:widget, metadata: ["1", 3]), + widget6 = build(:widget, metadata: {"json" => "object", "nested" => [{"stuff" => 3}]}), + _widget7 = build(:widget, metadata: "abc"), + _widget8 = build(:widget, metadata: nil) + ) + + # Verify that the Untypeds all round-trip through their indexing as expected. + # Also, verify that we can sort ascending by a Untyped field. + widgets = list_widgets_with(:metadata, order_by: [:metadata_ASC]) + expect(widgets.map { |w| w.fetch("metadata") }).to eq([nil] + sorted_metadata = [ + "abc", + "abc", + 3.75, + 9, + ["1", 3], + true, + {"json" => "object", "nested" => [{"stuff" => 3}]} + ]) + + # Verify that we can sort DESC by a Untyped field. + widgets = list_widgets_with(:metadata, order_by: [:metadata_DESC]) + expect(widgets.map { |w| w.fetch("metadata") }).to eq(sorted_metadata.reverse + [nil]) + + # Demonstrate basic filtering on a Untyped field + widgets = list_widgets_with(:metadata, filter: {"metadata" => {"equal_to_any_of" => [true, ["1", 3], nil]}}) + expect(widgets.map { |w| w.fetch("metadata") }).to contain_exactly( + true, + ["1", 3], + nil + ) + + # Demonstrate that the equality semantics of Untyped are what we expect. Two objects with the same entries + # but keys in a different order should be considered equal. + widgets1 = list_widgets_with(:metadata, filter: {"metadata" => {"equal_to_any_of" => [{"json" => "object", "nested" => [{"stuff" => 3}]}]}}) + widgets2 = list_widgets_with(:metadata, filter: {"metadata" => {"equal_to_any_of" => [{"nested" => [{"stuff" => 3}], "json" => "object"}]}}) + expect(widgets1).to eq(widgets2).and eq([string_hash_of(widget6, :id, :metadata)]) + + aggregations = list_widgets_with_aggregations(count_aggregation("metadata")) + expect(aggregations).to contain_exactly( + {grouped_by => {"metadata" => "abc"}, "count" => 2}, + {grouped_by => {"metadata" => 3.75}, "count" => 1}, + {grouped_by => {"metadata" => 9}, "count" => 1}, + {grouped_by => {"metadata" => ["1", 3]}, "count" => 1}, + {grouped_by => {"metadata" => true}, "count" => 1}, + {grouped_by => {"metadata" => {"json" => "object", "nested" => [{"stuff" => 3}]}}, "count" => 1}, + {grouped_by => {"metadata" => nil}, "count" => 1} + ) + end + + it "supports __typename at all levels of `{Type}Connection`" do + index_records( + build(:widget, amount_cents: 100, options: build(:widget_options, size: "SMALL", color: "BLUE"), created_at: "2019-06-01T12:00:00Z") + ) + results = list_widgets_and_aggregations_with_typename + + expect(results["__typename"]).to eq "Query" + + widgets = results.fetch("widgets") + expect(widgets["__typename"]).to eq(apply_derived_type_customizations("WidgetConnection")) + expect(widgets.dig("edges", 0, "__typename")).to eq(apply_derived_type_customizations("WidgetEdge")) + expect(widgets.dig("edges", 0, "node", "__typename")).to eq("Widget") + expect(widgets.dig("edges", 0, "node", "options", "__typename")).to eq("WidgetOptions") + expect(widgets.dig("edges", 0, "node", "inventor", "__typename")).to eq("Company").or eq("Person") + expect(widgets.dig(page_info, "__typename")).to eq("PageInfo") + + widget_aggregations = results.fetch(case_correctly("widget_aggregations")) + expect(widget_aggregations.dig("__typename")).to eq(apply_derived_type_customizations("WidgetAggregationConnection")) + expect(widget_aggregations.dig(page_info, "__typename")).to eq("PageInfo") + expect(widget_aggregations.dig("edges", 0, "__typename")).to eq(apply_derived_type_customizations("WidgetAggregationEdge")) + expect(widget_aggregations.dig("edges", 0, "node", "__typename")).to eq(apply_derived_type_customizations("WidgetAggregation")) + expect(widget_aggregations.dig("edges", 0, "node", grouped_by, "__typename")).to eq(apply_derived_type_customizations("WidgetGroupedBy")) + expect(widget_aggregations.dig("edges", 0, "node", grouped_by, "cost", "__typename")).to eq(apply_derived_type_customizations("MoneyGroupedBy")) + expect(widget_aggregations.dig("edges", 0, "node", aggregated_values, "__typename")).to eq(apply_derived_type_customizations("WidgetAggregatedValues")) + expect(widget_aggregations.dig("edges", 0, "node", aggregated_values, "cost", "__typename")).to eq(apply_derived_type_customizations("MoneyAggregatedValues")) + expect(widget_aggregations.dig("edges", 0, "node", aggregated_values, "cost", amount_cents, "__typename")).to eq(apply_derived_type_customizations("IntAggregatedValues")) + end + + it "supports storing and querying JsonSafeLong and LongString values" do + large_test_value = 2**62 + + original_widgets = [ + widget1 = build(:widget, weight_in_ng: 100, weight_in_ng_str: 100), + widget2 = build(:widget, weight_in_ng: 2**30, weight_in_ng_str: 2**30), + # The sum of these last 2 will exceed the max for the field type, allowing us to test `exact_sum` vs `approximate_sum`. + widget3 = build(:widget, weight_in_ng: 2**52, weight_in_ng_str: large_test_value), + widget4 = build(:widget, weight_in_ng: (2**52) + 10, weight_in_ng_str: (large_test_value + 10)) + ] + + index_records(*original_widgets) + + # Demonstrate that LongString fields allow larger values than JsonSafeLong fields. + expect { + index_records(build(:widget, weight_in_ng: large_test_value)) + }.to raise_error(Indexer::IndexingFailuresError, /maximum/) + + # Sort by JsonSafeLong field ASC + widgets = list_widgets_with(:weight_in_ng, + filter: {id: {equal_to_any_of: [widget1.fetch(:id), widget2.fetch(:id)]}}, + order_by: [:weight_in_ng_ASC]) + expect(widgets).to match([ + string_hash_of(widget1, :id, :weight_in_ng), + string_hash_of(widget2, :id, :weight_in_ng) + ]) + + # Filter by JsonSafeLong field + widgets = list_widgets_with(:weight_in_ng, + filter: {weight_in_ng: {gt: 2**35}}, + order_by: [:weight_in_ng_ASC]) + expect(widgets).to match([ + string_hash_of(widget3, :id, :weight_in_ng), + string_hash_of(widget4, :id, :weight_in_ng) + ]) + + expect { + # Attempting to filter with a JsonSafeLong that is too large should return an error + response = query_widgets_with(:weight_in_ng, + allow_errors: true, + filter: {weight_in_ng: {gt: JSON_SAFE_LONG_MAX + 1}}, + order_by: [:weight_in_ng_ASC]) + expect_error_related_to(response, apply_derived_type_customizations("JsonSafeLongFilterInput"), "gt", case_correctly("weight_in_ng")) + }.to log(a_string_including(apply_derived_type_customizations("JsonSafeLongFilterInput"), "gt", case_correctly("weight_in_ng"))) + + # Sort by JsonSafeLong field DESC + widgets = list_widgets_with(:weight_in_ng, order_by: [:weight_in_ng_DESC]) + expect(widgets).to match([ + string_hash_of(widget4, :id, :weight_in_ng), + string_hash_of(widget3, :id, :weight_in_ng), + string_hash_of(widget2, :id, :weight_in_ng), + string_hash_of(widget1, :id, :weight_in_ng) + ]) + + # Sort by LongString field ASC + widgets = list_widgets_with(:weight_in_ng_str, + filter: {id: {equal_to_any_of: [widget1.fetch(:id), widget2.fetch(:id)]}}, + order_by: [:weight_in_ng_str_ASC]) + expect(widgets).to match([ + string_hash_of(widget1, :id, weight_in_ng_str: "100"), + string_hash_of(widget2, :id, weight_in_ng_str: (2**30).to_s) + ]) + + # Filter by LongString field, passing a string value + long_string_filter_value = 2**35 + widgets = list_widgets_with(:weight_in_ng_str, + filter: {weight_in_ng_str: {gt: long_string_filter_value.to_s}}, + order_by: [:weight_in_ng_str_ASC]) + expect(widgets).to match([ + string_hash_of(widget3, :id, weight_in_ng_str: large_test_value.to_s), + string_hash_of(widget4, :id, weight_in_ng_str: (large_test_value + 10).to_s) + ]) + + # Attempting to filter with a LongString value that is too large should return an error + response = query_widgets_with(:weight_in_ng_str, + allow_errors: true, + filter: {weight_in_ng_str: {gt: LONG_STRING_MAX + 1}}, + order_by: [:weight_in_ng_str_ASC]) + expect_error_related_to(response, "LongString", "gt", case_correctly("weight_in_ng_str")) + + # Sort by JsonSafeLong field DESC + widgets = list_widgets_with(:weight_in_ng_str, order_by: [:weight_in_ng_str_DESC]) + expect(widgets).to match([ + string_hash_of(widget4, :id, weight_in_ng_str: (large_test_value + 10).to_s), + string_hash_of(widget3, :id, weight_in_ng_str: large_test_value.to_s), + string_hash_of(widget2, :id, weight_in_ng_str: (2**30).to_s), + string_hash_of(widget1, :id, weight_in_ng_str: "100") + ]) + + # Aggregate over all widgets. The sum should exceed the maximums for the scalar field type. + # This should make `exact_sum` nil. + aggregations = list_widgets_with_aggregations(all_weight_in_ng_aggregations) + weight_in_ngs = original_widgets.map { |w| w.fetch(case_correctly("weight_in_ng").to_sym) } + weight_in_ng_strs = original_widgets.map { |w| w.fetch(case_correctly("weight_in_ng_str").to_sym).to_i } + expect(aggregations.size).to eq(1) + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng", "approximate_sum")).to be_approximately(weight_in_ngs.sum) + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng_str", "approximate_sum")).to be_approximately(weight_in_ng_strs.sum) + # Value exceeds max JsonSafeLong, so exact_sum should be nil + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng", "exact_sum")).to be nil + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng_str", "exact_sum")).to be nil + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng", "approximate_avg")).to be_a(::Float).and be_approximately(weight_in_ngs.sum / weight_in_ngs.size) + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng_str", "approximate_avg")).to be_a(::Float).and be_approximately(weight_in_ng_strs.sum / weight_in_ng_strs.size) + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng", "exact_min")).to be_a(::Integer).and eq(weight_in_ngs.min) + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng_str", "exact_min")).to be_a(::Integer).and eq(weight_in_ng_strs.min) + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng_str", "approximate_min")).to be_a(::String).and eq(weight_in_ng_strs.min.to_s) + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng", "exact_max")).to be_a(::Integer).and eq(weight_in_ngs.max) + # Max value of `weight_in_ng_str` exceeds `JsonSafeLong` range, so exact should be nil but approx should be available. + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng_str", "exact_max")).to be nil # the max exceeds the JsonSafeLong range, so we can't get it exact + weight_in_ng_str_approx_max = value_at_path(aggregations.first, aggregated_values, "weight_in_ng_str", "approximate_max") + expect(weight_in_ng_str_approx_max).to be_a(::String) + expect(weight_in_ng_str_approx_max.to_f).to be_approximately(weight_in_ng_strs.max) + + # Aggregate over only the first two widgets (excluding the last 2, to ensure that the sum stays under the JsonSafeLong max) + # This should make `exact_sum` non-nil. + aggregations = list_widgets_with_aggregations( + all_weight_in_ng_aggregations(filter: {weight_in_ng_str: {lt: large_test_value.to_s}}) + ) + weight_in_ngs = weight_in_ngs.first(2) # drop last 2 weights + weight_in_ng_strs = weight_in_ng_strs.first(2) # drop last 2 weights + expect(aggregations.size).to eq(1) + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng", "approximate_sum")).to be_approximately(weight_in_ngs.sum) + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng_str", "approximate_sum")).to be_approximately(weight_in_ng_strs.sum) + # Value does not exceed max JsonSafeLong, so exact_sum should be present + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng", "exact_sum")).to be_a(::Integer).and eq(weight_in_ngs.sum) + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng_str", "exact_sum")).to be_a(::Integer).and eq(weight_in_ng_strs.sum) + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng", "approximate_avg")).to be_a(::Float).and be_approximately(weight_in_ngs.sum / weight_in_ngs.size) + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng_str", "approximate_avg")).to be_a(::Float).and be_approximately(weight_in_ng_strs.sum / weight_in_ng_strs.size) + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng", "exact_min")).to be_a(::Integer).and eq(weight_in_ngs.min) + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng_str", "exact_min")).to be_a(::Integer).and eq(weight_in_ng_strs.min) + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng_str", "approximate_min")).to be_a(::String).and eq(weight_in_ng_strs.min.to_s) + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng", "exact_max")).to be_a(::Integer).and eq(weight_in_ngs.max) + # Value does not exceed max JsonSafeLong, so approximate_max should be present + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng_str", "exact_max")).to be_a(::Integer).and eq(weight_in_ng_strs.max) + expect(value_at_path(aggregations.first, aggregated_values, "weight_in_ng_str", "approximate_max")).to be_a(::String).and eq(weight_in_ng_strs.max.to_s) + end + + it "supports storing and querying GeoLocation values" do + index_records( + build(:address, full_address: "space-needle", geo_location: {latitude: 47.62089914996321, longitude: -122.34924708967479}), + build(:address, full_address: "crystal-mtn", geo_location: {latitude: 46.93703464703253, longitude: -121.47398616597955}), + build(:address, full_address: "pike-place-mkt", geo_location: {latitude: 47.60909792583577, longitude: -122.33981115022492}) + ) + + # Fetch geo_location values. + addresses = list_addresses( + fields: "full_address geo_location { latitude longitude }", + order_by: [:full_address_ASC] + ) + + expect(addresses).to eq [ + {case_correctly("full_address") => "crystal-mtn", case_correctly("geo_location") => {"latitude" => 46.93703464703253, "longitude" => -121.47398616597955}}, + {case_correctly("full_address") => "pike-place-mkt", case_correctly("geo_location") => {"latitude" => 47.60909792583577, "longitude" => -122.33981115022492}}, + {case_correctly("full_address") => "space-needle", case_correctly("geo_location") => {"latitude" => 47.62089914996321, "longitude" => -122.34924708967479}} + ] + + downtown_seattle_location = {"latitude" => 47.6078024243176, "longitude" => -122.3345525727595} + + # Filtering on distance. + addresses = list_addresses( + fields: "full_address", + filter: {"geo_location" => {"near" => downtown_seattle_location.merge({"max_distance" => 3, "unit" => :MILE})}}, + order_by: [:full_address_ASC] + ) + + # Crystal Mountain is not within 3 miles of downtown Seattle but Pike Place Market and the Space Needle are. + expect(addresses).to eq [ + {case_correctly("full_address") => "pike-place-mkt"}, + {case_correctly("full_address") => "space-needle"} + ] + + # For now, other features (e,g. sorting and aggregating) are not supported on GeoLocation values. + end + end + + def value_at_path(hash, *field_parts) + field_parts = field_parts.map { |f| case_correctly(f) } + hash.dig(*field_parts) + end + + def be_approximately(value) + be_a(::Float).and be_within(0.00001).percent_of(value) + end + + def list_widgets_and_aggregations_with_typename + call_graphql_query(<<~QUERY).dig("data") + query { + __typename + + widgets { + __typename + edges { + __typename + node { + __typename + id + options { + __typename + size + } + inventor { + __typename + + ...on Person { + nationality + } + + ...on Company { + stock_ticker + } + } + } + } + page_info { + __typename + } + } + + widget_aggregations { + __typename + page_info { + __typename + } + edges { + __typename + node { + __typename + + grouped_by { + __typename + + cost { + currency + __typename + } + } + + aggregated_values { + __typename + + cost { + __typename + amount_cents { + __typename + approximate_sum + } + } + } + } + } + } + } + QUERY + end + + def all_weight_in_ng_aggregations(**agg_args) + <<~AGG + widget_aggregations#{graphql_args(agg_args)} { + edges { + node { + count + + aggregated_values { + weight_in_ng { + approximate_sum + exact_sum + approximate_avg + exact_min + exact_max + } + + weight_in_ng_str { + approximate_sum + exact_sum + approximate_avg + exact_min + approximate_min + exact_max + approximate_max + } + } + } + } + } + AGG + end + + def count_aggregation(*fields, **agg_args) + <<~AGG + widget_aggregations#{graphql_args(agg_args)} { + edges { + node { + grouped_by { + #{fields.join("\n")} + } + count + } + } + } + AGG + end + + def list_widgets_with_aggregations(widget_aggregations, **query_args) + call_graphql_query(<<~QUERY).dig("data", case_correctly("widget_aggregations"), "edges").map { |edge| edge["node"] } + query { + widgets#{graphql_args(query_args)} { + edges { + node { + id + name + amount_cents + options { + size + color + } + + inventor { + ... on Person { + name + nationality + } + + ... on Company { + name + stock_ticker + } + } + } + } + } + + #{widget_aggregations} + + # Also query `components.widgets` (even though we do not do anything with the returned data) + # to exercise an odd aggregations edge case where a nested relationship field exists with the + # same name as an earlier field that had aggregations. + components { + edges { + node { + widgets { + edges { + node { + id + } + } + } + } + } + } + } + QUERY + end + + def list_widgets_with(fieldname, **query_args) + query_widgets_with(fieldname, **query_args).dig("data", "widgets", "edges").map { |we| we.fetch("node") } + end + + def query_widgets_with(fieldname, allow_errors: false, **query_args) + call_graphql_query(<<~QUERY, allow_errors: allow_errors) + query { + widgets#{graphql_args(query_args)} { + edges { + node { + id + #{fieldname} + } + } + } + } + QUERY + end + + def list_addresses(fields:, gql: graphql, **query_args) + call_graphql_query(<<~QUERY, gql: gql).dig("data", "addresses", "edges").map { |we| we.fetch("node") } + query { + addresses#{graphql_args(query_args)} { + edges { + node { + #{fields} + } + } + } + } + QUERY + end + end +end diff --git a/elasticgraph-graphql/spec/acceptance/hidden_types_spec.rb b/elasticgraph-graphql/spec/acceptance/hidden_types_spec.rb new file mode 100644 index 00000000..807af6d5 --- /dev/null +++ b/elasticgraph-graphql/spec/acceptance/hidden_types_spec.rb @@ -0,0 +1,224 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "elasticgraph_graphql_acceptance_support" + +module ElasticGraph + RSpec.describe "ElasticGraph::GraphQL--hidden types" do + include_context "ElasticGraph GraphQL acceptance support" + + with_both_casing_forms do + context "when some indices are configured to query clusters that are not configured, or are configured with `query_cluster: nil`" do + let(:graphql_hiding_addresses_and_mechanical_parts) do + build_graphql do |config| + config.with(index_definitions: config.index_definitions.merge( + index_definition_name_for("addresses") => config_index_def_of(query_cluster: "unknown1"), + index_definition_name_for("mechanical_parts") => config_index_def_of(query_cluster: nil) + )) + end + end + + it "hides the GraphQL schema elements that require access to those indices" do + all_fields_by_type_name = fields_by_type_name_from(graphql) + restricted_fields_by_type_name = nil + + expect { + restricted_fields_by_type_name = fields_by_type_name_from(graphql_hiding_addresses_and_mechanical_parts) + }.to log_warning(a_string_including("2 GraphQL types were hidden", "Address", "MechanicalPart")) + + hidden_types = (all_fields_by_type_name.keys - restricted_fields_by_type_name.keys) + + hidden_fields = restricted_fields_by_type_name.each_with_object({}) do |(type, fields), hash| + missing_fields = all_fields_by_type_name.fetch(type) - fields + hash[type] = missing_fields if missing_fields.any? + end + + expect(hidden_types).to match_array(adjust_derived_type_names_as_needed( + all_types_related_to("Address") + + all_types_related_to("MechanicalPart") + + # `AddressTimestamps` and `GeoShape` are only used on `Address` so when `Address` is hidden, they are, too. + ["AddressTimestamps", "GeoShape"] + )) + + expect(hidden_fields).to eq( + "Query" => [case_correctly("address_aggregations"), "addresses", case_correctly("mechanical_part_aggregations"), case_correctly("mechanical_parts")], + "Manufacturer" => ["address"] + ) + + # Our mechanism for determining which types to hide on the basis of inaccessible indexes uses the type name. + # If/when our schema definition API started generating some new types for indexed types, the current logic + # may not correctly hide those types when the index is inaccessible. To guard against that, we are enumerating + # all of the types we expect to be present on both schemas here--that way, when the set of types on our test + # schema grows, we are forced to consider if the new type should be subject to the hidden type logic we test + # here. + # + # When the expectation below fails, please add the new types to this list here so long as the new types are + # not backed by the `addresses` or `mechanical_parts` indices. If they are backed by one of those indices, + # you'll need to account for the types in the expectations above (and fix the hidden type logic as well). + expected_types_present_on_both_schemas = GraphQL::Schema::BUILT_IN_TYPE_NAMES + + all_types_related_to("Widget") + + all_types_related_to("WidgetWorkspace") + + all_types_related_to("WidgetOrAddress") + + all_types_related_to("Component") + + all_types_related_to("Manufacturer") + + all_types_related_to("ElectricalPart") + + all_types_related_to("Part") + + all_types_related_to("NamedEntity") + + all_types_related_to("WidgetCurrency") + + all_types_related_to("Team") + + all_types_related_to("Sponsor") + + relay_types_related_to("String", include_list_filter: true) - ["StringSortOrderInput"] + + type_and_filters_for("Color", include_list: true, as_input_enum: true) + + type_and_filters_for("Date", include_list: true) + + type_and_filters_for("DateTime", include_list: true) + + type_and_filters_for("LocalTime") + + type_and_filters_for("JsonSafeLong", include_list: true) + + type_and_filters_for("Untyped") + + type_and_filters_for("LongString") + + type_and_filters_for("Material", as_input_enum: true) + + type_and_filters_for("Size", include_list: true, as_input_enum: true) + + type_and_filters_for("TeamNestedFields") + + type_and_filters_for("Affiliations") + + type_and_filters_for("ID", include_list: true) + + type_filter_and_non_indexed_aggregation_types_for("TeamDetails") + + type_filter_and_non_indexed_aggregation_types_for("AddressTimestamps") - ["AddressTimestamps"] + + type_filter_and_non_indexed_aggregation_types_for("Affiliations", include_fields_list_filter: true) + + type_filter_and_non_indexed_aggregation_types_for("CurrencyDetails") + + type_filter_and_non_indexed_aggregation_types_for("Inventor") + + type_filter_and_non_indexed_aggregation_types_for("NamedInventor") + + type_filter_and_non_indexed_aggregation_types_for("Money", include_list_filter: true, include_fields_list_filter: true) - ["MoneyListElementFilterInput"] + + type_filter_and_non_indexed_aggregation_types_for("Position") + + type_filter_and_non_indexed_aggregation_types_for("Player", include_list_filter: true, include_fields_list_filter: true) - ["PlayerListElementFilterInput"] + + type_filter_and_non_indexed_aggregation_types_for("PlayerSeason", include_list_filter: true, include_fields_list_filter: true) - ["PlayerSeasonListElementFilterInput"] + + type_filter_and_non_indexed_aggregation_types_for("TeamRecord", include_fields_list_filter: true) + + type_filter_and_non_indexed_aggregation_types_for("TeamSeason", include_list_filter: true, include_fields_list_filter: true) - ["TeamSeasonListElementFilterInput"] + + type_filter_and_non_indexed_aggregation_types_for("WidgetOptions") + + type_filter_and_non_indexed_aggregation_types_for("WidgetOptionSets") - ["WidgetOptionSetsGroupedBy"] + + type_filter_and_non_indexed_aggregation_types_for("WidgetCurrencyNestedFields") + + type_filter_and_non_indexed_aggregation_types_for("WorkspaceWidget") + + type_filter_and_non_indexed_aggregation_types_for("Sponsorship", include_list_filter: true, include_fields_list_filter: true) - ["SponsorshipListElementFilterInput"] + + ::GraphQL::Schema::BUILT_IN_TYPES.keys.flat_map { |k| type_and_filters_for(k) } - ["BooleanFilterInput"] + + %w[ + FloatAggregatedValues IntAggregatedValues JsonSafeLongAggregatedValues LongStringAggregatedValues NonNumericAggregatedValues + DateAggregatedValues DateTimeAggregatedValues LocalTimeAggregatedValues + Company Cursor PageInfo Person Query TextFilterInput GeoLocation + DateTimeGroupingOffsetInput DateTimeUnitInput DateTimeTimeOfDayFilterInput + DateGroupedBy DateGroupingGranularityInput DateGroupingOffsetInput DateGroupingTruncationUnitInput DateUnitInput + DateTimeGroupedBy DateTimeGroupingGranularityInput DateTimeGroupingTruncationUnitInput TimeZone + DayOfWeek DayOfWeekGroupingOffsetInput DistanceUnitInput GeoLocationFilterInput GeoLocationDistanceFilterInput + IntListFilterInput IntListElementFilterInput AggregationCountDetail + LocalTimeGroupingOffsetInput LocalTimeGroupingTruncationUnitInput LocalTimeUnitInput MatchesQueryFilterInput + MatchesPhraseFilterInput MatchesQueryAllowedEditsPerTermInput + ] + + # The sub-aggregation types are quite complicated and we just add them all here. + expected_types_present_on_both_schemas += %w[ + TeamAggregationSubAggregations + TeamMoneySubAggregation TeamMoneySubAggregationConnection + TeamPlayerSubAggregation TeamPlayerSubAggregationConnection + TeamTeamSeasonSubAggregation TeamTeamSeasonSubAggregationConnection + TeamAggregationCurrentPlayersObjectSubAggregations + TeamAggregationNestedFieldsSubAggregations + TeamAggregationNestedFields2SubAggregations + TeamAggregationSeasonsObjectPlayersObjectSubAggregations + TeamAggregationSeasonsObjectSubAggregations + TeamPlayerPlayerSeasonSubAggregation + TeamPlayerPlayerSeasonSubAggregationConnection + TeamPlayerSeasonSubAggregation + TeamPlayerSeasonSubAggregationConnection + TeamPlayerSubAggregationSubAggregations + TeamTeamSeasonPlayerPlayerSeasonSubAggregation + TeamTeamSeasonPlayerPlayerSeasonSubAggregationConnection + TeamTeamSeasonPlayerSeasonSubAggregation + TeamTeamSeasonPlayerSeasonSubAggregationConnection + TeamTeamSeasonPlayerSubAggregation + TeamTeamSeasonPlayerSubAggregationConnection + TeamTeamSeasonPlayerSubAggregationSubAggregations + TeamTeamSeasonSubAggregationPlayersObjectSubAggregations + TeamTeamSeasonSubAggregationSubAggregations + TeamSponsorshipSubAggregation + TeamSponsorshipSubAggregationConnection + TeamAggregationCurrentPlayersObjectAffiliationsSubAggregations + TeamAggregationSeasonsObjectPlayersObjectAffiliationsSubAggregations + TeamPlayerSponsorshipSubAggregation + TeamPlayerSponsorshipSubAggregationConnection + TeamPlayerSubAggregationAffiliationsSubAggregations + TeamTeamSeasonPlayerSponsorshipSubAggregation + TeamTeamSeasonPlayerSponsorshipSubAggregationConnection + TeamTeamSeasonPlayerSubAggregationAffiliationsSubAggregations + TeamTeamSeasonSponsorshipSubAggregation + TeamTeamSeasonSponsorshipSubAggregationConnection + TeamTeamSeasonSubAggregationPlayersObjectAffiliationsSubAggregations + ] + + expected_types_present_on_both_schemas = adjust_derived_type_names_as_needed(expected_types_present_on_both_schemas) + + actual_types_present_on_both_schemas = (all_fields_by_type_name.keys & restricted_fields_by_type_name.keys) + # We compare as multi-line strings here to get nice diffing from RSpec :). + expect(actual_types_present_on_both_schemas.sort.join("\n")).to eq(expected_types_present_on_both_schemas.uniq.sort.join("\n")) + end + + def adjust_derived_type_names_as_needed(type_names) + type_names.map { |type| apply_derived_type_customizations(type) } + end + + def fields_by_type_name_from(graphql) + types = call_graphql_query(<<~EOS, gql: graphql).fetch("data").fetch("__schema").fetch("types") + query { + __schema { + types { + name + fields { + name + } + } + } + } + EOS + + types.each_with_object({}) do |type, hash| + hash[type.fetch("name")] = (type.fetch("fields") || []).map { |f| f.fetch("name") } + end + end + + def all_types_related_to(type_name, include_list_filter: false) + relay_types_related_to(type_name, include_list_filter: include_list_filter) + + aggregation_types_related_to(type_name) + end + + def relay_types_related_to(type_name, include_list_filter: false) + ["Edge", "Connection", "SortOrderInput"].map do |suffix| + type_name + suffix + end + type_and_filters_for(type_name, include_list: include_list_filter) + end + + def type_and_filters_for(type_name, include_list: false, as_input_enum: false) + suffixes = ["", "FilterInput"] + suffixes += ["Input"] if as_input_enum + suffixes += ["ListFilterInput", "ListElementFilterInput"] if include_list + suffixes.map { |suffix| type_name + suffix } + end + + def type_filter_and_non_indexed_aggregation_types_for(type_name, include_list_filter: false, include_fields_list_filter: false) + suffixes = ["", "FilterInput", "GroupedBy", "AggregatedValues"] + suffixes += ["ListFilterInput", "ListElementFilterInput"] if include_list_filter + suffixes += ["FieldsListFilterInput"] if include_fields_list_filter + suffixes.map { |suffix| type_name + suffix } + end + + def aggregation_types_related_to(type_name) + suffixes = %w[ + Aggregation AggregationConnection AggregationEdge GroupedBy AggregatedValues + ] + + suffixes.map { |suffix| type_name + suffix } + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/acceptance/nested_relationships_spec.rb b/elasticgraph-graphql/spec/acceptance/nested_relationships_spec.rb new file mode 100644 index 00000000..8b6bc8d0 --- /dev/null +++ b/elasticgraph-graphql/spec/acceptance/nested_relationships_spec.rb @@ -0,0 +1,512 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "elasticgraph_graphql_acceptance_support" + +module ElasticGraph + RSpec.describe "ElasticGraph::GraphQL--nested relationships" do + include_context "ElasticGraph GraphQL acceptance support" + + context "with widget data indexed" do + with_both_casing_forms do + let(:manufacturer1) { build(:manufacturer) } + let(:manufacturer2) { build(:manufacturer) } + + let(:address1) { build(:address, full_address: "a1", manufacturer: manufacturer1, created_at: "2019-09-12T12:00:00Z") } + let(:address2) { build(:address, full_address: "a2", manufacturer: manufacturer2, created_at: "2019-09-11T12:00:00Z") } + + let(:part1) { build(:electrical_part, id: "p1", manufacturer: manufacturer1, created_at: "2019-06-01T00:00:00Z") } + let(:part2) { build(:electrical_part, id: "p2", manufacturer: manufacturer1, created_at: "2019-06-02T00:00:00Z") } + let(:part3) { build(:mechanical_part, id: "p3", manufacturer: manufacturer2, created_at: "2019-06-03T00:00:00Z") } + + let(:component1) { build(:component, parts: [part1, part2], name: "comp1", created_at: "2019-06-04T00:00:00Z") } + let(:component2) { build(:component, parts: [part1, part3], name: "comp2", created_at: "2019-06-03T00:00:00Z") } + let(:component3) { build(:component, parts: [part2, part3], name: "comp3", created_at: "2019-06-02T00:00:00Z") } + let(:component4) { build(:component, parts: [part2], name: "comp4", created_at: "2019-06-01T00:00:00Z") } + + let(:widget1) { build(:widget, name: "widget1", amount_cents: 100, components: [component1, component2, component4], created_at: "2019-06-02T00:00:00Z") } + let(:widget2) { build(:widget, name: "widget2", amount_cents: 200, components: [component3], created_at: "2019-06-01T00:00:00Z") } + + before do + index_records(manufacturer1, manufacturer2, address1, address2, part1, part2, part3, component1, component2, component3, component4, widget1, widget2) + end + + it "supports filtering relationships with additional filter conditions" do + component_args = {filter: {name: {equal_to_any_of: %w[comp1]}}} + results = query_components_and_dollar_widgets(component_args: component_args) + expect(results).to match( + case_correctly("nodes") => [{ + case_correctly("name") => "comp1", + case_correctly("dollar_widget") => { + case_correctly("name") => "widget1", + case_correctly("cost") => { + case_correctly("amount_cents") => 100 + } + } + }], + case_correctly("total_edge_count") => 1 + ) + + # verify that non-dollar widgets are filtered out + component_args2 = {filter: {name: {equal_to_any_of: %w[comp3]}}} + results2 = query_components_and_dollar_widgets(component_args: component_args2) + expect(results2).to match( + case_correctly("nodes") => [{ + case_correctly("name") => "comp3", + case_correctly("dollar_widget") => nil # dollar_widget is nil since widget 2 costs $2 + }], + case_correctly("total_edge_count") => 1 + ) + end + + it "supports loading bi-directional relationships, starting from either end", :expect_search_routing do + component_args_without_not = {filter: {name: {equal_to_any_of: %w[comp1 comp2 comp3]}}} + component_args_with_not = {filter: {name: {not: {equal_to_any_of: %w[comp4]}}}} + part_args = {order_by: [:id_DESC]} # ensure deterministic ordering of parts + address_args = {order_by: [:full_address_ASC]} # ensure deterministic ordering of addresses + + [component_args_without_not, component_args_with_not].each do |component_args| + expect { + expect(query_all_relationship_levels_from_widgets(component_args: component_args, part_args: part_args)).to match edges_of( + node_of(widget1, :name, components: edges_of( + node_of(component1, :name, parts: edges_of( + node_of(part2, :name, manufacturer: string_hash_of(manufacturer1, :name, address: string_hash_of(address1, :full_address))), + node_of(part1, :name, manufacturer: string_hash_of(manufacturer1, :name, address: string_hash_of(address1, :full_address))) + )), + node_of(component2, :name, parts: edges_of( + node_of(part3, :name, manufacturer: string_hash_of(manufacturer2, :name, address: string_hash_of(address2, :full_address))), + node_of(part1, :name, manufacturer: string_hash_of(manufacturer1, :name, address: string_hash_of(address1, :full_address))) + )) + )), + node_of(widget2, :name, components: edges_of( + node_of(component3, :name, parts: edges_of( + node_of(part3, :name, manufacturer: string_hash_of(manufacturer2, :name, address: string_hash_of(address2, :full_address))), + node_of(part2, :name, manufacturer: string_hash_of(manufacturer1, :name, address: string_hash_of(address1, :full_address))) + )) + )) + ) + }.to query_datastore("main", 5).times + + expect_to_have_routed_to_shards_with("main", + # Root `widgets` query isn't filtering on anything and uses no routing. + ["widgets_rollover__*", nil], + # `Widget.components` uses an `out` foreign key and routes on `id` + ["components", widget1.fetch(case_correctly(:component_ids)).sort.join(",")], + ["components", widget2.fetch(case_correctly(:component_ids)).sort.join(",")], + # `Component.parts` is an `out` foreign key, and routes on `id` + ["electrical_parts,mechanical_parts", component1.fetch(case_correctly(:part_ids)).sort.join(",")], + ["electrical_parts,mechanical_parts", component2.fetch(case_correctly(:part_ids)).sort.join(",")], + ["electrical_parts,mechanical_parts", component3.fetch(case_correctly(:part_ids)).sort.join(",")], + # `ElectricalPart.manufacturer`/`MechanicalPart.manufacturer` use an `out` foreign key and route on `id`. + ["manufacturers", part1.fetch(case_correctly(:manufacturer_id))], + ["manufacturers", part3.fetch(case_correctly(:manufacturer_id))], + # `Manufacturer.addresses` uses an `in` foreign key and therefore cannot route on `id`. + ["addresses", nil], ["addresses", nil]) + + datastore_requests_by_cluster_name["main"].clear + + expect { + expect(query_all_relationship_levels_from_addresses(component_args: component_args, part_args: part_args, address_args: address_args)).to match edges_of( + node_of(address1, :full_address, manufacturer: + string_hash_of(manufacturer1, :name, manufactured_parts: edges_of( + node_of(part2, :name, components: edges_of( + node_of(component1, :name, widget: string_hash_of(widget1, :name)), + node_of(component3, :name, widget: string_hash_of(widget2, :name)) + )), + node_of(part1, :name, components: edges_of( + node_of(component1, :name, widget: string_hash_of(widget1, :name)), + node_of(component2, :name, widget: string_hash_of(widget1, :name)) + )) + ))), + node_of(address2, :full_address, manufacturer: + string_hash_of(manufacturer2, :name, manufactured_parts: edges_of( + node_of(part3, :name, components: edges_of( + node_of(component2, :name, widget: string_hash_of(widget1, :name)), + node_of(component3, :name, widget: string_hash_of(widget2, :name)) + )) + ))) + ) + }.to query_datastore("main", 5).times + + expect_to_have_routed_to_shards_with("main", + # Root `addresses` query isn't filtering on anything and uses no routing. + ["addresses", nil], + # The `Address.manufacturer` relation uses an `out` foreign key and therefore uses routing on `id`. + ["manufacturers", address1.fetch(case_correctly(:manufacturer_id))], + ["manufacturers", address2.fetch(case_correctly(:manufacturer_id))], + # The `Manufacturer.manufactured_parts` relation uses an `in` foreign key and therefore cannot route on `id`. + ["electrical_parts,mechanical_parts", nil], ["electrical_parts,mechanical_parts", nil], + # The `ElectricalPart.components` and `MechanicalPart.components` relations use an `in` foreign key and therefore cannot route on `id` + ["components", nil], ["components", nil], ["components", nil], + # The `Component.widgets` relation uses an `in` foreign key and therefore cannot route on `id`. + ["widgets_rollover__*", nil], ["widgets_rollover__*", nil], ["widgets_rollover__*", nil]) + end + end + + it "supports pagination at any level using relay connections", :expect_search_routing do + results = query_widgets_and_components_including_page_info( + widget_args: {first: 1, order_by: [:amount_cents_ASC]}, + component_args: {first: 1, order_by: [:name_ASC]} + ) + + expect_to_have_routed_to_shards_with("main", + ["widgets_rollover__*", nil], + # `Widget.components` uses an `out` foreign key and routes on `id` + ["components", widget1.fetch(case_correctly(:component_ids)).sort.join(",")]) + + expect(results).to match_single_widget_and_component_result( + widget1, component1, + widgets_have_next_page: true, widgets_have_previous_page: false, widget_count: 2, + components_have_next_page: true, components_have_previous_page: false, component_count: 3 + ) + + # Demonstrate how negative `first` values behave. + expect { + response = query_widgets_and_components_including_page_info( + widget_args: {first: -2, order_by: [:amount_cents_ASC]}, + expect_errors: true + ) + expect(response["errors"]).to contain_exactly(a_hash_including("message" => "`first` cannot be negative, but is -2.")) + }.to log_warning a_string_including("`first` cannot be negative, but is -2.") + + # Demonstrate how broken cursors behave. + expect { + response = query_widgets_and_components_including_page_info( + widget_args: {first: 1, order_by: [:amount_cents_ASC]}, + component_args: {first: 1, after: [1, 2, 3], order_by: [:name_ASC]}, + expect_errors: true + ) + expect(response["errors"]).to contain_exactly(a_hash_including("message" => "Argument 'after' on Field 'components' has an invalid value ([1, 2, 3]). Expected type 'Cursor'.")) + }.to log_warning a_string_including("Argument 'after' on Field 'components' has an invalid value", "[1, 2, 3]") + + broken_cursor = results["edges"][0]["node"]["components"][case_correctly "page_info"][case_correctly "end_cursor"] + "-broken" + expect { + response = query_widgets_and_components_including_page_info( + widget_args: {first: 1, order_by: [:amount_cents_ASC]}, + component_args: {first: 1, after: broken_cursor, order_by: [:name_ASC]}, + expect_errors: true + ) + expect(response["errors"]).to contain_exactly(a_hash_including("message" => "Argument 'after' on Field 'components' has an invalid value (#{broken_cursor.inspect}). Expected type 'Cursor'.")) + }.to log_warning a_string_including("Argument 'after' on Field 'components' has an invalid value", broken_cursor) + + # get next page of components (but still on the first page of widgets) + results = query_widgets_and_components_including_page_info( + widget_args: {first: 1, order_by: [:amount_cents_ASC]}, + component_args: {first: 1, after: results["edges"][0]["node"]["components"][case_correctly "page_info"][case_correctly "end_cursor"], order_by: [:name_ASC]} + ) + + expect_to_have_routed_to_shards_with("main", + ["widgets_rollover__*", nil], + # `Widget.components` uses an `out` foreign key and routes on `id` + ["components", widget1.fetch(case_correctly(:component_ids)).sort.join(",")]) + + expect(results).to match_single_widget_and_component_result( + widget1, component2, + widgets_have_next_page: true, widgets_have_previous_page: false, widget_count: 2, + components_have_next_page: true, components_have_previous_page: true, component_count: 3 + ) + + # get next page of widgets + results = query_widgets_and_components_including_page_info( + widget_args: {first: 1, after: results[case_correctly "page_info"][case_correctly "end_cursor"], order_by: [:amount_cents_ASC]}, + component_args: {first: 1, order_by: [:name_ASC]} + ) + + expect_to_have_routed_to_shards_with("main", + ["widgets_rollover__*", nil], + # `Widget.components` uses an `out` foreign key and routes on `id` + ["components", widget2.fetch(case_correctly(:component_ids)).sort.join(",")]) + + expect(results).to match_single_widget_and_component_result( + widget2, component3, + widgets_have_next_page: false, widgets_have_previous_page: true, widget_count: 2, + components_have_next_page: false, components_have_previous_page: false, component_count: 1 + ) + + results = query_widget_pagination_info(widget_args: {first: 1}) + expect_to_have_routed_to_shards_with("main", ["widgets_rollover__*", nil]) + + expect(results).to match( + case_correctly("total_edge_count") => 2, + case_correctly("page_info") => { + case_correctly("has_next_page") => true, + case_correctly("has_previous_page") => false, + case_correctly("start_cursor") => /\w+/, + case_correctly("end_cursor") => /\w+/ + } + ) + end + + def case_correctly(string_or_sym) + return super(string_or_sym.to_s).to_sym if string_or_sym.is_a?(Symbol) + super + end + + def query_all_relationship_levels_from_widgets(component_args: {}, part_args: {}) + call_graphql_query(<<~QUERY).dig("data", "widgets") + query { + widgets { + edges { + node { + name + components#{graphql_args(component_args)} { + edges { + node { + name + parts#{graphql_args(part_args)} { + edges { + node { + ... on MechanicalPart { + name + manufacturer { + name + address { + full_address + } + } + } + ... on ElectricalPart { + name + manufacturer { + name + address { + full_address + } + } + } + } + } + } + } + } + } + } + } + } + } + QUERY + end + + def query_components_and_dollar_widgets(component_args: {}) + call_graphql_query(<<~QUERY).dig("data", "components") + query { + components#{graphql_args(component_args)} { + total_edge_count + nodes { + name + dollar_widget { + name + cost { + amount_cents + } + } + } + } + } + QUERY + end + + def query_widgets_and_components_including_page_info(component_args: {}, widget_args: {}, expect_errors: false) + response = call_graphql_query(<<~QUERY, allow_errors: expect_errors) + query { + widgets#{graphql_args(widget_args)} { + page_info { + has_next_page + has_previous_page + start_cursor + end_cursor + } + total_edge_count + edges { + cursor + node { + name + components#{graphql_args(component_args)} { + page_info { + has_next_page + has_previous_page + start_cursor + end_cursor + } + total_edge_count + edges { + cursor + node { + name + } + } + } + } + } + } + } + QUERY + + return response if expect_errors + response.dig("data", "widgets") + end + + def query_all_relationship_levels_from_addresses(component_args: {}, part_args: {}, address_args: {}) + call_graphql_query(<<~QUERY).dig("data", "addresses") + query { + addresses#{graphql_args(address_args)} { + edges { + node { + full_address + manufacturer { + name + manufactured_parts#{graphql_args(part_args)} { + edges { + node { + ... on MechanicalPart { + name + components#{graphql_args(component_args)} { + edges { + node { + name + widget { + name + } + } + } + } + } + ... on ElectricalPart { + name + components#{graphql_args(component_args)} { + edges { + node { + name + widget { + name + } + } + } + } + } + } + } + } + } + } + } + } + } + QUERY + end + end + end + + context "with team data indexed" do + with_both_casing_forms do + let(:sponsor1) { build(:sponsor, name: "business1") } + let(:sponsor2) { build(:sponsor, name: "business2") } + + let(:team1) { build(:team, current_name: "team1", sponsors: [sponsor1, sponsor2]) } + let(:team2) { build(:team, current_name: "team2", sponsors: [sponsor2]) } + before do + index_records(sponsor1, sponsor2, team1, team2) + end + + it "supports in relationships from fields nested in lists" do + sponsors_args = {filter: {name: {equal_to_any_of: %w[business1]}}} + results = query_sponsors(sponsor_args: sponsors_args) + + expect(results).to match( + case_correctly("nodes") => [ + { + case_correctly("name") => "business1", + case_correctly("affiliated_teams_from_object") => { + case_correctly("nodes") => [ + { + case_correctly("current_name") => "team1" + } + ] + }, + case_correctly("affiliated_teams_from_nested") => { + case_correctly("nodes") => [ + { + case_correctly("current_name") => "team1" + } + ] + } + } + ] + ) + end + end + end + def query_widget_pagination_info(widget_args: {}) + call_graphql_query(<<~QUERY).dig("data", "widgets") + query { + widgets#{graphql_args(widget_args)} { + total_edge_count + page_info { + has_next_page + has_previous_page + start_cursor + end_cursor + } + } + } + QUERY + end + + def query_sponsors(sponsor_args: {}) + call_graphql_query(<<~QUERY).dig("data", "sponsors") + query { + sponsors#{graphql_args(sponsor_args)} { + nodes { + name + affiliated_teams_from_object { + nodes { + current_name + } + } + affiliated_teams_from_nested { + nodes { + current_name + } + } + } + } + } + QUERY + end + + def match_single_widget_and_component_result(widget, component, + widgets_have_next_page:, widgets_have_previous_page:, widget_count:, + components_have_next_page:, components_have_previous_page:, component_count:) + match( + case_correctly("total_edge_count") => widget_count, + case_correctly("page_info") => { + case_correctly("has_next_page") => widgets_have_next_page, + case_correctly("has_previous_page") => widgets_have_previous_page, + case_correctly("start_cursor") => /\w+/, + case_correctly("end_cursor") => /\w+/ + }, + "edges" => [{ + "cursor" => /\w+/, + "node" => string_hash_of(widget, :name, components: { + case_correctly("total_edge_count") => component_count, + case_correctly("page_info") => { + case_correctly("has_next_page") => components_have_next_page, + case_correctly("has_previous_page") => components_have_previous_page, + case_correctly("start_cursor") => /\w+/, + case_correctly("end_cursor") => /\w+/ + }, + "edges" => [{ + "cursor" => /\w+/, + "node" => string_hash_of(component, :name) + }] + }) + }] + ) + end + end +end diff --git a/elasticgraph-graphql/spec/acceptance/schema_evolution_spec.rb b/elasticgraph-graphql/spec/acceptance/schema_evolution_spec.rb new file mode 100644 index 00000000..75489e76 --- /dev/null +++ b/elasticgraph-graphql/spec/acceptance/schema_evolution_spec.rb @@ -0,0 +1,80 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql" +require "elastic_graph/indexer" +require "elastic_graph/schema_definition/rake_tasks" +require "support/graphql" + +module ElasticGraph + RSpec.describe "Querying an evolving schema", :uses_datastore, :factories, :capture_logs, :in_temp_dir, :rake_task do + include GraphQLSupport + let(:path_to_schema) { "config/schema.rb" } + + before do + ::FileUtils.mkdir_p "config" + end + + it "treats a new list field as having a count of `0` on documents that were indexed before the field was defined" do + dump_schema_artifacts(json_schema_version: 1) + boot(Indexer).processor.process([build_upsert_event(:team, id: "t1", owners: [])], refresh_indices: true) + + dump_schema_artifacts(json_schema_version: 2, team_extras: <<~EOS) + t.field 'owners', '[String!]!' do |f| + f.mapping type: "object" + end + EOS + + data = call_graphql_query(<<~QUERY, gql: boot(GraphQL)).fetch("data") + query { + teams(filter: {owners: {count: {lt: 1}}}) { + nodes { id } + } + } + QUERY + + expect(data).to eq({"teams" => {"nodes" => [{"id" => "t1"}]}}) + end + + def dump_schema_artifacts(json_schema_version:, team_extras: "") + # This is a pared down schema definition of our normal test schema `Team` type. + ::File.write(path_to_schema, <<~EOS) + ElasticGraph.define_schema do |schema| + schema.json_schema_version #{json_schema_version} + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "league", "String" + t.field "formed_on", "Date" + t.field "past_names", "[String!]!" + #{team_extras} + t.index "teams" do |i| + i.route_with "league" + i.rollover :yearly, "formed_on" + end + end + end + EOS + + run_rake "schema_artifacts:dump" do |output| + SchemaDefinition::RakeTasks.new( + schema_element_name_form: :snake_case, + index_document_sizes: true, + path_to_schema: path_to_schema, + schema_artifacts_directory: "config/schema/artifacts", + enforce_json_schema_version: true, + output: output + ) + end + end + + def boot(klass) + klass.from_yaml_file(CommonSpecHelpers.test_settings_file) + end + end +end diff --git a/elasticgraph-graphql/spec/acceptance/search_spec.rb b/elasticgraph-graphql/spec/acceptance/search_spec.rb new file mode 100644 index 00000000..7fd60ebb --- /dev/null +++ b/elasticgraph-graphql/spec/acceptance/search_spec.rb @@ -0,0 +1,1087 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "elasticgraph_graphql_acceptance_support" + +module ElasticGraph + RSpec.describe "ElasticGraph::GraphQL--search" do + include_context "ElasticGraph GraphQL acceptance support" + + with_both_casing_forms do + it "indexes and queries (with filter support) records", :expect_search_routing, :expect_index_exclusions do + # Index widgets created at different months so that they go to different rollover indices, e.g. widgets_rollover__2019-06 and widgets_rollover__2019-07 + index_records( + widget1 = build( + :widget, + workspace_id: "workspace_1", + amount_cents: 100, + options: build(:widget_options, size: "SMALL", color: "BLUE"), + cost_currency: "USD", + name: "thing1", + created_at: "2019-06-02T06:00:00.000Z", + created_at_time_of_day: "06:00:00", + tags: ["abc", "def"], + fees: [{currency: "USD", amount_cents: 15}, {currency: "CAD", amount_cents: 30}] + ), + widget2 = build( + :widget, + workspace_id: "ignored_workspace_2", + amount_cents: 200, + options: build(:widget_options, size: "SMALL", color: "RED"), + cost_currency: "USD", + name: "thing2", + created_at: "2019-07-02T12:00:00.000Z", + created_at_time_of_day: "12:00:00.1", + tags: ["ghi", "jkl"], + fees: [{currency: "USD", amount_cents: 25}, {currency: "CAD", amount_cents: 40}] + ), + widget3 = build( + :widget, + workspace_id: "workspace_3", + amount_cents: 300, + cost_currency: "USD", + name: nil, # expected to be nil by some queries below + options: build(:widget_options, size: "MEDIUM", color: "RED"), + created_at: "2019-08-02T18:00:00.000Z", + created_at_time_of_day: "18:00:00.123", + tags: ["mno", "pqr"], + fees: [{currency: "USD", amount_cents: 35}, {currency: "CAD", amount_cents: 50}] + ) + ) + + unfiltered_widget_currencies = list_widget_currencies + expect(unfiltered_widget_currencies).to match([{ + "id" => "USD", + case_correctly("widget_names") => { + case_correctly("page_info") => { + case_correctly("end_cursor") => /\w+/, + case_correctly("start_cursor") => /\w+/, + case_correctly("has_next_page") => false, + case_correctly("has_previous_page") => false + }, + case_correctly("total_edge_count") => 2, + "edges" => [ + {"node" => "thing1", "cursor" => /\w+/}, + {"node" => "thing2", "cursor" => /\w+/} + ] + }, + case_correctly("widget_options") => { + "colors" => [enum_value("BLUE"), enum_value("RED")], + "sizes" => [enum_value("MEDIUM"), enum_value("SMALL")] + }, + case_correctly("widget_tags") => ["abc", "def", "ghi", "jkl", "mno", "pqr"], + case_correctly("widget_fee_currencies") => ["CAD", "USD"] + }]) + + expect(list_widget_currencies(filter: {id: {equal_to_any_of: ["USD"]}})).to eq(unfiltered_widget_currencies) + expect_to_have_routed_to_shards_with("main", ["widget_currencies_rollover__*", nil]) + + expect(list_widget_currencies(filter: {primary_continent: {equal_to_any_of: ["North America"]}})).to eq(unfiltered_widget_currencies) + expect_to_have_routed_to_shards_with("main", ["widget_currencies_rollover__*", "North America"]) + + filter = { + options: { + any_of: [ + {size: {equal_to_any_of: [enum_value(:MEDIUM)]}}, + { + size: {equal_to_any_of: [enum_value(:SMALL)]}, + color: {equal_to_any_of: [enum_value(:BLUE)]} + } + ] + } + } + widgets = list_widgets_with(<<~EOS, order_by: [:amount_cents_ASC], filter: filter) + #{case_correctly("amount_cents")} + tags + fees { + currency + #{case_correctly("amount_cents")} + } + EOS + + expect(widgets).to match([ + string_hash_of(widget1, :id, :amount_cents, :tags, :fees), + string_hash_of(widget3, :id, :amount_cents, :tags, :fees) + ]) + + widgets = list_widgets_with(:amount_cents, + filter: {id: {equal_to_any_of: [widget1.fetch(:id), widget2.fetch(:id), "", " ", "\n"]}}, # empty strings should be ignored. + order_by: [:amount_cents_ASC]) + + expect(widgets).to match([ + string_hash_of(widget1, :id, :amount_cents), + string_hash_of(widget2, :id, :amount_cents) + ]) + + widgets = list_widgets_with(:amount_cents, + filter: {amount_cents: {gt: 150}}, + order_by: [:amount_cents_ASC]) + + expect(widgets).to match([ + string_hash_of(widget2, :id, :amount_cents), + string_hash_of(widget3, :id, :amount_cents) + ]) + + # Verify that we can fetch, filter and sort by a graphql-only field which is an alias for a child field. + # (`Widget.size` is an alias for the indexed `Widget.options.size` field). + widgets = list_widgets_with(:size, order_by: [:size_DESC, :amount_cents_ASC]) + expect(widgets).to match([ + {"id" => widget1.fetch(:id), "size" => enum_value("SMALL")}, + {"id" => widget2.fetch(:id), "size" => enum_value("SMALL")}, + {"id" => widget3.fetch(:id), "size" => enum_value("MEDIUM")} + ]) + widgets = list_widgets_with(:size, order_by: [:size_ASC, :amount_cents_ASC]) + expect(widgets).to match([ + {"id" => widget3.fetch(:id), "size" => enum_value("MEDIUM")}, + {"id" => widget1.fetch(:id), "size" => enum_value("SMALL")}, + {"id" => widget2.fetch(:id), "size" => enum_value("SMALL")} + ]) + widgets = list_widgets_with(:size, filter: {size: {equal_to_any_of: [enum_value(:MEDIUM)]}}) + expect(widgets).to match([{"id" => widget3.fetch(:id), "size" => enum_value("MEDIUM")}]) + + # Verify that we can order by a nullable field and paginate over it. + widgets, page_info = list_widgets_and_page_info_with(:name, order_by: [:name_ASC], first: 1) + expect(widgets).to match([{"id" => widget3.fetch(:id), "name" => nil}]) # nil sorts first + widgets, page_info = list_widgets_and_page_info_with(:name, order_by: [:name_ASC], first: 1, after: page_info.dig(case_correctly("end_cursor"))) + expect(widgets).to match([string_hash_of(widget1, :id, :name)]) + widgets, page_info = list_widgets_and_page_info_with(:name, order_by: [:name_ASC], first: 1, after: page_info.dig(case_correctly("end_cursor"))) + expect(widgets).to match([string_hash_of(widget2, :id, :name)]) + expect(page_info).to include(case_correctly("has_next_page") => false) + + # Verify we can use the `not` filter operator correctly + widgets = list_widgets_with(:amount_cents, + filter: {options: {not: {color: {equal_to_any_of: [enum_value(:BLUE)]}}}}, + order_by: [:amount_cents_ASC]) + + expect(widgets).to match([ + string_hash_of(widget2, :id, :amount_cents), + string_hash_of(widget3, :id, :amount_cents) + ]) + + widgets = list_widgets_with(:amount_cents, + filter: {options: {not: {color: {equal_to_any_of: [nil]}}}}, + order_by: [:amount_cents_ASC]) + + expect(widgets).to match([ + string_hash_of(widget1, :id, :amount_cents), + string_hash_of(widget2, :id, :amount_cents), + string_hash_of(widget3, :id, :amount_cents) + ]) + + widgets = list_widgets_with(:amount_cents, + filter: {not: {amount_cents: {gt: 150}}}, + order_by: [:amount_cents_ASC]) + + expect(widgets).to match([ + string_hash_of(widget1, :id, :amount_cents) + ]) + + # Verify that we can filter on `DateTime` fields (and that `nil` DateTime filter values are ignored) + widgets = list_widgets_with(:created_at, + filter: {created_at: {gte: "2019-07-02T12:00:00Z", lt: nil}}, + order_by: [:created_at_ASC]) + expect_to_have_excluded_indices("main", [index_definition_name_for("widgets_rollover__before_2019")]) + + expect(widgets).to match([ + string_hash_of(widget2, :id, :created_at), + string_hash_of(widget3, :id, :created_at) + ]) + + # Verify the boundaries of `DateTime` field filtering. + widgets = list_widgets_with(:created_at, + filter: {created_at: {gte: "0001-01-01T00:00:00Z", lt: "9999-12-31T23:59:59.999Z"}}, + order_by: [:created_at_ASC]) + + expect(widgets).to match([ + string_hash_of(widget1, :id, :created_at), + string_hash_of(widget2, :id, :created_at), + string_hash_of(widget3, :id, :created_at) + ]) + + expected_msg = "DateTime: must be formatted as an ISO8601 DateTime string with a 4 digit year" + expect { + error = query_widgets_with(:created_at, + filter: {created_at: {lt: "10000-01-01T00:00:00Z"}}, + allow_errors: true).dig("errors", 0, "message") + expect(error).to include(expected_msg) + }.to log_warning(a_string_including(expected_msg)) + + # Verify that we can filter on `DateTime` time-of-day + widgets = list_widgets_with(:created_at, + filter: {created_at: {time_of_day: {gte: "02:00:00", lt: "13:00:00", time_zone: "America/Los_Angeles"}}}, + order_by: [:created_at_ASC]) + expect(widgets).to match([ + string_hash_of(widget2, :id, created_at: "2019-07-02T12:00:00.000Z"), # 2 am Pacific + string_hash_of(widget3, :id, created_at: "2019-08-02T18:00:00.000Z") # 10 am pacific + ]) + + # Demonstrate that `time_of_day.time_zone` defaults to UTC when not provided. + widgets = list_widgets_with(:created_at, + filter: {created_at: {time_of_day: {gte: "02:00:00", lt: "13:00:00"}}}, + order_by: [:created_at_ASC]) + expect(widgets).to match([ + string_hash_of(widget1, :id, created_at: "2019-06-02T06:00:00.000Z"), + string_hash_of(widget2, :id, created_at: "2019-07-02T12:00:00.000Z") + ]) + + # Verify that we can filter on `Date` fields (and that `nil` Date filter values are ignored) + widgets = list_widgets_with(:created_on, + filter: {created_on: {lte: "2019-07-02", gte: nil}}, + order_by: [:created_on_ASC]) + + expect(widgets).to match([ + string_hash_of(widget1, :id, :created_on), + string_hash_of(widget2, :id, :created_on) + ]) + + # Verify that a datetime filter that excludes all possible records returns nothing (and avoids + # querying the datastore entirely, since it is not needed). + expect { + widgets = list_widgets_with(:created_at, + # Make an empty time set with our `created_at` filter that can't match any widgets. + filter: {created_at: {gte: "2019-07-02T12:00:00Z", lt: "2019-07-02T12:00:00Z"}}, + order_by: [:created_at_ASC]) + expect(widgets).to eq [] + }.to make_no_datastore_calls("main") + + # Verify that a filter on the shard routing field that excludes all possible values returns + # nothing (and avoids querying the datastore entirely, since it is not needed). + expect { + widgets = list_widgets_with(:created_at, + filter: {workspace_id: {equal_to_any_of: []}}, + order_by: [:created_at_ASC]) + expect(widgets).to eq [] + }.to make_no_datastore_calls("main") + + # Verify that we can filter on `LocalTime` fields + widgets = list_widgets_with(:created_at_time_of_day, + filter: {created_at_time_of_day: {gte: "10:00:00"}}, + order_by: [:created_at_time_of_day_ASC]) + expect(widgets).to match([ + string_hash_of(widget2, :id, :created_at_time_of_day), + string_hash_of(widget3, :id, :created_at_time_of_day) + ]) + + widgets = list_widgets_with(:created_at_time_of_day, + filter: {created_at_time_of_day: {gt: "05:00:00", lt: "15:00:00"}}, + order_by: [:created_at_time_of_day_ASC]) + expect(widgets).to match([ + string_hash_of(widget1, :id, :created_at_time_of_day), + string_hash_of(widget2, :id, :created_at_time_of_day) + ]) + + # Demonstrate that filter field name translation works (`amount_cents2` + # is a GraphQL alias for the `amount_cents` field in the index). + widgets = list_widgets_with(:amount_cents2, + filter: {amount_cents2: {gt: 150}}, + order_by: [:amount_cents2_ASC]) + + expect(widgets).to match([ + string_hash_of(widget2, :id, amount_cents2: widget2.fetch(case_correctly("amount_cents").to_sym)), + string_hash_of(widget3, :id, amount_cents2: widget3.fetch(case_correctly("amount_cents").to_sym)) + ]) + + widgets = list_widgets_with(:amount_cents, order_by: [:cost_amount_cents_DESC]) + + expect(widgets).to match([ + string_hash_of(widget3, :id, :amount_cents), + string_hash_of(widget2, :id, :amount_cents), + string_hash_of(widget1, :id, :amount_cents) + ]) + + # Test `equal_to_any_of` with only `[nil]` + widgets = list_widgets_with(:amount_cents, + filter: {name: {equal_to_any_of: [nil]}}) + + expect(widgets).to match([ + string_hash_of(widget3, :id, :amount_cents) + ]) + + # Test `equal_to_any_of` with only `[nil]` in an `any_of` + widgets = list_widgets_with(:amount_cents, + filter: {any_of: [name: {equal_to_any_of: [nil]}]}) + + expect(widgets).to match([ + string_hash_of(widget3, :id, :amount_cents) + ]) + + # Test `equal_to_any_of` with `[nil, other_value]` + widgets = list_widgets_with(:amount_cents, + filter: {name: {equal_to_any_of: [nil, "thing2", "", " ", "\n"]}}, # empty strings should be ignored., + order_by: [:amount_cents_DESC]) + + expect(widgets).to match([ + string_hash_of(widget3, :id, :amount_cents), + string_hash_of(widget2, :id, :amount_cents) + ]) + + # Test `not` with `equal_to_any_of` with only `[nil]` + widgets = list_widgets_with(:amount_cents, + filter: {not: {name: {equal_to_any_of: [nil]}}}, + order_by: [:amount_cents_DESC]) + + expect(widgets).to match([ + string_hash_of(widget2, :id, :amount_cents), + string_hash_of(widget1, :id, :amount_cents) + ]) + + # Test `not` with `equal_to_any_of` with `[nil, other_value]` + widgets = list_widgets_with(:amount_cents, + filter: {not: {name: {equal_to_any_of: [nil, "thing1"]}}}) + + expect(widgets).to match([ + string_hash_of(widget2, :id, :amount_cents) + ]) + + # Test that a filter param set to 'nil' is accepted, and is treated + # the same as that filter param being omitted. + widgets = list_widgets_with(:amount_cents, + filter: {id: {equal_to_any_of: nil}}, + order_by: [:amount_cents_ASC]) + + expect(widgets).to match([ + string_hash_of(widget1, :id, :amount_cents), + string_hash_of(widget2, :id, :amount_cents), + string_hash_of(widget3, :id, :amount_cents) + ]) + + # The negation of an ignored filter is an ignored filter: `{not: {equal_to_any_of: nil}}` + # evaluates to {not: {}} which will be ignored, therefore the filter will match all documents. + widgets = list_widgets_with(:amount_cents, + filter: {id: {not: {equal_to_any_of: nil}}}, + order_by: [:amount_cents_ASC]) + + expect(widgets).to match([ + string_hash_of(widget1, :id, :amount_cents), + string_hash_of(widget2, :id, :amount_cents), + string_hash_of(widget3, :id, :amount_cents) + ]) + + # Test that sorting by the same field twice in different directions doesn't fail. + # (The extra sort should be effectively ignored). + widgets = list_widgets_with(:amount_cents, order_by: [:amount_cents_DESC, :amount_cents_ASC]) + + expect(widgets).to match([ + string_hash_of(widget3, :id, :amount_cents), + string_hash_of(widget2, :id, :amount_cents), + string_hash_of(widget1, :id, :amount_cents) + ]) + + widgets = list_widgets_via_widgets_or_addresses(filter: {id: {equal_to_any_of: [widget1.fetch(:id)]}}) + expect(widgets).to contain_exactly({"id" => widget1.fetch(:id)}) + + widgets = list_widgets_via_widgets_or_addresses(filter: {id: {not: {equal_to_any_of: [widget1.fetch(:id)]}}}) + expect(widgets).to contain_exactly({"id" => widget2.fetch(:id)}, {"id" => widget3.fetch(:id)}) + + # Test that we can query for widgets with ignored routing values + widgets = list_widgets_with(:workspace_id, + filter: {"workspace_id" => {"equal_to_any_of" => ["workspace_1", "ignored_workspace_2"]}}, + order_by: [:amount_cents_ASC]) + expect(widgets).to match([ + string_hash_of(widget1, :id, :workspace_id), + string_hash_of(widget2, :id, :workspace_id) + ]) + + widgets = list_widgets_with(:workspace_id, + filter: {"workspace_id" => {"not" => {"equal_to_any_of" => ["workspace_1", "ignored_workspace_2"]}}}, + order_by: [:amount_cents_ASC]) + expect(widgets).to match([ + string_hash_of(widget3, :id, :workspace_id) + ]) + + widgets = list_widgets_by_nodes_with(nil, allow_errors: false, first: 1) + expect(widgets).to match([ + string_hash_of(widget3, :id) + ]) + + unfiltered_widget_currencies = list_widget_currencies_by_nodes(first: 1) + expect(unfiltered_widget_currencies).to match([{ + "id" => "USD", + case_correctly("widget_names") => { + case_correctly("page_info") => { + case_correctly("end_cursor") => /\w+/, + case_correctly("start_cursor") => /\w+/, + case_correctly("has_next_page") => false, + case_correctly("has_previous_page") => false + }, + case_correctly("total_edge_count") => 2, + "nodes" => ["thing1", "thing2"] + }, + case_correctly("widget_options") => { + "colors" => [enum_value("BLUE"), enum_value("RED")], + "sizes" => [enum_value("MEDIUM"), enum_value("SMALL")] + }, + case_correctly("widget_tags") => ["abc", "def", "ghi", "jkl", "mno", "pqr"], + case_correctly("widget_fee_currencies") => ["CAD", "USD"] + }]) + + full_text_search_results = list_widgets_with(:name, filter: {"name_text" => {matches: "thing1"}}, order_by: [:name_ASC]) + expect(full_text_search_results).to match([ + string_hash_of(widget1, :id, :name) + ]) + + full_text_query_search_results = list_widgets_with(:name, filter: {"name_text" => {matches_query: {query: "thing1"}}}, order_by: [:name_ASC]) + expect(full_text_query_search_results).to match([ + string_hash_of(widget1, :id, :name), + string_hash_of(widget2, :id, :name) + ]) + + # Try passing an explicit `nil` for the `allowed_edits_per_term` parameter; it should get a GraphQL validation error instead of a runtime Exception. + response = query_widgets_with(:name, allow_errors: true, filter: {"name_text" => {matches_query: {query: "thing1", allowed_edits_per_term: nil}}}, order_by: [:name_ASC]) + expect(response["errors"].size).to eq(1) + expect(response.dig("errors", 0)).to include("message" => a_string_including("Argument '#{case_correctly "allowed_edits_per_term"}'", "has an invalid value (null).")) + + full_text_query_no_fuzziness_search_results = list_widgets_with(:name, filter: {"name_text" => {matches_query: {query: "thing1", allowed_edits_per_term: :NONE}}}, order_by: [:name_ASC]) + expect(full_text_query_no_fuzziness_search_results).to match([ + string_hash_of(widget1, :id, :name) + ]) + + phrase_search_results = list_widgets_with(:name, filter: {"name_text" => {matches_phrase: {phrase: "thin"}}}, order_by: [:name_ASC]) + expect(phrase_search_results).to match([ + string_hash_of(widget1, :id, :name), + string_hash_of(widget2, :id, :name) + ]) + end + + it "supports fetching interface fields" do + index_into( + graphql, + build(:widget, name: "w1", inventor: build(:person, name: "Bob", nationality: "Ukrainian")), + build(:widget, name: "w2", inventor: build(:company, name: "Clippy", stock_ticker: "CLIP")), + build(:component, name: "c1", created_at: "2021-01-01T12:30:00Z"), + build(:electrical_part, name: "e1"), + build(:mechanical_part, name: "m1"), + build(:manufacturer, name: "m2") + ) + + results = call_graphql_query(<<~EOS).dig("data", case_correctly("named_entities"), "edges").map { |e| e["node"] } + query { + named_entities(order_by: [name_ASC]) { + edges { + node { + name + + ... on Widget { + named_inventor { + name + + ... on Person { + nationality + } + + ... on Company { + stock_ticker + } + } + } + + ... on Component { + created_at + } + } + } + } + } + EOS + + expect(results).to eq [ + {"name" => "c1", case_correctly("created_at") => "2021-01-01T12:30:00.000Z"}, + {"name" => "e1"}, + {"name" => "m1"}, + {"name" => "m2"}, + {"name" => "w1", case_correctly("named_inventor") => {"name" => "Bob", "nationality" => "Ukrainian"}}, + {"name" => "w2", case_correctly("named_inventor") => {"name" => "Clippy", case_correctly("stock_ticker") => "CLIP"}} + ] + end + + describe "`list` filtering behavior" do + it "supports filtering on scalar lists, nested object lists, and embedded object lists" do + index_records( + build( + :team, + id: "t1", + details: build(:team_details, count: 5), + past_names: ["Pilots", "Pink Sox"], + won_championships_at: [], + forbes_valuations: [200_000_000, 5], + seasons: [build(:team_season, record: build(:team_record, wins: 50, losses: 12))], + current_players: [ + build(:player, name: "Babe Truth", nicknames: ["The Truth"], seasons: [ + build(:player_season, awards: ["MVP", "Rookie of the Year", "Cy Young"], games_played: 160), + build(:player_season, awards: ["Gold Glove"], games_played: 120) + ]) + ] + ), + build( + :team, + id: "t2", + details: build(:team_details, count: 15), + past_names: ["Pink Sox"], + won_championships_at: ["2013-11-27T02:30:00Z", "2013-11-27T22:30:00Z"], + forbes_valuations: [], + seasons: [ + build(:team_season, record: build(:team_record, wins: 100, losses: 12)), + build(:team_season, record: build(:team_record, wins: 3, losses: 60)) + ], + current_players: [ + build(:player, name: "Babe Truth", nicknames: ["The Babe", "Bambino"], seasons: [ + build(:player_season, awards: ["Silver Slugger"], games_played: 100) + ]), + build(:player, name: "Johnny Rocket", nicknames: ["The Rocket"], seasons: []) + ] + ), + build( + :team, + id: "t3", + details: build(:team_details, count: 4), + past_names: ["Pilots"], + won_championships_at: ["2003-10-27T19:30:00Z"], + forbes_valuations: [0, 50_000_000, 100_000_000], + seasons: [build(:team_season, record: build(:team_record, wins: 50, losses: 12))], + current_players: [ + build(:player, name: "Ichiro", nicknames: ["Bambino"], seasons: [ + build(:player_season, awards: ["MVP"], games_played: 50, year: nil), + build(:player_season, awards: ["RoY"], games_played: 90, year: nil) + ]), + build(:player, name: "Babe Truth", nicknames: ["The Wizard"], seasons: [ + build(:player_season, awards: ["Gold Glove"], games_played: 150) + ]) + ] + ), + build( + :team, + details: build(:team_details, count: 12), + id: "t4", + past_names: [], + won_championships_at: ["2005-10-27T12:30:00Z"], + forbes_valuations: [42], + seasons: [build(:team_season, record: build(:team_record, wins: 50, losses: 12))], + current_players: [] + ) + ) + + # Verify `any_satisfy: {...}` with all null predicates on a list-of-scalars field. + results = query_teams_with(filter: {past_names: {any_satisfy: {equal_to_any_of: nil}}}) + expect(results).to eq [{"id" => "t1"}, {"id" => "t2"}, {"id" => "t3"}, {"id" => "t4"}] + + # Verify `any_satisfy: {...}` with all null predicates on a nested field. + results = query_teams_with(filter: {current_players_nested: {any_satisfy: {name: {equal_to_any_of: nil}}}}) + expect(results).to eq [{"id" => "t1"}, {"id" => "t2"}, {"id" => "t3"}, {"id" => "t4"}] + + # Verify `any_satisfy: {...}` on a list-of-scalars field. + results = query_teams_with(filter: {past_names: {any_satisfy: {equal_to_any_of: ["Pilots", "Other"]}}}) + # t1 and t3 both have Pilots as a past name. + expect(results).to eq [{"id" => "t1"}, {"id" => "t3"}] + + # Verify `any_satisfy: {...}` on a list-of-numbers field with range operators. + results = query_teams_with(filter: {forbes_valuations: {any_satisfy: {gt: 50_000}}}) + # t1 and t3 both have a valuation > 50,000. + expect(results).to eq [{"id" => "t1"}, {"id" => "t3"}] + results = query_teams_with(filter: {forbes_valuations: {any_satisfy: {gt: 1, lt: 100}}}) + # t1 and t3 both have a valuation in the exclusive range 1 to 100. + expect(results).to eq [{"id" => "t1"}, {"id" => "t4"}] + + # Verify `not: {any_satisfy: ...}` on a list-of-scalars field. + results = query_teams_with(filter: {past_names: {not: {any_satisfy: {equal_to_any_of: ["Pilots", "Other"]}}}}) + # t2 matches because it does not have Pilots or Other in its list of names. + # t4 matches because its list of names is empty, and therefore does not have Pilots or Other in its list of names. + expect(results).to eq [{"id" => "t2"}, {"id" => "t4"}] + + # Verify `any_satisfy: {time_of_day: ...}` on a list-of-timestamps field. + results = query_teams_with(filter: {won_championships_at: {any_satisfy: {time_of_day: {gt: "15:00:00"}}}}) + expect(results).to eq [{"id" => "t2"}, {"id" => "t3"}] + + # Verify `any_satisfy: {any_of: [...]}` on a list-of-scalars field. + results = query_teams_with(filter: {forbes_valuations: {any_satisfy: {any_of: []}}}) + expect(results).to eq [] + results = query_teams_with(filter: {forbes_valuations: {any_satisfy: {any_of: [{gt: 50_000}]}}}) + # t1 and t3 both have a valuation > 50,000. + expect(results).to eq [{"id" => "t1"}, {"id" => "t3"}] + results = query_teams_with(filter: {forbes_valuations: {any_satisfy: {any_of: [{gt: 150_000_000}, {lt: 5}]}}}) + # t1 has 200_000_000; t3 has 0 + expect(results).to eq [{"id" => "t1"}, {"id" => "t3"}] + + # Verify we can use the `any_satisfy` filter operator on a list-of-nested objects correctly. + # Also, verify that the sub-objects are considered independently. Team t3 has a player with the name + # "Babe Truth" and a player with the nickname "Bambino", but they aren't the same player so it should + # not match. + results = query_teams_with(filter: {current_players_nested: {any_satisfy: { + name: {equal_to_any_of: ["Babe Truth"]}, + nicknames: {any_satisfy: {equal_to_any_of: ["Bambino"]}} + }}}) + expect(results).to eq [{"id" => "t2"}] + + # Verify we can use `not` on a single field within an `any_satisfy`. + results = query_teams_with(filter: {current_players_nested: {any_satisfy: { + name: {equal_to_any_of: ["Babe Truth"]}, + nicknames: {not: {any_satisfy: {equal_to_any_of: ["Bambino"]}}} + }}}) + expect(results).to eq [{"id" => "t1"}, {"id" => "t3"}] + + # Verify we can use `not` directly under `any_satisfy` on a nested field. + results = query_teams_with(filter: {current_players_nested: {any_satisfy: {not: { + name: {equal_to_any_of: ["Babe Truth"]}, + nicknames: {any_satisfy: {equal_to_any_of: ["Bambino"]}} + }}}}) + expect(results).to eq [{"id" => "t1"}, {"id" => "t2"}, {"id" => "t3"}] + + # Verify we can use `any_of` directly under `any_satisfy` on a nested field. + results = query_teams_with(filter: {current_players_nested: {any_satisfy: {any_of: [ + {name: {equal_to_any_of: ["Johnny Rocket"]}}, + {nicknames: {any_satisfy: {equal_to_any_of: ["The Truth"]}}} + ]}}}) + expect(results).to eq [{"id" => "t1"}, {"id" => "t2"}] + + # Verify `count` filtering on a root list-of-scalars field + results = query_teams_with(filter: {past_names: {count: {gt: 1}}}) + # t1 has 2 past_names. + expect(results).to eq [{"id" => "t1"}] + + # Verify `count` nil filtering on a root list-of-scalars field + results = query_teams_with(filter: {past_names: {count: {gt: nil}}}) + expect(results).to eq [{"id" => "t1"}, {"id" => "t2"}, {"id" => "t3"}, {"id" => "t4"}] + + # Verify `count` filtering on a list-of-nested field + results = query_teams_with(filter: {current_players_nested: {count: {gt: 1}}}) + # t2 and t3 have 2 players each. + expect(results).to eq [{"id" => "t2"}, {"id" => "t3"}] + + # Verify `count` nil filtering on a list-of-nested field + results = query_teams_with(filter: {current_players_nested: {count: {gt: nil}}}) + expect(results).to eq [{"id" => "t1"}, {"id" => "t2"}, {"id" => "t3"}, {"id" => "t4"}] + + # Verify `count` filtering on a list-of-object field + results = query_teams_with(filter: {current_players_object: {count: {gt: 1}}}) + # t2 and t3 have 2 players each. + expect(results).to eq [{"id" => "t2"}, {"id" => "t3"}] + + # Verify `count` on a scalar field under a list-of-object field + results = query_teams_with(filter: {current_players_object: {name: {count: {gt: 1}}}}) + # teams t2 and t3 have 2 players, each with a name. + expect(results).to eq [{"id" => "t2"}, {"id" => "t3"}] + + # Verify `count` on a scalar field under a list-of-object field does not count `nil` values as part of the field's total count + results = query_teams_with(filter: {current_players_object: {seasons_object: {year: {count: {lt: 2}}}}}) + # t1 has 1 player, with 2 seasons, each of which has `year` set, so it is not included. + # t2 has 2 players--one with 1 season, one with zero seasons--so it is included. + # t3 has 2 players with 3 total seasons; however on two of those, the year is `nil`, so the collection of player season years has only one element, so it's included. + # t4 has no players, so the count is effectively 0, so it is included. + expect(results).to eq [{"id" => "t2"}, {"id" => "t3"}, {"id" => "t4"}] + + # Verify that a `count` schema field (distinct from the `count` operator on a list field) can still be filtered on + results = query_teams_with(filter: {details: {count: {gt: 10}}}) + # t2 and t4 have details.count of 12 and 15 + expect(results).to eq [{"id" => "t2"}, {"id" => "t4"}] + results = query_teams_with(filter: {seasons_object: {count: {any_satisfy: {gt: 0}}}}) + # All 4 teams have `count` values on their `seasons`. + expect(results).to eq [{"id" => "t1"}, {"id" => "t2"}, {"id" => "t3"}, {"id" => "t4"}] + results = query_teams_with(filter: {seasons_object: {count: {count: {gt: 0}}}}) + # All 4 teams have `count` values on their `seasons`. + expect(results).to eq [{"id" => "t1"}, {"id" => "t2"}, {"id" => "t3"}, {"id" => "t4"}] + + # Verify the `count` of a subfield of an empty list is treated as 0 + results = query_teams_with(filter: {current_players_object: {nicknames: {count: {lt: 1}}}}) + # t4 has no players, so the count of `current_players_object.nicknames` is 0. + expect(results).to eq [{"id" => "t4"}] + + # Verify `any_satisfy` and `count` on a list-of-object-of-objects field + results = query_teams_with(filter: {current_players_object: {seasons_object: {awards: {any_satisfy: {equal_to_any_of: ["MVP"]}}}}}) + # t1 and t3 both have players who have won an MVP + expect(results).to eq [{"id" => "t1"}, {"id" => "t3"}] + results = query_teams_with(filter: {current_players_object: {seasons_object: {awards: {not: {any_satisfy: {equal_to_any_of: ["MVP"]}}}}}}) + # t2 and t4 both have no current players who have won an MVP award + expect(results).to eq [{"id" => "t2"}, {"id" => "t4"}] + results = query_teams_with(filter: {current_players_object: {seasons_object: {games_played: {any_satisfy: {any_of: [{gt: 150}, {lt: 100}]}}}}}) + # t1 and t3 have players with > 150 games played or < 100 games played + expect(results).to eq [{"id" => "t1"}, {"id" => "t3"}] + results = query_teams_with(filter: {current_players_object: {seasons_object: {awards: {count: {gte: 3}}}}}) + # t1 has 4 awards and t3 has 3 awards + expect(results).to eq [{"id" => "t1"}, {"id" => "t3"}] + + # Verify `any_satisfy` and `count` on a list-of-object-of-nested field + results = query_teams_with(filter: {current_players_object: {seasons_nested: {any_satisfy: {awards: {any_satisfy: {equal_to_any_of: ["MVP"]}}}}}}) + # t1 and t3 both have players who have won an MVP + expect(results).to eq [{"id" => "t1"}, {"id" => "t3"}] + results = query_teams_with(filter: {current_players_object: {seasons_nested: {any_satisfy: {awards: {not: {any_satisfy: {equal_to_any_of: ["MVP"]}}}}}}}) + # t1, t2 and t3 all have a player with a season in which they did not win MVP + expect(results).to eq [{"id" => "t1"}, {"id" => "t2"}, {"id" => "t3"}] + results = query_teams_with(filter: {current_players_object: {seasons_nested: {any_satisfy: {games_played: {any_of: [{gt: 150}, {lt: 100}]}}}}}) + # t1 and t3 have players with > 150 games played or < 100 games played + expect(results).to eq [{"id" => "t1"}, {"id" => "t3"}] + results = query_teams_with(filter: {current_players_object: {seasons_nested: {any_satisfy: {awards: {count: {gt: 2}}}}}}) + # t1 has a a player with a season with more than 2 awards + expect(results).to eq [{"id" => "t1"}] + + # Verify `any_satisfy` and `count` on a list-of-nested-of-nested field + results = query_teams_with(filter: {current_players_nested: {any_satisfy: {seasons_nested: {any_satisfy: {awards: {any_satisfy: {equal_to_any_of: ["MVP"]}}}}}}}) + # t1 and t3 both have players who have won an MVP + expect(results).to eq [{"id" => "t1"}, {"id" => "t3"}] + results = query_teams_with(filter: {current_players_nested: {any_satisfy: {seasons_nested: {any_satisfy: {awards: {not: {any_satisfy: {equal_to_any_of: ["MVP"]}}}}}}}}) + # t1, t2 and t3 all have a player with a season in which they did not win MVP + expect(results).to eq [{"id" => "t1"}, {"id" => "t2"}, {"id" => "t3"}] + results = query_teams_with(filter: {current_players_nested: {any_satisfy: {seasons_nested: {any_satisfy: {games_played: {any_of: [{gt: 150}, {lt: 100}]}}}}}}) + # t1 and t3 have players with > 150 games played or < 100 games played + expect(results).to eq [{"id" => "t1"}, {"id" => "t3"}] + results = query_teams_with(filter: {current_players_nested: {any_satisfy: {seasons_nested: {any_satisfy: {awards: {count: {gt: 2}}}}}}}) + # t1 has a a player with a season with more than 2 awards + expect(results).to eq [{"id" => "t1"}] + + # Verify `any_satisfy` and `count` on a list-of-nested-of-objects field + results = query_teams_with(filter: {current_players_nested: {any_satisfy: {seasons_object: {awards: {any_satisfy: {equal_to_any_of: ["MVP"]}}}}}}) + # t1 and t3 both have players who have won an MVP + expect(results).to eq [{"id" => "t1"}, {"id" => "t3"}] + results = query_teams_with(filter: {current_players_nested: {any_satisfy: {seasons_object: {awards: {not: {any_satisfy: {equal_to_any_of: ["MVP"]}}}}}}}) + # t2 and t3 both have a player who has never won the MVP award + expect(results).to eq [{"id" => "t2"}, {"id" => "t3"}] + results = query_teams_with(filter: {current_players_nested: {any_satisfy: {seasons_object: {games_played: {any_satisfy: {any_of: [{gt: 150}, {lt: 100}]}}}}}}) + # t1 and t3 have players with > 150 games played or < 100 games played + expect(results).to eq [{"id" => "t1"}, {"id" => "t3"}] + results = query_teams_with(filter: {current_players_nested: {any_satisfy: {seasons_object: {awards: {count: {gt: 2}}}}}}) + # t1 has a a player with a season with more than 2 awards + expect(results).to eq [{"id" => "t1"}] + + # Verify `all_of: [...]` with 2 `any_satisfy` sub-clauses. + results = query_teams_with(filter: {seasons_nested: {all_of: [ + # Note: we chose these fields (`record`, `wins`) because they use an alternate `name_in_index`, + # and we want to verify that field name translation under `all_of` works correctly. + {any_satisfy: {record: {wins: {gt: 95}}}}, + {any_satisfy: {record: {wins: {lt: 10}}}} + ]}}) + # Only t2 has a season with more than 95 wins and a season with less than 10 wins + expect(results).to eq [{"id" => "t2"}] + + # Verify `all_of: [{not: null}]` works as expected. + results = query_teams_with(filter: {seasons_nested: {all_of: [{not: nil}]}}) + # All teams should be returned since the `nil` part of the filter expression is pruned. + expect(results).to eq [{"id" => "t1"}, {"id" => "t2"}, {"id" => "t3"}, {"id" => "t4"}] + + # Verify `all_of: [{not: null}]` works as expected. + results = query_teams_with(filter: {seasons_nested: {all_of: [{all_of: nil}]}}) + # All teams should be returned since the `nil` part of the filter expression is pruned. + expect(results).to eq [{"id" => "t1"}, {"id" => "t2"}, {"id" => "t3"}, {"id" => "t4"}] + + # Verify `all_of: [{}]` works as expected. + results = query_teams_with(filter: {seasons_nested: {all_of: [{}]}}) + # All teams should be returned since the `nil` part of the filter expression is pruned. + expect(results).to eq [{"id" => "t1"}, {"id" => "t2"}, {"id" => "t3"}, {"id" => "t4"}] + end + + it "statically (through the schema) disallows some filter features that do not work well with `any_satisfy`" do + # `any_satisfy: {not: ...}` disallowed because we cannot implement it to work as a client would expect. + expect_error_from( + {past_names: {any_satisfy: {not: {equal_to_any_of: ["Pilots", "Other"]}}}}, + "InputObject '#{apply_derived_type_customizations("StringListElementFilterInput")}' doesn't accept argument 'not'" + ) + + # `any_satisfy: {equal_to_any_of: [null]}` disallowed because we cannot implement it to work as a client would expect + # That looks like it would match a list field with a `null` element, but the `exists` operator we use for `equal_to_any_of: [null]` + # doesn't support that: + # https://www.elastic.co/guide/en/elasticsearch/reference/8.9/query-dsl-exists-query.html + expect_error_from( + {past_names: {any_satisfy: {equal_to_any_of: [nil]}}}, + "Argument '#{case_correctly "equal_to_any_of"}' on InputObject '#{apply_derived_type_customizations("StringListElementFilterInput")}' has an invalid value ([null])" + ) + + # `any_satisfy: {[multiple predicates that translate to distinct clauses]}` disallowed because the datastore does not require them to all be true of the same value to match a document + expect_error_from( + {forbes_valuations: {any_satisfy: {gt: 100, equal_to_any_of: [5]}}}, + "`#{case_correctly "any_satisfy"}: {#{case_correctly "equal_to_any_of"}: [5], gt: 100}` is not supported because it produces multiple filtering clauses under `#{case_correctly "any_satisfy"}`" + ) + end + + def query_teams_with(expect_errors: false, **query_args) + results = call_graphql_query(<<~QUERY, allow_errors: expect_errors) + query { + teams#{graphql_args(query_args)} { + nodes { + id + } + } + } + QUERY + + expect_errors ? results.to_h : results.dig("data", "teams", "nodes") + end + + def expect_error_from(filter, *error_snippets) + expect { + results = query_teams_with(filter: filter, expect_errors: true) + expect(results.dig("errors", 0, "message")).to include(*error_snippets) + }.to log_warning(a_string_including(*error_snippets)) + end + end + + context "when multiple sources flow into the same index" do + it "automatically excludes documents that have not received data from their primary `__self` source" do + index_records( + build(:widget, id: "w1", name: "Pre-Thingy", component_ids: ["c23", "c47"]), + build(:component, id: "c23", name: "C") + ) + + nodes = call_graphql_query(<<~QUERY).dig("data", "components", "nodes") + query { + components { nodes { id } } + } + QUERY + + expect(nodes).to eq [{"id" => "c23"}] + end + end + + context "with nested fields" do + let(:widget1) { build(:widget, options: build(:widget_options, color: "RED"), inventor: build(:person)) } + let(:widget2) { build(:widget, options: build(:widget_options, color: "BLUE"), inventor: build(:company)) } + + before do + index_records(widget1, widget2) + end + + it "loads nested fields, with filtering support" do + expected_widget1 = string_hash_of(widget1, :id, :name, :amount_cents, + options: string_hash_of(widget1[:options], :size), + inventor: string_hash_of(widget1[:inventor], :name, :nationality)) + + expected_widget2 = string_hash_of(widget2, :id, :name, :amount_cents, + options: string_hash_of(widget2[:options], :size), + inventor: string_hash_of(widget2[:inventor], :name, :stock_ticker)) + + expect(list_widgets_with_options_and_inventor).to contain_exactly(expected_widget1, expected_widget2) + + expect(list_widgets_with_options_and_inventor( + filter: {options: {color: {equal_to_any_of: [enum_value(:RED)]}}} + )).to contain_exactly(expected_widget1) + + expect(list_widgets_with_options_and_inventor( + filter: {not: {options: {color: {equal_to_any_of: [enum_value(:RED)]}}}} + )).to contain_exactly(expected_widget2) + + # equal_to_any_of set to 'nil' should not cause any filtering on that value. + expect(list_widgets_with_options_and_inventor( + filter: {options: {color: {equal_to_any_of: nil}}} + )).to contain_exactly(expected_widget1, expected_widget2) + + expect(list_widgets_with_options_and_inventor( + filter: {options: {color: {not: {equal_to_any_of: nil}}}} + )).to contain_exactly(expected_widget1, expected_widget2) + + # not set to 'nil' should not cause any filtering on that value. + expect(list_widgets_with_options_and_inventor( + filter: {options: {color: {not: nil}}} + )).to contain_exactly(expected_widget1, expected_widget2) + + # On type unions you can filter on a subfield that is present on all subtypes... + expect(list_widgets_with_options_and_inventor( + filter: {inventor: {name: {equal_to_any_of: [widget1.fetch(:inventor).fetch(:name)]}}} + )).to contain_exactly(expected_widget1) + + expect(list_widgets_with_options_and_inventor( + filter: {inventor: {name: {not: {equal_to_any_of: [widget1.fetch(:inventor).fetch(:name)]}}}} + )).to contain_exactly(expected_widget2) + + stock_ticker_key = case_correctly("stock_ticker").to_sym + # ...or on a subfield that is present on only some subtypes... + expect(list_widgets_with_options_and_inventor( + filter: {inventor: {stock_ticker_key => {equal_to_any_of: [widget2.fetch(:inventor).fetch(stock_ticker_key)]}}} + )).to contain_exactly(expected_widget2) + + expect(list_widgets_with_options_and_inventor( + filter: {inventor: {stock_ticker_key => {not: {equal_to_any_of: [widget2.fetch(:inventor).fetch(stock_ticker_key)]}}}} + )).to contain_exactly(expected_widget1) + + # On interfaces you can filter on a subfield that is present on all subtypes... + expect(list_widgets_with_options_and_inventor( + filter: {named_inventor: {name: {equal_to_any_of: [widget1.fetch(:inventor).fetch(:name)]}}} + )).to contain_exactly(expected_widget1) + + expect(list_widgets_with_options_and_inventor( + filter: {named_inventor: {name: {not: {equal_to_any_of: [widget1.fetch(:inventor).fetch(:name)]}}}} + )).to contain_exactly(expected_widget2) + + stock_ticker_key = case_correctly("stock_ticker").to_sym + # ...or on a subfield that is present on only some subtypes... + expect(list_widgets_with_options_and_inventor( + filter: {named_inventor: {stock_ticker_key => {equal_to_any_of: [widget2.fetch(:inventor).fetch(stock_ticker_key)]}}} + )).to contain_exactly(expected_widget2) + + expect(list_widgets_with_options_and_inventor( + filter: {named_inventor: {stock_ticker_key => {not: {equal_to_any_of: [widget2.fetch(:inventor).fetch(stock_ticker_key)]}}}} + )).to contain_exactly(expected_widget1) + + # ...or on `__typename`. Well, you could if the GraphQL spec allowed input fields + # named `__typename`, but it does not (see http://spec.graphql.org/June2018/#sec-Input-Objects) + # so we do not yet support it. + # expect(list_widgets_with_options_and_inventor( + # filter: { inventor: { __typename: { equal_to_any_of: ["Company"] } } } + # )).to contain_exactly(expected_widget2) + end + end + + def list_widgets_with(fieldname, **query_args) + query_widgets_with(fieldname, **query_args).dig("data", "widgets", "edges").map { |we| we.fetch("node") } + end + + def list_widgets_and_page_info_with(fieldname, **query_args) + response = query_widgets_with(fieldname, **query_args).dig("data", "widgets") + + page_info = response.fetch(case_correctly("page_info")) + nodes = response.fetch("edges").map { |we| we.fetch("node") } + + [nodes, page_info] + end + + def list_widgets_via_widgets_or_addresses(**query_args) + call_graphql_query(<<~QUERY).dig("data", case_correctly("widgets_or_addresses"), "edges").map { |we| we.fetch("node") } + query { + widgets_or_addresses#{graphql_args(query_args)} { + edges { + node { + ...on Widget { + id + } + } + } + } + } + QUERY + end + + def list_widgets_by_nodes_with(fieldname, **query_args) + query_widgets_by_nodes_with(fieldname, **query_args).dig("data", "widgets", "nodes") + end + + def query_widgets_by_nodes_with(fieldname, allow_errors: false, **query_args) + call_graphql_query(<<~QUERY, allow_errors: allow_errors) + query { + widgets#{graphql_args(query_args)} { + nodes { + id + #{fieldname} + } + } + } + QUERY + end + + def query_widgets_with(fieldname, allow_errors: false, **query_args) + call_graphql_query(<<~QUERY, allow_errors: allow_errors) + query { + widgets#{graphql_args(query_args)} { + page_info { + end_cursor + has_next_page + } + edges { + node { + id + #{fieldname} + } + } + } + } + QUERY + end + + def list_widget_currencies(**query_args) + call_graphql_query(<<~QUERY).dig("data", case_correctly("widget_currencies"), "edges").map { |we| we.fetch("node") } + query { + widget_currencies#{graphql_args(query_args)} { + edges { + node { + id + widget_names { + page_info { + start_cursor + end_cursor + has_next_page + has_previous_page + } + + total_edge_count + + edges { + node + cursor + } + } + widget_tags + widget_fee_currencies + widget_options { + sizes + colors + } + } + } + } + } + QUERY + end + + def list_widget_currencies_by_nodes(**query_args) + call_graphql_query(<<~QUERY).dig("data", case_correctly("widget_currencies"), "nodes") + query { + widget_currencies#{graphql_args(query_args)} { + nodes { + id + widget_names { + page_info { + start_cursor + end_cursor + has_next_page + has_previous_page + } + + total_edge_count + + nodes + } + widget_tags + widget_fee_currencies + widget_options { + sizes + colors + } + } + } + } + QUERY + end + + def list_widgets_with_options_and_inventor(**widget_query_args) + call_graphql_query(<<~QUERY).dig("data", "widgets", "edges").map { |we| we.fetch("node") } + query { + widgets#{graphql_args(widget_query_args)} { + edges { + node { + id + name + amount_cents + + inventor { + ... on Person { + name + nationality + } + + ... on Company { + name + stock_ticker + } + } + + options { + size + } + } + } + } + } + QUERY + end + end + end +end diff --git a/elasticgraph-graphql/spec/acceptance/sub_aggregations_spec.rb b/elasticgraph-graphql/spec/acceptance/sub_aggregations_spec.rb new file mode 100644 index 00000000..d78be062 --- /dev/null +++ b/elasticgraph-graphql/spec/acceptance/sub_aggregations_spec.rb @@ -0,0 +1,1103 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/composite_grouping_adapter" +require "elastic_graph/graphql/aggregation/non_composite_grouping_adapter" +require_relative "elasticgraph_graphql_acceptance_support" + +module ElasticGraph + RSpec.describe "ElasticGraph::GraphQL--sub-aggregations" do + include_context "ElasticGraph GraphQL acceptance aggregation support" + + with_both_casing_forms do + shared_examples_for "sub-aggregation acceptance" do + it "returns the expected empty response when no documents have yet been indexed", :expect_search_routing do + reset_index_to_state_before_first_document_indexed + + # Note: this is technically a "root" aggregations query (not sub-aggregations), but it's another case we need to + # exercise when no documents have yet been indexed. So we're including it here. + teams_grouped_by_league = aggregate_teams_grouped_by_league + expect(teams_grouped_by_league).to eq [] + + player_count, season_count, season_player_count, season_player_season_count = aggregate_sibling_and_deeply_nested_counts + expect(player_count).to eq(count_detail_of(0)) + expect(season_count).to eq(count_detail_of(0)) + expect(season_player_count).to eq(count_detail_of(0)) + expect(season_player_season_count).to eq(count_detail_of(0)) + + team_seasons_nodes = aggregate_season_counts_grouped_by("year", "note") + expect(indexed_counts_from(team_seasons_nodes)).to eq({}) + end + + it "supports arbitrarily deeply nested sub-aggregations", :expect_search_routing, :expect_index_exclusions do + index_data + test_ungrouped_sub_aggregations + test_grouped_sub_aggregations + end + + def reset_index_to_state_before_first_document_indexed + main_datastore_client.delete_indices("team*") + admin.cluster_configurator.configure_cluster(StringIO.new) + end + + def index_data + index_records( + build( + :team, + current_name: "Yankees", + current_players: [build(:player, name: "Bob")], + formed_on: "1903-01-01", + seasons: [ + build(:team_season, year: 2020, record: build(:team_record, wins: 50, losses: 12, first_win_on: "2020-03-02", last_win_on: "2020-11-12"), notes: ["pandemic", "shortened"], started_at: "2020-01-15T12:02:20Z", players: [ + build(:player, name: "Ted", seasons: [build(:player_season, year: 2018, games_played: 20), build(:player_season, year: 2018, games_played: 30)]), + build(:player, name: "Dave", seasons: [build(:player_season, year: 2022, games_played: 10)]) + ]) + ] + ), + build( + :team, + current_name: "Dodgers", + formed_on: "1883-01-01", + current_players: [build(:player, name: "Kat", seasons: []), build(:player, name: "Sue", seasons: [])], + seasons: [ + build(:team_season, year: 2020, record: build(:team_record, wins: 30, losses: 22, first_win_on: "2020-05-02", last_win_on: "2020-12-12"), notes: ["pandemic", "covid"], started_at: "2020-01-15T12:02:20Z", players: []), + build(:team_season, year: 2021, record: build(:team_record, wins: nil, losses: 22, first_win_on: nil, last_win_on: "2021-12-12"), notes: [], started_at: "2021-02-16T13:03:30Z", players: []), + build(:team_season, year: 2022, record: build(:team_record, wins: 40, losses: 15, first_win_on: "2022-06-02", last_win_on: "2022-08-12"), notes: [], started_at: "2022-03-17T14:04:40Z", players: []), + build(:team_season, year: 2023, record: build(:team_record, wins: 50, losses: nil, first_win_on: "2023-01-06", last_win_on: nil), notes: [], started_at: "2023-04-18T12:02:59Z", players: []) + ] + ), + build( + :team, + current_name: "Red Sox", + formed_on: "1901-01-01", + current_players: [build(:player, name: "Ed", seasons: []), build(:player, name: "Ty", seasons: [])], + seasons: [ + build(:team_season, year: 2019, record: build(:team_record, wins: 40, losses: 7, first_win_on: "2019-04-02", last_win_on: "2019-07-12"), notes: ["old rules"], started_at: "2019-01-15T12:02:20Z", players: []) + ] + ), + build( + :team, + current_name: "Magenta Sox", + formed_on: "1921-01-01", + current_players: [], + seasons: [] + ) + ) + + # Ensure the cached `known_related_query_rollover_indices` is up-to-date with any new indices created by indexing these docs. + pre_cache_index_state(graphql) + end + + def test_ungrouped_sub_aggregations + # Demonstrate a successful empty result when we filter to no shard routing values. + seasons, seasons_player_seasons = count_seasons_and_season_player_seasons(team_aggregations: {filter: {league: {equal_to_any_of: []}}}) + expect(seasons).to eq(count_detail_of(0)) + expect(seasons_player_seasons).to eq(count_detail_of(0)) + + # Demonstrate a successful empty result when our rollover field filter excludes all values. + seasons, seasons_player_seasons = count_seasons_and_season_player_seasons(team_aggregations: {filter: {formed_on: {equal_to_any_of: []}}}) + expect(seasons).to eq(count_detail_of(0)) + expect(seasons_player_seasons).to eq(count_detail_of(0)) + + # Demonstrate a successful empty result when our rollover field filter only includes values outside the ranges of our indices. + seasons, seasons_player_seasons = count_seasons_and_season_player_seasons(team_aggregations: {filter: {formed_on: {gt: "7890-01-01"}}}) + expect(seasons).to eq(count_detail_of(0)) + expect(seasons_player_seasons).to eq(count_detail_of(0)) + + player_count, season_count, season_player_count, season_player_season_count = aggregate_sibling_and_deeply_nested_counts + expect(player_count).to eq(count_detail_of(5)) + expect(season_count).to eq(count_detail_of(6)) + expect(season_player_count).to eq(count_detail_of(2)) + expect(season_player_season_count).to eq(count_detail_of(3)) + + player_count, season_count = aggregate_count_under_extra_object_layer + expect(player_count).to eq(count_detail_of(5)) + expect(season_count).to eq(count_detail_of(6)) + + seasons_before_2021, seasons_before_2021_player_seasons = count_seasons_and_season_player_seasons( + seasons: {filter: {year: {lt: 2021}}} + ) + expect(seasons_before_2021).to eq(count_detail_of(3)) + expect(seasons_before_2021_player_seasons).to eq(count_detail_of(3)) + + seasons_after_2019, seasons_after_2019_player_seasons_before_2020 = count_seasons_and_season_player_seasons( + seasons: {filter: {year: {gt: 2019}}}, + season_player_seasons: {filter: {year: {lt: 2020}}} + ) + expect(seasons_after_2019).to eq(count_detail_of(5)) + expect(seasons_after_2019_player_seasons_before_2020).to eq(count_detail_of(2)) + + seasons_after_2019, seasons_after_2019_player_seasons_before_2020 = count_seasons_and_season_player_seasons( + seasons: {filter: {year: {gt: nil}}}, + season_player_seasons: {filter: {year: {lt: nil}}} + ) + expect(seasons_after_2019).to eq(count_detail_of(6)) + expect(seasons_after_2019_player_seasons_before_2020).to eq(count_detail_of(3)) + + # Test that aliases work as expected with sub-aggregations + verify_sub_aggregations_with_aliases + + # Test `first: positive-value` arg on an ungrouped sub-aggregation + seasons, season_player_seasons = count_seasons_and_season_player_seasons( + seasons: {first: 1}, + season_player_seasons: {first: 1} + ) + expect(seasons).to eq(count_detail_of(6)) + expect(season_player_seasons).to eq(count_detail_of(3)) + + # Test `first: 0` arg on an ungrouped sub-aggregation + seasons, season_player_seasons = count_seasons_and_season_player_seasons( + seasons: {first: 0}, + season_player_seasons: {first: 0} + ) + expect(seasons).to eq(nil) + expect(season_player_seasons).to eq(nil) + end + + def test_grouped_sub_aggregations + # Demonstrate a successful empty result when we filter to no documents. + team_seasons_nodes = aggregate_season_counts_grouped_by("year", team_aggregations_args: {filter: {current_name: {equal_to_any_of: [nil]}}}) + expect(team_seasons_nodes).to eq([]) + + # Demonstrate a successful empty result when we filter to no shard routing values. + team_seasons_nodes = aggregate_season_counts_grouped_by("year", team_aggregations_args: {filter: {league: {equal_to_any_of: []}}}) + expect(team_seasons_nodes).to eq([]) + + # Demonstrate a successful empty result when our rollover field filter excludes all values. + team_seasons_nodes = aggregate_season_counts_grouped_by("year", team_aggregations_args: {filter: {formed_on: {equal_to_any_of: []}}}) + expect(team_seasons_nodes).to eq([]) + + # Demonstrate a successful empty result when our rollover field filter only includes values outside the ranges of our indices. + team_seasons_nodes = aggregate_season_counts_grouped_by("year", team_aggregations_args: {filter: {formed_on: {gt: "7890-01-01"}}}) + expect(team_seasons_nodes).to eq([]) + + # Test a simple sub-aggregation grouping of one `terms` field + team_seasons_nodes = aggregate_season_counts_grouped_by("year") + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {"year" => 2019} => count_detail_of(1), + {"year" => 2020} => count_detail_of(2), + {"year" => 2021} => count_detail_of(1), + {"year" => 2022} => count_detail_of(1), + {"year" => 2023} => count_detail_of(1) + }) + + # Test what happens if all grouped by fields are excluded via a directive. + team_seasons_nodes = aggregate_season_counts_grouped_by("year @include(if: false)", "note @skip(if: true)") + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {} => count_detail_of(6) + }) + + # Test applying a filter on the rollover `Date` field. + team_seasons_nodes = aggregate_season_counts_grouped_by("year", team_aggregations_args: {filter: {formed_on: {gt: "1900-01-01"}}}) + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {"year" => 2019} => count_detail_of(1), + {"year" => 2020} => count_detail_of(1) + }) + expect_to_have_excluded_indices("main", [index_definition_name_for("teams_rollover__1883")]) + + # Test a sub-aggregation grouping of two `terms` field + team_seasons_nodes = aggregate_season_counts_grouped_by("year", "note") + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {"year" => 2020, "note" => "covid"} => count_detail_of(1), + {"year" => 2019, "note" => "old rules"} => count_detail_of(1), + {"year" => 2020, "note" => "pandemic"} => count_detail_of(2), + {"year" => 2020, "note" => "shortened"} => count_detail_of(1), + {"year" => 2021, "note" => nil} => count_detail_of(1), + {"year" => 2022, "note" => nil} => count_detail_of(1), + {"year" => 2023, "note" => nil} => count_detail_of(1) + }) + + verify_filtered_sub_aggregations_with_grouped_by + + # Group on some date fields (with no term grouping). + team_seasons_nodes = aggregate_season_counts_grouped_by("record { last_win_on { as_date(truncation_unit: MONTH) }, first_win_on { as_date(truncation_unit: MONTH) }}") + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {"record" => {case_correctly("first_win_on") => {case_correctly("as_date") => "2019-04-01"}, case_correctly("last_win_on") => {case_correctly("as_date") => "2019-07-01"}}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on") => {case_correctly("as_date") => "2020-03-01"}, case_correctly("last_win_on") => {case_correctly("as_date") => "2020-11-01"}}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on") => {case_correctly("as_date") => "2020-05-01"}, case_correctly("last_win_on") => {case_correctly("as_date") => "2020-12-01"}}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on") => {case_correctly("as_date") => "2022-06-01"}, case_correctly("last_win_on") => {case_correctly("as_date") => "2022-08-01"}}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on") => {case_correctly("as_date") => "2023-01-01"}, case_correctly("last_win_on") => {case_correctly("as_date") => nil}}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on") => {case_correctly("as_date") => nil}, case_correctly("last_win_on") => {case_correctly("as_date") => "2021-12-01"}}} => count_detail_of(1) + }) + + # Group on sub-fields of an object, some date fields, and some fields with alternate `name_in_index`. + team_seasons_nodes = aggregate_season_counts_grouped_by("record { wins, last_win_on { as_date(truncation_unit: MONTH) }, first_win_on { as_date(truncation_unit: MONTH) }}") + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {"record" => {case_correctly("first_win_on") => {case_correctly("as_date") => "2019-04-01"}, case_correctly("last_win_on") => {case_correctly("as_date") => "2019-07-01"}, "wins" => 40}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on") => {case_correctly("as_date") => "2020-03-01"}, case_correctly("last_win_on") => {case_correctly("as_date") => "2020-11-01"}, "wins" => 50}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on") => {case_correctly("as_date") => "2020-05-01"}, case_correctly("last_win_on") => {case_correctly("as_date") => "2020-12-01"}, "wins" => 30}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on") => {case_correctly("as_date") => "2022-06-01"}, case_correctly("last_win_on") => {case_correctly("as_date") => "2022-08-01"}, "wins" => 40}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on") => {case_correctly("as_date") => "2023-01-01"}, case_correctly("last_win_on") => {case_correctly("as_date") => nil}, "wins" => 50}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on") => {case_correctly("as_date") => nil}, case_correctly("last_win_on") => {case_correctly("as_date") => "2021-12-01"}, "wins" => nil}} => count_detail_of(1) + }) + + # Test using groupings on sibling and deeply nested sub-aggs. + verify_aggregate_sibling_and_deeply_nested_grouped_counts_and_aggregated_values + + # Test `first: positive-value` arg on a grouped sub-aggregation (single term) + team_seasons_nodes = aggregate_season_counts_grouped_by("year", first: 2) + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {"year" => 2020} => count_detail_of(2), + {"year" => 2019} => count_detail_of(1) + }) + + # Test `first: positive-value` arg on a grouped sub-aggregation (multiple terms) + team_seasons_nodes = aggregate_season_counts_grouped_by("year", "note", first: 2) + expect(indexed_counts_from(team_seasons_nodes)).to eq(first_two_counts_grouped_by_year_and_note) + + # Test `first: positive-value` arg on a grouped sub-aggregation (single term + multiple date fields) + team_seasons_nodes = aggregate_season_counts_grouped_by("record { wins, last_win_on { as_date(truncation_unit: MONTH) }, first_win_on { as_date(truncation_unit: MONTH) }}", first: 2) + expect(indexed_counts_from(team_seasons_nodes)).to eq(first_two_counts_grouped_by_wins_last_win_on_month_first_win_on_month) + + # Test `first: 0` arg on a grouped sub-aggregation (single term) + team_seasons_nodes = aggregate_season_counts_grouped_by("year", first: 0) + expect(team_seasons_nodes).to eq [] + + # Test `first: 0` arg on a grouped sub-aggregation (multiple terms) + team_seasons_nodes = aggregate_season_counts_grouped_by("year", "note", first: 0) + expect(team_seasons_nodes).to eq [] + + # Test `first: 0` arg on a grouped sub-aggregation (single term + multiple date fields) + team_seasons_nodes = aggregate_season_counts_grouped_by("record { wins, last_win_on { as_date(truncation_unit: MONTH) }, first_win_on { as_date(truncation_unit: MONTH) }}", first: 0) + expect(team_seasons_nodes).to eq [] + + # LEGACY Group on some date fields (with no term grouping). + team_seasons_nodes = aggregate_season_counts_grouped_by("record { last_win_on_legacy(granularity: MONTH), first_win_on_legacy(granularity: MONTH) }") + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {"record" => {case_correctly("first_win_on_legacy") => "2019-04-01", case_correctly("last_win_on_legacy") => "2019-07-01"}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on_legacy") => "2020-03-01", case_correctly("last_win_on_legacy") => "2020-11-01"}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on_legacy") => "2020-05-01", case_correctly("last_win_on_legacy") => "2020-12-01"}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on_legacy") => "2022-06-01", case_correctly("last_win_on_legacy") => "2022-08-01"}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on_legacy") => "2023-01-01", case_correctly("last_win_on_legacy") => nil}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on_legacy") => nil, case_correctly("last_win_on_legacy") => "2021-12-01"}} => count_detail_of(1) + }) + + # Group on sub-fields of an object, some date fields, and some fields with alternate `name_in_index`. + team_seasons_nodes = aggregate_season_counts_grouped_by("record { wins, last_win_on_legacy(granularity: MONTH), first_win_on_legacy(granularity: MONTH) }") + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {"record" => {case_correctly("first_win_on_legacy") => "2019-04-01", case_correctly("last_win_on_legacy") => "2019-07-01", "wins" => 40}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on_legacy") => "2020-03-01", case_correctly("last_win_on_legacy") => "2020-11-01", "wins" => 50}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on_legacy") => "2020-05-01", case_correctly("last_win_on_legacy") => "2020-12-01", "wins" => 30}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on_legacy") => "2022-06-01", case_correctly("last_win_on_legacy") => "2022-08-01", "wins" => 40}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on_legacy") => "2023-01-01", case_correctly("last_win_on_legacy") => nil, "wins" => 50}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on_legacy") => nil, case_correctly("last_win_on_legacy") => "2021-12-01", "wins" => nil}} => count_detail_of(1) + }) + + # Test using groupings on sibling and deeply nested sub-aggs. + verify_aggregate_sibling_and_deeply_nested_grouped_counts_and_aggregated_values + + # Test `first: positive-value` arg on a grouped sub-aggregation (single term) + team_seasons_nodes = aggregate_season_counts_grouped_by("year", first: 2) + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {"year" => 2020} => count_detail_of(2), + {"year" => 2019} => count_detail_of(1) + }) + + # Test `first: positive-value` arg on a grouped sub-aggregation (multiple terms) + team_seasons_nodes = aggregate_season_counts_grouped_by("year", "note", first: 2) + expect(indexed_counts_from(team_seasons_nodes)).to eq(first_two_counts_grouped_by_year_and_note) + + # Test `first: positive-value` arg on a grouped sub-aggregation (single term + multiple date fields) + team_seasons_nodes = aggregate_season_counts_grouped_by("record { wins, last_win_on_legacy(granularity: MONTH), first_win_on_legacy(granularity: MONTH) }", first: 2) + expect(indexed_counts_from(team_seasons_nodes)).to eq(legacy_first_two_counts_grouped_by_wins_last_win_on_month_first_win_on_month) + + # Test `first: 0` arg on a grouped sub-aggregation (single term) + team_seasons_nodes = aggregate_season_counts_grouped_by("year", first: 0) + expect(team_seasons_nodes).to eq [] + + # Test `first: 0` arg on a grouped sub-aggregation (multiple terms) + team_seasons_nodes = aggregate_season_counts_grouped_by("year", "note", first: 0) + expect(team_seasons_nodes).to eq [] + + # Test `first: 0` arg on a grouped sub-aggregation (single term + multiple date fields) + team_seasons_nodes = aggregate_season_counts_grouped_by("record { wins, last_win_on_legacy(granularity: MONTH), first_win_on_legacy(granularity: MONTH) }", first: 0) + expect(team_seasons_nodes).to eq [] + + # Verify date/time aggregations + # DateTime:as_date_time() + team_seasons_nodes = aggregate_season_counts_grouped_by("started_at {as_date_time(truncation_unit: DAY)}") + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {case_correctly("started_at") => {case_correctly("as_date_time") => "2019-01-15T00:00:00.000Z"}} => count_detail_of(1), + {case_correctly("started_at") => {case_correctly("as_date_time") => "2020-01-15T00:00:00.000Z"}} => count_detail_of(2), + {case_correctly("started_at") => {case_correctly("as_date_time") => "2021-02-16T00:00:00.000Z"}} => count_detail_of(1), + {case_correctly("started_at") => {case_correctly("as_date_time") => "2022-03-17T00:00:00.000Z"}} => count_detail_of(1), + {case_correctly("started_at") => {case_correctly("as_date_time") => "2023-04-18T00:00:00.000Z"}} => count_detail_of(1) + }) + + # DateTime: as_date() + team_seasons_nodes = aggregate_season_counts_grouped_by("started_at {as_date(truncation_unit: MONTH)}") + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {case_correctly("started_at") => {case_correctly("as_date") => "2019-01-01"}} => count_detail_of(1), + {case_correctly("started_at") => {case_correctly("as_date") => "2020-01-01"}} => count_detail_of(2), + {case_correctly("started_at") => {case_correctly("as_date") => "2021-02-01"}} => count_detail_of(1), + {case_correctly("started_at") => {case_correctly("as_date") => "2022-03-01"}} => count_detail_of(1), + {case_correctly("started_at") => {case_correctly("as_date") => "2023-04-01"}} => count_detail_of(1) + }) + + # DateTime: as_day_of_week() + team_seasons_nodes = aggregate_season_counts_grouped_by("started_at {as_day_of_week}") + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {case_correctly("started_at") => {case_correctly("as_day_of_week") => enum_value("TUESDAY")}} => count_detail_of(3), + {case_correctly("started_at") => {case_correctly("as_day_of_week") => enum_value("WEDNESDAY")}} => count_detail_of(2), + {case_correctly("started_at") => {case_correctly("as_day_of_week") => enum_value("THURSDAY")}} => count_detail_of(1) + }) + + # DateTime: as_time_of_day() truncated to SECOND + team_seasons_nodes = aggregate_season_counts_grouped_by("started_at {as_time_of_day(truncation_unit: SECOND)}") + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {case_correctly("started_at") => {case_correctly("as_time_of_day") => "12:02:20"}} => count_detail_of(3), + {case_correctly("started_at") => {case_correctly("as_time_of_day") => "12:02:59"}} => count_detail_of(1), + {case_correctly("started_at") => {case_correctly("as_time_of_day") => "13:03:30"}} => count_detail_of(1), + {case_correctly("started_at") => {case_correctly("as_time_of_day") => "14:04:40"}} => count_detail_of(1) + }) + + # DateTime: as_time_of_day() truncated to MINUTE + team_seasons_nodes = aggregate_season_counts_grouped_by("started_at {as_time_of_day(truncation_unit: MINUTE)}") + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {case_correctly("started_at") => {case_correctly("as_time_of_day") => "12:02:00"}} => count_detail_of(4), + {case_correctly("started_at") => {case_correctly("as_time_of_day") => "13:03:00"}} => count_detail_of(1), + {case_correctly("started_at") => {case_correctly("as_time_of_day") => "14:04:00"}} => count_detail_of(1) + }) + + # DateTime: as_time_of_day() truncated to HOUR + team_seasons_nodes = aggregate_season_counts_grouped_by("started_at {as_time_of_day(truncation_unit: HOUR)}") + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {case_correctly("started_at") => {case_correctly("as_time_of_day") => "12:00:00"}} => count_detail_of(4), + {case_correctly("started_at") => {case_correctly("as_time_of_day") => "13:00:00"}} => count_detail_of(1), + {case_correctly("started_at") => {case_correctly("as_time_of_day") => "14:00:00"}} => count_detail_of(1) + }) + + # Date: as_date() + team_seasons_nodes = aggregate_season_counts_grouped_by("record {last_win_on {as_date(truncation_unit: MONTH)}}") + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {"record" => {case_correctly("last_win_on") => {case_correctly("as_date") => "2019-07-01"}}} => count_detail_of(1), + {"record" => {case_correctly("last_win_on") => {case_correctly("as_date") => "2020-11-01"}}} => count_detail_of(1), + {"record" => {case_correctly("last_win_on") => {case_correctly("as_date") => "2020-12-01"}}} => count_detail_of(1), + {"record" => {case_correctly("last_win_on") => {case_correctly("as_date") => "2022-08-01"}}} => count_detail_of(1), + {"record" => {case_correctly("last_win_on") => {case_correctly("as_date") => nil}}} => count_detail_of(1), + {"record" => {case_correctly("last_win_on") => {case_correctly("as_date") => "2021-12-01"}}} => count_detail_of(1) + }) + + # Date: as_day_of_week() + team_seasons_nodes = aggregate_season_counts_grouped_by("record {last_win_on {as_day_of_week}}") + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {"record" => {case_correctly("last_win_on") => {case_correctly("as_day_of_week") => enum_value("THURSDAY")}}} => count_detail_of(1), + {"record" => {case_correctly("last_win_on") => {case_correctly("as_day_of_week") => enum_value("FRIDAY")}}} => count_detail_of(2), + {"record" => {case_correctly("last_win_on") => {case_correctly("as_day_of_week") => enum_value("SATURDAY")}}} => count_detail_of(1), + {"record" => {case_correctly("last_win_on") => {case_correctly("as_day_of_week") => enum_value("SUNDAY")}}} => count_detail_of(1), + {"record" => {case_correctly("last_win_on") => {case_correctly("as_day_of_week") => nil}}} => count_detail_of(1) + }) + + test_optimizable_aggregations_with_sub_aggregations + verify_2_subaggregations_on_same_field_under_different_parent_fields + + # Demonstrate that we can group subaggregations by fields that have a mixture of types with some `null` values mixed it. + # (At one point this lead to an exception). + team_seasons_nodes = aggregate_season_counts_grouped_by("note, record { losses}") + expect(team_seasons_nodes.map { |n| n.fetch(case_correctly("grouped_by")) }).to include( + {"note" => nil, "record" => {"losses" => nil}}, + {"note" => nil, "record" => {"losses" => 15}}, + {"note" => "covid", "record" => {"losses" => 22}} + ) + end + end + + context "when sub-aggregations use the `CompositeGroupingAdapter`" do + include_examples "sub-aggregation acceptance" + + def build_graphql(**options) + super( + sub_aggregation_grouping_adapter: GraphQL::Aggregation::CompositeGroupingAdapter, + **options + ) + end + + # This part of the test involves using groupings at two levels. That results in `composite` being used + # under `composite`, but Elasticsearch and OpenSearch return an error in that case (see below). Long term, + # we hope to find a solution for this. + def verify_aggregate_sibling_and_deeply_nested_grouped_counts_and_aggregated_values + error_msg = a_string_including("[composite] aggregation cannot be used with a parent aggregation of type: [CompositeAggregationFactory]") + + expect { + super + }.to raise_error(Errors::SearchFailedError, error_msg).and log_warning(error_msg) + end + + # Likewise, OpenSearch can't handle a composite agg under a filter agg, until they release this fix: + # https://github.com/opensearch-project/OpenSearch/pull/11499 + # + # ...but this works on Elasticsearch, and now works on OpenSearch 2.13+. + def verify_filtered_sub_aggregations_with_grouped_by + # :nocov: -- only one side of this conditional is covered in any given test suite run. + return super if datastore_backend == :elasticsearch + return super if ::Gem::Version.new(datastore_version) >= ::Gem::Version.new("2.13.0") + + error_msg = a_string_including("[composite] aggregation cannot be used with a parent aggregation of type: [FilterAggregatorFactory]") + + expect { + super + }.to raise_error(Errors::SearchFailedError, error_msg).and log_warning(error_msg) + # :nocov: + end + + # When the `CompositeGroupingAdapter` is used, aggregation groupings are sorted by the fields + # we group on. This results in a different first 2 from the `NonCompositeGroupingAdapter` case. + let(:first_two_counts_grouped_by_year_and_note) do + { + {"year" => 2019, "note" => "old rules"} => count_detail_of(1), + {"year" => 2020, "note" => "covid"} => count_detail_of(1) + } + end + let(:first_two_counts_grouped_by_wins_last_win_on_month_first_win_on_month) do + { + {"record" => {case_correctly("first_win_on") => {case_correctly("as_date") => nil}, case_correctly("last_win_on") => {case_correctly("as_date") => "2021-12-01"}, "wins" => nil}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on") => {case_correctly("as_date") => "2020-05-01"}, case_correctly("last_win_on") => {case_correctly("as_date") => "2020-12-01"}, "wins" => 30}} => count_detail_of(1) + } + end + let(:legacy_first_two_counts_grouped_by_wins_last_win_on_month_first_win_on_month) do + { + {"record" => {case_correctly("first_win_on_legacy") => nil, case_correctly("last_win_on_legacy") => "2021-12-01", "wins" => nil}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on_legacy") => "2020-05-01", case_correctly("last_win_on_legacy") => "2020-12-01", "wins" => 30}} => count_detail_of(1) + } + end + end + + context "when sub-aggregations use the `NonCompositeGroupingAdapter`" do + include_examples "sub-aggregation acceptance" + + def build_graphql(**options) + super( + sub_aggregation_grouping_adapter: GraphQL::Aggregation::NonCompositeGroupingAdapter, + **options + ) + end + + # When the `NonCompositeGroupingAdapter` is used, aggregation groupings are sorted desc by the + # counts. This results in a different first 2 from the `CompositeGroupingAdapter` case. + let(:first_two_counts_grouped_by_year_and_note) do + { + {"year" => 2020, "note" => "pandemic"} => count_detail_of(2), + {"year" => 2019, "note" => "old rules"} => count_detail_of(1) + } + end + let(:first_two_counts_grouped_by_wins_last_win_on_month_first_win_on_month) do + { + {"record" => {case_correctly("first_win_on") => {case_correctly("as_date") => "2019-04-01"}, case_correctly("last_win_on") => {case_correctly("as_date") => "2019-07-01"}, "wins" => 40}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on") => {case_correctly("as_date") => "2020-03-01"}, case_correctly("last_win_on") => {case_correctly("as_date") => "2020-11-01"}, "wins" => 50}} => count_detail_of(1) + } + end + let(:legacy_first_two_counts_grouped_by_wins_last_win_on_month_first_win_on_month) do + { + {"record" => {case_correctly("first_win_on_legacy") => "2019-04-01", case_correctly("last_win_on_legacy") => "2019-07-01", "wins" => 40}} => count_detail_of(1), + {"record" => {case_correctly("first_win_on_legacy") => "2020-03-01", case_correctly("last_win_on_legacy") => "2020-11-01", "wins" => 50}} => count_detail_of(1) + } + end + end + + def aggregate_sibling_and_deeply_nested_counts + response = call_graphql_query(<<~EOS) + query { + team_aggregations { + nodes { + sub_aggregations { + current_players_nested { + nodes { + count_detail { + ...count_aggregations + } + } + } + + seasons_nested { + nodes { + count_detail { + ...count_aggregations + } + + sub_aggregations { + players_nested { + nodes { + count_detail { + ...count_aggregations + } + + sub_aggregations { + seasons_nested { + nodes { + count_detail { + ...count_aggregations + } + } + } + } + } + } + } + } + } + } + } + } + } + + fragment count_aggregations on AggregationCountDetail { + approximate_value + exact_value + upper_bound + } + EOS + + team_node = get_single_aggregations_node_from(response, "team_aggregations", parent_field_name: "data") + player_node = get_single_aggregations_node_from(team_node, "current_players_nested") + season_node = get_single_aggregations_node_from(team_node, "seasons_nested") + season_player_node = get_single_aggregations_node_from(season_node, "players_nested") + season_player_season_node = get_single_aggregations_node_from(season_player_node, "seasons_nested") + + [ + player_node.fetch(case_correctly("count_detail")), + season_node.fetch(case_correctly("count_detail")), + season_player_node.fetch(case_correctly("count_detail")), + season_player_season_node.fetch(case_correctly("count_detail")) + ] + end + + def verify_filtered_sub_aggregations_with_grouped_by + team_seasons_nodes = aggregate_season_counts_grouped_by("year", filter: {year: {lt: 2021}}) + expect(indexed_counts_from(team_seasons_nodes)).to eq({ + {"year" => 2019} => count_detail_of(1), + {"year" => 2020} => count_detail_of(2) + }) + end + + def verify_2_subaggregations_on_same_field_under_different_parent_fields + team_aggs = call_graphql_query(<<~EOS).dig("data", case_correctly("team_aggregations")) + query { + team_aggregations { + nodes { + sub_aggregations { + nested_fields { + seasons { + nodes { + aggregated_values { + year { exact_min } + } + } + } + } + + nested_fields2 { + seasons { + nodes { + aggregated_values { + year { exact_min } + } + } + } + } + } + } + } + } + EOS + + expected_nested_fields_value = {"seasons" => { + "nodes" => [{ + case_correctly("aggregated_values") => { + "year" => {case_correctly("exact_min") => 2019} + } + }] + }} + + expect(team_aggs).to eq({ + "nodes" => [{ + case_correctly("sub_aggregations") => { + case_correctly("nested_fields") => expected_nested_fields_value, + case_correctly("nested_fields2") => expected_nested_fields_value + } + }] + }) + end + + def verify_aggregate_sibling_and_deeply_nested_grouped_counts_and_aggregated_values + team_aggs = call_graphql_query(<<~EOS).dig("data", case_correctly("team_aggregations")) + query { + team_aggregations { + nodes { + sub_aggregations { + current_players_nested { + nodes { + grouped_by { name } + count_detail { approximate_value } + aggregated_values { name { approximate_distinct_value_count } } + } + } + + seasons_nested { + nodes { + grouped_by { year } + count_detail { approximate_value } + aggregated_values { record { wins { approximate_avg, exact_max } } } + + sub_aggregations { + players_nested { + nodes { + grouped_by { name } + count_detail { approximate_value } + aggregated_values { name { approximate_distinct_value_count } } + + sub_aggregations { + seasons_nested { + nodes { + grouped_by { year } + count_detail { approximate_value } + aggregated_values { games_played { exact_max } } + } + } + } + } + } + } + } + } + } + } + } + } + EOS + + expect(team_aggs).to eq({ + case_correctly("nodes") => [ + { + case_correctly("sub_aggregations") => { + case_correctly("current_players_nested") => { + case_correctly("nodes") => [ + { + case_correctly("grouped_by") => {case_correctly("name") => "Bob"}, + case_correctly(case_correctly("count_detail")) => {case_correctly("approximate_value") => 1}, + case_correctly("aggregated_values") => {"name" => {case_correctly("approximate_distinct_value_count") => 1}} + }, + { + case_correctly("grouped_by") => {case_correctly("name") => "Ed"}, + case_correctly(case_correctly("count_detail")) => {case_correctly("approximate_value") => 1}, + case_correctly("aggregated_values") => {"name" => {case_correctly("approximate_distinct_value_count") => 1}} + }, + { + case_correctly("grouped_by") => {case_correctly("name") => "Kat"}, + case_correctly(case_correctly("count_detail")) => {case_correctly("approximate_value") => 1}, + case_correctly("aggregated_values") => {"name" => {case_correctly("approximate_distinct_value_count") => 1}} + }, + { + case_correctly("grouped_by") => {case_correctly("name") => "Sue"}, + case_correctly(case_correctly("count_detail")) => {case_correctly("approximate_value") => 1}, + case_correctly("aggregated_values") => {"name" => {case_correctly("approximate_distinct_value_count") => 1}} + }, + { + case_correctly("grouped_by") => {case_correctly("name") => "Ty"}, + case_correctly(case_correctly("count_detail")) => {case_correctly("approximate_value") => 1}, + case_correctly("aggregated_values") => {"name" => {case_correctly("approximate_distinct_value_count") => 1}} + } + ] + }, + case_correctly("seasons_nested") => { + case_correctly("nodes") => [ + { + case_correctly("grouped_by") => {case_correctly("year") => 2020}, + case_correctly(case_correctly("count_detail")) => {case_correctly("approximate_value") => 2}, + case_correctly("aggregated_values") => { + "record" => {"wins" => { + case_correctly("approximate_avg") => 40.0, + case_correctly("exact_max") => 50 + }} + }, + case_correctly("sub_aggregations") => { + case_correctly("players_nested") => { + case_correctly("nodes") => [ + { + case_correctly("grouped_by") => {case_correctly("name") => "Dave"}, + case_correctly(case_correctly("count_detail")) => {case_correctly("approximate_value") => 1}, + case_correctly("aggregated_values") => {"name" => {case_correctly("approximate_distinct_value_count") => 1}}, + case_correctly("sub_aggregations") => { + case_correctly("seasons_nested") => { + case_correctly("nodes") => [ + { + case_correctly("grouped_by") => {case_correctly("year") => 2022}, + case_correctly(case_correctly("count_detail")) => {case_correctly("approximate_value") => 1}, + case_correctly("aggregated_values") => {case_correctly("games_played") => {case_correctly("exact_max") => 10}} + } + ] + } + } + }, + { + case_correctly("grouped_by") => {case_correctly("name") => "Ted"}, + case_correctly(case_correctly("count_detail")) => {case_correctly("approximate_value") => 1}, + case_correctly("aggregated_values") => {"name" => {case_correctly("approximate_distinct_value_count") => 1}}, + case_correctly("sub_aggregations") => { + case_correctly("seasons_nested") => { + case_correctly("nodes") => [ + { + case_correctly("grouped_by") => {case_correctly("year") => 2018}, + case_correctly(case_correctly("count_detail")) => {case_correctly("approximate_value") => 2}, + case_correctly("aggregated_values") => {case_correctly("games_played") => {case_correctly("exact_max") => 30}} + } + ] + } + } + } + ] + } + } + }, + { + case_correctly("grouped_by") => {case_correctly("year") => 2019}, + case_correctly(case_correctly("count_detail")) => {case_correctly("approximate_value") => 1}, + case_correctly("aggregated_values") => { + "record" => {"wins" => { + case_correctly("approximate_avg") => 40.0, + case_correctly("exact_max") => 40 + }} + }, + case_correctly("sub_aggregations") => {case_correctly("players_nested") => {case_correctly("nodes") => []}} + }, + { + case_correctly("grouped_by") => {case_correctly("year") => 2021}, + case_correctly(case_correctly("count_detail")) => {case_correctly("approximate_value") => 1}, + case_correctly("aggregated_values") => { + "record" => {"wins" => { + case_correctly("approximate_avg") => nil, + case_correctly("exact_max") => nil + }} + }, + case_correctly("sub_aggregations") => {case_correctly("players_nested") => {case_correctly("nodes") => []}} + }, + { + case_correctly("grouped_by") => {case_correctly("year") => 2022}, + case_correctly(case_correctly("count_detail")) => {case_correctly("approximate_value") => 1}, + case_correctly("aggregated_values") => { + "record" => {"wins" => { + case_correctly("approximate_avg") => 40.0, + case_correctly("exact_max") => 40 + }} + }, + case_correctly("sub_aggregations") => {case_correctly("players_nested") => {case_correctly("nodes") => []}} + }, + { + case_correctly("grouped_by") => {case_correctly("year") => 2023}, + case_correctly(case_correctly("count_detail")) => {case_correctly("approximate_value") => 1}, + case_correctly("aggregated_values") => { + "record" => {"wins" => { + case_correctly("approximate_avg") => 50.0, + case_correctly("exact_max") => 50 + }} + }, + case_correctly("sub_aggregations") => {case_correctly("players_nested") => {case_correctly("nodes") => []}} + } + ] + } + } + } + ] + }) + end + + def aggregate_count_under_extra_object_layer + response = call_graphql_query(<<~EOS) + query { + team_aggregations { + nodes { + sub_aggregations { + nested_fields { + current_players { + nodes { + count_detail { + approximate_value + exact_value + upper_bound + } + } + } + + seasons { + nodes { + count_detail { + approximate_value + exact_value + upper_bound + } + } + } + } + } + } + } + } + EOS + + team_node = get_single_aggregations_node_from(response, "team_aggregations", parent_field_name: "data") + player_node = get_single_aggregations_node_from(team_node, "nested_fields", "current_players") + season_node = get_single_aggregations_node_from(team_node, "nested_fields", "seasons") + + [ + player_node.fetch(case_correctly("count_detail")), + season_node.fetch(case_correctly("count_detail")) + ] + end + + def verify_sub_aggregations_with_aliases + response = call_graphql_query(<<~EOS) + query { + teamaggs: team_aggregations { + ns: nodes { + subaggs: sub_aggregations { + before2021: seasons_nested(filter: {year: {lt: 2021}}) { + ns: nodes { + c: count_detail { val: approximate_value } + subaggs: sub_aggregations { + players: players_nested { + ns: nodes { + c: count_detail { val: approximate_value } + a: aggregated_values { + n1: name { val: approximate_distinct_value_count } + n2: name { val: approximate_distinct_value_count } + } + } + } + } + } + } + + after2021: seasons_nested(filter: {year: {gt: 2021}}) { + ns: nodes { + c: count_detail { val: approximate_value } + subaggs: sub_aggregations { + players: players_nested { + ns: nodes { + c: count_detail { val: approximate_value } + a: aggregated_values { + n1: name { val: approximate_distinct_value_count } + n2: name { val: approximate_distinct_value_count } + } + } + } + } + } + } + } + } + } + } + EOS + + expect(response.to_h).to eq({ + "data" => { + "teamaggs" => { + "ns" => [ + { + "subaggs" => { + "before2021" => { + "ns" => [ + { + "c" => {"val" => 3}, + "subaggs" => { + "players" => { + "ns" => [ + { + "c" => {"val" => 2}, + "a" => {"n1" => {"val" => 2}, "n2" => {"val" => 2}} + } + ] + } + } + } + ] + }, + "after2021" => { + "ns" => [ + { + "c" => {"val" => 2}, + "subaggs" => { + "players" => { + "ns" => [ + { + "c" => {"val" => 0}, + "a" => {"n1" => {"val" => 0}, "n2" => {"val" => 0}} + } + ] + } + } + } + ] + } + } + } + ] + } + } + }) + end + + def test_optimizable_aggregations_with_sub_aggregations + response = call_graphql_query(<<~EOS) + query { + t1: team_aggregations { + nodes { grouped_by { current_name } } + } + + t2: team_aggregations { + nodes { + sub_aggregations { + current_players_nested { + nodes { count_detail { approximate_value } } + } + } + } + } + } + EOS + + expect(response["data"]).to eq({ + "t1" => {"nodes" => [ + {case_correctly("grouped_by") => {case_correctly("current_name") => "Dodgers"}}, + {case_correctly("grouped_by") => {case_correctly("current_name") => "Magenta Sox"}}, + {case_correctly("grouped_by") => {case_correctly("current_name") => "Red Sox"}}, + {case_correctly("grouped_by") => {case_correctly("current_name") => "Yankees"}} + ]}, + "t2" => {"nodes" => [{ + case_correctly("sub_aggregations") => { + case_correctly("current_players_nested") => { + "nodes" => [ + {case_correctly("count_detail") => {case_correctly("approximate_value") => 5}} + ] + } + } + }]} + }) + end + + def count_seasons_and_season_player_seasons(seasons: {}, season_player_seasons: {}, team_aggregations: {}) + response = call_graphql_query(<<~EOS) + query { + team_aggregations#{graphql_args(team_aggregations)} { + nodes { + sub_aggregations { + seasons_nested#{graphql_args(seasons)} { + nodes { + count_detail { + approximate_value + exact_value + upper_bound + } + + sub_aggregations { + players_nested { + nodes { + sub_aggregations { + seasons_nested#{graphql_args(season_player_seasons)} { + nodes { + count_detail { + approximate_value + exact_value + upper_bound + } + } + } + } + } + } + } + } + } + } + } + } + } + EOS + + team_node = get_single_aggregations_node_from(response, "team_aggregations", parent_field_name: "data") + seasons_node = get_single_aggregations_node_from(team_node, "seasons_nested") + season_players_node = get_single_aggregations_node_from(seasons_node, "players_nested") if seasons_node + season_player_seasons_node = get_single_aggregations_node_from(season_players_node, "seasons_nested") if season_players_node + + [ + seasons_node&.fetch(case_correctly("count_detail")), + season_player_seasons_node&.fetch(case_correctly("count_detail")) + ] + end + + def aggregate_season_counts_grouped_by(*grouping_expressions, team_aggregations_args: {}, **args) + response = call_graphql_query(<<~EOS) + query { + team_aggregations#{graphql_args(team_aggregations_args)} { + nodes { + sub_aggregations { + seasons_nested#{graphql_args(args)} { + nodes { + grouped_by { + #{grouping_expressions.join("\n")} + } + + count_detail { ...count_aggregations } + } + } + } + } + } + } + + fragment count_aggregations on AggregationCountDetail { + approximate_value + exact_value + upper_bound + } + EOS + + team_node = get_single_aggregations_node_from(response, "team_aggregations", parent_field_name: "data") + get_aggregations_nodes_from(team_node, "seasons_nested") + end + + def aggregate_teams_grouped_by_league + call_graphql_query(<<~EOS).dig("data", case_correctly("team_aggregations"), "nodes") + query { + team_aggregations { + nodes { + grouped_by { league } + aggregated_values { forbes_valuations { approximate_sum } } + count + } + } + } + EOS + end + + def get_aggregations_nodes_from(response_data, *field_names, parent_field_name: "sub_aggregations") + field_names = field_names.map { |f| case_correctly(f) } + response_data.dig(case_correctly(parent_field_name), *field_names, "nodes") || [] + end + + def get_single_aggregations_node_from(response_data, *field_names, parent_field_name: "sub_aggregations") + nodes = get_aggregations_nodes_from(response_data, *field_names, parent_field_name: parent_field_name) + expect(nodes.size).to be < 2 + nodes.first + end + + def indexed_counts_from(nodes) + nodes.to_h do |node| + [node.fetch(case_correctly("grouped_by")), node.dig(case_correctly("count_detail"))] + end + end + + def count_detail_of(count) + { + case_correctly("approximate_value") => count, + case_correctly("exact_value") => count, + case_correctly("upper_bound") => count + } + end + end + end +end diff --git a/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/aggregation_pagination_spec.rb b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/aggregation_pagination_spec.rb new file mode 100644 index 00000000..a07a485d --- /dev/null +++ b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/aggregation_pagination_spec.rb @@ -0,0 +1,159 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_integration_support" +require_relative "pagination_shared_examples" +require "elastic_graph/graphql/aggregation/resolvers/relay_connection_builder" +require "support/aggregations_helpers" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "aggregation pagination" do + include_context "DatastoreQueryIntegrationSupport" + + include_examples "DatastoreQuery pagination--integration" do + include AggregationsHelpers + + let(:item1) { {"name" => "w1"} } + let(:item2) { {"name" => "w2"} } + let(:item3) { {"name" => "w3"} } + let(:item4) { {"name" => "w4"} } + let(:item5) { {"name" => "w5"} } + let(:item_with_null) { {"name" => nil} } + + before do + index_into( + graphql, + build(:widget, name: "w1", amount_cents: 10, created_at: "2022-01-01T00:00:00Z"), + build(:widget, name: "w2", amount_cents: 20, created_at: "2022-02-01T00:00:00Z"), + build(:widget, name: "w3", amount_cents: 30, created_at: "2022-03-01T00:00:00Z"), + build(:widget, name: "w2", amount_cents: 40, created_at: "2022-02-01T00:00:00Z"), + build(:widget, name: "w4", amount_cents: 50, created_at: "2022-04-01T00:00:00Z"), + build(:widget, name: "w5", amount_cents: 60, created_at: "2022-05-01T00:00:00Z") + ) + end + + def index_doc_with_null_value + index_into(graphql, build(:widget, name: nil)) + end + + it "can paginate an aggregations result that has no groupings or computations" do + paginated_search = ->(**options) { paginated_search(groupings: [], computations: [], **options) } + + items, page_info = paginated_search.call(first: 2) + expect(items.map(&:count)).to eq [6] + expect(page_info).to have_attributes(has_next_page: false, has_previous_page: false) + + items, page_info = paginated_search.call(first: 2, after: DecodedCursor::SINGLETON) + expect(items).to be_empty + expect(page_info).to have_attributes(has_next_page: false, has_previous_page: true) + + items, page_info = paginated_search.call(last: 2, before: DecodedCursor::SINGLETON) + expect(items).to be_empty + expect(page_info).to have_attributes(has_next_page: true, has_previous_page: false) + + items, page_info = paginated_search.call(after: DecodedCursor::SINGLETON, before: DecodedCursor::SINGLETON) + expect(items).to be_empty + expect(page_info).to have_attributes(has_next_page: false, has_previous_page: false) + end + + it "can paginate an aggregations result that has no groupings but has computations" do + paginated_search = ->(**options) { paginated_search(groupings: [], computations: [computation_of("amount_cents", :sum)], **options) } + + items, page_info = paginated_search.call(first: 2) + expect(items.map { |i| fetch_aggregated_values(i, "amount_cents", "sum") }).to eq [210] + expect(page_info).to have_attributes(has_next_page: false, has_previous_page: false) + + items, page_info = paginated_search.call(first: 2, after: DecodedCursor::SINGLETON) + expect(items).to be_empty + expect(page_info).to have_attributes(has_next_page: false, has_previous_page: true) + + items, page_info = paginated_search.call(last: 2, before: DecodedCursor::SINGLETON) + expect(items).to be_empty + expect(page_info).to have_attributes(has_next_page: true, has_previous_page: false) + + items, page_info = paginated_search.call(after: DecodedCursor::SINGLETON, before: DecodedCursor::SINGLETON) + expect(items).to be_empty + expect(page_info).to have_attributes(has_next_page: false, has_previous_page: false) + end + + it "can paginate an aggregations result that has a date histogram grouping" do + paginated_search = ->(**options) { paginated_search(groupings: [date_histogram_grouping_of("created_at", "month")], computations: [computation_of("amount_cents", :sum)], **options) } + + items, page_info = paginated_search.call(first: 2) + expect(items.to_h { |i| [fetch_grouped_by(i, "created_at"), fetch_aggregated_values(i, "amount_cents", "sum")] }).to eq({ + "2022-01-01T00:00:00.000Z" => 10, + "2022-02-01T00:00:00.000Z" => 60 + }) + expect(page_info).to have_attributes(has_next_page: true, has_previous_page: false) + + items, page_info = paginated_search.call(first: 2, after: items.last.cursor) + expect(items.to_h { |i| [fetch_grouped_by(i, "created_at"), fetch_aggregated_values(i, "amount_cents", "sum")] }).to eq({ + "2022-03-01T00:00:00.000Z" => 30, + "2022-04-01T00:00:00.000Z" => 50 + }) + expect(page_info).to have_attributes(has_next_page: true, has_previous_page: true) + + items, page_info = paginated_search.call(first: 2, after: items.last.cursor) + expect(items.to_h { |i| [fetch_grouped_by(i, "created_at"), fetch_aggregated_values(i, "amount_cents", "sum")] }).to eq({ + "2022-05-01T00:00:00.000Z" => 60 + }) + expect(page_info).to have_attributes(has_next_page: false, has_previous_page: true) + end + + def paginated_search(first: nil, after: nil, last: nil, before: nil, groupings: [field_term_grouping_of("name")], computations: [], filter_to: nil) + aggregation_query = aggregation_query_of( + groupings: groupings, computations: computations, + first: first, after: after, last: last, before: before, + max_page_size: graphql.config.max_page_size, + default_page_size: graphql.config.default_page_size + ) + + response = search_datastore( + document_pagination: {first: 0}, # make sure we don't ask for any documents, just aggregations. + aggregations: [aggregation_query], + total_document_count_needed: groupings.empty?, + filter: ({"name" => {"equal_to_any_of" => ids_of(filter_to)}} if filter_to) + ) + + connection = Aggregation::Resolvers::RelayConnectionBuilder.build_from_search_response( + query: aggregation_query, + search_response: response, + schema_element_names: graphql.runtime_metadata.schema_element_names + ) + + [connection.nodes, connection.page_info] + end + + def ids_of(*items) + items.flatten.map do |item| + if item.is_a?(Aggregation::Resolvers::Node) + fetch_grouped_by(item, "name") + else + item.fetch("name") + end + end + end + + def fetch_grouped_by(node, field) + node.bucket.fetch("key").fetch(field) + end + + def fetch_aggregated_values(node, *field_path, function_name) + key = Aggregation::Key::AggregatedValue.new( + aggregation_name: "aggregations", + field_path: field_path, + function_name: function_name + ) + + node.bucket.fetch(key.encode).fetch("value") + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/aggregations_spec.rb b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/aggregations_spec.rb new file mode 100644 index 00000000..3dd83ed1 --- /dev/null +++ b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/aggregations_spec.rb @@ -0,0 +1,494 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_aggregation_query_integration_support" +require "elastic_graph/graphql/aggregation/resolvers/relay_connection_builder" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "aggregations" do + include_context "DatastoreAggregationQueryIntegrationSupport" + + before do + index_into( + graphql, + build(:widget, amount_cents: 500, name: "Type 1", created_at: "2019-06-01T12:02:20Z"), + build(:widget, amount_cents: 200, name: "Type 2", created_at: "2019-06-02T13:03:30Z"), + build(:widget, amount_cents: 200, name: "Type 2", created_at: "2019-07-04T14:04:40Z"), + build(:widget, amount_cents: 100, name: "Type 3", created_at: "2019-07-03T12:02:59Z") + ) + end + + it "can run multiple aggregations in a single query" do + by_name = aggregation_query_of(name: "by_name", computations: [computation_of("amount_cents", :sum)], groupings: [field_term_grouping_of("name")]) + by_month = aggregation_query_of(name: "by_month", computations: [computation_of("amount_cents", :sum)], groupings: [date_histogram_grouping_of("created_at", "month")]) + just_sum = aggregation_query_of(name: "just_sum", computations: [computation_of("amount_cents", :sum)]) + min_and_max = aggregation_query_of(name: "min_and_max", computations: [computation_of("amount_cents", :min), computation_of("amount_cents", :max)]) + just_count = aggregation_query_of(name: "just_count", needs_doc_count: true) + + all_aggs = [by_name, by_month, just_sum, min_and_max, just_count] + + results = search_datastore(aggregations: [by_name, by_month, just_sum, min_and_max, just_count]) + + agg_nodes_by_agg_name = all_aggs.to_h do |agg| + connection = Aggregation::Resolvers::RelayConnectionBuilder.build_from_search_response( + schema_element_names: graphql.runtime_metadata.schema_element_names, + search_response: results, + query: agg + ) + + [agg.name, connection.nodes] + end + expect(agg_nodes_by_agg_name.keys).to contain_exactly("by_name", "by_month", "just_sum", "min_and_max", "just_count") + + amount_cents_sum = aggregated_value_key_of("amount_cents", "sum") + + amount_cents_sum = amount_cents_sum.with(aggregation_name: "by_name") + expect(agg_nodes_by_agg_name.fetch("by_name").map(&:bucket)).to eq [ + {amount_cents_sum.encode => {"value" => 500.0}, "doc_count" => 1, "key" => {"name" => "Type 1"}}, + {amount_cents_sum.encode => {"value" => 400.0}, "doc_count" => 2, "key" => {"name" => "Type 2"}}, + {amount_cents_sum.encode => {"value" => 100.0}, "doc_count" => 1, "key" => {"name" => "Type 3"}} + ] + + amount_cents_sum = amount_cents_sum.with(aggregation_name: "by_month") + expect(agg_nodes_by_agg_name.fetch("by_month").map(&:bucket)).to eq [ + {amount_cents_sum.encode => {"value" => 700.0}, "doc_count" => 2, "key" => {"created_at" => "2019-06-01T00:00:00.000Z"}}, + {amount_cents_sum.encode => {"value" => 300.0}, "doc_count" => 2, "key" => {"created_at" => "2019-07-01T00:00:00.000Z"}} + ] + + amount_cents_sum = amount_cents_sum.with(aggregation_name: "just_sum") + expect(agg_nodes_by_agg_name.fetch("just_sum").map(&:bucket)).to match [ + a_hash_including({amount_cents_sum.encode => {"value" => 1000.0}, "doc_count" => 4}) + ] + + amount_cents_sum = amount_cents_sum.with(aggregation_name: "min_and_max") + expect(agg_nodes_by_agg_name.fetch("min_and_max").map(&:bucket)).to match [ + a_hash_including({ + "doc_count" => 4, + amount_cents_sum.with(function_name: "max").encode => {"value" => 500.0}, + amount_cents_sum.with(function_name: "min").encode => {"value" => 100.0} + }) + ] + + expect(agg_nodes_by_agg_name.fetch("just_count").map(&:bucket)).to match [ + a_hash_including({"doc_count" => 4}) + ] + end + + it "can get aggregation computation results as a single bucket, regardless of how many are requested, but returns an empty list if 0 are requested" do + aggregations = aggregation_query_of(computations: [ + computation_of("amount_cents", :sum), + computation_of("amount_cents", :avg), + computation_of("amount_cents", :min), + computation_of("amount_cents", :max), + computation_of("amount_cents", :cardinality) + ]) + + results0 = search_datastore_aggregations(with_updated_paginator(aggregations, first: 0), total_document_count_needed: true) + results1 = search_datastore_aggregations(with_updated_paginator(aggregations, first: 1), total_document_count_needed: true) + results10 = search_datastore_aggregations(with_updated_paginator(aggregations, first: 10), total_document_count_needed: true) + + agg_key = aggregated_value_key_of("amount_cents", nil) + + expect(results0).to be_empty + expect(results1).to eq(results10).and contain_exactly( + { + "doc_count" => 4, + "key" => {}, + agg_key.with(function_name: "sum").encode => {"value" => 1000.0}, + agg_key.with(function_name: "min").encode => {"value" => 100.0}, + agg_key.with(function_name: "avg").encode => {"value" => 250.0}, + agg_key.with(function_name: "max").encode => {"value" => 500.0}, + agg_key.with(function_name: "cardinality").encode => {"value" => 3} + } + ) + end + + it "can get multiple buckets for a single aggregation computation and grouping, respecting the requested page size" do + aggregations = aggregation_query_of( + computations: [computation_of("amount_cents", :sum)], + groupings: [field_term_grouping_of("name")] + ) + + results0 = search_datastore_aggregations(with_updated_paginator(aggregations, first: 0)) + results1 = search_datastore_aggregations(with_updated_paginator(aggregations, first: 1)) + results3 = search_datastore_aggregations(with_updated_paginator(aggregations, first: 3)) + results10 = search_datastore_aggregations(with_updated_paginator(aggregations, first: 10)) + + amount_cents_sum = aggregated_value_key_of("amount_cents", "sum") + + expected_aggs = [ + {"key" => {"name" => "Type 2"}, "doc_count" => 2, amount_cents_sum.encode => {"value" => 400.0}}, + {"key" => {"name" => "Type 1"}, "doc_count" => 1, amount_cents_sum.encode => {"value" => 500.0}}, + {"key" => {"name" => "Type 3"}, "doc_count" => 1, amount_cents_sum.encode => {"value" => 100.0}} + ] + + expect(results0).to be_empty + expect(results1).to eq([expected_aggs[0]]).or eq([expected_aggs[1]]).or eq([expected_aggs[2]]) + expect(results3).to eq(results10).and match_array(expected_aggs) + end + + it "can get multiple buckets for multiple aggregation computations and groupings, respecting the requested page size" do + aggregations = aggregation_query_of( + computations: [computation_of("amount_cents", :sum), computation_of("amount_cents", :avg)], + groupings: [date_histogram_grouping_of("created_at", "day"), field_term_grouping_of("name")] + ) + + results0 = search_datastore_aggregations(with_updated_paginator(aggregations, first: 0)) + results1 = search_datastore_aggregations(with_updated_paginator(aggregations, first: 1)) + results4 = search_datastore_aggregations(with_updated_paginator(aggregations, first: 4)) + results10 = search_datastore_aggregations(with_updated_paginator(aggregations, first: 10)) + + amount_cents_sum = aggregated_value_key_of("amount_cents", "sum") + amount_cents_avg = aggregated_value_key_of("amount_cents", "avg") + + expected_aggs = [ + { + "key" => { + "created_at" => "2019-06-01T00:00:00.000Z", + "name" => "Type 1" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 500.0}, + amount_cents_avg.encode => {"value" => 500.0} + }, + { + "key" => { + "created_at" => "2019-06-02T00:00:00.000Z", + "name" => "Type 2" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 200.0}, + amount_cents_avg.encode => {"value" => 200.0} + }, + { + "key" => { + "created_at" => "2019-07-03T00:00:00.000Z", + "name" => "Type 3" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 100.0}, + amount_cents_avg.encode => {"value" => 100.0} + }, + { + "key" => { + "created_at" => "2019-07-04T00:00:00.000Z", + "name" => "Type 2" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 200.0}, + amount_cents_avg.encode => {"value" => 200.0} + } + ] + + expect(results0).to be_empty + expect(results1).to eq([expected_aggs[0]]).or eq([expected_aggs[1]]).or eq([expected_aggs[2]]).or eq([expected_aggs[3]]) + expect(results4).to eq(results10).and match_array(expected_aggs) + end + + it "returns correct results for an `as_day_of_week` grouping" do + aggregations = aggregation_query_of( + computations: [computation_of("amount_cents", :sum), computation_of("amount_cents", :avg)], + groupings: [ + as_day_of_week_grouping_of("created_at", runtime_metadata: graphql.runtime_metadata, graphql_subfield: "as_day_of_week"), + field_term_grouping_of("name") + ] + ) + + results = search_datastore_aggregations(with_updated_paginator(aggregations)) + + amount_cents_sum = aggregated_value_key_of("amount_cents", "sum") + amount_cents_avg = aggregated_value_key_of("amount_cents", "avg") + + expected_aggs = [ + { + "key" => { + "created_at.as_day_of_week" => "SATURDAY", + "name" => "Type 1" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 500.0}, + amount_cents_avg.encode => {"value" => 500.0} + }, + { + "key" => { + "created_at.as_day_of_week" => "SUNDAY", + "name" => "Type 2" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 200.0}, + amount_cents_avg.encode => {"value" => 200.0} + }, + { + "key" => { + "created_at.as_day_of_week" => "WEDNESDAY", + "name" => "Type 3" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 100.0}, + amount_cents_avg.encode => {"value" => 100.0} + }, + { + "key" => { + "created_at.as_day_of_week" => "THURSDAY", + "name" => "Type 2" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 200.0}, + amount_cents_avg.encode => {"value" => 200.0} + } + ] + + expect(results).to eq(results).and match_array(expected_aggs) + end + + it "returns correct results for an `as_day_of_week` grouping with `time_zone` set" do + aggregations = aggregation_query_of( + computations: [computation_of("amount_cents", :sum), computation_of("amount_cents", :avg)], + groupings: [ + as_day_of_week_grouping_of("created_at", time_zone: "Australia/Melbourne", runtime_metadata: graphql.runtime_metadata, graphql_subfield: "as_day_of_week") + ] + ) + + results = search_datastore_aggregations(with_updated_paginator(aggregations)) + + amount_cents_sum = aggregated_value_key_of("amount_cents", "sum") + amount_cents_avg = aggregated_value_key_of("amount_cents", "avg") + + expected_aggs = [ + { + "key" => { + "created_at.as_day_of_week" => "SATURDAY" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 500.0}, + amount_cents_avg.encode => {"value" => 500.0} + }, + { + "key" => { + "created_at.as_day_of_week" => "SUNDAY" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 200.0}, + amount_cents_avg.encode => {"value" => 200.0} + }, + { + "key" => { + "created_at.as_day_of_week" => "WEDNESDAY" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 100.0}, + amount_cents_avg.encode => {"value" => 100.0} + }, + { + "key" => { + "created_at.as_day_of_week" => "FRIDAY" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 200.0}, + amount_cents_avg.encode => {"value" => 200.0} + } + ] + + expect(results).to eq(results).and match_array(expected_aggs) + end + + it "returns correct results for an `as_day_of_week` grouping with `offset_ms` set" do + aggregations = aggregation_query_of( + computations: [computation_of("amount_cents", :sum), computation_of("amount_cents", :avg)], + groupings: [ + as_day_of_week_grouping_of("created_at", offset_ms: -86_400_000, runtime_metadata: graphql.runtime_metadata, graphql_subfield: "as_day_of_week") + ] + ) + + results = search_datastore_aggregations(with_updated_paginator(aggregations)) + + amount_cents_sum = aggregated_value_key_of("amount_cents", "sum") + amount_cents_avg = aggregated_value_key_of("amount_cents", "avg") + + expected_aggs = [ + { + "key" => { + "created_at.as_day_of_week" => "FRIDAY" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 500.0}, + amount_cents_avg.encode => {"value" => 500.0} + }, + { + "key" => { + "created_at.as_day_of_week" => "SATURDAY" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 200.0}, + amount_cents_avg.encode => {"value" => 200.0} + }, + { + "key" => { + "created_at.as_day_of_week" => "TUESDAY" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 100.0}, + amount_cents_avg.encode => {"value" => 100.0} + }, + { + "key" => { + "created_at.as_day_of_week" => "WEDNESDAY" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 200.0}, + amount_cents_avg.encode => {"value" => 200.0} + } + ] + + expect(results).to eq(results).and match_array(expected_aggs) + end + + it "returns correct results for an `as_time_of_day` grouping with a :second `interval`" do + aggregations = aggregation_query_of( + computations: [computation_of("amount_cents", :sum), computation_of("amount_cents", :avg)], + groupings: [ + as_time_of_day_grouping_of("created_at", "second", runtime_metadata: graphql.runtime_metadata, graphql_subfield: "as_time_of_day") + ] + ) + + results = search_datastore_aggregations(with_updated_paginator(aggregations)) + + amount_cents_sum = aggregated_value_key_of("amount_cents", "sum") + amount_cents_avg = aggregated_value_key_of("amount_cents", "avg") + + expected_aggs = [ + { + "key" => { + "created_at.as_time_of_day" => "12:02:20" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 500.0}, + amount_cents_avg.encode => {"value" => 500.0} + }, + { + "key" => { + "created_at.as_time_of_day" => "12:02:59" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 100.0}, + amount_cents_avg.encode => {"value" => 100.0} + }, + { + "key" => { + "created_at.as_time_of_day" => "13:03:30" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 200.0}, + amount_cents_avg.encode => {"value" => 200.0} + }, + { + "key" => { + "created_at.as_time_of_day" => "14:04:40" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 200.0}, + amount_cents_avg.encode => {"value" => 200.0} + } + ] + + expect(results).to eq(results).and match_array(expected_aggs) + end + + it "returns correct results for an `as_time_of_day` grouping with a :minute `interval` and `offset` set" do + aggregations = aggregation_query_of( + computations: [computation_of("amount_cents", :sum), computation_of("amount_cents", :avg)], + groupings: [ + as_time_of_day_grouping_of("created_at", "minute", offset_ms: 5_400_000, runtime_metadata: graphql.runtime_metadata, graphql_subfield: "as_time_of_day") + ] + ) + + results = search_datastore_aggregations(with_updated_paginator(aggregations)) + + amount_cents_sum = aggregated_value_key_of("amount_cents", "sum") + amount_cents_avg = aggregated_value_key_of("amount_cents", "avg") + + expected_aggs = [ + { + "key" => { + "created_at.as_time_of_day" => "13:32:00" + }, + "doc_count" => 2, + amount_cents_sum.encode => {"value" => 600.0}, + amount_cents_avg.encode => {"value" => 300.0} + }, + { + "key" => { + "created_at.as_time_of_day" => "14:33:00" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 200.0}, + amount_cents_avg.encode => {"value" => 200.0} + }, + { + "key" => { + "created_at.as_time_of_day" => "15:34:00" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 200.0}, + amount_cents_avg.encode => {"value" => 200.0} + } + ] + + expect(results).to eq(results).and match_array(expected_aggs) + end + + it "returns correct results for an `as_time_of_day` grouping with a :hour `interval` and `time_zone` set" do + aggregations = aggregation_query_of( + computations: [computation_of("amount_cents", :sum), computation_of("amount_cents", :avg)], + groupings: [ + as_time_of_day_grouping_of("created_at", "hour", time_zone: "Australia/Melbourne", runtime_metadata: graphql.runtime_metadata, graphql_subfield: "as_time_of_day") + ] + ) + + results = search_datastore_aggregations(with_updated_paginator(aggregations)) + + amount_cents_sum = aggregated_value_key_of("amount_cents", "sum") + amount_cents_avg = aggregated_value_key_of("amount_cents", "avg") + + expected_aggs = [ + { + "key" => { + "created_at.as_time_of_day" => "22:00:00" + }, + "doc_count" => 2, + amount_cents_sum.encode => {"value" => 600.0}, + amount_cents_avg.encode => {"value" => 300.0} + }, + { + "key" => { + "created_at.as_time_of_day" => "23:00:00" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 200.0}, + amount_cents_avg.encode => {"value" => 200.0} + }, + { + "key" => { + "created_at.as_time_of_day" => "00:00:00" + }, + "doc_count" => 1, + amount_cents_sum.encode => {"value" => 200.0}, + amount_cents_avg.encode => {"value" => 200.0} + } + ] + + expect(results).to eq(results).and match_array(expected_aggs) + end + + def with_updated_paginator(aggregation_query, **paginator_opts) + aggregation_query.with(paginator: aggregation_query.paginator.with(**paginator_opts)) + end + end + end +end diff --git a/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/datastore_aggregation_query_integration_support.rb b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/datastore_aggregation_query_integration_support.rb new file mode 100644 index 00000000..8c10b615 --- /dev/null +++ b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/datastore_aggregation_query_integration_support.rb @@ -0,0 +1,31 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_integration_support" +require "elastic_graph/graphql/aggregation/resolvers/relay_connection_builder" +require "support/aggregations_helpers" + +module ElasticGraph + class GraphQL + RSpec.shared_context "DatastoreAggregationQueryIntegrationSupport" do + include_context "DatastoreQueryIntegrationSupport" + + include AggregationsHelpers + + def search_datastore_aggregations(aggregation_query, **options) + connection = Aggregation::Resolvers::RelayConnectionBuilder.build_from_search_response( + schema_element_names: graphql.runtime_metadata.schema_element_names, + search_response: search_datastore(aggregations: [aggregation_query], **options), + query: aggregation_query + ) + + connection.edges.map(&:node).map(&:bucket) + end + end + end +end diff --git a/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/datastore_query_integration_support.rb b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/datastore_query_integration_support.rb new file mode 100644 index 00000000..193773b0 --- /dev/null +++ b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/datastore_query_integration_support.rb @@ -0,0 +1,61 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/key" +require "elastic_graph/graphql/datastore_query" +require "support/aggregations_helpers" +require "support/sort" + +module ElasticGraph + class GraphQL + # Note: we use `:capture_logs` here so that if any warnings are logged, a test fails. + # For example, we initially implemented aggregation `size` support in a way that + # triggered a `null_pointer_exception` in one shard, which caused the datastore to + # return a partial response that our router handled by logging a warning. The + # `:capture_logs` here caused the tests to fail until we fixed that issue. + RSpec.shared_context "DatastoreQueryIntegrationSupport", :uses_datastore, :factories, :capture_logs do + include AggregationsHelpers + include SortSupport + + let(:graphql) { build_graphql } + + def search_datastore(index_def_name: "widgets", aggregations: [], graphql: self.graphql, **options, &before_msearch) + index_def = graphql.datastore_core.index_definitions_by_name.fetch(index_def_name) + + query = graphql.datastore_query_builder.new_query( + search_index_definitions: [index_def], + requested_fields: ["id"], + sort: index_def.default_sort_clauses, + aggregations: aggregations.to_h { |agg| [agg.name, agg] }, + **options + ) + + perform_query(graphql, query, &before_msearch) + end + + def perform_query(graphql, query, &before_msearch) + query = query.then(&before_msearch || :itself) + + graphql + .datastore_search_router + .msearch([query]) + .values + .first + .tap do |response| + # To ensure that aggregations always satisfy the `QueryOptimizer` requirements, we validate all queries here. + aggregations = response.raw_data["aggregations"] + verify_aggregations_satisfy_optimizer_requirements(aggregations, for_query: query) + end + end + + def ids_of(*results) + results.flatten.map { |r| r.fetch("id") { r.fetch(:id) } } + end + end + end +end diff --git a/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/filtering_spec.rb b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/filtering_spec.rb new file mode 100644 index 00000000..037d16e5 --- /dev/null +++ b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/filtering_spec.rb @@ -0,0 +1,1225 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_integration_support" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "filtering" do + include_context "DatastoreQueryIntegrationSupport" + + specify "`equal_to_any_of`/`equalToAnyOf` filters to documents with a field matching any of the provided values" do + index_into( + graphql, + widget1 = build(:widget), + _widget2 = build(:widget), + widget3 = build(:widget) + ) + + results = search_with_filter("id", "equal_to_any_of", ids_of(widget1, widget3)) + + expect(results).to match_array(ids_of(widget1, widget3)) + + # Verify that empty strings (and other types like numbers) that don't match anything are ignored. + results2 = search_with_filter("id", "equal_to_any_of", ids_of(widget1, widget3) + ["", " ", "\n", 0]) + expect(results2).to eq(results) + end + + it "routes the request to specific shards when an `equal_to_any_of`/`equalToAnyOf` filter is used on a custom shard routing field", :expect_search_routing do + index_into( + graphql, + widget1 = build(:widget), + _widget2 = build(:widget), + widget3 = build(:widget) + ) + + workspace_ids = [widget1.fetch(:workspace_id), widget3.fetch(:workspace_id)] + + results = search_with_filter("workspace_id2", "equal_to_any_of", workspace_ids) + expect(results).to match_array(ids_of(widget1, widget3)) + + expect_to_have_routed_to_shards_with("main", ["widgets_rollover__*", workspace_ids.sort.join(",")]) + end + + it "avoids querying the datastore when a filter on the shard routing field excludes all values" do + pre_cache_index_state(graphql) # so that any datastore calls needed for the index state are made before we expect no calls. + + expect { + results = search_with_filter("workspace_id2", "equal_to_any_of", []) + expect(results).to eq([]) + }.to make_no_datastore_calls("main") + end + + it "still gets a well-formatted response, even when filtering to no routing values", :expect_search_routing do + results = search_with_filter("workspace_id2", "equal_to_any_of", []) + expect(results).to be_empty + end + + specify "comparison range operators filter to documents with a field satisfying the comparison" do + index_into( + graphql, + widget1 = build(:widget, amount_cents: 100), + widget2 = build(:widget, amount_cents: 150), + widget3 = build(:widget, amount_cents: 200) + ) + + expect(search_with_filter("amount_cents", "gt", 100)).to match_array(ids_of(widget2, widget3)) + expect(search_with_filter("amount_cents", "gte", 150)).to match_array(ids_of(widget2, widget3)) + expect(search_with_filter("amount_cents", "lt", 200)).to match_array(ids_of(widget1, widget2)) + expect(search_with_filter("amount_cents", "lte", 150)).to match_array(ids_of(widget1, widget2)) + expect(search_with_freeform_filter({"amount_cents" => {"gt" => 125, "lte" => 175}})).to match_array(ids_of(widget2)) + end + + specify "`near` works on a `GeoLocation` field to filter to nearby locations" do + # The lat/long values here were pulled from Google maps. + index_into( + graphql, + build(:address, id: "space_needle", geo_location: {latitude: 47.62089914996321, longitude: -122.34924708967479}), + build(:address, id: "crystal_mtn", geo_location: {latitude: 46.93703464703253, longitude: -121.47398616597955}), + build(:address, id: "pike_place_mkt", geo_location: {latitude: 47.60909792583577, longitude: -122.33981115022492}) + ) + + downtown_seattle_location = {"latitude" => 47.6078024243176, "longitude" => -122.3345525727595} + + about_3_miles_for_each_supported_unit = { + "MILE" => 3, + "FOOT" => 3 * 5_280, + "INCH" => 12 * 3 * 5_280, + "YARD" => 5_280, + "KILOMETER" => 4.828, + "METER" => 4_828, + "CENTIMETER" => 482_800, + "MILLIMETER" => 4_828_000, + "NAUTICAL_MILE" => 2.6 + } + expect(about_3_miles_for_each_supported_unit.keys).to match_array(graphql.runtime_metadata.enum_types_by_name.fetch("DistanceUnitInput").values_by_name.keys) + + about_3_miles_for_each_supported_unit.each do |unit, distance| + near_filter = downtown_seattle_location.merge( + "max_distance" => distance, + "unit" => enum_value("DistanceUnitInput", unit) + ) + + aggregate_failures "filtering with #{unit} unit" do + near_downtown_seattle = search_with_freeform_filter( + {"geo_location" => {"near" => near_filter}}, + index_def_name: "addresses" + ) + expect(near_downtown_seattle).to contain_exactly("space_needle", "pike_place_mkt") + + not_near_downtown_seattle = search_with_freeform_filter( + {"geo_location" => {"not" => {"near" => near_filter}}}, + index_def_name: "addresses" + ) + expect(not_near_downtown_seattle).to contain_exactly("crystal_mtn") + end + end + end + + describe "`all_of` operator" do + it "can be used to wrap multiple `any_satisfy` expressions to require multiple sub-filters to be satisfied by a list element" do + index_into( + graphql, + team1 = build(:team, forbes_valuations: [1_000_000, 100_000_000]), + team2 = build(:team, forbes_valuations: []), + team3 = build(:team, forbes_valuations: [3_000_000]) + ) + + results = search_with_filter("forbes_valuations", "any_satisfy", {"gt" => 2_000_000, "lt" => 5_000_000}) + # With a single (gt > 2M, lt < 5M) filter, only team3 (3M) qualifies... + expect(results).to match_array(ids_of(team3)) + + results = search_with_filter("forbes_valuations", "all_of", [ + {"any_satisfy" => {"gt" => 2_000_000}}, + {"any_satisfy" => {"lt" => 5_000_000}} + ]) + # ...but when you split them into separate `all_of` clauses, team1 (1M, 100M) also qualifies. + expect(results).to match_array(ids_of(team1, team3)) + + results = search_with_filter("forbes_valuations", "all_of", nil) + # all teamss qualify for `all_of: nil` + expect(results).to match_array(ids_of(team1, team2, team3)) + + results = search_with_filter("forbes_valuations", "all_of", []) + # all teamss qualify for `all_of: []` + expect(results).to match_array(ids_of(team1, team2, team3)) + end + + def search_datastore(**options, &before_msearch) + super(index_def_name: "teams", **options, &before_msearch) + end + end + + describe "`any_satisfy` filtering" do + context "when used on a list-of-scalars field" do + it "matches documents that have a list element matching the provided subfilter" do + index_into( + graphql, + team1 = build(:team, past_names: ["foo", "bar"]), + _team2 = build(:team, past_names: ["bar"]), + team3 = build(:team, past_names: ["bar", "bazz"]) + ) + + expect(search_with_filter( + "past_names", "any_satisfy", {"equal_to_any_of" => ["foo", "bazz"]} + )).to match_array(ids_of(team1, team3)) + end + + it "supports `not: {any_satisfy: ...}`, returning documents where their list field has no elements matching the provided subfilter" do + index_into( + graphql, + _team1 = build(:team, id: "t1", past_names: ["a", "b"]), + team2 = build(:team, id: "t2", past_names: ["b", "c"]), + team3 = build(:team, id: "t3", past_names: []), + _team4 = build(:team, id: "t4", past_names: ["a"]) + ) + + expect(search_with_filter( + "past_names", "not", {"any_satisfy" => {"equal_to_any_of" => ["a"]}} + )).to match_array(ids_of(team2, team3)) + end + + it "supports `any_satisfy: {time_of_day: ...}` filtering on a list-of-timestamps field" do + index_into( + graphql, + team1 = build(:team, id: "t1", won_championships_at: [], seasons: [ + build(:team_season, started_at: "2015-04-01T12:30:00Z", won_games_at: ["2015-04-08T15:30:00Z", "2015-04-09T16:30:00Z"]) + ]), + team2 = build(:team, id: "t2", won_championships_at: ["2013-11-27T02:30:00Z", "2013-11-27T22:30:00Z"], seasons: [ + build(:team_season, started_at: "2015-04-01T12:30:00Z", won_games_at: ["2015-04-08T02:30:00Z"]), + build(:team_season, started_at: "2015-04-01T02:30:00Z", won_games_at: ["2015-04-08T03:30:00Z", "2015-04-09T04:30:00Z"]) + ]), + team3 = build(:team, id: "t3", won_championships_at: ["2003-10-27T19:30:00Z"], seasons: []), + team4 = build(:team, id: "t4", won_championships_at: ["2005-10-27T12:30:00Z"], seasons: [ + build(:team_season, started_at: "2015-04-01T19:30:00Z", won_games_at: []) + ]) + ) + + # On a list-of-scalars field on the root doc + expect(search_with_filter( + "won_championships_at", "any_satisfy", {"time_of_day" => {"gt" => "15:00:00", "time_zone" => "UTC"}} + )).to match_array(ids_of(team2, team3)) + + # On a list-of-scalars field on the root doc with a non-UTC timezone + expect(search_with_filter( + "won_championships_at", "any_satisfy", {"time_of_day" => {"lte" => "08:00:00", "time_zone" => "America/Los_Angeles"}} + )).to match_array(ids_of(team4)) + + # On a singleton scalar field under a nested field + expect(search_with_filter( + "seasons_nested", "any_satisfy", {"started_at" => {"time_of_day" => {"gt" => "18:00:00", "time_zone" => "UTC"}}} + )).to match_array(ids_of(team4)) + + # On a list-of-scalars field under a nested field + expect(search_with_filter( + "seasons_nested", "any_satisfy", {"won_games_at" => {"any_satisfy" => {"time_of_day" => {"gt" => "14:00:00", "time_zone" => "UTC"}}}} + )).to match_array(ids_of(team1)) + + # On a singleton scalar field under an object field + expect(search_with_filter( + "seasons_object", "started_at", {"time_of_day" => {"gt" => "18:00:00", "time_zone" => "UTC"}} + )).to match_array(ids_of(team4)) + + # On a list-of-scalars field under an object field + expect(search_with_filter( + "seasons_object", "won_games_at", {"any_satisfy" => {"time_of_day" => {"gt" => "14:00:00", "time_zone" => "UTC"}}} + )).to match_array(ids_of(team1)) + end + + it "supports `any_satisfy: {...}` with range operators on a list-of-numbers field" do + index_into( + graphql, + _team1 = build(:team, id: "t1", forbes_valuations: []), + team2 = build(:team, id: "t2", forbes_valuations: [10_000_000]), + team3 = build(:team, id: "t3", forbes_valuations: [0, 500_000_000]), + team4 = build(:team, id: "t4", forbes_valuations: [5000, 250_000_000]) + ) + + expect(search_with_filter( + "forbes_valuations", "any_satisfy", {"gt" => 100_000_000} + )).to match_array(ids_of(team3, team4)) + + expect(search_with_filter( + "forbes_valuations", "any_satisfy", {"lt" => 100_000_000} + )).to match_array(ids_of(team2, team3, team4)) + + expect(search_with_filter( + "forbes_valuations", "any_satisfy", {"gt" => 10_000, "lt" => 500_000_000} + )).to match_array(ids_of(team2, team4)) + + expect(search_with_filter( + # We don't expect users to use all 4 range operators, but if they do it should work! + "forbes_valuations", "any_satisfy", {"gt" => 10_000, "gte" => 20_000, "lt" => 500_000_000, "lte" => 100_000_000} + )).to match_array(ids_of(team2)) + end + end + + context "when used on a list-of-nested-objects field" do + it "can correctly consider each nested object independently, correctly matching on multiple filters" do + index_into( + graphql, + _team1 = build(:team, current_players: [build(:player, name: "Babe Truth", nicknames: ["The Truth"])]), + team2 = build(:team, current_players: [ + build(:player, name: "Babe Truth", nicknames: ["The Babe", "Bambino"]), + build(:player, name: "Johnny Rocket", nicknames: ["The Rocket"]) + ]), + _team3 = build(:team, current_players: [ + build(:player, name: "Ichiro", nicknames: ["Bambino"]), + build(:player, name: "Babe Truth", nicknames: ["The Wizard"]) + ]), + _team4 = build(:team, current_players: []) + ) + + results = search_with_filter("current_players_nested", "any_satisfy", { + "name" => {"equal_to_any_of" => ["Babe Truth"]}, + "nicknames" => {"any_satisfy" => {"equal_to_any_of" => ["Bambino"]}} + }) + + expect(results).to match_array(ids_of(team2)) + end + + it "correctly ignores nil filters under `any_satisfy`" do + results = search_with_filter("current_players_nested", "any_satisfy", { + "name" => {"equal_to_any_of" => nil}, + "nicknames" => {"any_satisfy" => {"equal_to_any_of" => nil}} + }) + + # Results are empty since nothing has been indexed here. The original regression this test guards against + # is an exception produced by Elasticsearch/OpenSearch, so we primarily care about it not raising an + # exception here. + expect(results).to be_empty + end + end + + def search_datastore(**options, &before_msearch) + super(index_def_name: "teams", **options, &before_msearch) + end + end + + # Note: a `count` filter gets translated into `__counts` (to distinguish it from a schema field named `count`), + # and that translation happens as the query is being built, so we use `__counts` in our example filters here. + describe "`count` filtering on a list" do + it "matches documents with a root list count matching the filter" do + index_into( + graphql, + _team1 = build(:team, past_names: ["a", "b", "c", "d"]), + team2 = build(:team, past_names: ["a", "b", "c"]), + _team3 = build(:team, past_names: ["a", "b"]) + ) + + results = search_with_freeform_filter({"past_names" => {LIST_COUNTS_FIELD => {"gt" => 2, "lt" => 4}}}) + expect(results).to match_array(ids_of(team2)) + end + + it "matches documents with a list count under an object field" do + index_into( + graphql, + _team1 = build(:team, details: {uniform_colors: ["a", "b", "c", "d"]}), + team2 = build(:team, details: {uniform_colors: ["a", "b", "c"]}), + _team1 = build(:team, details: {uniform_colors: ["a", "b"]}) + ) + + results = search_with_freeform_filter({"details" => {"uniform_colors" => {LIST_COUNTS_FIELD => {"gt" => 2, "lt" => 4}}}}) + expect(results).to match_array(ids_of(team2)) + end + + it "correctly matches a list field under a list-of-nested objects" do + index_into( + graphql, + team1 = build(:team, seasons: [ + build(:team_season, notes: ["a", "b", "c"]) + ]), + _team2 = build(:team, seasons: [ + build(:team_season, notes: ["a", "b"]), + build(:team_season, notes: ["a", "b", "c", "d"]) + ]), + _team3 = build(:team, seasons: []) + ) + + results = search_with_freeform_filter({"seasons_nested" => {"any_satisfy" => {"notes" => {LIST_COUNTS_FIELD => {"gt" => 2, "lt" => 4}}}}}) + expect(results).to match_array(ids_of(team1)) + end + + it "treats a filter on a `count` schema field like a filter on any other schema field" do + index_into( + graphql, + team1 = build(:team, details: {count: 50}), + _team2 = build(:team, details: {count: 40}) + ) + + results = search_with_freeform_filter({"details" => {"count" => {"gt" => 45}}}) + expect(results).to match_array(ids_of(team1)) + end + + context "when filtering in a way that matches a count of zero" do + it "matches documents that have no record of the count since that indicates they were indexed before the list field was defined, and they therefore have no values for it" do + index_into( + graphql, + team1 = build(:team) + ) + + results = search_with_freeform_filter({"some_new_field_defined_after_indexing" => {LIST_COUNTS_FIELD => {"lt" => 1}}}) + expect(results).to match_array(ids_of(team1)) + results = search_with_freeform_filter({"some_new_field_defined_after_indexing" => {LIST_COUNTS_FIELD => {"gt" => -1}}}) + expect(results).to match_array(ids_of(team1)) + results = search_with_freeform_filter({"some_new_field_defined_after_indexing" => {LIST_COUNTS_FIELD => {"lte" => 0}}}) + expect(results).to match_array(ids_of(team1)) + results = search_with_freeform_filter({"some_new_field_defined_after_indexing" => {LIST_COUNTS_FIELD => {"gte" => 0}}}) + expect(results).to match_array(ids_of(team1)) + results = search_with_freeform_filter({"some_new_field_defined_after_indexing" => {LIST_COUNTS_FIELD => {"equal_to_any_of" => [3, 0, 10]}}}) + expect(results).to match_array(ids_of(team1)) + + results = search_with_freeform_filter({"some_new_field_defined_after_indexing" => {LIST_COUNTS_FIELD => {"lt" => 0}}}) + expect(results).to be_empty + results = search_with_freeform_filter({"some_new_field_defined_after_indexing" => {LIST_COUNTS_FIELD => {"gt" => 0}}}) + expect(results).to be_empty + results = search_with_freeform_filter({"some_new_field_defined_after_indexing" => {LIST_COUNTS_FIELD => {"lte" => -1}}}) + expect(results).to be_empty + results = search_with_freeform_filter({"some_new_field_defined_after_indexing" => {LIST_COUNTS_FIELD => {"gte" => 1}}}) + expect(results).to be_empty + results = search_with_freeform_filter({"some_new_field_defined_after_indexing" => {LIST_COUNTS_FIELD => {"equal_to_any_of" => [3, 10]}}}) + expect(results).to be_empty + end + end + + def search_datastore(**options, &before_msearch) + super(index_def_name: "teams", **options, &before_msearch) + end + end + + describe "`time_of_day` filtering", :builds_admin do + # When working on the logic of the `filter/by_time_of_day.painless` script, it makes the feedback cycle slow + # to have to run `bundle exec rake schema_artifacts:dump` after each change in the script before it can be used + # by the test here. To have a faster feedback cycle, I added this before hook that uses a custom (empty) schema + # definition, runs our script configurators, and updates the `static_script_ids_by_scoped_name` to the new script + # id. With this before hook in place, changes in the script immediately go into effect when we run the tests here. + # + # However, if you're not actively editing the script it's not needed to run this extra bit of setup (and is + # undesirable since it slows things down a bit). + # + # So, here we detect if the script file has changed since the last time RSpec ran. + rspec_results_file = ::File.join(CommonSpecHelpers::REPO_ROOT, "tmp", "rspec", "stats.txt") + filter_by_time_of_day_file = ::File.join([ + CommonSpecHelpers::REPO_ROOT, + "elasticgraph-schema_definition", + "lib", + "elastic_graph", + "schema_definition", + "scripting", + "scripts", + "filter", + "by_time_of_day.painless" + ]) + + if ::File.exist?(rspec_results_file) && ::File.mtime(rspec_results_file) < ::File.mtime(filter_by_time_of_day_file) + # :nocov: -- often skipped + before do + admin = build_admin(schema_definition: ->(schema) {}) + + admin + .cluster_configurator.send(:script_configurators_for, $stdout) + .each(&:configure!) + + graphql.runtime_metadata.static_script_ids_by_scoped_name.replace( + admin.schema_artifacts.runtime_metadata.static_script_ids_by_scoped_name + ) + end + # :nocov: + end + + it "supports gt/gte/lt/lte/equal_to_any_of operators, honoring the given time zone" do + index_into( + graphql, + # In 2022, DST started on 2022-03-13, so on this date the Pacific Time offset was -8 hours. + widget_02am = build(:widget, id: "02am", created_at: "2022-03-12T10:23:10Z"), # Local time in Pacific: 02:23:10 + # ...on on this date the Pacific Time offset was -7 hours. + widget_03am = build(:widget, id: "03am", created_at: "2022-03-13T10:23:10Z"), # Local time in Pacific: 03:23:10 + # ...and on this date the Pacific Time offset was -7 hours. + widget_08pm = build(:widget, id: "08pm", created_at: "2022-04-12T03:05:00Z"), # Local time in Pacific: 20:05:00 + # In 2022, DST ended on 2022-11-06, so on this date the Pacific Time offset was -7 hours. + widget_11am = build(:widget, id: "11am", created_at: "2022-11-05T18:45:23.987Z"), # Local time in Pacific: 11:45:23.987 + # ...and on this date the Pacific Time offset was -8 hours. + widget_10am = build(:widget, id: "10am", created_at: "2022-11-07T18:45:23.987Z") # Local time in Pacific: 10:45:23.987 + ) + + # First, test gte. Use a LocalTime value exactly equal to one of the timestamps... + results = search_with_freeform_filter({ + "created_at" => {"time_of_day" => {"gte" => "10:45:23.987", "time_zone" => "America/Los_Angeles"}} + }) + expect(results).to match_array(ids_of(widget_10am, widget_11am, widget_08pm)) + # ...and then go 1 ms later to show that result is excluded. + results = search_with_freeform_filter({ + "created_at" => {"time_of_day" => {"gte" => "10:45:23.988", "time_zone" => "America/Los_Angeles"}} + }) + expect(results).to match_array(ids_of(widget_11am, widget_08pm)) + + # Next, test gt. Use a LocalTime 1 ms less than one of the timestamps... + results = search_with_freeform_filter({ + "created_at" => {"time_of_day" => {"gt" => "10:45:23.986", "time_zone" => "America/Los_Angeles"}} + }) + expect(results).to match_array(ids_of(widget_10am, widget_11am, widget_08pm)) + # ...and then go 1 ms later to show that result is excluded. + results = search_with_freeform_filter({ + "created_at" => {"time_of_day" => {"gt" => "10:45:23.987", "time_zone" => "America/Los_Angeles"}} + }) + expect(results).to match_array(ids_of(widget_11am, widget_08pm)) + + # Next, test lte. Use a LocalTime value exactly equal to one of the timestamps... + results = search_with_freeform_filter({ + "created_at" => {"time_of_day" => {"lte" => "11:45:23.987", "time_zone" => "America/Los_Angeles"}} + }) + expect(results).to match_array(ids_of(widget_11am, widget_10am, widget_02am, widget_03am)) + # ...and then go 1 ms earlier to show that result is excluded. + results = search_with_freeform_filter({ + "created_at" => {"time_of_day" => {"lte" => "11:45:23.986", "time_zone" => "America/Los_Angeles"}} + }) + expect(results).to match_array(ids_of(widget_10am, widget_02am, widget_03am)) + + # Next, test lt. Use a LocalTime 1 ms more than one of the timestamps... + results = search_with_freeform_filter({ + "created_at" => {"time_of_day" => {"lt" => "11:45:23.988", "time_zone" => "America/Los_Angeles"}} + }) + expect(results).to match_array(ids_of(widget_11am, widget_10am, widget_02am, widget_03am)) + # ...and then go 1 ms earlier to show that result is excluded. + results = search_with_freeform_filter({ + "created_at" => {"time_of_day" => {"lt" => "11:45:23.987", "time_zone" => "America/Los_Angeles"}} + }) + expect(results).to match_array(ids_of(widget_10am, widget_02am, widget_03am)) + + # Next, test `equal_to_any_of` with 2 values that match records and one that doesn't. + results = search_with_freeform_filter({ + "created_at" => {"time_of_day" => { + "equal_to_any_of" => ["20:05:00", "03:23:10", "23:12:12"], + "time_zone" => "America/Los_Angeles" + }} + }) + expect(results).to match_array(ids_of(widget_08pm, widget_03am)) + + # Finally, combine multiple operators in one filter to show that works. + results = search_with_freeform_filter({ + "created_at" => {"time_of_day" => { + "gte" => "02:30:00", + "lt" => "12:00:00", + "time_zone" => "America/Los_Angeles" + }} + }) + expect(results).to match_array(ids_of(widget_03am, widget_10am, widget_11am)) + end + + it "works on nested DateTime scalar fields" do + index_into( + graphql, + # In 2022, DST started on 2022-03-13, so on this date the Pacific Time offset was -8 hours. + address_02am = build(:address, id: "02am", timestamps: {created_at: "2022-03-12T10:23:10Z"}), # Local time in Pacific: 02:23:10 + # ...and on this date the Pacific Time offset was -7 hours. + address_08pm = build(:address, id: "08pm", timestamps: {created_at: "2022-04-12T03:05:00Z"}) # Local time in Pacific: 20:05:00 + ) + + results = search_with_freeform_filter({ + "timestamps" => {"created_at" => {"time_of_day" => {"gte" => "12:00:00", "time_zone" => "America/Los_Angeles"}}} + }, index_def_name: "addresses") + expect(results).to match_array(ids_of(address_08pm)) + + results = search_with_freeform_filter({ + "timestamps" => {"created_at" => {"time_of_day" => {"lte" => "12:00:00", "time_zone" => "America/Los_Angeles"}}} + }, index_def_name: "addresses") + expect(results).to match_array(ids_of(address_02am)) + end + + it "does not match documents that have `null` for their timestamp field value" do + index_into( + graphql, + build(:address, timestamps: {created_at: nil}) + ) + + results = search_with_freeform_filter({ + "timestamps" => {"created_at" => {"time_of_day" => {"gte" => "00:00:00", "time_zone" => "America/Los_Angeles"}}} + }, index_def_name: "addresses") + expect(results).to be_empty + + results = search_with_freeform_filter({ + "timestamps" => {"created_at" => {"time_of_day" => {"lte" => "23:59:59.999", "time_zone" => "America/Los_Angeles"}}} + }, index_def_name: "addresses") + expect(results).to be_empty + end + + it "works correctly with compound filter operators like `not` and `any_of`" do + index_into( + graphql, + # In 2022, DST started on 2022-03-13, so on this date the Pacific Time offset was -8 hours. + widget_02am = build(:widget, id: "02am", created_at: "2022-03-12T10:23:10Z"), # Local time in Pacific: 02:23:10 + # ...and on this date the Pacific Time offset was -7 hours. + widget_08pm = build(:widget, id: "08pm", created_at: "2022-04-12T03:05:00Z"), # Local time in Pacific: 20:05:00 + # In 2022, DST ended on 2022-11-06, so on this date the Pacific Time offset was -7 hours. + widget_11am = build(:widget, id: "11am", created_at: "2022-11-05T18:45:23.987Z") # Local time in Pacific: 11:45:23.987 + ) + + results = search_with_freeform_filter({ + "created_at" => {"time_of_day" => {"gte" => "12:00:00", "time_zone" => "America/Los_Angeles"}} + }) + expect(results).to match_array(ids_of(widget_08pm)) + + # Try `not` outside of `time_of_day`: it should be the inverse of the above + results = search_with_freeform_filter({ + "created_at" => {"not" => {"time_of_day" => {"gte" => "12:00:00", "time_zone" => "America/Los_Angeles"}}} + }) + expect(results).to match_array(ids_of(widget_02am, widget_11am)) + + # Try `any_of` outside of `time_of_day`: it should OR together conditions correctly. + results = search_with_freeform_filter({ + "created_at" => {"any_of" => [ + {"time_of_day" => {"gte" => "12:00:00", "time_zone" => "America/Los_Angeles"}}, + {"time_of_day" => {"lte" => "10:00:00", "time_zone" => "America/Los_Angeles"}} + ]} + }) + expect(results).to match_array(ids_of(widget_08pm, widget_02am)) + + # Note: `any_of` and `not` are intentionally not supported inside a `time_of_day` filter at this time. + end + + it "matches all documents when the filter includes no operators" do + index_into( + graphql, + widget = build(:widget, id: "02am") + ) + + results = search_with_freeform_filter({ + "created_at" => {"time_of_day" => {"time_zone" => "America/Los_Angeles"}} + }) + expect(results).to match_array(ids_of(widget)) + + results = search_with_freeform_filter({ + "created_at" => {"time_of_day" => {}} + }) + expect(results).to match_array(ids_of(widget)) + end + end + + context "when range operators are used with timestamp strings" do + it "applies the filtering and also excludes rollover indices from being searched which cannot contain any matching documents", :expect_index_exclusions do + index_into( + graphql, + widget1 = build(:widget, created_at: (t1 = "2019-05-10T12:00:00Z")), + widget2 = build(:widget, created_at: (t2 = "2019-05-15T12:00:00Z")), + widget3 = build(:widget, created_at: (t3 = "2019-05-20T12:00:00Z")) + ) + + expect(search_with_filter("created_at", "gt", t1)).to match_array(ids_of(widget2, widget3)) + expect_to_have_excluded_indices("main", ["widgets_rollover__before_2019"]) + + expect(search_with_filter("created_at", "gte", t2)).to match_array(ids_of(widget2, widget3)) + expect_to_have_excluded_indices("main", ["widgets_rollover__before_2019"]) + + expect(search_with_filter("created_at", "lt", t3)).to match_array(ids_of(widget1, widget2)) + expect_to_have_excluded_indices("main", ["widgets_rollover__2020", "widgets_rollover__2021", "widgets_rollover__after_2021"]) + + expect(search_with_filter("created_at", "lte", t2)).to match_array(ids_of(widget1, widget2)) + expect_to_have_excluded_indices("main", ["widgets_rollover__2020", "widgets_rollover__2021", "widgets_rollover__after_2021"]) + + expect { + expect(search_with_freeform_filter({"created_at" => {"gte" => t2, "lt" => t2}})).to eq [] + }.to make_no_datastore_calls("main") + end + + context "when the configuration does not agree with the indices that exist in the datastore", :expect_index_exclusions do + let(:graphql) do + index_defs = ::Hash.new do |hash, index_def_name| + hash[index_def_name] = config_index_def_of + end + + standard_test_custom_timestamp_ranges = CommonSpecHelpers + .parsed_test_settings_yaml + .dig("datastore", "index_definitions", "widgets", "custom_timestamp_ranges") + .map { |r| r["index_name_suffix"] } + + expect(standard_test_custom_timestamp_ranges).to contain_exactly("before_2019", "after_2021") + index_defs["widgets"] = config_index_def_of( + # Here we omit the `after_2021` custom timestamp range that our test config normally defines + # (and for which an index already exists in the datastore). + custom_timestamp_ranges: [ + { + index_name_suffix: "before_2019", + lt: "2019-01-01T00:00:00Z", + setting_overrides: {} + } + ], + # Here we define an extra index which does not exist in the datastore. + setting_overrides_by_timestamp: { + "2047-01-01T00:00:00Z" => {} + } + ) + + build_graphql(index_definitions: index_defs) + end + + it "does not attempt to exclude non-existent indices from being searched since the datastore returns an error if we do that" do + index_into( + build_graphql, # must build a fresh `graphql` instance because the indexer will fail due to the configured indices that haven't been created. + widget1 = build(:widget, created_at: (t1 = "2019-05-10T12:00:00Z")) + ) + + expect(search_with_filter("created_at", "gte", t1)).to match_array(ids_of(widget1)) + expect_to_have_excluded_indices("main", ["widgets_rollover__before_2019"]) + end + end + end + + specify "`matches` filters using full text search" do + index_into( + graphql, + widget1 = build(:widget, name_text: "a blue thing"), + widget2 = build(:widget, name_text: "a red thing"), + _widget3 = build(:widget, name_text: "entirely different name"), + widget4 = build(:widget, name_text: "a thing that is blue"), + widget5 = build(:widget, name_text: "a blue device") + ) + + results = search_with_filter("name_text", "matches", "blue thing") + + expect(results).to match_array(ids_of(widget1, widget2, widget4, widget5)) + end + + specify "`matches` filters using full text search with flexible options" do + index_into( + graphql, + widget1 = build(:widget, name_text: "a blue thing"), + widget2 = build(:widget, name_text: "a red thing"), + _widget3 = build(:widget, name_text: "entirely different name"), + widget4 = build(:widget, name_text: "a thing that is blue"), + widget5 = build(:widget, name_text: "a blue device") + ) + + results = search_with_filter( + "name_text", + "matches_query", + { + "query" => "blue thang", + # `DYNAMIC` is the GraphQL default + "allowed_edits_per_term" => enum_value("MatchesQueryAllowedEditsPerTermInput", "DYNAMIC"), + # `false` is the GraphQL default + "require_all_terms" => false + } + ) + expect(results).to match_array(ids_of(widget1, widget2, widget4, widget5)) + + results = search_with_filter( + "name_text", + "matches_query", + { + "query" => "blue thang", + "allowed_edits_per_term" => enum_value("MatchesQueryAllowedEditsPerTermInput", "NONE"), + # `false` is the GraphQL default + "require_all_terms" => false + } + ) + # Only matches based on "blue" now since "thang" cannot be edited to "thing". + # widget2 ("a red thing") is not included as a result. + expect(results).to match_array(ids_of(widget1, widget4, widget5)) + + results = search_with_filter( + "name_text", + "matches_query", + { + "query" => "blue thing", + # `DYNAMIC` is the GraphQL default + "allowed_edits_per_term" => enum_value("MatchesQueryAllowedEditsPerTermInput", "DYNAMIC"), + "require_all_terms" => true + } + ) + # widget2 has "thing" but not "blue", so is excluded. + # widget5 has "blue" but not "thing", so is excluded. + expect(results).to match_array(ids_of(widget1, widget4)) + end + + specify "`matches_phrase` filters on an entire phrase" do + index_into( + graphql, + _widget1 = build(:widget, name_text: "a blue thing"), + _widget2 = build(:widget, name_text: "a red thing"), + _widget3 = build(:widget, name_text: "entirely different name"), + _widget4 = build(:widget, name_text: "a thing that is blue"), + widget5 = build(:widget, name_text: "a blue device"), + _widget6 = build(:widget, name_text: "a blue or green device") + ) + + results = search_with_filter("name_text", "matches_phrase", {"phrase" => "blue device"}) + + expect(results).to match_array(ids_of(widget5)) + end + + specify "`matches_phrase` filters on an entire phrase also with prefix" do + index_into( + graphql, + widget1 = build(:widget, name_text: "entirely new thing"), + _widget2 = build(:widget, name_text: "entirely new device") + ) + + results = search_with_filter("name_text", "matches_phrase", {"phrase" => "entirely new t"}) + + expect(results).to match_array(ids_of(widget1)) + end + + specify "`any_of` supports ORing multiple filters for flat fields" do + index_into( + graphql, + widget1 = build(:widget, amount_cents: 50), + widget2 = build(:widget, amount_cents: 100), + _widget3 = build(:widget, amount_cents: 150), + _widget4 = build(:widget, amount_cents: 200), + _widget5 = build(:widget, amount_cents: 250) + ) + + results = search_with_filter("amount_cents", "any_of", [ + {"equal_to_any_of" => [50]}, + {"equal_to_any_of" => [100]} + ]) + + expect(results).to match_array(ids_of(widget1, widget2)) + end + + specify "`any_of` supports ORing multiple filters for nested fields" do + index_into( + graphql, + widget1 = build(:widget, options: {size: "SMALL", color: "BLUE"}), + _widget2 = build(:widget, options: {size: "LARGE", color: "RED"}), + widget3 = build(:widget, options: {size: "MEDIUM", color: "RED"}) + ) + + results = search_with_freeform_filter({ + "options" => { + "any_of" => [ + {"size" => {"equal_to_any_of" => [size_of("SMALL")]}}, + {"size" => {"equal_to_any_of" => [size_of("MEDIUM")]}} + ] + } + }) + + expect(results).to match_array(ids_of(widget1, widget3)) + end + + specify "`any_of` supports both term and text searches" do + index_into( + graphql, + _widget1 = build(:widget, name_text: "a blue thing", options: {size: "SMALL"}), + widget2 = build(:widget, name_text: "a red thing", options: {size: "MEDIUM"}), + _widget3 = build(:widget, name_text: "a black thing", options: {size: "MEDIUM"}), + _widget4 = build(:widget, name_text: "another red thing", options: {size: "SMALL"}), + _widget5 = build(:widget, name_text: "a green thing", options: {size: "SMALL"}) + ) + + results = search_with_freeform_filter({ + "options" => { + "size" => {"equal_to_any_of" => [size_of("MEDIUM")]} + }, + "name_text" => { + "any_of" => [ + {"matches" => "red"}, + {"matches" => "blue"} + ] + } + }) + + expect(results).to match_array(ids_of(widget2)) + end + + specify "`any_of` supports multiple levels of itself" do + index_into( + graphql, + widget1 = build(:widget, name_text: "a blue thing", options: {size: "SMALL"}), + widget2 = build(:widget, name_text: "a red thing", options: {size: "MEDIUM"}), + widget3 = build(:widget, name_text: "a green thing", options: {size: "SMALL"}), + _widget4 = build(:widget, name_text: "another red thing", options: {size: "SMALL"}), + widget5 = build(:widget, name_text: "another green thing", options: {size: "SMALL"}) + ) + + results = search_with_freeform_filter({ + "any_of" => [ + { + "name_text" => { + "any_of" => [ + {"matches" => "blue"}, + {"matches" => "green"} + ] + } + }, + { + "options" => {"size" => {"equal_to_any_of" => [size_of("MEDIUM")]}} + } + ] + }) + + expect(results).to match_array(ids_of(widget1, widget2, widget3, widget5)) + end + + specify "`any_of` used at nested cousin nodes works correctly" do + index_into( + graphql, + # should not match; while cost matches (twice, actually), options does not match. + _widget1 = build(:widget, options: {size: "SMALL", color: "GREEN"}, cost: {currency: "USD", amount_cents: 100}), + # should match: cost.currency and options.size both match + widget2 = build(:widget, options: {size: "MEDIUM", color: "GREEN"}, cost: {currency: "USD", amount_cents: 200}), + # should not match: while options.color matches, cost does not match + _widget3 = build(:widget, options: {size: "SMALL", color: "RED"}, cost: {currency: "GBP", amount_cents: 200}), + # should match: cost.amount_cents and options.color both match + widget4 = build(:widget, options: {size: "SMALL", color: "RED"}, cost: {currency: "GBP", amount_cents: 100}), + # should not match; while options matches (twice, actually), cost does not match. + _widget5 = build(:widget, options: {size: "MEDIUM", color: "RED"}, cost: {currency: "GBP", amount_cents: 200}) + ) + + results = search_with_freeform_filter({ + "cost" => { + "any_of" => [ + {"currency" => {"equal_to_any_of" => ["USD"]}}, + {"amount_cents" => {"equal_to_any_of" => [100]}} + ] + }, + "options" => { + "any_of" => [ + {"size" => {"equal_to_any_of" => [size_of("MEDIUM")]}}, + {"color" => {"equal_to_any_of" => [color_of("RED")]}} + ] + } + }) + + expect(results).to match_array(ids_of(widget2, widget4)) + end + + specify "`equal_to_any_of: []` or `any_of: []` matches no documents, but `any_predicate: nil` or `field: {}` is ignored, matching all documents" do + index_into( + graphql, + widget1 = build(:widget), + widget2 = build(:widget) + ) + + expect(search_with_filter("id", "equal_to_any_of", [])).to eq [] + expect(search_with_filter("id", "any_of", [])).to eq [] + + expect(search_with_filter("id", "equal_to_any_of", nil)).to eq ids_of(widget1, widget2) + expect(search_with_filter("amount_cents", "lt", nil)).to eq ids_of(widget1, widget2) + expect(search_with_filter("amount_cents", "lte", nil)).to eq ids_of(widget1, widget2) + expect(search_with_filter("amount_cents", "gt", nil)).to eq ids_of(widget1, widget2) + expect(search_with_filter("amount_cents", "gte", nil)).to eq ids_of(widget1, widget2) + expect(search_with_filter("amount_cents", "any_of", nil)).to eq ids_of(widget1, widget2) + expect(search_with_filter("name_text", "matches", nil)).to eq ids_of(widget1, widget2) + expect(search_with_filter("name_text", "matches_query", nil)).to eq ids_of(widget1, widget2) + expect(search_with_filter("name_text", "matches_phrase", nil)).to eq ids_of(widget1, widget2) + expect(search_with_freeform_filter({"id" => {}})).to eq ids_of(widget1, widget2) + end + + it "`equal_to_any_of:` with `nil` matches documents with null values or not null values" do + index_into( + graphql, + widget1 = build(:widget, amount_cents: 1000), + widget2 = build(:widget, amount_cents: nil), + widget3 = build(:widget, amount_cents: 2500) + ) + + expect(search_with_filter("amount_cents", "equal_to_any_of", [nil])).to eq ids_of(widget2) + expect(search_with_filter("amount_cents", "equal_to_any_of", [nil, 2500])).to match_array(ids_of(widget2, widget3)) + + inner_not_result1 = search_with_freeform_filter({"amount_cents" => {"not" => {"equal_to_any_of" => [nil]}}}) + outer_not_result1 = search_with_freeform_filter({"not" => {"amount_cents" => {"equal_to_any_of" => [nil]}}}) + expect(inner_not_result1).to eq(outer_not_result1).and match_array(ids_of(widget1, widget3)) + + inner_not_result2 = search_with_freeform_filter({"amount_cents" => {"not" => {"equal_to_any_of" => [nil, 1000]}}}) + outer_not_result2 = search_with_freeform_filter({"not" => {"amount_cents" => {"equal_to_any_of" => [nil, 1000]}}}) + expect(inner_not_result2).to eq(outer_not_result2).and eq ids_of(widget3) + end + + it "`equal_to_any_of:` with `nil` nested within `any_of` matches documents with null values" do + index_into( + graphql, + build(:widget, amount_cents: 1000), + widget2 = build(:widget, amount_cents: nil), + build(:widget, amount_cents: 2500) + ) + + expect( + ids_of(search_datastore(filter: + {"any_of" => [{"amount_cents" => {"equal_to_any_of" => [nil]}}]}).to_a) + ).to eq ids_of(widget2) + end + + describe "`not`" do + it "negates the inner filter expression" do + index_into( + graphql, + widget1 = build(:widget), + widget2 = build(:widget), + widget3 = build(:widget) + ) + + inner_not_result = search_with_freeform_filter({"id" => {"not" => {"equal_to_any_of" => ids_of(widget1, widget3)}}}) + outer_not_result = search_with_freeform_filter({"not" => {"id" => {"equal_to_any_of" => ids_of(widget1, widget3)}}}) + + expect(inner_not_result).to eq(outer_not_result).and match_array(ids_of(widget2)) + end + + it "can negate multiple inner filter predicates" do + index_into( + graphql, + widget1 = build(:widget, amount_cents: 100), + _widget2 = build(:widget, amount_cents: 205), + widget3 = build(:widget, amount_cents: 400) + ) + + inner_not_result = search_with_freeform_filter({"amount_cents" => {"not" => { + "gte" => 200, + "lt" => 300 + }}}) + + outer_not_result = search_with_freeform_filter({"not" => {"amount_cents" => { + "gte" => 200, + "lt" => 300 + }}}) + + expect(inner_not_result).to eq(outer_not_result).and match_array(ids_of(widget1, widget3)) + end + + it "negates a complex compound inner filter expression" do + index_into( + graphql, + widget1 = build(:widget, options: {size: "SMALL", color: "GREEN"}, cost: {currency: "USD", amount_cents: 100}), + _widget2 = build(:widget, options: {size: "MEDIUM", color: "GREEN"}, cost: {currency: "USD", amount_cents: 200}), + widget3 = build(:widget, options: {size: "SMALL", color: "RED"}, cost: {currency: "GBP", amount_cents: 200}), + _widget4 = build(:widget, options: {size: "SMALL", color: "RED"}, cost: {currency: "GBP", amount_cents: 100}), + widget5 = build(:widget, options: {size: "MEDIUM", color: "RED"}, cost: {currency: "GBP", amount_cents: 200}) + ) + + result = search_with_freeform_filter({"not" => { + "cost" => { + "any_of" => [ + {"currency" => {"equal_to_any_of" => ["USD"]}}, + {"amount_cents" => {"equal_to_any_of" => [100]}} + ] + }, + "options" => { + "any_of" => [ + {"size" => {"equal_to_any_of" => [size_of("MEDIUM")]}}, + {"color" => {"equal_to_any_of" => [color_of("RED")]}} + ] + } + }}) + + expect(result).to match_array(ids_of(widget1, widget3, widget5)) + end + + it "works correctly when included alongside other filtering operators" do + index_into( + graphql, + _widget1 = build(:widget, amount_cents: 100), + _widget2 = build(:widget, amount_cents: 205), + widget3 = build(:widget, amount_cents: 400) + ) + + inner_not_result = search_with_freeform_filter({"amount_cents" => { + "gt" => 200, + "not" => {"equal_to_any_of" => [205]} + }}) + + outer_not_result = search_with_freeform_filter({ + "amount_cents" => { + "gt" => 200 + }, + "not" => { + "amount_cents" => { + "equal_to_any_of" => [205] + } + } + }) + + expect(inner_not_result).to eq(outer_not_result).and match_array(ids_of(widget3)) + end + + it "works correctly when included alongside an `any_of`" do + index_into( + graphql, + widget1 = build(:widget, amount_cents: 100), + _widget2 = build(:widget, amount_cents: 205), + _widget3 = build(:widget, amount_cents: 400), + widget4 = build(:widget, amount_cents: 550) + ) + + inner_not_result = search_with_freeform_filter({"amount_cents" => { + "not" => {"equal_to_any_of" => [205]}, + "any_of" => [ + {"gt" => 500}, + {"lt" => 300} + ] + }}) + + outer_not_result = search_with_freeform_filter({ + "not" => { + "amount_cents" => {"equal_to_any_of" => [205]} + }, + "amount_cents" => { + "any_of" => [ + {"gt" => 500}, + {"lt" => 300} + ] + } + }) + + expect(inner_not_result).to eq(outer_not_result).and match_array(ids_of(widget1, widget4)) + end + + it "works correctly when `not` is within `any_of`" do + index_into( + graphql, + widget1 = build(:widget, amount_cents: 100), + widget2 = build(:widget, amount_cents: 205), + _widget3 = build(:widget, amount_cents: 400), + widget4 = build(:widget, amount_cents: 550) + ) + + inner_not_result = search_with_freeform_filter({"amount_cents" => { + "any_of" => [ + {"not" => {"equal_to_any_of" => [400]}}, + {"gt" => 500}, + {"lt" => 300} + ] + }}) + + outer_not_result = search_with_freeform_filter({ + "not" => { + "amount_cents" => {"equal_to_any_of" => [400]} + }, + "amount_cents" => { + "any_of" => [ + {"gt" => 500}, + {"lt" => 300} + ] + } + }) + + expect(inner_not_result).to eq(outer_not_result).and match_array(ids_of(widget1, widget2, widget4)) + end + + it "filters to no documents when filtering to `expression AND NOT (expression)`" do + index_into( + graphql, + _widget1 = build(:widget, amount_cents: 100), + _widget2 = build(:widget, amount_cents: 205), + _widget3 = build(:widget, amount_cents: 400) + ) + + inner_not_result = search_with_freeform_filter({"amount_cents" => { + "equal_to_any_of" => [205], + "not" => {"equal_to_any_of" => [205]} + }}) + + outer_not_result = search_with_freeform_filter({ + "not" => { + "amount_cents" => {"equal_to_any_of" => [205]} + }, + "amount_cents" => { + "equal_to_any_of" => [205] + } + }) + + expect(inner_not_result).to eq(outer_not_result).and eq [] + end + + it "handles nested `not`s" do + index_into( + graphql, + widget1 = build(:widget, amount_cents: 100), + widget2 = build(:widget, amount_cents: 205), + widget3 = build(:widget, amount_cents: 400) + ) + + inner_not_result = search_with_freeform_filter({"amount_cents" => { + "not" => { + "not" => {"equal_to_any_of" => [205]} + } + }}) + + outer_not_result = search_with_freeform_filter({"not" => { + "amount_cents" => { + "not" => {"equal_to_any_of" => [205]} + } + }}) + + triple_nested_not = search_with_freeform_filter({"amount_cents" => { + "not" => { + "not" => { + "not" => {"equal_to_any_of" => [205]} + } + } + }}) + + expect(inner_not_result).to eq(outer_not_result).and match_array(ids_of(widget2)) + expect(triple_nested_not).to match_array(ids_of(widget1, widget3)) + end + + it "is ignored when set to nil" do + index_into( + graphql, + widget1 = build(:widget, amount_cents: 100), + widget2 = build(:widget, amount_cents: 205), + widget3 = build(:widget, amount_cents: 400) + ) + + inner_not_result1 = search_with_freeform_filter({"amount_cents" => { + "not" => nil + }}) + + outer_not_result1 = search_with_freeform_filter({"not" => { + "amount_cents" => nil + }}) + + inner_not_result2 = search_with_freeform_filter({"amount_cents" => { + "not" => nil, + "lt" => 200 + }}) + + outer_not_result2 = search_with_freeform_filter({ + "not" => { + "amount_cents" => nil + }, + "amount_cents" => { + "lt" => 200 + } + }) + + inner_not_result3 = search_with_freeform_filter({"amount_cents" => { + "not" => {"equal_to_any_of" => nil} + }}) + + outer_not_result3 = search_with_freeform_filter({"not" => { + "amount_cents" => {"equal_to_any_of" => nil} + }}) + + expect(inner_not_result1).to eq(outer_not_result1).and match_array(ids_of(widget1, widget2, widget3)) + expect(inner_not_result2).to eq(outer_not_result2).and match_array(ids_of(widget1)) + expect(inner_not_result3).to eq(outer_not_result3).and match_array(ids_of(widget1, widget2, widget3)) + end + end + + def search_with_freeform_filter(filter, **options) + ids_of(search_datastore(filter: filter, sort: [], **options).to_a) + end + + def search_with_filter(field, operator, value) + ids_of(search_datastore(filter: {field => {operator => value}}, sort: []).to_a) + end + + def enum_value(type_name, value_name) + graphql.schema.type_named(type_name).enum_value_named(value_name) + end + + def size_of(value_name) + enum_value("SizeInput", value_name) + end + + def color_of(value_name) + enum_value("ColorInput", value_name) + end + end + end +end diff --git a/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/misc_spec.rb b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/misc_spec.rb new file mode 100644 index 00000000..c8043307 --- /dev/null +++ b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/misc_spec.rb @@ -0,0 +1,151 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_integration_support" +require "elastic_graph/errors" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "misc" do + include_context "DatastoreQueryIntegrationSupport" + + # This test must always run with VCR disabled, because it depends on sending + # a real slow query and then timing out on the client side. + describe "deadline behavior", :no_vcr do + around do |ex| + GC.disable # ensure GC pauses don't make the `search_datastore` call take longer + ex.run + GC.enable # re-enable GC. + end + + before do + # Prevent `list_indices_matching` datastore request from interacting with the timeout + # test below, by pre-caching its results. + pre_cache_index_state(graphql) + end + + it "aborts a slow query on the basis of the deadline" do + # To force a slow query, we update the body before submitting the search to have a slow script. + # Note: 100K is the most loop iterations Elasticsearch allows: + # https://github.com/elastic/elasticsearch/blob/v7.12.0/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java#L52-L57 + # + # On my machine, this query takes 3-4 seconds. (But the timeout will make the test finish much faster!) + intentionally_slow_client_class = ::Class.new(::SimpleDelegator) do + def msearch(body:, **args, &block) + body.last[:script_fields] = { + slow: { + script: <<~EOS + String text = ""; + + for (int i = 0; i < 100000; i++) { + text += "abcdef"; + } + + return "done"; + EOS + } + } + + super(body: body, **args, &block) + end + end + + client = intentionally_slow_client_class.new(main_datastore_client) + graphql = build_graphql(clients_by_name: {"main" => client}) + + index_into(graphql, build(:widget), build(:widget)) + + expect { + search_datastore(graphql: graphql) do |query| + # Compute the deadline just before the query is executed, since the deadline is based on the + # system monotonic clock and we want a very tight deadline. Since graphql dependencies + # are loaded and initialized lazily, if we eagerly calculate the deadline before calling + # `search_datastore`, the lazy initialization might cause us to have already passed our deadline + # before the router is ready to submit the query to the datastore. That's a slightly different + # case and code path than timing out while waiting on the datastore query, so we want to + # guard against that here by delaying the computation of the deadline until the last possible + # moment. + query.with(monotonic_clock_deadline: graphql.monotonic_clock.now_in_ms + 200) + end + }.to raise_error(Errors::RequestExceededDeadlineError, a_string_including("request exceeded timeout")) + # Really it should take ~50 ms, but give extra time for ruby VM overhead. + # On CI we've seen it take 1000-1100 ms but never move than that. Meanwhile, + # without the `monotonic_clock_deadline:` arg, the query takes 3-4 seconds, so + # it taking under 1200 ms still demonstrates the query is being aborted early. + .and take_less_than(1200).milliseconds + .and log_warning(a_string_including("request exceeded timeout")) + end + end + + context "when the indexed type has nested `default_sort_fields`" do + let(:graphql) do + build_graphql(schema_definition: lambda do |schema| + schema.object_type "Money" do |t| + t.field "currency", "String!" + t.field "amount_cents", "Int!" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime" + t.field "workspace_id", "ID", name_in_index: "workspace_id2" + t.field "cost", "Money" + + t.index "widgets" do |i| + i.rollover :yearly, "created_at" + i.route_with "workspace_id" + i.default_sort "cost.amount_cents", :desc + end + end + end) + end + + it "correctly sorts on that nested field" do + index_into( + graphql, + widget1 = build(:widget, cost: {currency: "USD", amount_cents: 100}), + widget2 = build(:widget, cost: {currency: "USD", amount_cents: 300}), + widget3 = build(:widget, cost: {currency: "USD", amount_cents: 200}) + ) + + expect(ids_of(search_datastore.to_a)).to eq(ids_of([widget2, widget3, widget1])) + end + end + + context "on a rollover index when no concrete indices yet exist (e.g. before indexing the first document)" do + let(:graphql) do + build_graphql(schema_definition: lambda do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime!" + t.index unique_index_name do |i| + i.rollover :monthly, "created_at" + end + end + end) + end + + it "returns an empty result" do + widgets_def = graphql.datastore_core.index_definitions_by_name.fetch(unique_index_name) + + query = graphql.datastore_query_builder.new_query( + search_index_definitions: [widgets_def], + requested_fields: ["id"] + ) + + index_names = main_datastore_client.list_indices_matching("*") + expect(index_names).not_to include(a_string_including(unique_index_name)) + + results = perform_query(graphql, query) + + expect(results.to_a).to eq [] + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/pagination_shared_examples.rb b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/pagination_shared_examples.rb new file mode 100644 index 00000000..b4c83301 --- /dev/null +++ b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/pagination_shared_examples.rb @@ -0,0 +1,267 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + RSpec.shared_examples_for "DatastoreQuery pagination--integration" do + let(:graphql) { build_graphql(default_page_size: 3) } + + it "returns the first `default_page_size` items when no pagination args are provided" do + items, page_info = paginated_search + + expect(ids_of(items)).to eq ids_of(item1, item2, item3) + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + end + + it "can paginate forward and backward when sorting ascending" do + items, page_info = paginated_search(first: 2) + expect(ids_of(items)).to eq ids_of(item1, item2) + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + items, page_info = paginated_search(first: 2, after: items.last.cursor) + expect(ids_of(items)).to eq ids_of(item3, item4) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + + items, page_info = paginated_search(last: 2) + expect(ids_of(items)).to eq ids_of(item4, item5) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + + items, page_info = paginated_search(last: 2, before: items.first.cursor) + expect(ids_of(items)).to eq ids_of(item2, item3) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + end + + it "sorts ascending and paginates correctly when a node has `null` for the field being sorted on" do + index_doc_with_null_value + + # Forward paginate with ascending sort... + items, page_info = paginated_search(first: 4) + expect(ids_of(items)).to eq ids_of(item_with_null, item1, item2, item3) + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + items, page_info = paginated_search(first: 4, after: items.last.cursor) + expect(ids_of(items)).to eq ids_of(item4, item5) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + + items, page_info = paginated_search(first: 4, after: items.last.cursor) + expect(ids_of(items)).to be_empty + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + + # Backward paginate with ascending sort... + items, page_info = paginated_search(last: 4) + expect(ids_of(items)).to eq ids_of(item2, item3, item4, item5) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + + items, page_info = paginated_search(last: 4, before: items.first.cursor) + expect(ids_of(items)).to eq ids_of(item_with_null, item1) + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + items, page_info = paginated_search(last: 4, before: items.first.cursor) + expect(ids_of(items)).to be_empty + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + # Forward paginate with ascending sort 1 node at a time (to ensure that the cursor references the node with a `null` value)... + items, page_info = paginated_search(first: 1) + expect(ids_of(items)).to eq ids_of(item_with_null) + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + [item1, item2, item3, item4].each do |item| + items, page_info = paginated_search(first: 1, after: items.last.cursor) + expect(ids_of(items)).to eq ids_of(item) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + end + + items, page_info = paginated_search(first: 1, after: items.last.cursor) + expect(ids_of(items)).to eq ids_of(item5) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + + items, page_info = paginated_search(first: 1, after: items.last.cursor) + expect(ids_of(items)).to be_empty + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + + # Backwards paginate with ascending sort 1 node at a time (to ensure that the cursor references the node with a `null` value)... + items, page_info = paginated_search(last: 1) + expect(ids_of(items)).to eq ids_of(item5) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + + [item4, item3, item2, item1].each do |item| + items, page_info = paginated_search(last: 1, before: items.first.cursor) + expect(ids_of(items)).to eq ids_of(item) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + end + + items, page_info = paginated_search(last: 1, before: items.first.cursor) + expect(ids_of(items)).to eq ids_of(item_with_null) + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + items, page_info = paginated_search(last: 1, before: items.first.cursor) + expect(ids_of(items)).to be_empty + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + end + + it "can jump to the middle using just `before` or `after` without `first` and `last`" do + items, page_info = paginated_search(before: cursor_of(item5)) + expect(ids_of(items)).to eq ids_of(item2, item3, item4) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + + items, page_info = paginated_search(after: cursor_of(item1)) + expect(ids_of(items)).to eq ids_of(item2, item3, item4) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + end + + it "returns the last y of the first x when `first: x, last: y` is provided" do + items, page_info = paginated_search(first: 4, last: 2) + expect(ids_of(items)).to eq ids_of(item3, item4) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + + items, page_info = paginated_search(first: 2, last: 4) + expect(ids_of(items)).to eq ids_of(item1, item2) + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + items, page_info = paginated_search(first: 3, after: cursor_of(item1), last: 2) + expect(ids_of(items)).to eq ids_of(item3, item4) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + + items, page_info = paginated_search(first: 2, after: cursor_of(item1), last: 3) + expect(ids_of(items)).to eq ids_of(item2, item3) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + + items, page_info = paginated_search(first: 3, last: 2, before: cursor_of(item4)) + expect(ids_of(items)).to eq ids_of(item2, item3) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + + items, page_info = paginated_search(first: 3, after: cursor_of(item1), last: 2, before: cursor_of(item4)) + expect(ids_of(items)).to eq ids_of(item2, item3) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + end + + it "excludes documents with a cursor on or after `before` when either `first` or `after` are provided (which requires us to search the index from the start)" do + items, page_info = paginated_search(first: 3, before: cursor_of(item5)) + expect(ids_of(items)).to eq ids_of(item1, item2, item3) + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + items, page_info = paginated_search(first: 3, before: cursor_of(item3)) + expect(ids_of(items)).to eq ids_of(item1, item2) + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + items, page_info = paginated_search(after: cursor_of(item1), before: cursor_of(item4)) + expect(ids_of(items)).to eq ids_of(item2, item3) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + + items, page_info = paginated_search(first: 3, last: 2, before: cursor_of(item3)) + expect(ids_of(items)).to eq ids_of(item1, item2) + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + items, page_info = paginated_search(first: 3, after: cursor_of(item1), before: cursor_of(item4)) + expect(ids_of(items)).to eq ids_of(item2, item3) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + + items, page_info = paginated_search(first: 3, after: cursor_of(item2), last: 3, before: cursor_of(item5)) + expect(ids_of(items)).to eq ids_of(item3, item4) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + end + + it "excludes document with a cursor on or before `after` when `last` is provided without `first` (which requires us to search the index from the end)" do + items, page_info = paginated_search(after: cursor_of(item2), last: 4) + expect(ids_of(items)).to eq ids_of(item3, item4, item5) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + + items, page_info = paginated_search(after: cursor_of(item2), last: 3, before: cursor_of(item5)) + expect(ids_of(items)).to eq ids_of(item3, item4) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + end + + it "correctly returns no results when `before` and `after` cancel out or one of `first` and `last` are 0" do + items, page_info = paginated_search(after: cursor_of(item3), before: cursor_of(item4)) + expect(items).to eq [] + # Note: as explained in the comment on `Paginator#build_page_info`, it's not clear what the + # "correct" value for `has_previous_page`/`has_next_page` page is when we are dealing with an empty + # page, so here (and throughout this example)_ we just specify that it is a boolean of some sort. + expect(page_info).to have_attributes(has_previous_page: a_boolean, has_next_page: a_boolean) + + items, page_info = paginated_search(first: 0, after: cursor_of(item1)) + expect(items).to eq [] + expect(page_info).to have_attributes(has_previous_page: a_boolean, has_next_page: a_boolean) + + items, page_info = paginated_search(first: 0, after: cursor_of(item2)) + expect(items).to eq [] + expect(page_info).to have_attributes(has_previous_page: a_boolean, has_next_page: a_boolean) + + items, page_info = paginated_search(last: 0, before: cursor_of(item4)) + expect(items).to eq [] + expect(page_info).to have_attributes(has_previous_page: a_boolean, has_next_page: a_boolean) + + items, page_info = paginated_search(last: 0, first: 0) + expect(items).to eq [] + expect(page_info).to have_attributes(has_previous_page: a_boolean, has_next_page: a_boolean) + end + + it "returns correct values for `has_next_page` and `has_previous_page` when the page only one element" do + items, page_info = paginated_search(first: 1, after: cursor_of(item4)) + expect(ids_of(items)).to eq ids_of(item5) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + + items, page_info = paginated_search(last: 1, before: cursor_of(item2)) + expect(ids_of(items)).to eq ids_of(item1) + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + items, page_info = paginated_search(after: cursor_of(item2), before: cursor_of(item4)) + expect(ids_of(items)).to eq ids_of(item3) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + + items, page_info = paginated_search(first: 3, last: 1) + expect(ids_of(items)).to eq ids_of(item3) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + + items, page_info = paginated_search(first: 5, last: 1) + expect(ids_of(items)).to eq ids_of(item5) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + end + + it "correctly handles the a page of one item when that item's cursor is used for `before` and/or `after`" do + items, page_info = paginated_search(filter_to: item1) + expect(ids_of(items)).to eq ids_of(item1) + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: false) + + items, page_info = paginated_search(filter_to: item1, before: cursor_of(item1)) + expect(ids_of(items)).to be_empty + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + items, page_info = paginated_search(filter_to: item1, after: cursor_of(item1)) + expect(ids_of(items)).to be_empty + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + + items, page_info = paginated_search(filter_to: item1, before: cursor_of(item1), after: cursor_of(item1)) + expect(ids_of(items)).to be_empty + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: false) + end + + it "raises `GraphQL::ExecutionError` when the cursor is lacking required information" do + broken_cursor = DecodedCursor.new({"not" => "valid"}) + + expect { + paginated_search(first: 1, after: broken_cursor) + }.to raise_error ::GraphQL::ExecutionError, a_string_including("`#{broken_cursor.encode}` is not a valid cursor") + end + + it "raises errors when given a negative `first` or `last` option" do + expect { + paginated_search(last: -1) + }.to raise_error(::GraphQL::ExecutionError, a_string_including("last", "negative", "-1")) + + expect { + paginated_search(first: -7) + }.to raise_error(::GraphQL::ExecutionError, a_string_including("first", "negative", "-7")) + end + + def cursor_of(item) + DecodedCursor.new(item) + end + end + end +end diff --git a/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/pagination_spec.rb b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/pagination_spec.rb new file mode 100644 index 00000000..6ff9bada --- /dev/null +++ b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/pagination_spec.rb @@ -0,0 +1,205 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_integration_support" +require_relative "pagination_shared_examples" +require "elastic_graph/graphql/resolvers/relay_connection/search_response_adapter_builder" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "document pagination" do + include_context "DatastoreQueryIntegrationSupport" + + let(:time1) { "2019-06-20T12:00:00Z" } + let(:time2) { "2019-06-25T12:00:00Z" } + + let(:item1) { build(:widget, amount_cents: 100, id: "widget1", created_at: time1) } + let(:item2) { build(:widget, amount_cents: 150, id: "widget2", created_at: time2) } + let(:item3) { build(:widget, amount_cents: 200, id: "widget3", created_at: time1) } + let(:item4) { build(:widget, amount_cents: 250, id: "widget4", created_at: time2) } + let(:item5) { build(:widget, amount_cents: 300, id: "widget5", created_at: time1) } + let(:item_with_null) { build(:widget, amount_cents: nil, id: "widget_with_null", created_at: time1) } + + let(:sort_list) { [{"amount_cents" => {"order" => "asc"}}] } + let(:decoded_cursor_factory) { decoded_cursor_factory_for(sort_list) } + + before do + index_into(graphql, item1, item2, item3, item4, item5) + end + + include_examples "DatastoreQuery pagination--integration" do + def index_doc_with_null_value + index_into(graphql, item_with_null) + end + + describe "pagination behaviors unique to document pagination" do + # doesn't apply to aggregation pagination because it doesn't support a `pagination:` argument + # (instead it has them "unwrapped"). + it "treats `document_pagination: nil`, `document_pagination: {}`, and no `document_pagination` arg the same" do + items1, page_info1 = paginated_search + expect(ids_of(items1)).to eq ids_of(item1, item2, item3) + expect(page_info1).to have_attributes(has_previous_page: false, has_next_page: true) + + items2, page_info2 = paginated_search(document_pagination: nil) + expect(items2).to eq items1 + expect(page_info2).to eq page_info1 + + items3, page_info3 = paginated_search(document_pagination: {}) + expect(items3).to eq items1 + expect(page_info3).to eq page_info1 + end + + # We don't yet support specifying sorting on aggregation pagination, so this doesn't apply to that case. + it "can paginate forward and backward when sorting descending" do + paginated_search = ->(**options) { paginated_search(sort: [{"amount_cents" => {"order" => "desc"}}], **options) } + + items, page_info = paginated_search.call(first: 2) + expect(ids_of(items)).to eq ids_of(item5, item4) + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + items, page_info = paginated_search.call(first: 2, after: items.last.cursor) + expect(ids_of(items)).to eq ids_of(item3, item2) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + + items, page_info = paginated_search.call(last: 2) + expect(ids_of(items)).to eq ids_of(item2, item1) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + + items, page_info = paginated_search.call(last: 2, before: items.first.cursor) + expect(ids_of(items)).to eq ids_of(item4, item3) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + end + + # We don't yet support specifying sorting on aggregation pagination, so this doesn't apply to that case. + it "sorts descending and paginates correctly when a node has `null` for the field being sorted on" do + index_doc_with_null_value + + paginated_search = ->(**options) { paginated_search(sort: [{"amount_cents" => {"order" => "desc"}}], **options) } + + # Forward paginate with descending sort... + items, page_info = paginated_search.call(first: 4) + expect(ids_of(items)).to eq ids_of(item5, item4, item3, item2) + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + items, page_info = paginated_search.call(first: 4, after: items.last.cursor) + expect(ids_of(items)).to eq ids_of(item1, item_with_null) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + + items, page_info = paginated_search.call(first: 4, after: items.last.cursor) + expect(ids_of(items)).to be_empty + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + + # Backward paginate with descending sort... + items, page_info = paginated_search.call(last: 4) + expect(ids_of(items)).to eq ids_of(item3, item2, item1, item_with_null) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + + items, page_info = paginated_search.call(last: 4, before: items.first.cursor) + expect(ids_of(items)).to eq ids_of(item5, item4) + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + items, page_info = paginated_search.call(last: 4, before: items.first.cursor) + expect(ids_of(items)).to be_empty + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + # Forward paginate with descending sort 1 node at a time (to ensure that the cursor references the node with a `null` value)... + items, page_info = paginated_search.call(first: 1) + expect(ids_of(items)).to eq ids_of(item5) + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + [item4, item3, item2, item1].each do |item| + items, page_info = paginated_search.call(first: 1, after: items.last.cursor) + expect(ids_of(items)).to eq ids_of(item) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + end + + items, page_info = paginated_search.call(first: 1, after: items.last.cursor) + expect(ids_of(items)).to eq ids_of(item_with_null) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + + items, page_info = paginated_search.call(first: 1, after: items.last.cursor) + expect(ids_of(items)).to be_empty + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + + # Backward paginate with descending sort 1 node at a time (to ensure that the cursor references the node with a `null` value)... + items, page_info = paginated_search.call(last: 1) + expect(ids_of(items)).to eq ids_of(item_with_null) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: false) + + [item1, item2, item3, item4].each do |item| + items, page_info = paginated_search.call(last: 1, before: items.first.cursor) + expect(ids_of(items)).to eq ids_of(item) + expect(page_info).to have_attributes(has_previous_page: true, has_next_page: true) + end + + items, page_info = paginated_search.call(last: 1, before: items.first.cursor) + expect(ids_of(items)).to eq ids_of(item5) + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + + items, page_info = paginated_search.call(last: 1, before: items.first.cursor) + expect(ids_of(items)).to be_empty + expect(page_info).to have_attributes(has_previous_page: false, has_next_page: true) + end + + # We don't yet support specifying sorting on aggregation pagination, so this doesn't apply to that case. + it "properly supports `before` and `after` when mixing ascending and descending sort clauses" do + sort = [{"created_at" => {"order" => "asc"}}, {"amount_cents" => {"order" => "desc"}}] + # sort order: item5, item3, item1, item4, item2 + reverse_sort = [{"created_at" => {"order" => "desc"}}, {"amount_cents" => {"order" => "asc"}}] + decoded_cursor_factory = decoded_cursor_factory_for(sort) + + items, _page_info = paginated_search(sort: sort, after: cursor_of(item5, decoded_cursor_factory: decoded_cursor_factory), before: cursor_of(item2, decoded_cursor_factory: decoded_cursor_factory)) + expect(ids_of(items)).to eq ids_of(item3, item1, item4) + + items, _page_info = paginated_search(sort: reverse_sort, after: cursor_of(item2, decoded_cursor_factory: decoded_cursor_factory), before: cursor_of(item5, decoded_cursor_factory: decoded_cursor_factory)) + expect(ids_of(items)).to eq ids_of(item4, item1, item3) + end + + # Aggregation pagination doesn't need to worry about this because grouping guarantees uniqueness of the keys that are encoded as cursors. + it "ensures each edge has a unique cursor value, even when they have the same values for the requested sort fields, to avoid ambiguity" do + index_into(graphql, item1b = item1.merge(id: item1.fetch(:id) + "b")) + + documents, _page_info = paginated_search(first: 2).to_a + + expect(ids_of(documents)).to eq ids_of(item1, item1b) + expect(documents.map(&:cursor).uniq.count).to eq 2 + end + end + end + + def paginated_search(first: nil, after: nil, last: nil, before: nil, document_pagination: nil, sort: sort_list, filter_to: nil) + document_pagination ||= {first: first, after: after, last: last, before: before}.compact + query = nil + response = search_datastore( + sort: sort, + document_pagination: document_pagination, + filter: ({"id" => {"equal_to_any_of" => ids_of(filter_to)}} if filter_to) + ) { |q| query = q } + + adapter = Resolvers::RelayConnection::SearchResponseAdapterBuilder.build_from( + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: :snake_case, overrides: {}), + search_response: response, + query: query + ) + + [adapter.nodes, adapter.page_info] + end + + def cursor_of(widget, decoded_cursor_factory: self.decoded_cursor_factory) + values = decoded_cursor_factory.sort_fields.to_h do |field| + value = widget.fetch(field.to_sym) + # The datastore use integers (milliseconds since epoch) as sort values for timestamp fields. + value = ::Time.parse(value).to_i * 1000 if field.end_with?("_at") + [field, value] + end + + DecodedCursor.new(values) + end + end + end +end diff --git a/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/requested_fields_spec.rb b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/requested_fields_spec.rb new file mode 100644 index 00000000..0116d4ee --- /dev/null +++ b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/requested_fields_spec.rb @@ -0,0 +1,46 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_integration_support" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "#requested_fields" do + include_context "DatastoreQueryIntegrationSupport" + + specify "only the fields requested are returned by the datastore (including embedded object fields), ignoring unknown fields" do + index_into(graphql, build(:widget)) + + results = search_datastore(requested_fields: ["name", "id", "options.size", "foobar"]) + + expect(results.first.payload.keys).to contain_exactly("name", "id", "options") + expect(results.first["options"].keys).to contain_exactly("size") + end + + specify "returns only id field in payload when no fields are requested (but still returns the right number of documents, provided `individual_docs_needed` is true)" do + index_into(graphql, widget1 = build(:widget), widget2 = build(:widget)) + + results = search_datastore(requested_fields: [], individual_docs_needed: true) + + expect(results.size).to eq 2 # 2 results should be returned to support `PageInfo` working correctly. + expect(results.map(&:payload)).to contain_exactly( + {"id" => widget1.fetch(:id)}, + {"id" => widget2.fetch(:id)} + ) + end + + specify "returns no fields and no documents when no fields are requested (provided `individual_docs_needed` is not forced to true)" do + index_into(graphql, build(:widget), build(:widget)) + + results = search_datastore(requested_fields: []) + + expect(results.size).to eq 0 + end + end + end +end diff --git a/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/sub_aggregations_spec.rb b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/sub_aggregations_spec.rb new file mode 100644 index 00000000..5d3c30db --- /dev/null +++ b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/sub_aggregations_spec.rb @@ -0,0 +1,1149 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_aggregation_query_integration_support" +require "support/sub_aggregation_support" +require "time" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "sub-aggregations" do + using Aggregation::SubAggregationRefinements + include_context "DatastoreAggregationQueryIntegrationSupport" + include_context "sub-aggregation support", Aggregation::NonCompositeGroupingAdapter + + before do + index_into( + graphql, + build( + :team, + current_players: [build(:player, name: "Bob", seasons: [ + build(:player_season, year: 2020), + build(:player_season, year: 2021) + ])], + seasons: [ + build(:team_season, year: 2022, record: build(:team_record, wins: 50), notes: ["new rules"], players: [ + build(:player, seasons: [build(:player_season, year: 2015), build(:player_season, year: 2016)]), + build(:player, seasons: [build(:player_season, year: 2015)]) + ]) + ] + ), + build( + :team, + current_players: [build(:player, name: "Tom", seasons: []), build(:player, name: "Ted", seasons: [])], + seasons: [] + ), + build( + :team, + current_players: [build(:player, name: "Dan", seasons: []), build(:player, name: "Ben", seasons: [])], + seasons: [ + build(:team_season, year: 2022, record: build(:team_record, wins: 40), notes: ["new rules"], players: []), + build(:team_season, year: 2021, record: build(:team_record, wins: 60), notes: ["old rules"], players: []), + build(:team_season, year: 2020, record: build(:team_record, wins: 30), notes: ["old rules", "pandemic"], players: []) + ] + ), + build( + :team, + current_players: [], + seasons: [] + ) + ) + end + + it "supports multiple sibling sub-aggregations" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["current_players_nested"], query: sub_aggregation_query_of(name: "current_players_nested", first: 12)), + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", first: 5)) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq [{ + "doc_count" => 0, + "key" => {}, + "teams:current_players_nested" => {"doc_count" => 5, "meta" => outer_meta(size: 12)}, + "teams:seasons_nested" => {"doc_count" => 4, "meta" => outer_meta(size: 5)} + }] + end + + it "returns a well-structured response even when filtering to no shard routing values", :expect_search_routing do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["current_players_nested"], query: sub_aggregation_query_of(name: "current_players_nested", first: 12)), + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", first: 5)) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams", filter: {"league" => {"equal_to_any_of" => []}}) + + expect(results).to eq [{ + "doc_count" => 0, + "key" => {}, + "teams:current_players_nested" => {"doc_count" => 0, "meta" => outer_meta(size: 12)}, + "teams:seasons_nested" => {"doc_count" => 0, "meta" => outer_meta(size: 5)} + }] + end + + it "returns a well-structured response even when filtering on the rollover field to an empty set of values", :expect_index_exclusions do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["current_players_nested"], query: sub_aggregation_query_of(name: "current_players_nested", first: 12)), + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", first: 5)) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams", filter: {"formed_on" => {"equal_to_any_of" => []}}) + + expect(results).to eq [{ + "doc_count" => 0, + "key" => {}, + "teams:current_players_nested" => {"doc_count" => 0, "meta" => outer_meta(size: 12)}, + "teams:seasons_nested" => {"doc_count" => 0, "meta" => outer_meta(size: 5)} + }] + end + + it "returns a well-structured response even when filtering on the rollover field with criteria that excludes all existing indices", :expect_index_exclusions do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["current_players_nested"], query: sub_aggregation_query_of(name: "current_players_nested", first: 12)), + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", first: 5)) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams", filter: {"formed_on" => {"gt" => "7890-01-01"}}) + + expect(results).to eq [{ + "doc_count" => 0, + "key" => {}, + "teams:current_players_nested" => {"doc_count" => 0, "meta" => outer_meta(size: 12)}, + "teams:seasons_nested" => {"doc_count" => 0, "meta" => outer_meta(size: 5)} + }] + end + + it "supports sub-aggregations of sub-aggregations of sub-aggregations" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + first: 15, + sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested", "players_nested"], query: sub_aggregation_query_of( + name: "players_nested", + first: 16, + sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested", "players_nested", "seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + first: 17 + )) + ] + )) + ] + )) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq [{ + "doc_count" => 0, + "key" => {}, + "teams:seasons_nested" => { + "doc_count" => 4, + "meta" => outer_meta(size: 15), + "teams:seasons_nested:seasons_nested.players_nested" => { + "doc_count" => 2, + "meta" => outer_meta(size: 16), + "teams:seasons_nested:players_nested:seasons_nested.players_nested.seasons_nested" => { + "doc_count" => 3, + "meta" => outer_meta(size: 17) + } + } + } + }] + end + + it "supports nesting sub-aggreations under an extra object layer" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["the_nested_fields", "current_players"], query: sub_aggregation_query_of(name: "current_players")) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq [{ + "doc_count" => 0, + "key" => {}, + "teams:the_nested_fields.current_players" => {"doc_count" => 5, "meta" => outer_meta} + }] + end + + it "supports filtered sub-aggregations" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["current_players_nested"], query: sub_aggregation_query_of(name: "current_players_nested", filter: { + "name" => {"equal_to_any_of" => %w[Dan Ted Bob]} + })) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq [{ + "doc_count" => 0, + "key" => {}, + "teams:current_players_nested" => { + "meta" => outer_meta({"bucket_path" => ["current_players_nested:filtered"]}), + "doc_count" => 5, + "current_players_nested:filtered" => {"doc_count" => 3} + } + }] + end + + it "ignores empty filters" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["current_players_nested"], query: sub_aggregation_query_of(name: "current_players_nested", filter: { + "name" => {"equal_to_any_of" => nil} + })) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq [{ + "doc_count" => 0, + "key" => {}, + "teams:current_players_nested" => {"doc_count" => 5, "meta" => outer_meta} + }] + end + + it "supports sub-aggregations under a filtered sub-aggregation" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["current_players_nested"], query: sub_aggregation_query_of( + name: "current_players_nested", + filter: {"name" => {"equal_to_any_of" => %w[Dan Ted Bob]}}, + sub_aggregations: [nested_sub_aggregation_of( + path_in_index: ["current_players_nested", "seasons_nested"], + query: sub_aggregation_query_of(name: "seasons_nested") + )] + )) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq [{ + "doc_count" => 0, + "key" => {}, + "teams:current_players_nested" => { + "meta" => outer_meta({"bucket_path" => ["current_players_nested:filtered"]}), + "doc_count" => 5, + "current_players_nested:filtered" => { + "doc_count" => 3, + "teams:current_players_nested:current_players_nested.seasons_nested" => {"doc_count" => 2, "meta" => outer_meta} + } + } + }] + end + + it "supports filtered sub-aggregations under a filtered sub-aggregation" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["current_players_nested"], query: sub_aggregation_query_of( + name: "current_players_nested", + filter: {"name" => {"equal_to_any_of" => %w[Dan Ted Bob]}}, + sub_aggregations: [nested_sub_aggregation_of( + path_in_index: ["current_players_nested", "seasons_nested"], + query: sub_aggregation_query_of(name: "seasons_nested", filter: {"year" => {"gt" => 2020}}) + )] + )) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq [{ + "doc_count" => 0, + "key" => {}, + "teams:current_players_nested" => { + "meta" => outer_meta({"bucket_path" => ["current_players_nested:filtered"]}), + "doc_count" => 5, + "current_players_nested:filtered" => { + "doc_count" => 3, + "teams:current_players_nested:current_players_nested.seasons_nested" => { + "meta" => outer_meta({"bucket_path" => ["seasons_nested:filtered"]}), + "doc_count" => 2, + "seasons_nested:filtered" => {"doc_count" => 1} + } + } + } + }] + end + + context "with computations" do + it "can compute ungrouped aggregated values" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", computations: [ + computation_of("seasons_nested", "year", :min, computed_field_name: "exact_min"), + computation_of("seasons_nested", "the_record", "win_count", :max, computed_field_name: "exact_max"), + computation_of("seasons_nested", "the_record", "win_count", :avg, computed_field_name: "approximate_avg") + ])) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq [{ + "doc_count" => 0, + "key" => {}, + "teams:seasons_nested" => { + "doc_count" => 4, + "meta" => outer_meta, + "seasons_nested:seasons_nested.the_record.win_count:approximate_avg" => {"value" => 45}, + "seasons_nested:seasons_nested.the_record.win_count:exact_max" => {"value" => 60.0}, + "seasons_nested:seasons_nested.year:exact_min" => {"value" => 2020.0} + } + }] + end + + it "can compute filtered aggregated values" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + filter: {"year" => {"gt" => 2021}}, + computations: [ + computation_of("seasons_nested", "year", :min, computed_field_name: "exact_min"), + computation_of("seasons_nested", "the_record", "win_count", :max, computed_field_name: "exact_max"), + computation_of("seasons_nested", "the_record", "win_count", :avg, computed_field_name: "approximate_avg") + ] + )) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq [{ + "doc_count" => 0, + "key" => {}, + "teams:seasons_nested" => { + "doc_count" => 4, + "meta" => outer_meta({"bucket_path" => ["seasons_nested:filtered"]}), + "seasons_nested:filtered" => { + "doc_count" => 2, + "seasons_nested:seasons_nested.the_record.win_count:approximate_avg" => {"value" => 45}, + "seasons_nested:seasons_nested.the_record.win_count:exact_max" => {"value" => 50.0}, + "seasons_nested:seasons_nested.year:exact_min" => {"value" => 2022.0} + } + } + }] + end + + it "can compute grouped aggregated values" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + computations: [ + computation_of("seasons_nested", "year", :min, computed_field_name: "exact_min"), + computation_of("seasons_nested", "the_record", "win_count", :max, computed_field_name: "exact_max"), + computation_of("seasons_nested", "the_record", "win_count", :avg, computed_field_name: "approximate_avg") + ], + groupings: [ + field_term_grouping_of("seasons_nested", "notes") + ] + )) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq [{ + "doc_count" => 0, + "key" => {}, + "teams:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.notes"]}), + "doc_count" => 4, + "seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + term_bucket("new rules", 2, { + "seasons_nested:seasons_nested.year:exact_min" => {"value" => 2022.0}, + "seasons_nested:seasons_nested.the_record.win_count:exact_max" => {"value" => 50.0}, + "seasons_nested:seasons_nested.the_record.win_count:approximate_avg" => {"value" => 45.0} + }), + term_bucket("old rules", 2, { + "seasons_nested:seasons_nested.year:exact_min" => {"value" => 2020.0}, + "seasons_nested:seasons_nested.the_record.win_count:exact_max" => {"value" => 60.0}, + "seasons_nested:seasons_nested.the_record.win_count:approximate_avg" => {"value" => 45.0} + }), + term_bucket("pandemic", 1, { + "seasons_nested:seasons_nested.year:exact_min" => {"value" => 2020.0}, + "seasons_nested:seasons_nested.the_record.win_count:exact_max" => {"value" => 30.0}, + "seasons_nested:seasons_nested.the_record.win_count:approximate_avg" => {"value" => 30.0} + }) + ] + } + }.with_missing_value_bucket(0, { + "seasons_nested:seasons_nested.year:exact_min" => {"value" => nil}, + "seasons_nested:seasons_nested.the_record.win_count:exact_max" => {"value" => nil}, + "seasons_nested:seasons_nested.the_record.win_count:approximate_avg" => {"value" => nil} + }) + }] + end + + it "can compute aggregated values on multiple levels of sub-aggregations" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + computations: [computation_of("seasons_nested", "year", :min)], + sub_aggregations: [nested_sub_aggregation_of(path_in_index: ["seasons_nested", "players_nested"], query: sub_aggregation_query_of( + name: "players_nested", + computations: [computation_of("seasons_nested", "players_nested", "name", :cardinality)], + sub_aggregations: [nested_sub_aggregation_of(path_in_index: ["seasons_nested", "players_nested", "seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + computations: [computation_of("seasons_nested", "players_nested", "seasons_nested", "year", :max)] + ))] + ))] + )) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq [{ + "doc_count" => 0, + "key" => {}, + "teams:seasons_nested" => { + "doc_count" => 4, + "meta" => outer_meta, + "seasons_nested:seasons_nested.year:min" => {"value" => 2020.0}, + "teams:seasons_nested:seasons_nested.players_nested" => { + "doc_count" => 2, + "meta" => outer_meta, + "players_nested:seasons_nested.players_nested.name:cardinality" => {"value" => 2}, + "teams:seasons_nested:players_nested:seasons_nested.players_nested.seasons_nested" => { + "doc_count" => 3, + "meta" => outer_meta, + "seasons_nested:seasons_nested.players_nested.seasons_nested.year:max" => {"value" => 2016.0} + } + } + } + }] + end + end + + context "with groupings (using the `NonCompositeGroupingAdapter`)" do + include_context "sub-aggregation support", Aggregation::NonCompositeGroupingAdapter + + it "can group sub-aggregations on a single non-date field" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", groupings: [ + field_term_grouping_of("seasons_nested", "year") + ])) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq([{ + "doc_count" => 0, + "key" => {}, + "teams:seasons_nested" => { + "doc_count" => 4, + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}), + "seasons_nested.year" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.year"], "key_path" => ["key"]}), + "buckets" => [ + term_bucket(2022, 2), + term_bucket(2020, 1), + term_bucket(2021, 1) + ], + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0 + } + }.with_missing_value_bucket(0) + }]) + end + + it "can group sub-aggregations on multiple non-date fields" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", groupings: [ + field_term_grouping_of("seasons_nested", "year"), + field_term_grouping_of("seasons_nested", "notes") + ])) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq([{ + "doc_count" => 0, + "key" => {}, + "teams:seasons_nested" => { + "doc_count" => 4, + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}), + "seasons_nested.year" => { + "meta" => inner_terms_meta({ + "buckets_path" => ["seasons_nested.notes"], + "key_path" => ["key"], + "grouping_fields" => ["seasons_nested.year"] + }), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + term_bucket(2022, 2, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [term_bucket("new rules", 2)] + }}.with_missing_value_bucket(0)), + term_bucket(2020, 1, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [term_bucket("old rules", 1), term_bucket("pandemic", 1)] + }}.with_missing_value_bucket(0)), + term_bucket(2021, 1, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [term_bucket("old rules", 1)] + }}.with_missing_value_bucket(0)) + ] + } + }.with_missing_value_bucket(0, { + "seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + } + }.with_missing_value_bucket(0)) + }]) + end + + it "limits the size of `terms` grouping based on the sub-aggregation `size`" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + first: 1, + groupings: [field_term_grouping_of("seasons_nested", "year")] + )) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq([{ + "doc_count" => 0, + "key" => {}, + "teams:seasons_nested" => { + "doc_count" => 4, + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}, size: 1), + "seasons_nested.year" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.year"], "key_path" => ["key"]}), + "buckets" => [term_bucket(2022, 2)], + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 2 + } + }.with_missing_value_bucket(0) + }]) + end + + it "limits the size of a `terms` grouping based on the sub-aggregation `size`" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + first: 1, + groupings: [field_term_grouping_of("seasons_nested", "year"), field_term_grouping_of("seasons_nested", "notes")] + )) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq([{ + "doc_count" => 0, + "key" => {}, + "teams:seasons_nested" => { + "doc_count" => 4, + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}, size: 1), + "seasons_nested.year" => { + "meta" => inner_terms_meta({"buckets_path" => ["seasons_nested.notes"], "grouping_fields" => ["seasons_nested.year"], "key_path" => ["key"]}), + "buckets" => [ + term_bucket(2022, 2, { + "seasons_nested.notes" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.notes"], "key_path" => ["key"]}), + "buckets" => [ + term_bucket("new rules", 2) + ], + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0 + } + }.with_missing_value_bucket(0)) + ], + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 2 + } + }.with_missing_value_bucket(0, { + "seasons_nested.notes" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.notes"], "key_path" => ["key"]}), + "buckets" => [], + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0 + } + }.with_missing_value_bucket(0)) + }]) + end + + it "avoids performing a sub-aggregation when the sub-aggregation is requesting an empty page" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + first: 0, + groupings: [field_term_grouping_of("seasons_nested", "year"), field_term_grouping_of("seasons_nested", "notes")] + )) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq([{"doc_count" => 0, "key" => {}}]) + end + + it "can group sub-aggregations on a single date field" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", groupings: [ + date_histogram_grouping_of("seasons_nested", "started_at", "year") + ])) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq([{ + "doc_count" => 0, + "key" => {}, + "teams:seasons_nested" => { + "doc_count" => 4, + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at"]}), + "seasons_nested.started_at" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.started_at"], "key_path" => ["key_as_string"]}), + "buckets" => [ + date_histogram_bucket(2020, 1), + date_histogram_bucket(2021, 1), + date_histogram_bucket(2022, 2) + ] + } + }.with_missing_value_bucket(0) + }]) + end + + it "can group sub-aggregations on a multiple date fields" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", groupings: [ + date_histogram_grouping_of("seasons_nested", "started_at", "year"), + date_histogram_grouping_of("seasons_nested", "won_games_at", "year") + ])) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq([{ + "doc_count" => 0, + "key" => {}, + "teams:seasons_nested" => { + "doc_count" => 4, + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at"]}), + "seasons_nested.started_at" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.started_at"], "buckets_path" => ["seasons_nested.won_games_at"], "key_path" => ["key_as_string"]}), + "buckets" => [ + date_histogram_bucket(2020, 1, {"seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.won_games_at"], "key_path" => ["key_as_string"]}), + "buckets" => [date_histogram_bucket(2020, 1)] + }}.with_missing_value_bucket(0)), + date_histogram_bucket(2021, 1, {"seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.won_games_at"], "key_path" => ["key_as_string"]}), + "buckets" => [date_histogram_bucket(2021, 1)] + }}.with_missing_value_bucket(0)), + date_histogram_bucket(2022, 2, {"seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.won_games_at"], "key_path" => ["key_as_string"]}), + "buckets" => [date_histogram_bucket(2022, 2)] + }}.with_missing_value_bucket(0)) + ] + } + }.with_missing_value_bucket(0, {"seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.won_games_at"], "key_path" => ["key_as_string"]}), + "buckets" => [] + }}.with_missing_value_bucket(0)) + }]) + end + + it "can group sub-aggregations on a single non-date field and a single date field" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", groupings: [ + date_histogram_grouping_of("seasons_nested", "started_at", "year"), + field_term_grouping_of("seasons_nested", "notes") + ])) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq([{ + "doc_count" => 0, + "key" => {}, + "teams:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at"]}), + "doc_count" => 4, + "seasons_nested.started_at" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.notes"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.started_at"]}), + "buckets" => [ + date_histogram_bucket(2020, 1, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + term_bucket("old rules", 1), + term_bucket("pandemic", 1) + ] + }}.with_missing_value_bucket(0)), + date_histogram_bucket(2021, 1, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + term_bucket("old rules", 1) + ] + }}.with_missing_value_bucket(0)), + date_histogram_bucket(2022, 2, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + term_bucket("new rules", 2) + ] + }}.with_missing_value_bucket(0)) + ] + } + }.with_missing_value_bucket(0, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + }}.with_missing_value_bucket(0)) + }]) + end + + it "can group sub-aggregations on multiple non-date fields and a single date field" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", groupings: [ + date_histogram_grouping_of("seasons_nested", "started_at", "year"), + field_term_grouping_of("seasons_nested", "year"), + field_term_grouping_of("seasons_nested", "notes") + ])) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq([{ + "doc_count" => 0, + "key" => {}, + "teams:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at"]}), + "doc_count" => 4, + "seasons_nested.started_at" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.year"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.started_at"]}), + "buckets" => [ + date_histogram_bucket(2020, 1, {"seasons_nested.year" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"], "buckets_path" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + term_bucket(2020, 1, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + term_bucket("old rules", 1), + term_bucket("pandemic", 1) + ] + }}.with_missing_value_bucket(0)) + ] + }}.with_missing_value_bucket(0, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + }}.with_missing_value_bucket(0))), + date_histogram_bucket(2021, 1, {"seasons_nested.year" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"], "buckets_path" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + term_bucket(2021, 1, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + term_bucket("old rules", 1) + ] + }}.with_missing_value_bucket(0)) + ] + }}.with_missing_value_bucket(0, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + }}.with_missing_value_bucket(0))), + date_histogram_bucket(2022, 2, {"seasons_nested.year" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"], "buckets_path" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + term_bucket(2022, 2, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + term_bucket("new rules", 2) + ] + }}.with_missing_value_bucket(0)) + ] + }}.with_missing_value_bucket(0, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + }}.with_missing_value_bucket(0))) + ] + } + }.with_missing_value_bucket(0, {"seasons_nested.year" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"], "buckets_path" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + }}.with_missing_value_bucket(0, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + }}.with_missing_value_bucket(0))) + }]) + end + + it "can group sub-aggregations on multiple non-date fields and multiple date fields" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", groupings: [ + date_histogram_grouping_of("seasons_nested", "started_at", "year"), + date_histogram_grouping_of("seasons_nested", "won_games_at", "year"), + field_term_grouping_of("seasons_nested", "year"), + field_term_grouping_of("seasons_nested", "notes") + ])) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq([{ + "doc_count" => 0, + "key" => {}, + "teams:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at"]}), + "doc_count" => 4, + "seasons_nested.started_at" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.won_games_at"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.started_at"]}), + "buckets" => [ + date_histogram_bucket(2020, 1, {"seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.year"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_games_at"]}), + "buckets" => [ + date_histogram_bucket(2020, 1, {"seasons_nested.year" => { + "meta" => inner_terms_meta({"buckets_path" => ["seasons_nested.notes"], "key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + term_bucket(2020, 1, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + term_bucket("old rules", 1), + term_bucket("pandemic", 1) + ] + }}.with_missing_value_bucket(0)) + ] + }}.with_missing_value_bucket(0, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + }}.with_missing_value_bucket(0))) + ] + }}.with_missing_value_bucket(0, {"seasons_nested.year" => { + "meta" => inner_terms_meta({"buckets_path" => ["seasons_nested.notes"], "key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + }}.with_missing_value_bucket(0, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + }}.with_missing_value_bucket(0)))), + date_histogram_bucket(2021, 1, {"seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.year"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_games_at"]}), + "buckets" => [ + date_histogram_bucket(2021, 1, {"seasons_nested.year" => { + "meta" => inner_terms_meta({"buckets_path" => ["seasons_nested.notes"], "key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + term_bucket(2021, 1, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + term_bucket("old rules", 1) + ] + }}.with_missing_value_bucket(0)) + ] + }}.with_missing_value_bucket(0, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + }}.with_missing_value_bucket(0))) + ] + }}.with_missing_value_bucket(0, {"seasons_nested.year" => { + "meta" => inner_terms_meta({"buckets_path" => ["seasons_nested.notes"], "key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + }}.with_missing_value_bucket(0, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + }}.with_missing_value_bucket(0)))), + date_histogram_bucket(2022, 2, {"seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.year"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_games_at"]}), + "buckets" => [ + date_histogram_bucket(2022, 2, {"seasons_nested.year" => { + "meta" => inner_terms_meta({"buckets_path" => ["seasons_nested.notes"], "key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + term_bucket(2022, 2, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + term_bucket("new rules", 2) + ] + }}.with_missing_value_bucket(0)) + ] + }}.with_missing_value_bucket(0, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + }}.with_missing_value_bucket(0))) + ] + }}.with_missing_value_bucket(0, {"seasons_nested.year" => { + "meta" => inner_terms_meta({"buckets_path" => ["seasons_nested.notes"], "key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + }}.with_missing_value_bucket(0, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + }}.with_missing_value_bucket(0)))) + ] + } + }.with_missing_value_bucket(0, {"seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.year"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_games_at"]}), + "buckets" => [] + }}.with_missing_value_bucket(0, {"seasons_nested.year" => { + "meta" => inner_terms_meta({"buckets_path" => ["seasons_nested.notes"], "key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + }}.with_missing_value_bucket(0, {"seasons_nested.notes" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.notes"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [] + }}.with_missing_value_bucket(0)))) + }]) + end + + it "accounts for an extra filtering layer in the `buckets_path` meta" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + groupings: [field_term_grouping_of("seasons_nested", "year")], + filter: {"year" => {"gt" => 2020}} + )) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results).to eq([{ + "doc_count" => 0, + "key" => {}, + "teams:seasons_nested" => { + "doc_count" => 4, + "meta" => outer_meta({"buckets_path" => ["seasons_nested:filtered", "seasons_nested.year"]}), + "seasons_nested:filtered" => { + "doc_count" => 3, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.year"], "key_path" => ["key"]}), + "buckets" => [ + term_bucket(2022, 2), + term_bucket(2021, 1) + ], + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0 + } + }.with_missing_value_bucket(0) + } + }]) + end + + it "ignores date histogram buckets that have no documents in them" do + index_into( + graphql, + build( + :team, + seasons: [ + build(:team_season, year: 2000, notes: ["old rules"], players: []) + ] + ) + ) + + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", groupings: [ + date_histogram_grouping_of("seasons_nested", "started_at", "year") + ])) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results.dig(0, "teams:seasons_nested", "seasons_nested.started_at", "buckets")).to eq([ + date_histogram_bucket(2000, 1), + # Notice no buckets between 2000 and 2020 in spite of the gaps. + date_histogram_bucket(2020, 1), + date_histogram_bucket(2021, 1), + date_histogram_bucket(2022, 2) + ]) + end + + it "gets `doc_count_error_upper_bound` based on the `needs_doc_count_error` query flag" do + buckets_for_true, buckets_for_false = [true, false].map do |needs_doc_count_error| + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + needs_doc_count_error: needs_doc_count_error, + groupings: [field_term_grouping_of("seasons_nested", "year")] + )) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + results.dig(0, "teams:seasons_nested", "seasons_nested.year", "buckets") + end + + expect(buckets_for_true).to eq [ + {"doc_count" => 2, "key" => 2022, "doc_count_error_upper_bound" => 0}, + {"doc_count" => 1, "key" => 2020, "doc_count_error_upper_bound" => 0}, + {"doc_count" => 1, "key" => 2021, "doc_count_error_upper_bound" => 0} + ] + + expect(buckets_for_false).to eq [ + {"doc_count" => 2, "key" => 2022}, + {"doc_count" => 1, "key" => 2020}, + {"doc_count" => 1, "key" => 2021} + ] + end + end + + context "with groupings (using the `CompositeGroupingAdapter`)" do + include_context "sub-aggregation support", Aggregation::CompositeGroupingAdapter + + it "supports multiple groupings, aggregated values, and count" do + query = aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + needs_doc_count: true, + computations: [ + computation_of("seasons_nested", "year", :min, computed_field_name: "exact_min"), + computation_of("seasons_nested", "the_record", "win_count", :max, computed_field_name: "exact_max"), + computation_of("seasons_nested", "the_record", "win_count", :avg, computed_field_name: "approximate_avg") + ], + groupings: [ + date_histogram_grouping_of("seasons_nested", "started_at", "year"), + field_term_grouping_of("seasons_nested", "year"), + field_term_grouping_of("seasons_nested", "notes") + ] + )) + ]) + + results = search_datastore_aggregations(query, index_def_name: "teams") + + expect(results.dig(0, "teams:seasons_nested")).to eq({ + "meta" => outer_meta({"buckets_path" => ["seasons_nested"]}), + "doc_count" => 4, + "seasons_nested" => { + "after_key" => { + "seasons_nested.started_at" => "2022-01-01T00:00:00.000Z", + "seasons_nested.year" => 2022, + "seasons_nested.notes" => "new rules" + }, + "buckets" => [ + { + "key" => { + "seasons_nested.started_at" => "2020-01-01T00:00:00.000Z", + "seasons_nested.year" => 2020, + "seasons_nested.notes" => "old rules" + }, + "doc_count" => 1, + "seasons_nested:seasons_nested.year:exact_min" => {"value" => 2020.0}, + "seasons_nested:seasons_nested.the_record.win_count:exact_max" => {"value" => 30.0}, + "seasons_nested:seasons_nested.the_record.win_count:approximate_avg" => {"value" => 30.0} + }, + { + "key" => { + "seasons_nested.started_at" => "2020-01-01T00:00:00.000Z", + "seasons_nested.year" => 2020, + "seasons_nested.notes" => "pandemic" + }, + "doc_count" => 1, + "seasons_nested:seasons_nested.year:exact_min" => {"value" => 2020.0}, + "seasons_nested:seasons_nested.the_record.win_count:exact_max" => {"value" => 30.0}, + "seasons_nested:seasons_nested.the_record.win_count:approximate_avg" => {"value" => 30.0} + }, + { + "key" => { + "seasons_nested.started_at" => "2021-01-01T00:00:00.000Z", + "seasons_nested.year" => 2021, + "seasons_nested.notes" => "old rules" + }, + "doc_count" => 1, + "seasons_nested:seasons_nested.year:exact_min" => {"value" => 2021.0}, + "seasons_nested:seasons_nested.the_record.win_count:exact_max" => {"value" => 60.0}, + "seasons_nested:seasons_nested.the_record.win_count:approximate_avg" => {"value" => 60.0} + }, + { + "key" => { + "seasons_nested.started_at" => "2022-01-01T00:00:00.000Z", + "seasons_nested.year" => 2022, + "seasons_nested.notes" => "new rules" + }, + "doc_count" => 2, + "seasons_nested:seasons_nested.year:exact_min" => {"value" => 2022.0}, + "seasons_nested:seasons_nested.the_record.win_count:exact_max" => {"value" => 50.0}, + "seasons_nested:seasons_nested.the_record.win_count:approximate_avg" => {"value" => 45.0} + } + ] + } + }) + end + end + + def term_bucket(key_or_keys, count, extra_fields = {}) + { + "key" => key_or_keys, + "doc_count" => count + }.compact.merge(extra_fields) + end + + def date_histogram_bucket(year, count, extra_fields = {}) + time = ::Time.iso8601("#{year}-01-01T00:00:00Z") + { + "key" => time.to_i * 1000, + "key_as_string" => time.iso8601(3), + "doc_count" => count + }.merge(extra_fields) + end + end + end +end diff --git a/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/total_document_count_needed_spec.rb b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/total_document_count_needed_spec.rb new file mode 100644 index 00000000..34e2a75f --- /dev/null +++ b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/total_document_count_needed_spec.rb @@ -0,0 +1,35 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_integration_support" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "#total_document_count_needed" do + include_context "DatastoreQueryIntegrationSupport" + + specify "returns doc count when it is requested" do + index_into(graphql, build(:widget)) + + results = search_datastore(total_document_count_needed: true) + + expect(results.total_document_count).to eq(1) + end + + specify "raises an exception when total document count is not requested but accessed" do + index_into(graphql, build(:widget)) + + results = search_datastore(total_document_count_needed: false) + + expect { + results.total_document_count + }.to raise_error(Errors::CountUnavailableError) + end + end + end +end diff --git a/elasticgraph-graphql/spec/integration/elastic_graph/graphql/resolvers/list_records_spec.rb b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/resolvers/list_records_spec.rb new file mode 100644 index 00000000..ffedb26f --- /dev/null +++ b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/resolvers/list_records_spec.rb @@ -0,0 +1,97 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/resolvers/list_records" + +module ElasticGraph + class GraphQL + module Resolvers + RSpec.describe ListRecords, :factories, :uses_datastore, :resolver do + context "when the field being resolved is a relay connection field" do + let(:graphql) { build_graphql } + + it "wraps the datastore response in a relay connection adapter when the field is a relay connection field" do + expect(resolve(:Query, :widgets)).to be_a RelayConnection::GenericAdapter + end + end + + describe "sorting" do + let(:graphql) { build_graphql } + + let(:widget1) { build(:widget, id: "w1", amount_cents: 100, created_at: "2019-06-01T00:00:00Z") } + let(:widget2) { build(:widget, id: "w2", amount_cents: 200, created_at: "2019-06-02T00:00:00Z") } + let(:widget3) { build(:widget, id: "w3", amount_cents: 200, created_at: "2019-06-03T00:00:00Z") } + + before do + index_into(graphql, widget1, widget2, widget3) + end + + it "respects a list of `order_by` options, supporting ascending and descending sorts" do + results = resolve_nodes(:Query, :widgets, order_by: ["amount_cents_DESC", "created_at_ASC"]) + expect(results.map { |r| r.fetch("id") }).to eq ["w2", "w3", "w1"] + + results = resolve_nodes(:Query, :widgets, order_by: ["amount_cents_ASC", "created_at_DESC"]) + expect(results.map { |r| r.fetch("id") }).to eq ["w1", "w3", "w2"] + end + end + + describe "filtering" do + let(:graphql) { build_graphql } + + let(:widget1) { build(:widget, name: "w1", amount_cents: 100, id: "w1", created_at: "2021-01-01T12:00:00Z") } + let(:widget2a) { build(:widget, name: "w2", amount_cents: 100, id: "w2a", created_at: "2021-02-01T12:00:00Z") } + let(:widget2b) { build(:widget, name: "w2", amount_cents: 200, id: "w2b", created_at: "2021-02-01T12:00:00Z") } + let(:widget3) { build(:widget, name: "w3", amount_cents: 300, id: "w3", created_at: "2021-03-01T12:00:00Z") } + + before do + index_into(graphql, widget1, widget2a, widget2b, widget3) + end + + it "supports filtering by a list of one value" do + results = resolve_nodes(:Query, :widgets, filter: {id: {equal_to_any_of: [widget1.fetch(:id)]}}) + expect(results.map { |r| r.fetch("id") }).to contain_exactly(widget1.fetch(:id)) + + results = resolve_nodes(:Query, :widgets, filter: {name: {equal_to_any_of: ["w2"]}}) + expect(results.map { |r| r.fetch("id") }).to contain_exactly(widget2a.fetch(:id), widget2b.fetch(:id)) + end + + it "supports filtering by a list of multiple values" do + results = resolve_nodes(:Query, :widgets, filter: {id: {equal_to_any_of: [widget1.fetch(:id), widget3.fetch(:id)]}}) + expect(results.map { |r| r.fetch("id") }).to contain_exactly(widget1.fetch(:id), widget3.fetch(:id)) + + results = resolve_nodes(:Query, :widgets, filter: {name: {equal_to_any_of: ["w2", "w1"]}}) + expect(results.map { |r| r.fetch("id") }).to contain_exactly(widget1.fetch(:id), widget2a.fetch(:id), widget2b.fetch(:id)) + end + + it "supports default sort" do + results = resolve_nodes(:Query, :widgets, filter: {id: {equal_to_any_of: ["w3", "w2b", "w1"]}}) + expect(results.map { |r| r.fetch("id") }).to eq([widget3.fetch(:id), widget2b.fetch(:id), widget1.fetch(:id)]) + end + + it "supports combining filters on more than one field" do + results = resolve_nodes(:Query, :widgets, filter: { + name: {equal_to_any_of: ["w2", "w3"]}, + amount_cents: {equal_to_any_of: [100, 400]} + }) + + expect(results.map { |r| r.fetch("id") }).to contain_exactly(widget2a.fetch(:id)) + end + end + + # Override `resolve` to force `id` to always be requested since the specs rely on it. + def resolve(*args, **options) + super(*args, query_overrides: {requested_fields: ["id"]}, **options) + end + + def resolve_nodes(...) + resolve(...).edges.map(&:node) + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/integration/elastic_graph/graphql/resolvers/nested_relationships_spec.rb b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/resolvers/nested_relationships_spec.rb new file mode 100644 index 00000000..8e895cfd --- /dev/null +++ b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/resolvers/nested_relationships_spec.rb @@ -0,0 +1,748 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/resolvers/nested_relationships" + +module ElasticGraph + class GraphQL + module Resolvers + RSpec.describe NestedRelationships, :factories, :uses_datastore, :capture_logs, :resolver do + # :expect_search_routing because the relation we use here uses an outbaund foreign key, which + # is implemented via a filter on `id` (the search routing field) + context "when the field being resolved is a relay connection field", :expect_search_routing do + let(:graphql) do + build_graphql(schema_definition: lambda do |schema| + schema.object_type "Component" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "created_at", "DateTime!" + t.relates_to_one "widget", "Widget!", via: "component_ids", dir: :in + t.index "components" do |i| + i.default_sort "created_at", :desc + end + end + + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "workspace_id", "ID", name_in_index: "workspace_id2" + t.field "created_at", "DateTime!" + t.relates_to_many "components", "Component", via: "component_ids", dir: :out, singular: "component" + t.index "widgets" do |i| + i.rollover :yearly, "created_at" + i.route_with "workspace_id" + i.default_sort "created_at", :desc + end + end + end) + end + + let(:component1) { build(:component) } + + it "wraps the datastore response in a relay connection adapter when the field is a relay connection field" do + result = resolve(:Widget, :components, {"component_ids" => [component1.fetch(:id)]}) + + expect(result).to be_a RelayConnection::GenericAdapter + end + + it "wraps the datastore response in a relay connection adapter when the foreign key field is missing" do + expect { + result = resolve(:Widget, :components, {"name" => "a"}) + + expect(result).to be_a(RelayConnection::GenericAdapter) + expect(result.edges).to be_empty + }.to log a_string_including("Widget(id: ).components", "component_ids is missing from the document") + end + end + + describe "a relates_to_many/relates_to_one bidirectional relationship with an array foreign key from the one to the many" do + let(:graphql) do + build_graphql(schema_definition: lambda do |schema| + schema.object_type "Money" do |t| + t.field "currency", "String!" + t.field "amount_cents", "Int" + end + + schema.object_type "Component" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "created_at", "DateTime!" + t.relates_to_one "widget", "Widget!", via: "component_ids", dir: :in + t.relates_to_one "dollar_widget", "Widget", via: "component_ids", dir: :in do |rel| + rel.additional_filter cost: {amount_cents: {equal_to_any_of: [100]}} + end + t.relates_to_many "dollar_widgets", "Widget", via: "component_ids", dir: :in, singular: "dollar_widget" do |rel| + rel.additional_filter cost: {amount_cents: {equal_to_any_of: [100]}} + end + t.index "components" do |i| + i.default_sort "created_at", :desc + end + end + + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "workspace_id", "ID", name_in_index: "workspace_id2" + t.field "created_at", "DateTime!" + t.field "cost", "Money" + t.relates_to_many "components", "Component", via: "component_ids", dir: :out, singular: "component" + t.index "widgets" do |i| + i.rollover :yearly, "created_at" + i.route_with "workspace_id" + i.default_sort "created_at", :desc + end + end + end) + end + + let(:component1) { build(:component, created_at: "2019-06-01T00:00:00Z") } + let(:component2) { build(:component, created_at: "2019-06-02T00:00:00Z") } + let(:component3) { build(:component) } + let(:widget1) { build(:widget, amount_cents: 100, components: [component1, component2], created_at: "2019-06-01T00:00:00Z") } + let(:widget2) { build(:widget, components: [component2], created_at: "2019-06-02T00:00:00Z") } + let(:widget3) { build(:widget, amount_cents: 200, components: [component3], created_at: "2019-06-03T00:00:00Z") } + + # :expect_search_routing because the relation we use here uses an outbaund foreign key, which + # is implemented via a filter on `id` (the search routing field) + context "loading the relates_to_many", :expect_search_routing do + before do + index_into(graphql, component1, component2, component3, widget1, widget2, widget3) + end + + it "loads the relationship and respects defaultSort" do + result = resolve_nodes(:Widget, :components, {"component_ids" => [component1.fetch(:id), component2.fetch(:id)]}) + + expect(result.map { |c| c.fetch("id") }).to eq([component2.fetch(:id), component1.fetch(:id)]) + end + + it "respects a list of `order_by` options, supporting ascending and descending sorts" do + result = resolve_nodes(:Widget, :components, + {"component_ids" => [component2.fetch(:id), component3.fetch(:id), component1.fetch(:id)]}, + order_by: ["created_at_ASC"]) + expect(result.map { |c| c.fetch("id") }).to eq([component1.fetch(:id), component2.fetch(:id), component3.fetch(:id)]) + + result = resolve_nodes(:Widget, :components, + {"component_ids" => [component2.fetch(:id), component3.fetch(:id), component1.fetch(:id)]}, + order_by: ["created_at_DESC"]) + expect(result.map { |c| c.fetch("id") }).to eq([component3.fetch(:id), component2.fetch(:id), component1.fetch(:id)]) + end + + it "tolerates not finding some ids" do + result = resolve_nodes(:Widget, :components, {"component_ids" => [component1.fetch(:id), build(:component).fetch(:id)]}) + + expect(result.map { |c| c.fetch("id") }).to contain_exactly(component1.fetch(:id)) + end + + it "returns an empty list when given a blank list of ids" do + result = resolve_nodes(:Widget, :components, {"component_ids" => []}) + + expect(result).to be_empty + end + + it "returns a list of a single record (and logs a warning) when the foreign key field is a scalar instead of a list" do + expect { + result = resolve_nodes(:Widget, :components, {"component_ids" => component1.fetch(:id), "id" => "123"}) + + expect(result.map { |c| c.fetch("id") }).to contain_exactly(component1.fetch(:id)) + }.to log a_string_including("Widget(id: 123).components", "component_ids: scalar instead of a list") + end + + it "returns an empty list (and logs a warning) when the foreign key field is missing" do + expect { + result = resolve_nodes(:Widget, :components, {"name" => "a"}) + + expect(result).to be_empty + }.to log a_string_including("Widget(id: ).components", "component_ids is missing from the document") + end + + it "returns a list of records matching additional filter conditions" do + result = resolve_nodes(:Component, :dollar_widgets, {"id" => component1.fetch(:id)}, requested_fields: ["id", "cost.amount_cents"]) + + expect(result.map { |c| c.fetch("id") }).to contain_exactly(widget1.fetch(:id)) + expect(result.map { |c| c.fetch("cost").fetch("amount_cents") }).to contain_exactly(100) + end + + it "returns an empty list when records with matching conditions are not found" do + result = resolve_nodes(:Component, :dollar_widgets, {"id" => component3.fetch(:id)}, requested_fields: ["id", "cost.amount_cents"]) + + expect(result).to be_empty + end + end + + context "loading the relates_to_one" do + before do + index_into(graphql, widget1, widget2, widget3) + end + + it "loads the relationship" do + result = resolve(:Component, :widget, {"id" => component1.fetch(:id)}) + + expect(result.fetch("id")).to eq widget1.fetch(:id) + end + + it "tolerates not finding the id" do + result = resolve(:Component, :widget, {"id" => build(:component).fetch(:id)}) + + expect(result).to eq nil + end + + it "returns nil when given a nil id" do + result = resolve(:Component, :widget, {"id" => nil}) + + expect(result).to eq nil + end + + it "returns one record (and logs a warning) when querying the datastore produces a list of records instead of a single one" do + expect { + # Note: inclusion of nested field `cost.amount_cents` is necessary to exercise an edge case that originally resulted in an exception. + result = resolve(:Component, :widget, {"id" => component2.fetch(:id)}, requested_fields: ["id", "cost.amount_cents"]) + + expect(result.fetch("id")).to eq(widget1.fetch(:id)).or eq(widget2.fetch(:id)) + expect(result.fetch("cost").fetch("amount_cents")).to eq(widget1.fetch(:cost).fetch(:amount_cents)).or eq(widget2.fetch(:cost).fetch(:amount_cents)) + }.to log a_string_including("Component(id: #{component2.fetch(:id)}).widget", "got list of more than one item instead of a scalar from the datastore search query") + end + + it "returns one record (and logs a warning) when the id field is a list instead of a scalar" do + ids = [component1.fetch(:id), component3.fetch(:id)] + expect { + result = resolve(:Component, :widget, {"id" => ids}) + + expect(result.fetch("id")).to eq(widget1.fetch(:id)).or eq(widget3.fetch(:id)) + }.to log a_string_including("Component(id: #{ids}).widget", "id: list of more than one item instead of a scalar") + end + + it "returns nil (and logs a warning) when the id field is missing" do + expect { + result = resolve(:Component, :widget, {"name" => "foo"}) + + expect(result).to eq nil + }.to log a_string_including("Component(id: ).widget", "id is missing from the document") + end + + it "returns one record matching additional filter conditions" do + result = resolve(:Component, :dollar_widget, {"id" => component1.fetch(:id)}, requested_fields: ["id", "cost.amount_cents"]) + + expect(result.fetch("id")).to eq(widget1.fetch(:id)) + expect(result.fetch("cost").fetch("amount_cents")).to eq(widget1.fetch(:cost).fetch(:amount_cents)) + end + + it "returns nil when a record with matching conditions is not found" do + result = resolve(:Component, :dollar_widget, {"id" => component3.fetch(:id)}, requested_fields: ["id", "cost.amount_cents"]) + + expect(result).to eq nil + end + end + end + + describe "a relates_to_many/relates_to_one bidirectional relationship with a scalar foreign key from the many to the one" do + let(:graphql) do + build_graphql(schema_definition: lambda do |schema| + schema.object_type "ElectricalPart" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "created_at", "DateTime!" + t.relates_to_one "manufacturer", "Manufacturer", via: "manufacturer_id", dir: :out + t.index "electrical_parts" do |i| + i.default_sort "created_at", :desc + end + end + + schema.object_type "MechanicalPart" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "created_at", "DateTime!" + t.relates_to_one "manufacturer", "Manufacturer", via: "manufacturer_id", dir: :out + t.index "mechanical_parts" do |i| + i.default_sort "created_at", :desc + end + end + + schema.union_type "Part" do |t| + t.subtypes "ElectricalPart", "MechanicalPart" + end + + schema.object_type "Manufacturer" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "created_at", "DateTime!" + t.relates_to_many "manufactured_parts", "Part", via: "manufacturer_id", dir: :in, singular: "manufactured_part" + t.index "manufacturers" + end + end) + end + + let(:manufacturer1) { build(:manufacturer, created_at: "2019-06-01T00:00:00Z") } + let(:manufacturer2) { build(:manufacturer, created_at: "2019-06-02T00:00:00Z") } + let(:part1) { build(:part, manufacturer: manufacturer1, created_at: "2019-06-01T00:00:00Z") } + let(:part2) { build(:part, manufacturer: manufacturer1, created_at: "2019-06-02T00:00:00Z") } + let(:part3) { build(:part, manufacturer: manufacturer2, created_at: "2019-06-03T00:00:00Z") } + + context "loading the relates_to_many" do + before do + index_into(graphql, part1, part2, part3) + end + + it "loads the relationship and respects defaultSort" do + result = resolve_nodes(:Manufacturer, :manufactured_parts, {"id" => manufacturer1.fetch(:id)}) + + expect(result.map { |c| c.fetch("id") }).to eq([part2.fetch(:id), part1.fetch(:id)]) + end + + it "respects a list of `order_by` options, supporting ascending and descending sorts" do + result = resolve_nodes(:Manufacturer, :manufactured_parts, + {"id" => manufacturer1.fetch(:id)}, + order_by: ["created_at_ASC"]) + expect(result.map { |c| c.fetch("id") }).to eq([part1.fetch(:id), part2.fetch(:id)]) + + result = resolve_nodes(:Manufacturer, :manufactured_parts, + {"id" => manufacturer1.fetch(:id)}, + order_by: ["created_at_DESC"]) + expect(result.map { |c| c.fetch("id") }).to eq([part2.fetch(:id), part1.fetch(:id)]) + end + + it "tolerates not finding records for the given id" do + result = resolve_nodes(:Manufacturer, :manufactured_parts, {"id" => build(:manufacturer).fetch(:id)}) + + expect(result).to be_empty + end + + it "returns an empty list when given a nil id" do + result = resolve_nodes(:Manufacturer, :manufactured_parts, {"id" => nil}) + + expect(result).to be_empty + end + + it "returns one of the two lists of matches (and logs a warning) when the id field is a list instead of a scalar" do + ids = [manufacturer1.fetch(:id), manufacturer2.fetch(:id)] + expect { + result = resolve_nodes(:Manufacturer, :manufactured_parts, {"id" => ids}) + + expect(result.map { |c| c.fetch("id") }).to contain_exactly(part1.fetch(:id), part2.fetch(:id)).or eq([part3.fetch(:id)]) + }.to log a_string_including("Manufacturer(id: #{ids}).manufactured_parts", "id: list of more than one item instead of a scalar") + end + + it "returns an empty list (and logs a warning) when the id field is missing" do + expect { + result = resolve_nodes(:Manufacturer, :manufactured_parts, {"name" => "foo"}) + + expect(result).to be_empty + }.to log a_string_including("Manufacturer(id: ).manufactured_parts", "id is missing from the document") + end + end + + # :expect_search_routing because the relation we use here uses an outbaund foreign key, which + # is implemented via a filter on `id` (the search routing field) + context "loading the relates_to_one", :expect_search_routing do + before do + index_into(graphql, manufacturer1, manufacturer2) + end + + it "loads the relationship" do + result = resolve(:ElectricalPart, :manufacturer, {"manufacturer_id" => manufacturer1.fetch(:id)}) + + expect(result.fetch("id")).to eq manufacturer1.fetch(:id) + end + + it "tolerates not finding a record the given id" do + result = resolve(:ElectricalPart, :manufacturer, {"manufacturer_id" => build(:manufacturer).fetch(:id)}) + + expect(result).to eq nil + end + + it "returns nil when given a nil id" do + result = resolve(:ElectricalPart, :manufacturer, {"manufacturer_id" => nil}) + + expect(result).to eq nil + end + + it "returns one of the referenced documents (and logs a warning) when the foreign key field is a list instead of a scalar" do + expect { + manufacturer_ids = [manufacturer1.fetch(:id), manufacturer2.fetch(:id)] + result = resolve(:ElectricalPart, :manufacturer, {"id" => "123", "manufacturer_id" => manufacturer_ids}) + + expect(result.fetch("id")).to eq(manufacturer1.fetch(:id)).or eq(manufacturer2.fetch(:id)) + }.to log a_string_including("Part(id: 123).manufacturer", "manufacturer_id: list of more than one item instead of a scalar") + end + + it "returns nil (and logs a warning) when the foreign key field is missing" do + expect { + result = resolve(:ElectricalPart, :manufacturer, {"id" => "123"}) + + expect(result).to eq nil + }.to log a_string_including("Part(id: 123).manufacturer", "manufacturer_id is missing from the document") + end + end + end + + describe "a relates_to_many/relates_to_many bidirectional relationship with an array foreign key from a many to a many" do + let(:graphql) do + build_graphql(schema_definition: lambda do |schema| + schema.object_type "Component" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "created_at", "DateTime!" + t.relates_to_many "parts", "Part", via: "part_ids", dir: :out, singular: "part" + t.index "components" do |i| + i.default_sort "created_at", :desc + end + end + + schema.object_type "ElectricalPart" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "created_at", "DateTime!" + t.relates_to_many "components", "Component", via: "part_ids", dir: :in, singular: "component" + t.index "electrical_parts" do |i| + i.default_sort "created_at", :desc + end + end + + schema.object_type "MechanicalPart" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "created_at", "DateTime!" + t.relates_to_many "components", "Component", via: "part_ids", dir: :in, singular: "component" + t.index "mechanical_parts" do |i| + i.default_sort "created_at", :desc + end + end + + schema.union_type "Part" do |t| + t.subtypes "ElectricalPart", "MechanicalPart" + end + end) + end + + # Here we explicitly create parts of both unioned types. That ensures that both indices are created + # in the datastore. Otherwise, some of the tests below may run with one or the other index missing, + # if all 4 parts are randomly created of the same type. While we can easily work around that with + # The datastore's `search` using `ignore_unavailable: true`, the same thing is not supported for `msearch`, + # which we are migrating to. We do not expect this to ever be a problem outside tests, and we plan + # to solve this in a more robust way by explicitly putting mappings into the datastore, but for now + # this is simple work around. + let(:part1) { build(:mechanical_part, name: "p1", created_at: "2019-06-01T00:00:00Z") } + let(:part2) { build(:mechanical_part, name: "p2", created_at: "2019-06-02T00:00:00Z") } + let(:part3) { build(:electrical_part, name: "p3", created_at: "2019-06-03T00:00:00Z") } + let(:part4) { build(:electrical_part, name: "p4", created_at: "2019-06-04T00:00:00Z") } + let(:component1) { build(:component, parts: [part1, part2], name: "c1", created_at: "2019-06-01T00:00:00Z") } + let(:component2) { build(:component, parts: [part1, part3], name: "c2", created_at: "2019-06-02T00:00:00Z") } + let(:component3) { build(:component, parts: [part2, part3], name: "c3", created_at: "2019-06-03T00:00:00Z") } + let(:component4) { build(:component, name: "c4") } + + # :expect_search_routing because the relation we use here uses an outbaund foreign key, which + # is implemented via a filter on `id` (the search routing field) + context "loading the relates_to_many with an outbound foreign key", :expect_search_routing do + before do + index_into(graphql, part1, part2, part3, part4) + end + + it "loads the relationship and respects defaultSort" do + result = resolve_nodes(:Component, :parts, {"part_ids" => [part1.fetch(:id), part2.fetch(:id)]}) + + expect(result.map { |c| c.fetch("id") }).to eq([part2.fetch(:id), part1.fetch(:id)]) + end + + it "respects a list of `order_by` options, supporting ascending and descending sorts" do + result = resolve_nodes(:Component, :parts, + {"part_ids" => [part1.fetch(:id), part2.fetch(:id)]}, + order_by: ["created_at_ASC"]) + expect(result.map { |c| c.fetch("id") }).to eq([part1.fetch(:id), part2.fetch(:id)]) + + result = resolve_nodes(:Component, :parts, + {"part_ids" => [part1.fetch(:id), part2.fetch(:id)]}, + order_by: ["created_at_DESC"]) + expect(result.map { |c| c.fetch("id") }).to eq([part2.fetch(:id), part1.fetch(:id)]) + end + + it "tolerates not finding some ids" do + result = resolve_nodes(:Component, :parts, {"part_ids" => [part1.fetch(:id), build(:part).fetch(:id)]}) + + expect(result.map { |c| c.fetch("id") }).to contain_exactly(part1.fetch(:id)) + end + + it "returns an empty list when given a blank list of ids" do + result = resolve_nodes(:Component, :parts, {"part_ids" => []}) + + expect(result).to be_empty + end + + it "returns a list of the matching document (and logs a warning) when the foreign key field is a scalar instead of a list" do + expect { + result = resolve_nodes(:Component, :parts, {"id" => "123", "part_ids" => part1.fetch(:id)}) + + expect(result.map { |p| p.fetch("id") }).to eq [part1.fetch(:id)] + }.to log a_string_including("Component(id: 123).parts", "part_ids: scalar instead of a list") + end + + it "returns an empty list" do + expect { + result = resolve_nodes(:Component, :parts, {"id" => "123"}) + + expect(result).to be_empty + }.to log a_string_including("Component(id: 123).parts", "part_ids is missing from the document") + end + + it "supports filtering on a non-id field" do + results = resolve_nodes(:Component, :parts, {"part_ids" => [part1.fetch(:id), part2.fetch(:id)]}, + filter: {name: {equal_to_any_of: ["p1", "p4"]}}) + + expect(results.map { |c| c.fetch("id") }).to contain_exactly(part1.fetch(:id)) + end + + it "supports filtering on id" do + results = resolve_nodes(:Component, :parts, {"part_ids" => [part1.fetch(:id), part2.fetch(:id)]}, + filter: {id: {equal_to_any_of: [part1.fetch(:id), part4.fetch(:id)]}}) + + expect(results.map { |c| c.fetch("id") }).to contain_exactly(part1.fetch(:id)) + end + end + + context "loading the relates_to_many with an inbound foreign key" do + before do + index_into(graphql, component1, component2, component3, component4) + end + + it "loads the relationship and supports defaultSort" do + result = resolve_nodes(:ElectricalPart, :components, {"id" => part1.fetch(:id)}) + + expect(result.map { |c| c.fetch("id") }).to eq([component2.fetch(:id), component1.fetch(:id)]) + end + + it "respects a list of `order_by` options, supporting ascending and descending sorts" do + result = resolve_nodes(:ElectricalPart, :components, {"id" => part1.fetch(:id)}, order_by: ["created_at_ASC"]) + expect(result.map { |c| c.fetch("id") }).to eq([component1.fetch(:id), component2.fetch(:id)]) + + result = resolve_nodes(:ElectricalPart, :components, {"id" => part1.fetch(:id)}, order_by: ["created_at_DESC"]) + expect(result.map { |c| c.fetch("id") }).to eq([component2.fetch(:id), component1.fetch(:id)]) + end + + it "tolerates not finding records for the given id" do + result = resolve_nodes(:ElectricalPart, :components, {"id" => build(:part).fetch(:id)}) + + expect(result).to be_empty + end + + it "returns an empty list when given a nil id" do + result = resolve_nodes(:ElectricalPart, :components, {"id" => nil}) + + expect(result).to be_empty + end + + it "returns one of the two lists of matches (and logs a warning) when the id field is a list instead of a scalar" do + ids = [part1.fetch(:id), part2.fetch(:id)] + expect { + result = resolve_nodes(:ElectricalPart, :components, {"id" => ids}) + + expect(result.map { |c| c.fetch("id") }).to \ + contain_exactly(component1.fetch(:id), component2.fetch(:id)).or \ + contain_exactly(component1.fetch(:id), component3.fetch(:id)) + }.to log a_string_including("ElectricalPart(id: #{ids}).components", "id: list of more than one item instead of a scalar") + end + + it "returns an empty list (and logs a warning) when the id field is missing" do + expect { + result = resolve_nodes(:ElectricalPart, :components, {"name" => "foo"}) + + expect(result).to be_empty + }.to log a_string_including("ElectricalPart(id: ).components", "id is missing from the document") + end + + it "supports filtering on a non-id field" do + results = resolve_nodes(:ElectricalPart, :components, {"id" => part1.fetch(:id)}, + filter: {name: {equal_to_any_of: ["c1", "c4"]}}) + + expect(results.map { |c| c.fetch("id") }).to contain_exactly(component1.fetch(:id)) + end + + it "supports filtering on id", :expect_search_routing do + component_ids = [component1.fetch(:id), component4.fetch(:id)] + + results = resolve_nodes(:ElectricalPart, :components, {"id" => part1.fetch(:id)}, + filter: {id: {equal_to_any_of: component_ids}}) + + expect(results.map { |c| c.fetch("id") }).to contain_exactly(component1.fetch(:id)) + expect_to_have_routed_to_shards_with("main", ["components", component_ids.sort.join(",")]) + end + end + end + + describe "a relates_to_one/relates_to_one bidirectional relationship with a scalar foreign key from a one to a one" do + let(:graphql) do + build_graphql(schema_definition: lambda do |schema| + schema.object_type "Manufacturer" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.relates_to_one "address", "Address!", via: "manufacturer_id", dir: :in + t.index "manufacturers" + end + + schema.object_type "Address" do |t| + t.field "id", "ID!" + t.field "full_address", "String" + t.relates_to_one "manufacturer", "Manufacturer!", via: "manufacturer_id", dir: :out + t.index "addresses" + end + end) + end + + let(:manufacturer1) { build(:manufacturer) } + let(:manufacturer2) { build(:manufacturer) } + let(:manufacturer3) { build(:manufacturer) } + + let(:address1) { build(:address, manufacturer: manufacturer1) } + let(:address2) { build(:address, manufacturer: manufacturer2) } + let(:address3) { build(:address, manufacturer: manufacturer2) } + let(:address4) { build(:address, manufacturer: manufacturer3) } + + # :expect_search_routing because the relation we use here uses an outbaund foreign key, which + # is implemented via a filter on `id` (the search routing field) + context "loading the relates_to_one with the outbound foreign key", :expect_search_routing do + before do + index_into(graphql, manufacturer1, manufacturer2) + end + + it "loads the relationship" do + result = resolve(:Address, :manufacturer, {"manufacturer_id" => manufacturer1.fetch(:id)}) + + expect(result.fetch("id")).to eq manufacturer1.fetch(:id) + end + + it "tolerates not finding a record for the given id" do + result = resolve(:Address, :manufacturer, {"manufacturer_id" => build(:manufacturer).fetch(:id)}) + + expect(result).to eq nil + end + + it "returns nil when given a nil id" do + result = resolve(:Address, :manufacturer, {"manufacturer_id" => nil}) + + expect(result).to eq nil + end + + it "returns one of the referenced documents (and logs a warning) when the foreign key field is a list instead of a scalar" do + expect { + manufacturer_ids = [manufacturer1.fetch(:id), manufacturer2.fetch(:id)] + result = resolve(:Address, :manufacturer, {"id" => "123", "manufacturer_id" => manufacturer_ids}) + + expect(result.fetch("id")).to eq(manufacturer1.fetch(:id)).or eq(manufacturer2.fetch(:id)) + }.to log a_string_including("Address(id: 123).manufacturer", "manufacturer_id: list of more than one item instead of a scalar") + end + + it "returns nil (and logs a warning) when the foreign key field is missing" do + expect { + result = resolve(:Address, :manufacturer, {"id" => "123"}) + + expect(result).to eq nil + }.to log a_string_including("Address(id: 123).manufacturer", "manufacturer_id is missing from the document") + end + end + + context "loading the relates_to_one with the inbound foreign key" do + before do + index_into(graphql, address1, address2, address3, address4) + end + + it "loads the relationship" do + result = resolve(:Manufacturer, :address, {"id" => manufacturer1.fetch(:id)}) + + expect(result.fetch("id")).to eq address1.fetch(:id) + end + + it "tolerates not finding a record for the given id" do + result = resolve(:Manufacturer, :address, {"id" => build(:manufacturer).fetch(:id)}) + + expect(result).to eq nil + end + + it "returns nil when given a nil id" do + result = resolve(:Manufacturer, :address, {"id" => nil}) + + expect(result).to eq nil + end + + it "returns one of the matching records (and logs a warning) when querying the datastore produces a list of records instead of a single one" do + expect { + result = resolve(:Manufacturer, :address, {"id" => manufacturer2.fetch(:id)}) + + expect(result.fetch("id")).to eq(address2.fetch(:id)).or eq(address3.fetch(:id)) + }.to log a_string_including("Manufacturer(id: #{manufacturer2.fetch(:id)}).address", "got list of more than one item instead of a scalar from the datastore search query") + end + + it "returns one of the matching records (and logs a warning) when the id field is a list instead of a scalar" do + ids = [manufacturer1.fetch(:id), manufacturer3.fetch(:id)] + expect { + result = resolve(:Manufacturer, :address, {"id" => ids}) + + expect(result.fetch("id")).to eq(address1.fetch(:id)).or eq(address4.fetch(:id)) + }.to log a_string_including("Manufacturer(id: #{ids}).address", "id: list of more than one item instead of a scalar") + end + + it "returns nil (and logs a warning) when the id field is missing" do + expect { + result = resolve(:Manufacturer, :address, {"name" => "foo"}) + + expect(result).to eq nil + }.to log a_string_including("Manufacturer(id: ).address", "id is missing from the document") + end + end + end + + describe "a relates_to_many unidirectional relationship with a nested array foreign key from the one to the many" do + let(:graphql) { build_graphql } + let(:sponsor1) { build(:sponsor) } + let(:sponsor2) { build(:sponsor) } + + let(:team1) { build(:team, sponsors: [sponsor1, sponsor2]) } + let(:team2) { build(:team, sponsors: [sponsor1]) } + let(:team3) { build(:team, sponsors: [sponsor2]) } + + context "loading the relates_to_many in relationship with fields in nested list objects" do + before do + index_into(graphql, sponsor1, sponsor2, team1, team2, team3) + end + + it "loads the relationship from a nested field in an object list" do + result = resolve_nodes(:Sponsor, :affiliated_teams_from_object, {"id" => sponsor1.fetch(:id)}, requested_fields: ["id"]) + + expect(result.map { |t| t.fetch("id") }).to contain_exactly(team1.fetch(:id), team2.fetch(:id)) + end + + it "loads the relationship from a nested field in a nested list" do + result = resolve_nodes(:Sponsor, :affiliated_teams_from_nested, {"id" => sponsor1.fetch(:id)}, requested_fields: ["id"]) + + expect(result.map { |t| t.fetch("id") }).to contain_exactly(team1.fetch(:id), team2.fetch(:id)) + end + end + end + + # we override `resolve` (defined in the `resolver support` shared context) in order + # to enforce that no `resolve` call ever queries the datastore more than once, as part + # of preventing N+1 queries. + # Also, we force `id` as a requested field here since the specs rely on it always being requested. + def resolve(*args, requested_fields: ["id"], **options) + result = nil + + # Perform any cached calls to the datastore to happen before our `query_datastore` + # matcher below which tries to assert which specific requests get made, since index definitions + # have caching behavior that can make the presence or absence of that request slightly non-deterministic. + pre_cache_index_state(graphql) + + expect { + result = super(*args, query_overrides: {requested_fields: requested_fields}, **options) + }.to query_datastore("main", 0).times.or query_datastore("main", 1).time + + result + end + + def resolve_nodes(...) + resolve(...).edges.map(&:node) + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/integration/elastic_graph/graphql/resolvers/query_source_spec.rb b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/resolvers/query_source_spec.rb new file mode 100644 index 00000000..14969a8d --- /dev/null +++ b/elasticgraph-graphql/spec/integration/elastic_graph/graphql/resolvers/query_source_spec.rb @@ -0,0 +1,57 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/query_details_tracker" +require "elastic_graph/graphql/resolvers/query_source" +require "graphql/dataloader" + +module ElasticGraph + class GraphQL + module Resolvers + RSpec.describe QuerySource, :factories, :uses_datastore do + let(:graphql) { build_graphql } + + it "batches up multiple queries, returning a chainable promise for each" do + index_into( + graphql, + widget = build(:widget), + component = build(:component) + ) + + widgets_def = graphql.datastore_core.index_definitions_by_name.fetch("widgets") + components_def = graphql.datastore_core.index_definitions_by_name.fetch("components") + + widget_query = graphql.datastore_query_builder.new_query( + search_index_definitions: [widgets_def], + requested_fields: ["id"] + ) + + component_query = graphql.datastore_query_builder.new_query( + search_index_definitions: [components_def], + requested_fields: ["id"] + ) + + # Perform any cached calls to the datastore to prevent them from alter interacting with the + # `query_datastore` assertion below. + pre_cache_index_state(graphql) + query_tracker = QueryDetailsTracker.empty + + expect { + widget_results, component_results = ::GraphQL::Dataloader.with_dataloading do |dataloader| + dataloader.with(QuerySource, graphql.datastore_search_router, query_tracker) + .load_all([widget_query, component_query]) + end + + expect(widget_results.first.fetch("id")).to eq widget.fetch(:id) + expect(component_results.first.fetch("id")).to eq component.fetch(:id) + }.to query_datastore(components_def.cluster_to_query, 1).time + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/spec_helper.rb b/elasticgraph-graphql/spec/spec_helper.rb new file mode 100644 index 00000000..222428f7 --- /dev/null +++ b/elasticgraph-graphql/spec/spec_helper.rb @@ -0,0 +1,57 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-graphql`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. + +module ElasticGraph + class GraphQL + SPEC_ROOT = __dir__ + end + + module GraphQLSpecHelpers + def build_datastore_core(**options, &block) + options = {for_context: :graphql}.merge(options) + super(**options, &block) + end + end +end + +RSpec.configure do |config| + config.define_derived_metadata(absolute_file_path: %r{/elasticgraph-graphql/}) do |meta| + meta[:builds_graphql] = true + end + + config.when_first_matching_example_defined(:resolver) { require_relative "support/resolver" } + config.when_first_matching_example_defined(:query_adapter) { require_relative "support/query_adapter" } + config.prepend ElasticGraph::GraphQLSpecHelpers, absolute_file_path: %r{/elasticgraph-graphql/} +end + +RSpec::Matchers.define :take_less_than do |max_expected_duration| + require "elastic_graph/support/monotonic_clock" + clock = ElasticGraph::Support::MonotonicClock.new + + chain(:milliseconds) {} + supports_block_expectations + + match do |block| + start = clock.now_in_ms + block.call + stop = clock.now_in_ms + + @actual_duration = stop - start + @actual_duration < max_expected_duration + end + + failure_message do + # :nocov: -- only executed when a `take_less_than` expectation fails. + "expected block to #{description}, but took #{@actual_duration} milliseconds" + # :nocov: + end +end diff --git a/elasticgraph-graphql/spec/support/aggregations_helpers.rb b/elasticgraph-graphql/spec/support/aggregations_helpers.rb new file mode 100644 index 00000000..e662e437 --- /dev/null +++ b/elasticgraph-graphql/spec/support/aggregations_helpers.rb @@ -0,0 +1,194 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/composite_grouping_adapter" +require "elastic_graph/graphql/aggregation/computation" +require "elastic_graph/graphql/aggregation/date_histogram_grouping" +require "elastic_graph/graphql/aggregation/field_term_grouping" +require "elastic_graph/graphql/aggregation/key" +require "elastic_graph/graphql/aggregation/nested_sub_aggregation" +require "elastic_graph/graphql/aggregation/non_composite_grouping_adapter" +require "elastic_graph/graphql/aggregation/path_segment" +require "elastic_graph/graphql/aggregation/query" +require "elastic_graph/graphql/aggregation/script_term_grouping" +require "elastic_graph/schema_artifacts/runtime_metadata/computation_detail" +require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" + +module ElasticGraph + module AggregationsHelpers + def computation_of(*field_names_in_index, function, computed_field_name: function.to_s, field_names_in_graphql_query: field_names_in_index) + source_field_path = build_field_path(names_in_index: field_names_in_index, names_in_graphql_query: field_names_in_graphql_query) + + GraphQL::Aggregation::Computation.new( + source_field_path: source_field_path, + computed_index_field_name: computed_field_name, + detail: SchemaArtifacts::RuntimeMetadata::ComputationDetail.new( + function: function, + empty_bucket_value: (function == :sum || function == :cardinality) ? 0 : nil + ) + ) + end + + def date_histogram_grouping_of( + *field_names_in_index, + interval, + time_zone: "UTC", + offset: nil, + graphql_subfield: nil, + field_names_in_graphql_query: field_names_in_index + [graphql_subfield].compact + ) + field_path = build_field_path(names_in_index: field_names_in_index, names_in_graphql_query: field_names_in_graphql_query) + GraphQL::Aggregation::DateHistogramGrouping.new(field_path, interval, time_zone, offset) + end + + def as_time_of_day_grouping_of( + *field_names_in_index, + interval, + offset_ms: 0, + time_zone: "UTC", + graphql_subfield: nil, + field_names_in_graphql_query: field_names_in_index + [graphql_subfield].compact, + script_id: nil, + runtime_metadata: nil + ) + script_id ||= runtime_metadata.static_script_ids_by_scoped_name.fetch("field/as_time_of_day") + params = {"interval" => interval, "offset_ms" => offset_ms, "time_zone" => time_zone}.compact + script_term_grouping_of(*field_names_in_index, script_id: script_id, field_names_in_graphql_query: field_names_in_graphql_query, params: params) + end + + def as_day_of_week_grouping_of( + *field_names_in_index, + offset_ms: 0, + time_zone: "UTC", + graphql_subfield: nil, + field_names_in_graphql_query: field_names_in_index + [graphql_subfield].compact, + script_id: nil, + runtime_metadata: nil + ) + script_id ||= runtime_metadata.static_script_ids_by_scoped_name.fetch("field/as_day_of_week") + params = {"time_zone" => time_zone, "offset_ms" => offset_ms}.compact + script_term_grouping_of(*field_names_in_index, script_id: script_id, field_names_in_graphql_query: field_names_in_graphql_query, params: params) + end + + def script_term_grouping_of( + *field_names_in_index, + script_id:, + field_names_in_graphql_query: field_names_in_index, + params: {} + ) + field_path = build_field_path(names_in_index: field_names_in_index, names_in_graphql_query: field_names_in_graphql_query) + GraphQL::Aggregation::ScriptTermGrouping.new(field_path: field_path, script_id: script_id, params: params) + end + + def field_term_grouping_of(*field_names_in_index, field_names_in_graphql_query: field_names_in_index) + field_path = build_field_path(names_in_index: field_names_in_index, names_in_graphql_query: field_names_in_graphql_query) + GraphQL::Aggregation::FieldTermGrouping.new(field_path: field_path) + end + + def nested_sub_aggregation_of(path_in_index: nil, query: nil, path_in_graphql_query: path_in_index) + field_path = build_field_path(names_in_index: path_in_index, names_in_graphql_query: path_in_graphql_query) + GraphQL::Aggregation::NestedSubAggregation.new(nested_path: field_path, query: query) + end + + def sub_aggregation_query_of(grouping_adapter: GraphQL::Aggregation::NonCompositeGroupingAdapter, **options) + aggregation_query_of(grouping_adapter: grouping_adapter, **options) + end + + # Default values for `default_page_size` and `max_page_size` come from `config/settings/test.yaml.template` + def aggregation_query_of( + name: "aggregations", + grouping_adapter: GraphQL::Aggregation::CompositeGroupingAdapter, + computations: [], + groupings: [], + sub_aggregations: [], + needs_doc_count: false, + needs_doc_count_error: false, + filter: nil, + first: nil, + after: nil, + last: nil, + before: nil, + default_page_size: 50, + max_page_size: 500 + ) + schema_element_names = SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: :snake_case, overrides: {}) + sub_aggs_hash = sub_aggregations.to_h { |sa| [sa.nested_path_key, sa] } + + # Verify that we didn't lose any sub-aggregations by having conflicting keys + expect(sub_aggs_hash.values).to match_array(sub_aggregations) + + GraphQL::Aggregation::Query.new( + name: name, + grouping_adapter: grouping_adapter, + computations: computations.to_set, + groupings: groupings.to_set, + sub_aggregations: sub_aggs_hash, + needs_doc_count: needs_doc_count, + needs_doc_count_error: needs_doc_count_error, + filter: filter, + paginator: GraphQL::DatastoreQuery::Paginator.new( + first: first, + after: after, + last: last, + before: before, + default_page_size: default_page_size, + max_page_size: max_page_size, + schema_element_names: schema_element_names + ) + ) + end + + def aggregated_value_key_of(*field_path, function_name, aggregation_name: "aggregations") + GraphQL::Aggregation::Key::AggregatedValue.new( + aggregation_name: aggregation_name, + field_path: field_path, + function_name: function_name + ) + end + + # The `QueryOptimizer` assumes that `Aggregation::Query` will always produce aggregation keys + # using `Aggregation::Query#name` such that `Aggregation::Key.extract_aggregation_name_from` is able + # to extract the original name from response keys. If that is violated, it will not work properly and + # subtle bugs can result. + # + # This helper method is used from our unit and integration tests for `DatastoreQuery` to verify that + # that requirement is satisfied. + def verify_aggregations_satisfy_optimizer_requirements(aggregations, for_query:) + return if aggregations.nil? + + actual_agg_names = aggregations.keys.map do |key| + GraphQL::Aggregation::Key.extract_aggregation_name_from(key) + end + + expected_agg_names = for_query.aggregations.values.flat_map do |agg| + # For groupings/computations, we expect a single key if we have any groupings; + # if not, we expect one per computation since each computation will go directly + # at the root. + count = agg.groupings.empty? ? agg.computations.size : 1 + + # In addition, each sub-aggregation gets its own key. + count += agg.sub_aggregations.size + + [agg.name] * count + end + + expect(actual_agg_names).to match_array(expected_agg_names) + end + + private + + def build_field_path(names_in_index:, names_in_graphql_query:) + names_in_graphql_query.zip(names_in_index).map do |name_in_graphql_query, name_in_index| + GraphQL::Aggregation::PathSegment.new( + name_in_graphql_query: name_in_graphql_query, + name_in_index: name_in_index + ) + end + end + end +end diff --git a/elasticgraph-graphql/spec/support/client_resolvers.rb b/elasticgraph-graphql/spec/support/client_resolvers.rb new file mode 100644 index 00000000..15663525 --- /dev/null +++ b/elasticgraph-graphql/spec/support/client_resolvers.rb @@ -0,0 +1,33 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + module ClientResolvers + class ViaHTTPHeader < ::Struct.new(:header_name) + def initialize(config) + super(header_name: config.fetch("header_name")) + end + + def resolve(http_request) + if (status_code = http_request.normalized_headers["X-CLIENT-RESOLVER-RESPOND-WITH"]) + HTTPResponse.error(status_code.to_i, "Rejected by client resolver") + else + Client.new( + source_description: header_name, + name: http_request.normalized_headers[header_name] + ) + end + end + end + + class Invalid + end + end + end +end diff --git a/elasticgraph-graphql/spec/support/graphql.rb b/elasticgraph-graphql/spec/support/graphql.rb new file mode 100644 index 00000000..4ca63d88 --- /dev/null +++ b/elasticgraph-graphql/spec/support/graphql.rb @@ -0,0 +1,27 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/graphql_formatter" + +module GraphQLSupport + def graphql_args(hash) + ElasticGraph::Support::GraphQLFormatter.format_args(**hash) + end + + def call_graphql_query(query, gql: graphql, allow_errors: false, **options) + gql.graphql_query_executor.execute(query, **options).tap do |response| + expect(response["errors"]).to(eq([]).or(eq(nil))) unless allow_errors + end + end + + def expect_error_related_to(response, *error_message_snippets) + expect(response["errors"].size).to eq(1) + expect(response["errors"].to_s).to include(*error_message_snippets) + expect(response["data"]).to be nil + end +end diff --git a/elasticgraph-graphql/spec/support/query_adapter.rb b/elasticgraph-graphql/spec/support/query_adapter.rb new file mode 100644 index 00000000..21832521 --- /dev/null +++ b/elasticgraph-graphql/spec/support/query_adapter.rb @@ -0,0 +1,178 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/resolvers/query_adapter" +require "rspec/expectations" + +module QueryAdapterSpecSupport + # An object that helps us test our query adapters by injecting an alternate + # `graphql_adapter` when building the `ElasticGraph::GraphQL` instance. The + # alternate `graphql_adapter` builds a query for each field. This is returned to + # the caller so that it can make assertions about what GraphQL::DatastoreQuery instances get + # built. + # + # Note: the `graphql_adapter` returns simple default values for each resolved field + # (empty object, nil, etc), which means that it really only works for schemas that allow + # all scalar fields to be nullable. + class QueryProbe + include ::RSpec::Matchers + + attr_reader :graphql + + def initialize(list_item_count: 1) + @list_item_count = list_item_count + @graphql = yield(graphql_adapter: self) + + # Only 2 resolvers yield to `Resolvers::GraphQLAdapter` to get a query built. Here we + # put those into an array so that we can mimic the behavior in this `QueryProbe` and only + # build an `DatastoreQuery` for the same fields. + resolvers_module = ::ElasticGraph::GraphQL::Resolvers + @resolvers_that_yield_for_datastore_query = @graphql.graphql_resolvers.select do |resolver| + case resolver + when resolvers_module::ListRecords, resolvers_module::NestedRelationships + true + else + false + end + end + end + + def datastore_queries_by_field_for(query) + @datastore_queries_by_field = ::Hash.new { |h, k| h[k] = [] } + response = @graphql.graphql_query_executor.execute(query) + expect(response.to_h["errors"]).to eq nil + @datastore_queries_by_field + end + + def errors_for(query) + @datastore_queries_by_field = ::Hash.new { |h, k| h[k] = [] } + response = @graphql.graphql_query_executor.execute(query) + expect(response.to_h).to include("errors") + response["errors"] + end + + # Necessary for when we use `QueryProbe` on a query that requests a union or interface type. + def resolve_type(union_type, object, context) + # Just pick one of the possible types. + object.fetch(:type).unwrap_fully.subtypes.first.graphql_type + end + + def call(parent_type, field, object, args, context) + context[:schema_element_names] = @graphql.schema.element_names + schema_field = @graphql.schema.field_named(parent_type.graphql_name, field.name) + + lookahead = args[:lookahead] + args = schema_field.args_to_schema_form(args.except(:lookahead)) + + if @resolvers_that_yield_for_datastore_query.any? { |res| res.can_resolve?(field: schema_field, object: object) } + field_key = "#{schema_field.parent_type.name}.#{lookahead.ast_nodes.first.alias || schema_field.name}" + @datastore_queries_by_field[field_key] << query_adapter.build_query_from( + field: schema_field, + args: args, + lookahead: lookahead, + context: context + ) + end + + default_value_for(schema_field) + end + + def coerce_input(type, value, ctx) + value + end + + def coerce_result(type, value, ctx) + value + end + + def default_value_for(schema_field) + object = {type: schema_field.type} # to support `#resolve_type`. + + if schema_field.type.list? + # Return a list of one item to be the GraphQL engine tries to resolve + # the subfields (presumably, it wouldn't attempt them if it was an empty list) + # :nocov: (branch) -- all our tests that use this so far are for fields that wrap objects, not scalars. + [schema_field.type.unwrap_fully.object? ? object : default_scalar_value_for(schema_field.type)] * @list_item_count + # :nocov: (branch) + elsif schema_field.type.object? + object + else + default_scalar_value_for(schema_field.type) + end + end + + DEFAULT_SCALAR_VALUES = { + Int: 37, + Float: 37.0, + String: "37", + ID: "37", + Cursor: "37", + JsonSafeLong: 37, + LongString: "37", + Boolean: false, + Date: "2021-08-23", + DateTime: "2021-08-23T12:00:00Z", + DayOfWeek: "MONDAY", + LocalTime: "12:00:00" + } + + def default_scalar_value_for(type) + DEFAULT_SCALAR_VALUES.fetch(type.unwrap_fully.name) + end + + def query_adapter + @query_adapter ||= ::ElasticGraph::GraphQL::Resolvers::QueryAdapter.new( + datastore_query_builder: graphql.datastore_query_builder, + datastore_query_adapters: graphql.datastore_query_adapters + ) + end + end + + # Executes the provided `graphql_query` string against the provided schema string + # in order to probe the execution to build a hash of { field => GraphQL::DatastoreQuery }. + def datastore_queries_by_field_for(graphql_query, schema_artifacts:, list_item_count: 1, **graphql_opts) + graphql_and_datastore_queries_by_field_for( + graphql_query, + schema_artifacts: schema_artifacts, + list_item_count: list_item_count, + **graphql_opts + ).last + end + + # Executes the provided `graphql_query` string against the provided schema string + # in order to probe the execution to build a hash of { field => GraphQL::DatastoreQuery }. + # + # Returns the GraphQL instance (for things that need to work with other GraphQL dependencies) and that hash. + def graphql_and_datastore_queries_by_field_for(graphql_query, schema_artifacts:, list_item_count: 1, **graphql_opts) + probe = QueryProbe.new(list_item_count: list_item_count) do |graphql_adapter:| + build_graphql(schema_artifacts: schema_artifacts, graphql_adapter: graphql_adapter, **graphql_opts) + end + + [probe.graphql, probe.datastore_queries_by_field_for(graphql_query)] + end + + # Executes the provided `graphql_query` string against the provided schema string + # in order to probe the execution to get the datastore query for the schema field + # identified by `type` and `field`. + def datastore_query_for(schema_artifacts:, graphql_query:, type:, field:, **graphql_opts) + queries = datastore_queries_by_field_for(graphql_query, schema_artifacts: schema_artifacts, **graphql_opts) + field_queries = queries.fetch("#{type}.#{field}") + expect(field_queries.size).to eq 1 + field_queries.first + end + + def graphql_errors_for(schema_artifacts:, graphql_query:, **graphql_opts) + QueryProbe.new do |graphql_adapter:| + build_graphql(schema_artifacts: schema_artifacts, graphql_adapter: graphql_adapter, **graphql_opts) + end.errors_for(graphql_query) + end +end + +RSpec.configure do |rspec| + rspec.include QueryAdapterSpecSupport, :query_adapter +end diff --git a/elasticgraph-graphql/spec/support/resolver.rb b/elasticgraph-graphql/spec/support/resolver.rb new file mode 100644 index 00000000..50e1e927 --- /dev/null +++ b/elasticgraph-graphql/spec/support/resolver.rb @@ -0,0 +1,101 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/query_details_tracker" +require "elastic_graph/graphql/resolvers/query_adapter" +require "elastic_graph/graphql/resolvers/query_source" +require "graphql/dataloader" + +module ResolverHelperMethods + def resolve(type_name, field_name, document = nil, **options) + field = graphql.schema.field_named(type_name, field_name) + query_overrides = options.fetch(:query_overrides) { {} } + args = field.args_to_schema_form(options.except(:query_overrides, :lookahead)) + lookahead = options[:lookahead] || GraphQL::Execution::Lookahead::NULL_LOOKAHEAD + query_details_tracker = ElasticGraph::GraphQL::QueryDetailsTracker.empty + + ::GraphQL::Dataloader.with_dataloading do |dataloader| + context = ::GraphQL::Query::Context.new( + query: nil, + schema: graphql.schema.graphql_schema, + values: { + schema_element_names: graphql.runtime_metadata.schema_element_names, + dataloader: dataloader, + elastic_graph_query_tracker: query_details_tracker, + datastore_search_router: graphql.datastore_search_router + } + ) + + expect(document).to satisfy { |doc| resolver.can_resolve?(field: field, object: doc) } + + query = nil + query_builder = -> { + query ||= query_adapter.build_query_from(field: field, lookahead: lookahead, args: args, context: context).with(**query_overrides) + } + + begin + # In the 2.1.0 release of the GraphQL gem, `GraphQL::Pagination::Connection#initialize` expects a particular thread local[^1]. + # Here we initialize the thread local in a similar way to how the GraphQL gem does it[^2]. + # + # [^1]: https://github.com/rmosolgo/graphql-ruby/blob/v2.1.0/lib/graphql/pagination/connection.rb#L94-L96 + # [^2]: https://github.com/rmosolgo/graphql-ruby/blob/v2.1.0/lib/graphql/execution/interpreter/runtime.rb#L935-L941 + ::Thread.current[:__graphql_runtime_info] = ::Hash.new { |h, k| h[k] = ::GraphQL::Execution::Interpreter::Runtime::CurrentState.new } + resolver.resolve(field: field, object: document, context: context, args: args, lookahead: lookahead, &query_builder) + ensure + ::Thread.current[:__graphql_runtime_info] = nil + end + end + end +end + +# Provides support for integration testing resolvers. It assumes: +# - You have exposed `let(:graphql)` in your host example group. +# - You are `describe`ing the resolver class (it uses `described_class`) +# - All the initialization args for the resolver class are keyword args and are available off of `graphql`. +# +# The provided `resolve` method calls the resolver directly instead of going through the resolver adapter. +RSpec.shared_context "resolver support" do + include ResolverHelperMethods + + resolver_dependencies = described_class + .instance_method(:initialize) + .parameters + .map do |type, name| + if [:keyreq, :key].include?(type) + name + else + # :nocov: -- only executed when a resolver's `initialize` is incorrectly defined to use positional args instead of kw args + raise "All resolver init args must be keyword args, but `#{described_class}#initialize` accepts non-kwarg `#{name}`" + # :nocov: + end + end + + subject(:resolver) do + dependencies = resolver_dependencies.each_with_object({}) do |dep_name, deps| + deps[dep_name] = + if dep_name == :schema_element_names + graphql.runtime_metadata.schema_element_names + else + graphql.public_send(dep_name) + end + end + + described_class.new(**dependencies) + end + + let(:query_adapter) do + ElasticGraph::GraphQL::Resolvers::QueryAdapter.new( + datastore_query_builder: graphql.datastore_query_builder, + datastore_query_adapters: graphql.datastore_query_adapters + ) + end +end + +RSpec.configure do |c| + c.include_context "resolver support", :resolver +end diff --git a/elasticgraph-graphql/spec/support/scalar_coercion_adapter.rb b/elasticgraph-graphql/spec/support/scalar_coercion_adapter.rb new file mode 100644 index 00000000..39ecfc01 --- /dev/null +++ b/elasticgraph-graphql/spec/support/scalar_coercion_adapter.rb @@ -0,0 +1,136 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/graphql_formatter" +require "forwardable" +require "graphql" + +# Provides test harness support for testing scalar coercion adapters. Makes +# it easy to exercise the `coerce_input`/`coerce_result` methods of +# a scalar adapter and see how that behavior is visible from a client's +# point of view. +RSpec.shared_context "scalar coercion adapter support" do |scalar_type_name, schema_definition: nil| + before(:context) do + normal_graphql_adapter = build_graphql(schema_definition: schema_definition, clients_by_name: {}).schema.send(:resolver) + @test_adapter = ScalarCoercionAdapterTestGraphQLAdapter.new(normal_graphql_adapter) + + @graphql = build_graphql(graphql_adapter: @test_adapter, clients_by_name: {}, schema_definition: lambda do |schema| + schema_definition&.call(schema) + schema.raw_sdl <<~EOS + type Query { + echo(arg: #{scalar_type_name}): #{scalar_type_name} + } + EOS + end) + + @query = <<~QUERY + query TestQuery($value: #{scalar_type_name}) { + echo(arg: $value) + } + QUERY + end + + def execute_query_with_variable_value(value) + @test_adapter.return_value = nil + @graphql.graphql_query_executor.execute(@query, variables: {value: value}).to_h + end + + def execute_query_with_inline_query_value(value) + @test_adapter.return_value = nil + query = "query { echo#{ElasticGraph::Support::GraphQLFormatter.format_args(arg: value)} }" + @graphql.graphql_query_executor.execute(query).to_h + end + + def execute_query_returning(value) + @test_adapter.return_value = value + @graphql.graphql_query_executor.execute(@query).to_h + end + + def expect_input_value_to_be_accepted(value, as: value, only_test_variable: false) + response = execute_query_with_variable_value(value) + + expect(response).not_to include("errors") + expect(response).to eq({"data" => {"echo" => nil}}) + expect(@test_adapter.last_arg_value).to eq(as) + + unless only_test_variable + response = execute_query_with_inline_query_value(value) + + expect(response).not_to include("errors") + expect(response).to eq({"data" => {"echo" => nil}}) + expect(@test_adapter.last_arg_value).to eq(as) + end + end + + # Use `define_method` instead of `def` to have access to `scalar_type_name` + define_method :expect_input_value_to_be_rejected do |value, *error_snippets, expect_error_to_lack: [], only_test_variable: false| + response = execute_query_with_variable_value(value) + + expect(response["data"]).to be nil + expect(response.dig("errors", 0, "message")).to include(scalar_type_name) + expect(response.dig("errors", 0, "extensions", "value")).to eq(value) + + explanation = response.dig("errors", 0, "extensions", "problems", 0, "explanation") + if error_snippets.any? + expect(explanation).to include(*error_snippets) + end + + if expect_error_to_lack.any? + expect(explanation).not_to include(*expect_error_to_lack) + end + + unless only_test_variable + response = execute_query_with_inline_query_value(value) + + message = response.dig("errors", 0, "message") + expect(response["data"]).to be nil + expect(message).to include( + scalar_type_name, + ElasticGraph::Support::GraphQLFormatter.serialize(value), + *error_snippets + ) + + if expect_error_to_lack.any? + expect(message).not_to include(*expect_error_to_lack) + end + end + end + + def expect_result_to_be_returned(value, as: value) + response = execute_query_returning(value) + + expect(response).to eq({"data" => {"echo" => as}}) + end + + def expect_result_to_be_replaced_with_nil(value) + response = execute_query_returning(value) + + expect(response).to eq({"data" => {"echo" => nil}}) + end +end + +# An GraphQL adapter meant to be used in place of a real EG adapter +# just so it can replace the `call` logic with something super simple +# that (1) returns a specific value and (2) records the arg value +# that got passed to `call`. +# +# It wraps a real adapter in order to delegate `coerce_input` and `coerce_result` to it. +class ScalarCoercionAdapterTestGraphQLAdapter + extend ::Forwardable + attr_accessor :return_value, :last_arg_value + def_delegators :@wrapped_graphql_adapter, :coerce_input, :coerce_result + + def initialize(wrapped_graphql_adapter) + @wrapped_graphql_adapter = wrapped_graphql_adapter + end + + def call(parent_type, field, object, args, context) + self.last_arg_value = args[:arg] + return_value + end +end diff --git a/elasticgraph-graphql/spec/support/sort.rb b/elasticgraph-graphql/spec/support/sort.rb new file mode 100644 index 00000000..f84cab37 --- /dev/null +++ b/elasticgraph-graphql/spec/support/sort.rb @@ -0,0 +1,41 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/decoded_cursor" +require "elastic_graph/graphql/datastore_query" + +module ElasticGraph + module SortSupport + TIEBREAKER_SORT_CLAUSES = GraphQL::DatastoreQuery::TIEBREAKER_SORT_CLAUSES + + def sort_list_with_missing_option_for(*sort_clauses) + sort_list_for(*sort_clauses).map do |clause| + clause.transform_values do |options| + options.merge("missing" => (options.fetch("order") == "asc") ? "_first" : "_last") + end + end + end + + def sort_list_for(*sort_clauses) + flattened_sort_clauses = sort_clauses.flatten + # :nocov: -- currently we don't have any tests that explicitly specify `id` as the sort field, but we might in the future + flattened_sort_clauses + ((flattened_sort_clauses.any? { |s| s.key?("id") }) ? [] : TIEBREAKER_SORT_CLAUSES) + # :nocov: + end + + def decoded_cursor_factory_for(*sort_clauses_or_fields) + sort_clauses_or_fields = sort_clauses_or_fields.flatten + if sort_clauses_or_fields.first.is_a?(Hash) # it is a list of clauses + GraphQL::DecodedCursor::Factory.from_sort_list(sort_list_for(*sort_clauses_or_fields)) + else + # cursor doesn't care about sort direction (asc VS desc), so we just assign `asc` as a dummy value + decoded_cursor_factory_for(sort_clauses_or_fields.map { |field| {field => {"order" => "asc"}} }) + end + end + end +end diff --git a/elasticgraph-graphql/spec/support/sub_aggregation_support.rb b/elasticgraph-graphql/spec/support/sub_aggregation_support.rb new file mode 100644 index 00000000..b353d2fe --- /dev/null +++ b/elasticgraph-graphql/spec/support/sub_aggregation_support.rb @@ -0,0 +1,67 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/key" + +module ElasticGraph + class GraphQL + module Aggregation + RSpec.shared_context "sub-aggregation support" do |grouping_adapter| + define_method :outer_meta do |hash = {}, size: 50| + {"size" => size, "adapter" => grouping_adapter.meta_name}.merge(hash) + end + + def inner_terms_meta(hash = {}) + {"merge_into_bucket" => {}}.merge(hash) + end + + def inner_date_meta(hash = {}) + {"merge_into_bucket" => {"doc_count_error_upper_bound" => 0}}.merge(hash) + end + + define_method :sub_aggregation_query_of do |**options| + super(grouping_adapter: grouping_adapter, **options) + end + end + + module SubAggregationRefinements + refine ::Hash do + # Helper method that can be used to add a missing value bucket aggregation to an + # existing aggregation hash. Defined as a refinement to support a chainable syntax + # in order to minimize churn in our specs at the point we added missing value buckets. + def with_missing_value_bucket(count, extras = {}) + grouped_field = SubAggregationRefinements.grouped_field_from(self) + + missing_value_bucket = extras.merge({ + "doc_count" => count, + "meta" => (extras.empty? ? nil : dig(grouped_field, "meta")) + }.compact) + + merge({ + Aggregation::Key.missing_value_bucket_key(grouped_field) => missing_value_bucket + }) + end + end + + extend ::RSpec::Matchers + + def self.grouped_field_from(agg_hash) + grouped_field_candidates = agg_hash.except( + "doc_count", "meta", "key", "key_as_string", + "doc_count_error_upper_bound", + "sum_other_doc_count" + ).keys + + # We expect only one candidate; here we use an expectation that will show them all if there are more. + expect(grouped_field_candidates).to eq([grouped_field_candidates.first]) + grouped_field_candidates.first + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/computation_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/computation_spec.rb new file mode 100644 index 00000000..67cc3849 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/computation_spec.rb @@ -0,0 +1,54 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/computation" +require "support/aggregations_helpers" + +module ElasticGraph + class GraphQL + module Aggregation + RSpec.describe Computation do + include AggregationsHelpers + + describe "#key" do + it "returns the computed field name prefixed with the field path and the aggregation name" do + computation = computation_of("foo", "bar", :avg, computed_field_name: "average") + + expect(computation.key(aggregation_name: "my_aggs")).to eq aggregated_value_key_of("foo", "bar", "average", aggregation_name: "my_aggs").encode + end + + it "uses GraphQL query field names when they differ from the name of the field in the index" do + computation = computation_of("foo", "bar", :avg, computed_field_name: "average", field_names_in_graphql_query: ["oof", "rab"]) + + expect(computation.key(aggregation_name: "my_aggs")).to eq aggregated_value_key_of("oof", "rab", "average", aggregation_name: "my_aggs").encode + end + end + + describe "#clause" do + it 'builds a datastore aggregation computation clause in the form: {function => {"field" => field_name}}' do + computation = computation_of("foo", "bar", :avg) + + expect(computation.clause).to eq({"avg" => {"field" => "foo.bar"}}) + end + + it "uses the names of the fields in the index rather than the GraphQL query field names when they differ" do + computation = computation_of("foo", "bar", :avg, field_names_in_graphql_query: ["oof", "rab"]) + + expect(computation.clause).to eq({"avg" => {"field" => "foo.bar"}}) + end + + it "allows a `name_in_index` that references a child field" do + computation = computation_of("foo.c", "bar.d", :avg, field_names_in_graphql_query: ["oof", "rab"]) + + expect(computation.clause).to eq({"avg" => {"field" => "foo.c.bar.d"}}) + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/date_histogram_grouping_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/date_histogram_grouping_spec.rb new file mode 100644 index 00000000..6bfd1af8 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/date_histogram_grouping_spec.rb @@ -0,0 +1,150 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/date_histogram_grouping" +require "elastic_graph/constants" +require "support/aggregations_helpers" + +module ElasticGraph + class GraphQL + module Aggregation + RSpec.describe DateHistogramGrouping do + include AggregationsHelpers + + describe "#key" do + it "returns the encoded field path" do + date_histogram_grouping = date_histogram_grouping_of("foo", "bar", "day") + + expect(date_histogram_grouping.key).to eq "foo.bar" + end + + it "uses GraphQL query field names when they differ from the name of the field in the index" do + grouping = date_histogram_grouping_of("foo", "bar", "day", field_names_in_graphql_query: ["oof", "rab"]) + + expect(grouping.key).to eq "oof.rab" + end + end + + describe "#encoded_index_field_path" do + it "returns the encoded field path" do + grouping = date_histogram_grouping_of("foo", "bar", "day") + + expect(grouping.encoded_index_field_path).to eq "foo.bar" + end + + it "uses the names in the index when they differ from the GraphQL names" do + grouping = date_histogram_grouping_of("oof", "rab", "day", field_names_in_graphql_query: ["foo", "bar"]) + + expect(grouping.encoded_index_field_path).to eq "oof.rab" + end + + it "allows a `name_in_index` that references a child field" do + grouping = date_histogram_grouping_of("foo.c", "bar.d", "day", field_names_in_graphql_query: ["foo", "bar"]) + + expect(grouping.encoded_index_field_path).to eq "foo.c.bar.d" + end + end + + describe "#composite_clause" do + it 'builds a datastore aggregation date histogram grouping clause in the form: {"date_histogram" => {"field" => field_name, "calendar_interval" => interval}}' do + date_histogram_grouping = date_histogram_grouping_of("foo", "bar", "day") + + expect(date_histogram_grouping.composite_clause).to eq({ + "date_histogram" => { + "field" => "foo.bar", + "format" => DATASTORE_DATE_TIME_FORMAT, + "calendar_interval" => "day", + "time_zone" => "UTC" + } + }) + end + + it "uses the names of the fields in the index rather than the GraphQL query field names when they differ" do + grouping = date_histogram_grouping_of("foo", "bar", "day", field_names_in_graphql_query: ["oof", "rab"]) + + expect(grouping.composite_clause.dig("date_histogram", "field")).to eq("foo.bar") + end + + %w[year quarter month week day hour minute].each do |calendar_interval| + it "supports a `calendar_interval` of `#{calendar_interval.inspect}`" do + date_histogram_grouping = date_histogram_grouping_of("foo", "bar", calendar_interval) + + expect(date_histogram_grouping.composite_clause).to eq({ + "date_histogram" => { + "field" => "foo.bar", + "format" => DATASTORE_DATE_TIME_FORMAT, + "calendar_interval" => calendar_interval.to_s, + "time_zone" => "UTC" + } + }) + end + end + + {"second" => "1s", "millisecond" => "1ms"}.each do |interval_name, fixed_interval_value| + it "supports a `fixed_interval` of `#{fixed_interval_value.inspect}` for #{interval_name.inspect}" do + date_histogram_grouping = date_histogram_grouping_of("foo", "bar", interval_name) + + expect(date_histogram_grouping.composite_clause).to eq({ + "date_histogram" => { + "field" => "foo.bar", + "format" => DATASTORE_DATE_TIME_FORMAT, + "fixed_interval" => fixed_interval_value, + "time_zone" => "UTC" + } + }) + end + end + + it "allows the default time zone of `UTC` to be overridden" do + date_histogram_grouping = date_histogram_grouping_of("foo", "bar", "day") + expect(date_histogram_grouping.composite_clause.dig("date_histogram", "time_zone")).to eq("UTC") + + date_histogram_grouping = date_histogram_grouping_of("foo", "bar", "day", time_zone: "America/Los_Angeles") + expect(date_histogram_grouping.composite_clause.dig("date_histogram", "time_zone")).to eq("America/Los_Angeles") + end + + it "omits `timezone` if the value is `nil`" do + date_histogram_grouping = date_histogram_grouping_of("foo", "bar", "day", time_zone: nil) + + expect(date_histogram_grouping.composite_clause.fetch("date_histogram").keys).to contain_exactly("calendar_interval", "field", "format") + end + + it "includes `offset` if set" do + date_histogram_grouping = date_histogram_grouping_of("foo", "bar", "day", time_zone: "UTC", offset: "4h") + + expect(date_histogram_grouping.composite_clause).to eq({ + "date_histogram" => { + "field" => "foo.bar", + "format" => DATASTORE_DATE_TIME_FORMAT, + "calendar_interval" => "day", + "time_zone" => "UTC", + "offset" => "4h" + } + }) + end + + it "merges in the provided grouping options" do + grouping = date_histogram_grouping_of("foo", "bar", "day") + + clause = grouping.composite_clause(grouping_options: {"optA" => 1, "optB" => false}) + + expect(clause["date_histogram"]).to include({"optA" => 1, "optB" => false}) + end + + it "throws a clear exception when given an unsupported interval" do + date_histogram_grouping = date_histogram_grouping_of("foo", "bar", :fortnight) + + expect { + date_histogram_grouping.composite_clause + }.to raise_error ArgumentError, a_string_including("unsupported interval", "fortnight") + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/field_path_encoder_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/field_path_encoder_spec.rb new file mode 100644 index 00000000..ce6a4db3 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/field_path_encoder_spec.rb @@ -0,0 +1,56 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/field_path_encoder" + +module ElasticGraph + class GraphQL + module Aggregation + RSpec.describe FieldPathEncoder do + let(:path_delimiter) { FieldPathEncoder::DELIMITER } + + describe "#encode" do + it "encodes a list of field names as a single path string with dots separating the parts" do + encoded = FieldPathEncoder.encode(["a", "b"]) + expect(encoded).to eq("a.b") + end + + it "raises an `Errors::InvalidArgumentValueError` if a field name includes the field path delimiter" do + expect { + FieldPathEncoder.encode(["my_#{path_delimiter}field"]) + }.to raise_error(Errors::InvalidArgumentValueError, a_string_including("contains delimiter")) + end + end + + describe "#join" do + it "encodes a list of field names as a single path string with dots separating the parts" do + encoded = FieldPathEncoder.join(["a", "b"]) + expect(encoded).to eq("a.b") + end + + it "encodes a list of field paths as a single path string with dots separating the parts" do + encoded = FieldPathEncoder.join(["a.b", "c.d"]) + expect(encoded).to eq("a.b.c.d") + end + end + + it "encodes and decodes a field path with a single field name part for a root field" do + field_names = FieldPathEncoder.decode(FieldPathEncoder.encode(["my_field"])) + + expect(field_names).to eq(["my_field"]) + end + + it "encodes and decodes a field path with a listed of field name parts for a nested field" do + field_names = FieldPathEncoder.decode(FieldPathEncoder.encode(["transaction", "amountMoney", "amount"])) + + expect(field_names).to eq(["transaction", "amountMoney", "amount"]) + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/field_term_grouping_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/field_term_grouping_spec.rb new file mode 100644 index 00000000..df3bd395 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/field_term_grouping_spec.rb @@ -0,0 +1,90 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/field_term_grouping" +require "support/aggregations_helpers" + +module ElasticGraph + class GraphQL + module Aggregation + RSpec.describe FieldTermGrouping do + include AggregationsHelpers + + describe "#key" do + it "returns the encoded field path" do + term_grouping = field_term_grouping_of("foo", "bar") + + expect(term_grouping.key).to eq "foo.bar" + end + + it "uses GraphQL query field names when they differ from the name of the field in the index" do + grouping = field_term_grouping_of("foo", "bar", field_names_in_graphql_query: ["oof", "rab"]) + + expect(grouping.key).to eq "oof.rab" + end + end + + describe "#encoded_index_field_path" do + it "returns the encoded field path" do + grouping = field_term_grouping_of("foo", "bar") + + expect(grouping.encoded_index_field_path).to eq "foo.bar" + end + + it "uses the names in the index when they differ from the GraphQL names" do + grouping = field_term_grouping_of("oof", "rab", field_names_in_graphql_query: ["foo", "bar"]) + + expect(grouping.encoded_index_field_path).to eq "oof.rab" + end + + it "allows a `name_in_index` that references a child field" do + grouping = field_term_grouping_of("foo.c", "bar.d", field_names_in_graphql_query: ["foo", "bar"]) + + expect(grouping.encoded_index_field_path).to eq "foo.c.bar.d" + end + end + + describe "#composite_clause" do + it 'builds a datastore aggregation term grouping clause in the form: {"terms" => {"field" => field_name}}' do + term_grouping = field_term_grouping_of("foo", "bar") + + expect(term_grouping.composite_clause).to eq({"terms" => { + "field" => "foo.bar" + }}) + end + + it "uses the names of the fields in the index rather than the GraphQL query field names when they differ" do + grouping = field_term_grouping_of("foo", "bar", field_names_in_graphql_query: ["oof", "rab"]) + + expect(grouping.composite_clause.dig("terms", "field")).to eq("foo.bar") + end + + it "merges in the provided grouping options" do + grouping = field_term_grouping_of("foo", "bar") + + clause = grouping.composite_clause(grouping_options: {"optA" => 1, "optB" => false}) + + expect(clause["terms"]).to include({"optA" => 1, "optB" => false}) + end + end + + describe "#inner_meta" do + it "returns inner meta" do + grouping = field_term_grouping_of("foo", "bar") + expect(grouping.inner_meta).to eq({ + "key_path" => [ + "key" + ], + "merge_into_bucket" => {} + }) + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/key_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/key_spec.rb new file mode 100644 index 00000000..b8c78bf6 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/key_spec.rb @@ -0,0 +1,128 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/key" + +module ElasticGraph + class GraphQL + module Aggregation + RSpec.describe Key do + let(:delimiter) { Key::DELIMITER } + let(:path_delimiter) { FieldPathEncoder::DELIMITER } + + describe Key::AggregatedValue do + it "allows an `AggregatedValue` for an unnested field to be encoded" do + key = Key::AggregatedValue.new( + aggregation_name: "my_aggs", + field_path: ["my_field"], + function_name: "sum" + ) + + encoded = key.encode + expect(encoded).to eq "my_aggs:my_field:sum" + end + + it "allows an `AggregatedValue` for an nested field to be encoded" do + key = Key::AggregatedValue.new( + aggregation_name: "my_aggs", + field_path: ["transaction", "amountMoney", "amount"], + function_name: "sum" + ) + + encoded = key.encode + expect(encoded).to eq("my_aggs:transaction.amountMoney.amount:sum") + end + + it "raises an `Errors::InvalidArgumentValueError` if a field name includes the delimiter" do + expect { + Key::AggregatedValue.new( + aggregation_name: "my_aggs", + field_path: ["my_#{delimiter}field"], + function_name: "sum" + ) + }.to raise_error(Errors::InvalidArgumentValueError, a_string_including("contains delimiter")) + end + + it "raises an `Errors::InvalidArgumentValueError` if function name includes the delimiter" do + expect { + Key::AggregatedValue.new( + aggregation_name: "my_aggs", + field_path: ["my_field"], + function_name: "su#{delimiter}m" + ) + }.to raise_error(Errors::InvalidArgumentValueError, a_string_including("contains delimiter")) + end + + it "raises an `Errors::InvalidArgumentValueError` if aggregation name includes the delimiter" do + expect { + Key::AggregatedValue.new( + aggregation_name: "my#{delimiter}aggs", + field_path: ["my_field"], + function_name: "sum" + ) + }.to raise_error(Errors::InvalidArgumentValueError, a_string_including("contains delimiter")) + end + + it "raises an `Errors::InvalidArgumentValueError` if a field name includes the field path delimiter" do + expect { + Key::AggregatedValue.new( + aggregation_name: "my_aggs", + field_path: ["my_#{path_delimiter}field"], + function_name: "sum" + ) + }.to raise_error(Errors::InvalidArgumentValueError, a_string_including("contains delimiter")) + end + + describe "#encode" do + it "returns a non-empty string" do + encoded = Key::AggregatedValue.new( + aggregation_name: "my_aggs", + field_path: ["my_field"], + function_name: "sum" + ).encode + + expect(encoded).to be_a(String) + expect(encoded).not_to eq("") + end + end + + it "returns the original `field_path` from `#field_path` in spite of it being stored internally as an encoded path" do + key = Key::AggregatedValue.new( + aggregation_name: "my_aggs", + field_path: ["my_field", "sub_field"], + function_name: "sum" + ) + + expect(key.field_path).to eq(["my_field", "sub_field"]) + expect(key.encoded_field_path).to eq("my_field.sub_field") + end + end + + describe "#extract_aggregation_name_from" do + it "returns the aggregation name portion of an encoded key" do + aggregated_value_key = Key::AggregatedValue.new( + aggregation_name: "my_aggs", + field_path: ["my_field"], + function_name: "sum" + ) + + agg_name = Key.extract_aggregation_name_from(aggregated_value_key.encode) + + expect(agg_name).to eq "my_aggs" + end + + it "returns a string that's not an encoded key as-is" do + agg_name = Key.extract_aggregation_name_from("by_size") + + expect(agg_name).to eq "by_size" + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/query_adapter_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/query_adapter_spec.rb new file mode 100644 index 00000000..5d046568 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/query_adapter_spec.rb @@ -0,0 +1,2194 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/query_adapter" +require "support/aggregations_helpers" +require "support/graphql" + +module ElasticGraph + class GraphQL + module Aggregation + RSpec.describe QueryAdapter, :query_adapter do + include AggregationsHelpers + + attr_accessor :schema_artifacts + + before(:context) do + self.schema_artifacts = generate_schema_artifacts { |schema| define_schema(schema) } + end + + shared_examples_for "a query selecting nodes under aggregations" do |before_nodes:, after_nodes:| + it "can build an aggregations object with 2 computations and no groupings" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + aggregated_values { + amount_cents { + approximate_avg + exact_max + approximate_distinct_value_count + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of(name: "widget_aggregations", computations: [ + computation_of("amount_cents", :avg, computed_field_name: "approximate_avg"), + computation_of("amount_cents", :max, computed_field_name: "exact_max"), + computation_of("amount_cents", :cardinality, computed_field_name: "approximate_distinct_value_count") + ])]) + end + + context "when the aggregated field has a `name_in_index` override" do + before(:context) do + self.schema_artifacts = generate_schema_artifacts { |schema| define_schema(schema, amount_cents_opts: {name_in_index: "amt_cts"}) } + end + + it "respects the override in the generated aggregation hash" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + aggregated_values { + amount_cents { + approximate_avg + exact_max + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of(name: "widget_aggregations", computations: [ + computation_of("amt_cts", :avg, computed_field_name: "approximate_avg", field_names_in_graphql_query: ["amount_cents"]), + computation_of("amt_cts", :max, computed_field_name: "exact_max", field_names_in_graphql_query: ["amount_cents"]) + ])]) + end + end + + it "can build an ungrouped aggregation hash from nested query fields" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + aggregated_values { + cost { + amount_cents { + approximate_avg + exact_max + } + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of(name: "widget_aggregations", computations: [ + computation_of("cost", "amount_cents", :avg, computed_field_name: "approximate_avg"), + computation_of("cost", "amount_cents", :max, computed_field_name: "exact_max") + ])]) + end + + context "when a parent of the aggregated field has a `name_in_index` override" do + before(:context) do + self.schema_artifacts = generate_schema_artifacts { |schema| define_schema(schema, cost_opts: {name_in_index: "the_cost"}) } + end + + it "respects the override in the generated aggregation hash" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + aggregated_values { + cost { + amount_cents { + approximate_avg + exact_max + } + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of(name: "widget_aggregations", computations: [ + computation_of("the_cost", "amount_cents", :avg, computed_field_name: "approximate_avg", field_names_in_graphql_query: ["cost", "amount_cents"]), + computation_of("the_cost", "amount_cents", :max, computed_field_name: "exact_max", field_names_in_graphql_query: ["cost", "amount_cents"]) + ])]) + end + end + + context "with `sub_aggregations`" do + before(:context) do + self.schema_artifacts = CommonSpecHelpers.stock_schema_artifacts(for_context: :graphql) + end + + it "can build sub-aggregations" do + aggregations = aggregations_from_datastore_query(:Query, :team_aggregations, <<~QUERY) + query { + team_aggregations { + #{before_nodes} + sub_aggregations { + current_players_nested { + nodes { + count_detail { + approximate_value + } + } + } + + seasons_nested { + nodes { + count_detail { + approximate_value + } + } + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of(name: "team_aggregations", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["current_players_nested"], query: sub_aggregation_query_of(name: "current_players_nested", needs_doc_count: true)), + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", needs_doc_count: true)) + ])]) + end + + it "uses the injected grouping adapter on sub-aggregations" do + build_sub_agg = lambda do |adapter| + aggregations = aggregations_from_datastore_query(:Query, :team_aggregations, <<~QUERY, sub_aggregation_grouping_adapter: adapter) + query { + team_aggregations { + #{before_nodes} + sub_aggregations { + current_players_nested { + nodes { + count_detail { + approximate_value + } + } + } + + seasons_nested { + nodes { + count_detail { + approximate_value + } + } + } + } + #{after_nodes} + } + } + QUERY + + aggregations.first.sub_aggregations.values.first + end + + expect(build_sub_agg.call(NonCompositeGroupingAdapter).query.grouping_adapter).to eq NonCompositeGroupingAdapter + expect(build_sub_agg.call(CompositeGroupingAdapter).query.grouping_adapter).to eq CompositeGroupingAdapter + end + + it "determines it needs the doc count error if `upper_bound` or `exact_value` are requested, but not if `approximate_value` is requested" do + needs_doc_count_error_by_count_field = %w[exact_value approximate_value upper_bound].to_h do |count_field| + aggs = aggregations_from_datastore_query(:Query, :team_aggregations, <<~QUERY) + query { + team_aggregations { + #{before_nodes} + sub_aggregations { + current_players_nested { + nodes { + count_detail { + #{count_field} + } + } + } + } + #{after_nodes} + } + } + QUERY + + [count_field, aggs.first.sub_aggregations.values.first.query.needs_doc_count_error] + end + + expect(needs_doc_count_error_by_count_field).to eq({ + "exact_value" => true, + "approximate_value" => false, + "upper_bound" => true + }) + end + + it "builds a multi-part nested `path` for a `nested` field under extra object layers" do + aggregations = aggregations_from_datastore_query(:Query, :team_aggregations, <<~QUERY) + query { + team_aggregations { + #{before_nodes} + sub_aggregations { + nested_fields { + current_players { + nodes { + count_detail { + approximate_value + } + } + } + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of(name: "team_aggregations", sub_aggregations: [ + nested_sub_aggregation_of( + path_in_index: ["the_nested_fields", "current_players"], + query: sub_aggregation_query_of(name: "current_players", needs_doc_count: true), + path_in_graphql_query: ["nested_fields", "current_players"] + ) + ])]) + end + + it "supports sub-aggregation fields having an alternate `name_in_index`" do + aggregations = aggregations_from_datastore_query(:Query, :team_aggregations, <<~QUERY) + query { + team_aggregations { + #{before_nodes} + sub_aggregations { + nested_fields { + seasons { + nodes { + count_detail { + approximate_value + } + } + } + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of(name: "team_aggregations", sub_aggregations: [ + nested_sub_aggregation_of( + path_in_index: ["the_nested_fields", "the_seasons"], + query: sub_aggregation_query_of(name: "seasons", needs_doc_count: true), + path_in_graphql_query: ["nested_fields", "seasons"] + ) + ])]) + end + + it "supports sub-aggregations of sub-aggregations" do + aggregations = aggregations_from_datastore_query(:Query, :team_aggregations, <<~QUERY) + query { + team_aggregations { + #{before_nodes} + sub_aggregations { + current_players_nested { + nodes { + count_detail { + approximate_value + } + + sub_aggregations { + seasons_nested { + nodes { + count_detail { + approximate_value + } + } + } + } + } + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of(name: "team_aggregations", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["current_players_nested"], query: sub_aggregation_query_of( + name: "current_players_nested", + needs_doc_count: true, + sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["current_players_nested", "seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", needs_doc_count: true)) + ] + )) + ])]) + end + + it "can handle sub-aggregation fields of the same name under parents of a different name" do + aggregations = aggregations_from_datastore_query(:Query, :team_aggregations, <<~QUERY) + query { + team_aggregations { + #{before_nodes} + sub_aggregations { + nested_fields { + seasons { + nodes { + aggregated_values { + year { exact_min } + } + } + } + } + + nested_fields2 { + seasons { + nodes { + aggregated_values { + year { exact_min } + } + } + } + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq [aggregation_query_of(name: "team_aggregations", sub_aggregations: [ + nested_sub_aggregation_of( + path_in_graphql_query: ["nested_fields", "seasons"], + path_in_index: ["the_nested_fields", "the_seasons"], + query: sub_aggregation_query_of( + name: "seasons", + computations: [ + computation_of( + "the_nested_fields", "the_seasons", "year", :min, + computed_field_name: "exact_min", + field_names_in_graphql_query: ["nested_fields", "seasons", "year"] + ) + ] + ) + ), + nested_sub_aggregation_of( + path_in_graphql_query: ["nested_fields2", "seasons"], + path_in_index: ["nested_fields2", "the_seasons"], + query: sub_aggregation_query_of( + name: "seasons", + computations: [ + computation_of( + "nested_fields2", "the_seasons", "year", :min, + computed_field_name: "exact_min", + field_names_in_graphql_query: ["nested_fields2", "seasons", "year"] + ) + ] + ) + ) + ])] + end + + it "allows aliases to be used on a sub-aggregations field to request multiple differing sub-aggregations" do + aggregations = aggregations_from_datastore_query(:Query, :team_aggregations, <<~QUERY) + query { + team_aggregations { + #{before_nodes} + sub_aggregations { + players1: current_players_nested { + nodes { + count_detail { + approximate_value + } + } + } + + players2: current_players_nested { + nodes { + count_detail { + exact_value + } + } + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of(name: "team_aggregations", sub_aggregations: [ + nested_sub_aggregation_of( + path_in_index: ["current_players_nested"], + path_in_graphql_query: ["players1"], + query: sub_aggregation_query_of(name: "players1", needs_doc_count: true) + ), + nested_sub_aggregation_of( + path_in_index: ["current_players_nested"], + path_in_graphql_query: ["players2"], + query: sub_aggregation_query_of(name: "players2", needs_doc_count_error: true, needs_doc_count: true) + ) + ])]) + end + + it "builds a `filter` on a sub-aggregation when present on the GraphQL query" do + aggregations = aggregations_from_datastore_query(:Query, :team_aggregations, <<~QUERY) + query { + team_aggregations { + #{before_nodes} + sub_aggregations { + seasons_nested(filter: {year: {gt: 2000}}) { + nodes { + count_detail { + approximate_value + } + } + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of(name: "team_aggregations", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + needs_doc_count: true, + filter: {"year" => {"gt" => 2000}} + )) + ])]) + end + + it "builds `groupings` at any level of sub-aggregation when `groupedBy` is present in the query at that level" do + aggregations = aggregations_from_datastore_query(:Query, :team_aggregations, <<~QUERY) + query { + team_aggregations { + #{before_nodes} + sub_aggregations { + seasons_nested { + nodes { + grouped_by { + year + note + started_at { + as_date_time(truncation_unit: YEAR) + } + } + + sub_aggregations { + players_nested { + nodes { + grouped_by { + name + } + } + } + } + } + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of(name: "team_aggregations", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + groupings: [ + field_term_grouping_of("seasons_nested", "year"), + field_term_grouping_of("seasons_nested", "notes", field_names_in_graphql_query: ["seasons_nested", "note"]), + date_histogram_grouping_of("seasons_nested", "started_at", "year", field_names_in_graphql_query: ["seasons_nested", "started_at", "as_date_time"]) + ], + sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested", "players_nested"], query: sub_aggregation_query_of( + name: "players_nested", + groupings: [field_term_grouping_of("seasons_nested", "players_nested", "name")] + )) + ] + )) + ])]) + end + + it "builds legacy date time `groupings` at any level of sub-aggregation when `groupedBy` is present in the query at that level" do + aggregations = aggregations_from_datastore_query(:Query, :team_aggregations, <<~QUERY) + query { + team_aggregations { + #{before_nodes} + sub_aggregations { + seasons_nested { + nodes { + grouped_by { + year + note + started_at_legacy(granularity: YEAR) + } + + sub_aggregations { + players_nested { + nodes { + grouped_by { + name + } + } + } + } + } + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of(name: "team_aggregations", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + groupings: [ + field_term_grouping_of("seasons_nested", "year"), + field_term_grouping_of("seasons_nested", "notes", field_names_in_graphql_query: ["seasons_nested", "note"]), + date_histogram_grouping_of("seasons_nested", "started_at", "year", field_names_in_graphql_query: ["seasons_nested", "started_at_legacy"]) + ], + sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested", "players_nested"], query: sub_aggregation_query_of( + name: "players_nested", + groupings: [field_term_grouping_of("seasons_nested", "players_nested", "name")] + )) + ] + )) + ])]) + end + + it "does not support multiple aliases on `nodes` or `grouped_by` because if different `grouped_by` fields are selected we can have conflicting grouping requirements" do + error = single_graphql_error_for(<<~QUERY) + query { + team_aggregations { + #{before_nodes} + sub_aggregations { + seasons_nested { + by_year: nodes { + grouped_by { year } + count_detail { approximate_value } + } + + by_note: nodes { + grouped_by { note } + count_detail { approximate_value } + } + } + } + #{after_nodes} + } + } + QUERY + + expect(error["message"]).to include("more than one `nodes` selection under `seasons_nested` (`by_year`, `by_note`),") + + error = single_graphql_error_for(<<~QUERY) + query { + team_aggregations { + #{before_nodes} + sub_aggregations { + seasons_nested { + nodes { + by_year: grouped_by { year } + by_note: grouped_by { note } + count_detail { approximate_value } + } + } + } + #{after_nodes} + } + } + QUERY + + expect(error["message"]).to include("more than one `grouped_by` selection under `seasons_nested` (`by_year`, `by_note`),") + end + + it "builds `computations` at any level of sub-aggregation when `aggregated_values` is present in the query at that level" do + aggregations = aggregations_from_datastore_query(:Query, :team_aggregations, <<~QUERY) + query { + team_aggregations { + #{before_nodes} + sub_aggregations { + seasons_nested { + nodes { + aggregated_values { + year { exact_min } + record { + wins { + exact_max + approximate_avg + } + } + } + + sub_aggregations { + players_nested { + nodes { + aggregated_values { + nicknames { + approximate_distinct_value_count + } + } + } + } + } + } + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of(name: "team_aggregations", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + computations: [ + computation_of("seasons_nested", "year", :min, computed_field_name: "exact_min"), + computation_of("seasons_nested", "the_record", "win_count", :max, computed_field_name: "exact_max", field_names_in_graphql_query: ["seasons_nested", "record", "wins"]), + computation_of("seasons_nested", "the_record", "win_count", :avg, computed_field_name: "approximate_avg", field_names_in_graphql_query: ["seasons_nested", "record", "wins"]) + ], + sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested", "players_nested"], query: sub_aggregation_query_of( + name: "players_nested", + computations: [ + computation_of("seasons_nested", "players_nested", "nicknames", :cardinality, computed_field_name: "approximate_distinct_value_count") + ] + )) + ] + )) + ])]) + end + + describe "paginator.desired_page_size" do + include GraphQLSupport + + it "is set based on the `first:` argument" do + sub_agg_query = build_sub_aggregation_query({first: 30}, default_page_size: 47, max_page_size: 212) + + expect(sub_agg_query.paginator.desired_page_size).to eq(30) + end + + it "is set to the configured `default_page_size` if not specified in the query" do + sub_agg_query = build_sub_aggregation_query({}, default_page_size: 47, max_page_size: 212) + + expect(sub_agg_query.paginator.desired_page_size).to eq(47) + end + + it "is set to the configured `default_page_size` if set to `null` in the query" do + sub_agg_query = build_sub_aggregation_query({first: nil}, default_page_size: 47, max_page_size: 212) + + expect(sub_agg_query.paginator.desired_page_size).to eq(47) + end + + it "is limited based on the configured `max_page_size`" do + sub_agg_query = build_sub_aggregation_query({first: 300}, default_page_size: 47, max_page_size: 212) + + expect(sub_agg_query.paginator.desired_page_size).to eq(212) + end + + it "return an error if `first` is specified as a negative number in the query" do + sub_agg_query = build_sub_aggregation_query({first: -10}, default_page_size: 47, max_page_size: 212) + + expect { + sub_agg_query.paginator.desired_page_size + }.to raise_error ::GraphQL::ExecutionError, "`first` cannot be negative, but is -10." + end + + def build_sub_aggregation_query(args, default_page_size:, max_page_size:) + aggregations = aggregations_from_datastore_query( + :Query, + :team_aggregations, + query_for(**args), + default_page_size: default_page_size, + max_page_size: max_page_size + ) + + expect(aggregations.size).to eq(1) + expect(aggregations.first.sub_aggregations.size).to eq(1) + aggregations.first.sub_aggregations.values.first.query + end + + define_method :query_for do |**args| + <<~QUERY + query { + team_aggregations { + #{before_nodes} + sub_aggregations { + seasons_nested#{graphql_args(args)} { + nodes { + grouped_by { year } + count_detail { approximate_value } + } + } + } + #{after_nodes} + } + } + QUERY + end + end + + context "when filtering on a field that has an alternate `name_in_index`" do + before(:context) do + self.schema_artifacts = generate_schema_artifacts { |schema| define_schema(schema) } + end + + it "respects the `name_in_index` when parsing a filter" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + sub_aggregations { + costs(filter: {amount: {gt: 2000}}) { + nodes { + count_detail { + approximate_value + } + } + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of(name: "widget_aggregations", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["costs"], query: sub_aggregation_query_of( + name: "costs", + needs_doc_count: true, + filter: {"amount_in_index" => {"gt" => 2000}} + )) + ])]) + end + end + end + + context "aggregations with `legacy_grouping_schema`" do + before(:context) do + self.schema_artifacts = generate_schema_artifacts { |schema| define_schema(schema, legacy_grouping_schema: true) } + end + it "can build an aggregations object with multiple groupings and computations" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + size + color + created_at(granularity: DAY) + } + + aggregated_values { + amount_cents { + approximate_avg + exact_max + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [ + computation_of("amount_cents", :avg, computed_field_name: "approximate_avg"), + computation_of("amount_cents", :max, computed_field_name: "exact_max") + ], + groupings: [ + field_term_grouping_of("size"), + field_term_grouping_of("color"), + date_histogram_grouping_of("created_at", "day") + ] + )]) + end + + it "respects the `time_zone` option" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + created_at(granularity: DAY, time_zone: "America/Los_Angeles") + } + + count + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [], + needs_doc_count: true, + groupings: [ + date_histogram_grouping_of("created_at", "day", time_zone: "America/Los_Angeles") + ] + )]) + end + + it "respects the `offset` option" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + created_at(granularity: DAY, offset: {amount: -12, unit: HOUR}) + } + + count + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [], + needs_doc_count: true, + groupings: [ + date_histogram_grouping_of("created_at", "day", time_zone: "UTC", offset: "-12h") + ] + )]) + end + + it "supports `Date` field groupings, allowing them to have no `time_zone` argument" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + created_on(granularity: MONTH) + } + + count + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [], + needs_doc_count: true, + groupings: [ + date_histogram_grouping_of("created_on", "month", time_zone: nil) + ] + )]) + end + + it "supports `offsetDays` on `Date` field groupings" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + created_on(granularity: MONTH, offset_days: -12) + } + + count + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [], + needs_doc_count: true, + groupings: [ + date_histogram_grouping_of("created_on", "month", time_zone: nil, offset: "-12d") + ] + )]) + end + end + + context "aggregations without `legacy_grouping_schema`" do + it "can build an aggregations object with multiple groupings and computations" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + size + color + created_at { + as_date_time(truncation_unit: DAY) + } + } + + aggregated_values { + amount_cents { + approximate_avg + exact_max + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [ + computation_of("amount_cents", :avg, computed_field_name: "approximate_avg"), + computation_of("amount_cents", :max, computed_field_name: "exact_max") + ], + groupings: [ + field_term_grouping_of("size"), + field_term_grouping_of("color"), + date_histogram_grouping_of("created_at", "day", graphql_subfield: "as_date_time") + ] + )]) + end + + it "can build an aggregations object with every `DateTimeGroupedBy` subfield" do + # Verify that the fields we request in the query below are in fact all the subfields + expect(sub_fields_of("DateTimeGroupedBy")).to contain_exactly("as_date_time", "as_date", "as_time_of_day", "as_day_of_week") + + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + created_at { + as_date_time(truncation_unit: DAY) + as_date(truncation_unit: YEAR, time_zone: "America/Los_Angeles", offset: {amount: 30, unit: DAY}) + as_time_of_day(truncation_unit: SECOND, time_zone: "America/Los_Angeles", offset: {amount: 60, unit: MINUTE}) + as_day_of_week(time_zone: "America/Los_Angeles", offset: {amount: 3, unit: HOUR}) + } + } + + count + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [], + needs_doc_count: true, + groupings: [ + date_histogram_grouping_of("created_at", "day", time_zone: "UTC", graphql_subfield: "as_date_time"), + date_histogram_grouping_of("created_at", "year", time_zone: "America/Los_Angeles", offset: "30d", graphql_subfield: "as_date"), + as_time_of_day_grouping_of("created_at", "second", time_zone: "America/Los_Angeles", offset_ms: 3_600_000, graphql_subfield: "as_time_of_day"), + as_day_of_week_grouping_of("created_at", time_zone: "America/Los_Angeles", offset_ms: 10_800_000, graphql_subfield: "as_day_of_week") + ] + )]) + end + + it "sets defaults correctly when `as_day_of_week` for time_zone and offset_ms" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + created_at { + as_day_of_week + } + } + + count + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [], + needs_doc_count: true, + groupings: [ + as_day_of_week_grouping_of("created_at", time_zone: "UTC", offset_ms: 0, graphql_subfield: "as_day_of_week") + ] + )]) + end + + it "sets defaults correctly when `as_time_of_day` for time_zone and offset_ms" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + created_at { + as_time_of_day(truncation_unit: HOUR) + } + } + + count + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [], + needs_doc_count: true, + groupings: [ + as_time_of_day_grouping_of("created_at", "hour", time_zone: "UTC", offset_ms: 0, graphql_subfield: "as_time_of_day") + ] + )]) + end + + it "respects the `time_zone` option" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + created_at { + as_date_time(truncation_unit: DAY, time_zone: "America/Los_Angeles") + } + } + + count + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [], + needs_doc_count: true, + groupings: [ + date_histogram_grouping_of("created_at", "day", time_zone: "America/Los_Angeles", graphql_subfield: "as_date_time") + ] + )]) + end + + it "respects the `offset` option" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + created_at { + as_date_time(truncation_unit: DAY, offset: {amount: -12, unit: HOUR}) + } + } + + count + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [], + needs_doc_count: true, + groupings: [ + date_histogram_grouping_of("created_at", "day", time_zone: "UTC", offset: "-12h", graphql_subfield: "as_date_time") + ] + )]) + end + + it "supports `Date` field groupings, allowing them to have no `time_zone` argument" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + created_on { + as_date(truncation_unit: MONTH) + } + } + + count + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [], + needs_doc_count: true, + groupings: [ + date_histogram_grouping_of("created_on", "month", time_zone: "UTC", graphql_subfield: "as_date") + ] + )]) + end + + it "can build an aggregations object with every `DateGroupedBy` subfield" do + # Verify that the fields we request in the query below are in fact all the subfields + expect(sub_fields_of("DateGroupedBy")).to contain_exactly("as_date", "as_day_of_week") + + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + created_on { + as_date(truncation_unit: YEAR, offset: {amount: 30, unit: DAY}) + as_day_of_week(offset: {amount: 1, unit: DAY}) + } + } + + count + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [], + needs_doc_count: true, + groupings: [ + date_histogram_grouping_of("created_on", "year", time_zone: "UTC", offset: "30d", graphql_subfield: "as_date"), + as_day_of_week_grouping_of("created_on", offset_ms: 86_400_000, graphql_subfield: "as_day_of_week") + ] + )]) + end + + it "supports `offset` on `Date` field groupings" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + created_on { + as_date(truncation_unit: MONTH, offset: {amount: -12, unit: DAY}) + } + } + + count + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [], + needs_doc_count: true, + groupings: [ + date_histogram_grouping_of("created_on", "month", time_zone: "UTC", offset: "-12d", graphql_subfield: "as_date") + ] + )]) + end + end + + it "omits grouping fields that have a `@skip(if: true)` directive" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + size @skip(if: true) + color @skip(if: false) + created_at { + as_date_time(truncation_unit: DAY) + } + } + + aggregated_values { + amount_cents { + exact_max + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [ + computation_of("amount_cents", :max, computed_field_name: "exact_max") + ], + groupings: [ + field_term_grouping_of("color"), + date_histogram_grouping_of("created_at", "day", graphql_subfield: "as_date_time") + ] + )]) + end + + it "omits grouping fields that have an `@include(if: false)` directive" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + size @include(if: true) + color @include(if: false) + created_at { + as_date_time(truncation_unit: DAY) + } + } + + aggregated_values { + amount_cents { + exact_max + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [ + computation_of("amount_cents", :max, computed_field_name: "exact_max") + ], + groupings: [ + field_term_grouping_of("size"), + date_histogram_grouping_of("created_at", "day", graphql_subfield: "as_date_time") + ] + )]) + end + + it "omits computation fields that have a `@skip(if: true)` directive" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + size + } + + aggregated_values { + amount_cents { + approximate_avg @skip(if: true) + exact_max @skip(if: false) + exact_min + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [ + computation_of("amount_cents", :max, computed_field_name: "exact_max"), + computation_of("amount_cents", :min, computed_field_name: "exact_min") + ], + groupings: [ + field_term_grouping_of("size") + ] + )]) + end + + it "omits computation fields that have a `@include(if: true)` directive" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + size + } + + aggregated_values { + amount_cents { + approximate_avg @include(if: true) + exact_max @include(if: false) + exact_min + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [ + computation_of("amount_cents", :avg, computed_field_name: "approximate_avg"), + computation_of("amount_cents", :min, computed_field_name: "exact_min") + ], + groupings: [ + field_term_grouping_of("size") + ] + )]) + end + + it "sets `needs_doc_count` to false if the count field has a `@skip(if: true)` directive" do + skip_false = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY).first + query { + widget_aggregations { + #{before_nodes} + grouped_by { + size + } + + count @skip(if: false) + #{after_nodes} + } + } + QUERY + + expect(skip_false.needs_doc_count).to eq true + + skip_true = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY).first + query { + widget_aggregations { + #{before_nodes} + grouped_by { + size + } + + count @skip(if: true) + #{after_nodes} + } + } + QUERY + + expect(skip_true.needs_doc_count).to eq false + end + + it "sets `needs_doc_count` to false if the count field has an `@include(if: false)` directive" do + include_false = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY).first + query { + widget_aggregations { + #{before_nodes} + grouped_by { + size + } + + count @include(if: false) + #{after_nodes} + } + } + QUERY + + expect(include_false.needs_doc_count).to eq false + + include_true = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY).first + query { + widget_aggregations { + #{before_nodes} + grouped_by { + size + } + + count @include(if: true) + #{after_nodes} + } + } + QUERY + + expect(include_true.needs_doc_count).to eq true + end + + it "handles multiple aggregation aliases directly on the aggregation field (e.g. to support grouping on different dimensions)" do + aggs_map = aggregations_by_field_name_for(<<~QUERY) + query { + by_day: widget_aggregations(first: 1) { + #{before_nodes} + grouped_by { + created_at { + as_date_time(truncation_unit: DAY) + } + } + + aggregated_values { + amount_cents { + approximate_avg + } + } + #{after_nodes} + } + + by_month: widget_aggregations(last: 3) { + #{before_nodes} + grouped_by { + created_at { + as_date_time(truncation_unit: MONTH) + } + } + + aggregated_values { + amount_cents { + exact_max + } + } + #{after_nodes} + } + + just_count: widget_aggregations { + #{before_nodes} + count + #{after_nodes} + } + + just_sum: widget_aggregations { + #{before_nodes} + aggregated_values { + amount_cents { + exact_sum + } + } + #{after_nodes} + } + } + QUERY + + aggs_map = aggs_map.transform_keys { |qualified_field_name| qualified_field_name.split(".").last } + + expect(aggs_map.keys).to contain_exactly("by_day", "by_month", "just_count", "just_sum") + + expect(aggs_map["by_day"]).to eq aggregation_query_of( + name: "by_day", + groupings: [date_histogram_grouping_of("created_at", "day", graphql_subfield: "as_date_time")], + computations: [computation_of("amount_cents", :avg, computed_field_name: "approximate_avg")], + first: 1 + ) + + expect(aggs_map["by_month"]).to eq aggregation_query_of( + name: "by_month", + groupings: [date_histogram_grouping_of("created_at", "month", graphql_subfield: "as_date_time")], + computations: [computation_of("amount_cents", :max, computed_field_name: "exact_max")], + last: 3 + ) + + expect(aggs_map["just_count"]).to eq aggregation_query_of( + name: "just_count", + needs_doc_count: true + ) + + expect(aggs_map["just_sum"]).to eq aggregation_query_of( + name: "just_sum", + computations: [computation_of("amount_cents", :sum, computed_field_name: "exact_sum")] + ) + end + + it "does not support multiple unaliased aggregations because if different `grouped_by` fields are selected under `node` we can have conflicting grouping requirements" do + error = single_graphql_error_for(<<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { size } + count + #{after_nodes} + } + + widget_aggregations { + #{before_nodes} + grouped_by { color } + count + #{after_nodes} + } + } + QUERY + + expect(error["message"]).to include("more than one `widget_aggregations` selection with the same name (`widget_aggregations`, `widget_aggregations`)") + end + + it "does not support multiple aggregations with the same alias because if different `grouped_by` fields are selected under `node` we can have conflicting grouping requirements" do + error = single_graphql_error_for(<<~QUERY) + query { + w: widget_aggregations { + #{before_nodes} + grouped_by { size } + count + #{after_nodes} + } + + w: widget_aggregations { + #{before_nodes} + grouped_by { color } + count + #{after_nodes} + } + } + QUERY + + expect(error["message"]).to include("more than one `widget_aggregations` selection with the same name (`w`, `w`)") + end + + it "does not support multiple aliases on `grouped_by` because we need a single list of fields to group by" do + error = single_graphql_error_for(<<~QUERY) + query { + aggs: widget_aggregations { + #{before_nodes} + by_size: grouped_by { size } + by_color: grouped_by { color } + count + #{after_nodes} + } + } + QUERY + + expect(error["message"]).to include("more than one `grouped_by` selection under `aggs` (`by_size`, `by_color`)") + end + + it "does not support multiple unaliased `grouped_by`s because we need a single list of fields to group by" do + error = single_graphql_error_for(<<~QUERY) + query { + aggs: widget_aggregations { + #{before_nodes} + grouped_by { size } + grouped_by { color } + count + #{after_nodes} + } + } + QUERY + + expect(error["message"]).to include("more than one `grouped_by` selection under `aggs` (`grouped_by`, `grouped_by`)") + end + + it "supports multiple aliases on `count` since that doesn't interfere with grouping" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + count1: count + count2: count + + aggregated_values { + amount_cents { + approximate_avg + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [computation_of("amount_cents", :avg, computed_field_name: "approximate_avg")], + needs_doc_count: true + )]) + end + + it "supports multiple aliases on `aggregated_values` since that doesn't interfere with grouping" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + values1: aggregated_values { + amount_cents { + approximate_avg + } + } + + values2: aggregated_values { + amount_cents { + approximate_avg + } + } + + values3: aggregated_values { + amount_cents { + exact_sum + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [ + computation_of("amount_cents", :avg, computed_field_name: "approximate_avg"), + computation_of("amount_cents", :sum, computed_field_name: "exact_sum") + ] + )]) + end + + it "supports multiple aliases on subfields of `aggregated_values` since that doesn't interfere with grouping" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + aggregated_values { + ac1: amount_cents { + approximate_avg + } + + ac2: amount_cents { + aa1: approximate_avg + aa2: approximate_avg + exact_sum + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [ + computation_of("amount_cents", :avg, computed_field_name: "approximate_avg", field_names_in_graphql_query: ["ac1"]), + computation_of("amount_cents", :avg, computed_field_name: "approximate_avg", field_names_in_graphql_query: ["ac2"]), + computation_of("amount_cents", :sum, computed_field_name: "exact_sum", field_names_in_graphql_query: ["ac2"]) + ] + )]) + end + + it "supports multiple aliases on subfields of `grouped_by` since that doesn't interfere with grouping" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + count + + grouped_by { + size1: size + size2: size + color + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + needs_doc_count: true, + groupings: [ + field_term_grouping_of("size", field_names_in_graphql_query: ["size1"]), + field_term_grouping_of("size", field_names_in_graphql_query: ["size2"]), + field_term_grouping_of("color") + ] + )]) + end + + it "can build an aggregations object with multiple computations and groupings (including on a nested scalar field)" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + size + created_at { + as_date_time(truncation_unit: YEAR) + } + cost { + currency + } + } + + aggregated_values { + amount_cents { + exact_max + } + + cost { + amount_cents { + exact_max + approximate_avg + } + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [ + computation_of("amount_cents", :max, computed_field_name: "exact_max"), + computation_of("cost", "amount_cents", :max, computed_field_name: "exact_max"), + computation_of("cost", "amount_cents", :avg, computed_field_name: "approximate_avg") + ], + groupings: [ + field_term_grouping_of("size"), + field_term_grouping_of("cost", "currency"), + date_histogram_grouping_of("created_at", "year", graphql_subfield: "as_date_time") + ] + )]) + end + + it "only builds aggregations for relay connection fields with an aggregations subfield, even when the same field name is used elsewhere" do + fields = fields_with_aggregations_for(<<~QUERY) + query { + widget_aggregations { + #{before_nodes} + grouped_by { + size + color + created_at { + as_date_time(truncation_unit: DAY) + } + } + + aggregated_values { + amount_cents { + approximate_avg + exact_max + } + } + #{after_nodes} + } + + components { + #{before_nodes} + widgets { + edges { + node { + id + } + } + } + #{after_nodes} + } + } + QUERY + + expect(fields).to eq ["Query.widget_aggregations"] + end + + it "does not build any groupings or computations for the aggregation `count` field" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + count + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of(name: "widget_aggregations", computations: [], groupings: [], needs_doc_count: true)]) + end + + it "supports aliases for aggregated_values, grouped_by, and count" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + #{before_nodes} + the_grouped_by: grouped_by { + size + } + + the_count: count + + the_aggregated_values: aggregated_values { + amount_cents { + approximate_avg + } + } + #{after_nodes} + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + computations: [ + computation_of("amount_cents", :avg, computed_field_name: "approximate_avg") + ], + needs_doc_count: true, + groupings: [ + field_term_grouping_of("size") + ] + )]) + end + + describe "needs_doc_count" do + it "is true when the count field is requested" do + aggs = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY).first + query { + widget_aggregations { + #{before_nodes} + count + + aggregated_values { + amount_cents { + approximate_avg + exact_max + } + } + #{after_nodes} + } + } + QUERY + + expect(aggs.needs_doc_count).to be true + end + + it "is false when the count field is not requested" do + aggs = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY).first + query { + widget_aggregations { + #{before_nodes} + aggregated_values { + amount_cents { + approximate_avg + exact_max + } + } + #{after_nodes} + } + } + QUERY + + expect(aggs.needs_doc_count).to be false + end + end + end + + it "works correctly when nothing under `node` is requested (e.g. just `page_info`)" do + aggregations = aggregations_from_datastore_query(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + page_info { + has_next_page + end_cursor + } + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "widget_aggregations", + needs_doc_count: false + )]) + end + + context "when the query uses `edges { node { ... } }`" do + include_examples "a query selecting nodes under aggregations", + before_nodes: "edges { node {", + after_nodes: "} }" + + it "does not support multiple unaliased `edges` because if different `grouped_by` fields are selected under `node` we can have conflicting grouping requirements" do + error = single_graphql_error_for(<<~QUERY) + query { + widget_aggregations { + edges { + node { + grouped_by { size } + count + } + } + + edges { + node { + grouped_by { color } + count + } + } + } + } + QUERY + + expect(error["message"]).to include("more than one `edges` selection under `widget_aggregations` (`edges`, `edges`),") + end + + it "does not support multiple aliases on `edges` because if different `grouped_by` fields are selected under `node` we can have conflicting grouping requirements" do + error = single_graphql_error_for(<<~QUERY) + query { + widget_aggregations { + by_size: edges { + node { + grouped_by { size } + count + } + } + + by_color: edges { + node { + grouped_by { color } + count + } + } + } + } + QUERY + + expect(error["message"]).to include("more than one `edges` selection under `widget_aggregations` (`by_size`, `by_color`),") + end + + it "does not support multiple aliases on `node` because if different `grouped_by` fields are selected we can have conflicting grouping requirements" do + error = single_graphql_error_for(<<~QUERY) + query { + widget_aggregations { + edges { + by_size: node { + grouped_by { size } + count + } + + node { + grouped_by { color } + count + } + } + } + } + QUERY + + expect(error["message"]).to include("more than one `node` selection under `widget_aggregations` (`by_size`, `node`),") + end + + it "does not support multiple unaliased `node`s because if different `grouped_by` fields are selected we can have conflicting grouping requirements" do + error = single_graphql_error_for(<<~QUERY) + query { + widget_aggregations { + edges { + node { + grouped_by { size } + count + } + + node { + grouped_by { color } + count + } + } + } + } + QUERY + + expect(error["message"]).to include("more than one `node` selection under `widget_aggregations` (`node`, `node`),") + end + + it "supports a single alias on every layer of nesting of a full query, since that doesn't interfere with grouping" do + aggregations = aggregations_from_datastore_query(:Query, :wa, <<~QUERY) + query { + wa: widget_aggregations { + e: edges { + n: node { + g: grouped_by { + s: size + c: color + } + + c: count + + av: aggregated_values { + ac: amount_cents { + aa: approximate_avg + em: exact_max + } + } + } + } + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "wa", + computations: [ + computation_of("amount_cents", :avg, computed_field_name: "approximate_avg", field_names_in_graphql_query: ["ac"]), + computation_of("amount_cents", :max, computed_field_name: "exact_max", field_names_in_graphql_query: ["ac"]) + ], + groupings: [ + field_term_grouping_of("size", field_names_in_graphql_query: ["s"]), + field_term_grouping_of("color", field_names_in_graphql_query: ["c"]) + ], + needs_doc_count: true + )]) + end + end + + context "when the query uses `nodes { }` and `edges { node { } }`" do + it "returns an error to guard against conflicting grouping requirements" do + error = single_graphql_error_for(<<~QUERY) + query { + widget_aggregations { + nodes { + grouped_by { size } + count + } + + edges { + node { + grouped_by { color } + count + } + } + } + } + QUERY + + expect(error["message"]).to include("more than one node selection (`nodes`, `edges.node`)") + end + end + + context "when the query uses `nodes { }`" do + include_examples "a query selecting nodes under aggregations", + before_nodes: "nodes {", + after_nodes: "}" + + it "does not support multiple unaliased `nodes` because if different `grouped_by` fields are selected under `node` we can have conflicting grouping requirements" do + error = single_graphql_error_for(<<~QUERY) + query { + widget_aggregations { + nodes { + grouped_by { size } + count + } + + nodes { + grouped_by { color } + count + } + } + } + QUERY + + expect(error["message"]).to include("more than one `nodes` selection under `widget_aggregations` (`nodes`, `nodes`),") + end + + it "does not support multiple aliases on `nodes` because if different `grouped_by` fields are selected we can have conflicting grouping requirements" do + error = single_graphql_error_for(<<~QUERY) + query { + widget_aggregations { + by_size: nodes { + grouped_by { size } + count + } + + by_color: nodes { + grouped_by { color } + count + } + } + } + QUERY + + expect(error["message"]).to include("more than one `nodes` selection under `widget_aggregations` (`by_size`, `by_color`),") + end + + it "supports a single alias on every layer of nesting of a full query, since that doesn't interfere with grouping" do + aggregations = aggregations_from_datastore_query(:Query, :wa, <<~QUERY) + query { + wa: widget_aggregations { + n: nodes { + g: grouped_by { + s: size + c: color + } + + c: count + + av: aggregated_values { + ac: amount_cents { + aa: approximate_avg + em: exact_max + } + } + } + } + } + QUERY + + expect(aggregations).to eq([aggregation_query_of( + name: "wa", + computations: [ + computation_of("amount_cents", :avg, computed_field_name: "approximate_avg", field_names_in_graphql_query: ["ac"]), + computation_of("amount_cents", :max, computed_field_name: "exact_max", field_names_in_graphql_query: ["ac"]) + ], + groupings: [ + field_term_grouping_of("size", field_names_in_graphql_query: ["s"]), + field_term_grouping_of("color", field_names_in_graphql_query: ["c"]) + ], + needs_doc_count: true + )]) + end + end + + def aggregations_from_datastore_query(type, field, graphql_query, **graphql_opts) + datastore_query_for( + schema_artifacts: schema_artifacts, + graphql_query: graphql_query, + type: type, + field: field, + **graphql_opts + ).aggregations.values + end + + def aggregations_by_field_name_for(query_string) + queries = datastore_queries_by_field_for(query_string, schema_artifacts: schema_artifacts) + + queries.filter_map do |field, field_queries| + expect(field_queries.size).to eq(1) + field_query = field_queries.first + + if field_query.aggregations.any? + expect(field_query.aggregations.size).to eq(1) + [field, field_query.aggregations.values.first] + end + end.to_h + end + + def fields_with_aggregations_for(query_string) + aggregations_by_field_name_for(query_string).keys + end + + def graphql_errors_for(query_string, **graphql_opts) + super(schema_artifacts: schema_artifacts, graphql_query: query_string, **graphql_opts) + end + + def single_graphql_error_for(query_string) + errors = graphql_errors_for(query_string) + expect(errors.size).to eq 1 + errors.first + end + + def define_schema(schema, amount_cents_opts: {}, cost_opts: {}, legacy_grouping_schema: false) + schema.object_type "Money" do |t| + t.field "currency", "String!" + t.field "amount_cents", "Int!" + t.field "amount", "Int", name_in_index: "amount_in_index" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "name", "String" + t.field "created_at", "DateTime", legacy_grouping_schema: legacy_grouping_schema + t.field "created_on", "Date", legacy_grouping_schema: legacy_grouping_schema + t.field "size", "String" + t.field "color", "String" + t.field "amount_cents", "Int", **amount_cents_opts + t.field "cost", "Money", **cost_opts + t.field "costs", "[Money!]!" do |f| + f.mapping type: "nested" + end + t.field "component_ids", "[ID!]!" + + t.index "widgets" + end + + schema.object_type "Component" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.relates_to_many "widgets", "Widget", via: "component_ids", dir: :in, singular: "widget" + t.index "components" + end + end + + def sub_fields_of(type_name) + ::GraphQL::Schema + .from_definition(schema_artifacts.graphql_schema_string) + .types + .fetch(type_name) + .fields + .keys + end + + def as_day_of_week_grouping_of(*field_names_in_index, **args) + super(*field_names_in_index, runtime_metadata: schema_artifacts.runtime_metadata, **args) + end + + def as_time_of_day_grouping_of(*field_names_in_index, interval, **args) + super(*field_names_in_index, interval, runtime_metadata: schema_artifacts.runtime_metadata, **args) + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/query_optimizer_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/query_optimizer_spec.rb new file mode 100644 index 00000000..3a485353 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/query_optimizer_spec.rb @@ -0,0 +1,243 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/query_optimizer" +require "support/aggregations_helpers" + +module ElasticGraph + class GraphQL + module Aggregation + RSpec.describe QueryOptimizer, ".optimize_queries", :capture_logs do + include AggregationsHelpers + + let(:graphql) { build_graphql } + let(:widgets_def) { graphql.datastore_core.index_definitions_by_name.fetch("widgets") } + let(:components_def) { graphql.datastore_core.index_definitions_by_name.fetch("components") } + let(:executed_queries) { [] } + + it "merges aggregations queries together when they only differ in their `aggs`, allowing us to send fewer requests to the datastore" do + by_size_agg = aggregation_query_of( + name: "by_size", + computations: [computation_of("amountMoney", "amount", :sum)], + groupings: [field_term_grouping_of("options", "size")] + ) + by_color_agg = aggregation_query_of( + name: "by_color", + computations: [computation_of("amountMoney", "amount", :sum)], + groupings: [field_term_grouping_of("options", "color")] + ) + just_sum_agg = aggregation_query_of( + name: "just_sum", + computations: [computation_of("amountMoney", "amount", :sum)] + ) + + base_query = new_query(filter: {"age" => {"equal_to_any_of" => [0]}}) + + by_size = with_aggs(base_query, [by_size_agg]) + by_color = with_aggs(base_query, [by_color_agg]) + just_sum = with_aggs(base_query, [just_sum_agg]) + + results_by_query = optimize_queries(by_size, by_color, just_sum) + + expect(executed_queries).to contain_exactly(with_aggs(base_query, [ + with_prefix(by_size_agg, 1), + with_prefix(by_color_agg, 2), + with_prefix(just_sum_agg, 3) + ])) + expect(results_by_query.keys).to contain_exactly(by_size, by_color, just_sum) + expect(results_by_query[by_size]).to eq(raw_response_with_aggregations(build_agg_response_for(by_size_agg))) + expect(results_by_query[by_color]).to eq(raw_response_with_aggregations(build_agg_response_for(by_color_agg))) + expect(results_by_query[just_sum]).to eq(raw_response_with_aggregations(build_agg_response_for(just_sum_agg))) + expect(merged_query_logs.size).to eq(1) + expect(merged_query_logs.first).to include( + "query_count" => 3, + "aggregation_count" => 3, + "aggregation_names" => ["1_by_size", "2_by_color", "3_just_sum"] + ) + end + + it "can handle duplicated aggregation names" do + by_size_agg = aggregation_query_of( + name: "my_agg", + computations: [computation_of("amountMoney", "amount", :sum)], + groupings: [field_term_grouping_of("options", "size")] + ) + by_color_agg = aggregation_query_of( + name: "my_agg", + computations: [computation_of("amountMoney", "amount", :sum)], + groupings: [field_term_grouping_of("options", "color")] + ) + just_sum_agg = aggregation_query_of( + name: "my_agg", + computations: [computation_of("amountMoney", "amount", :sum)] + ) + + base_query = new_query(filter: {"age" => {"equal_to_any_of" => [0]}}) + + by_size = with_aggs(base_query, [by_size_agg]) + by_color = with_aggs(base_query, [by_color_agg]) + just_sum = with_aggs(base_query, [just_sum_agg]) + + results_by_query = optimize_queries(by_size, by_color, just_sum) + + expect(executed_queries).to contain_exactly(with_aggs(base_query, [ + with_prefix(by_size_agg, 1), + with_prefix(by_color_agg, 2), + with_prefix(just_sum_agg, 3) + ])) + expect(results_by_query.keys).to contain_exactly(by_size, by_color, just_sum) + expect(results_by_query[by_size]).to eq(raw_response_with_aggregations(build_agg_response_for(by_size_agg))) + expect(results_by_query[by_color]).to eq(raw_response_with_aggregations(build_agg_response_for(by_color_agg))) + expect(results_by_query[just_sum]).to eq(raw_response_with_aggregations(build_agg_response_for(just_sum_agg))) + expect(merged_query_logs.size).to eq(1) + expect(merged_query_logs.first).to include( + "query_count" => 3, + "aggregation_count" => 3, + "aggregation_names" => ["1_my_agg", "2_my_agg", "3_my_agg"] + ) + end + + it "can merge non-aggregation queries that are identical as well" do + q1 = new_query(filter: {"age" => {"equal_to_any_of" => [0]}}) + q2 = new_query(filter: {"age" => {"equal_to_any_of" => [0]}}) + expect(q1).to eq(q2) + + results_by_query = optimize_queries(q1, q2) + + expect(executed_queries).to contain_exactly(q1) + expect(results_by_query[q1]).to eq build_response_for(q1) + expect(results_by_query[q2]).to eq build_response_for(q2) + expect(merged_query_logs.size).to eq(1) + expect(merged_query_logs.first).to include( + "query_count" => 2, + "aggregation_count" => 0, + "aggregation_names" => [] + ) + end + + it "keeps queries separate when they have non-aggregation differences" do + optimize_queries( + base_query = new_query(filter: {"age" => {"equal_to_any_of" => [0]}}, individual_docs_needed: true), + alt_filter = base_query.with(filter: {"age" => {"equal_to_any_of" => [1]}}), + alt_pagination = base_query.with(document_pagination: {first: 1}), + alt_individual_docs_needed = base_query.with(individual_docs_needed: !base_query.individual_docs_needed), + alt_sort = base_query.with(sort: [{"age" => {"order" => "desc"}}]), + alt_requested_fields = base_query.with(requested_fields: ["name"]), + alt_index = base_query.with(search_index_definitions: [components_def]) + ) + + expect(executed_queries).to contain_exactly( + base_query, + alt_filter, + alt_pagination, + alt_individual_docs_needed, + alt_sort, + alt_requested_fields, + alt_index + ) + + expect(merged_query_logs).to be_empty + end + + it "does not mess with the aggregation name when no merging happens" do + by_size_agg = aggregation_query_of( + name: "by_size", + computations: [computation_of("amountMoney", "amount", :sum)], + groupings: [field_term_grouping_of("options", "size")] + ) + by_color_agg = aggregation_query_of( + name: "by_color", + computations: [computation_of("amountMoney", "amount", :sum)], + groupings: [field_term_grouping_of("options", "color")] + ) + + base_query = new_query(filter: {"age" => {"equal_to_any_of" => [0]}}) + + by_size = base_query.with( + filter: {"age" => {"equal_to_any_of" => [0]}}, + aggregations: {by_size_agg.name => by_size_agg} + ) + by_color = base_query.with( + filter: {"age" => {"equal_to_any_of" => [1]}}, + aggregations: {by_color_agg.name => by_color_agg} + ) + + results_by_query = optimize_queries(by_size, by_color) + + expect(executed_queries).to contain_exactly(by_size, by_color) + expect(results_by_query.keys).to contain_exactly(by_size, by_color) + expect(results_by_query[by_size]).to eq(raw_response_with_aggregations(build_agg_response_for(by_size_agg))) + expect(results_by_query[by_color]).to eq(raw_response_with_aggregations(build_agg_response_for(by_color_agg))) + expect(merged_query_logs).to be_empty + end + + it "returns an empty hash and never yields if given an empty list of queries" do + expect { |probe| + result = QueryOptimizer.optimize_queries([], &probe) + expect(result).to eq({}) + }.not_to yield_control + end + + def optimize_queries(*queries) + QueryOptimizer.optimize_queries(queries) do |header_body_tuples_by_query| + header_body_tuples_by_query.to_h do |query, _| + executed_queries << query + [query, build_response_for(query)] + end + end + end + + def new_query(**options) + graphql.datastore_query_builder.new_query(search_index_definitions: [widgets_def], **options) + end + + def build_response_for(query) + return DatastoreResponse::SearchResponse::RAW_EMPTY if query.aggregations.empty? + + aggregations = query.aggregations.values.map do |agg| + build_agg_response_for(agg) + end.reduce(:merge) + + raw_response_with_aggregations(aggregations) + end + + def build_agg_response_for(agg) + metrics = agg.computations.to_h do |comp| + [comp.key(aggregation_name: agg.name), {"value" => 17}] + end + + if agg.groupings.any? + response_key = agg.groupings.to_h do |grouping| + [grouping.key, "some-value"] + end + + {agg.name => {"buckets" => [metrics.merge("key" => response_key)]}} + else + metrics + end + end + + def raw_response_with_aggregations(aggregations) + DatastoreResponse::SearchResponse::RAW_EMPTY.merge("aggregations" => aggregations) + end + + def merged_query_logs + logged_jsons_of_type("AggregationQueryOptimizerMergedQueries") + end + + def with_prefix(aggregation, num) + aggregation.with(name: "#{num}_#{aggregation.name}") + end + + def with_aggs(query, aggs) + query.with(aggregations: aggs.to_h { |a| [a.name, a] }) + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/aggregation_resolver_support.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/aggregation_resolver_support.rb new file mode 100644 index 00000000..05c00eb6 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/aggregation_resolver_support.rb @@ -0,0 +1,46 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/resolvers/node" +require "support/aggregations_helpers" + +module ElasticGraph + class GraphQL + module Aggregation + RSpec.shared_context "aggregation resolver support" do + include AggregationsHelpers + let(:graphql) { build_graphql } + + def resolve_target_nodes( + inner_query, + target_buckets: [], + hit_count: nil, + aggs: {"target" => {"buckets" => target_buckets}}, + path: ["data", "target", "nodes"] + ) + allow(datastore_client).to receive(:msearch).and_return({"responses" => [datastore_response_payload_with_aggs(aggs, hit_count)]}) + + response = graphql.graphql_query_executor.execute("query { #{inner_query} }") + expect(response["errors"]).to eq([]).or eq(nil) + response.dig(*path) + end + + def datastore_response_payload_with_aggs(aggregations, hit_count) + { + "took" => 25, + "timed_out" => false, + "_shards" => {"total" => 30, "successful" => 30, "skipped" => 0, "failed" => 0}, + "hits" => {"total" => {"value" => hit_count, "relation" => "eq"}, "max_score" => nil, "hits" => []}, + "aggregations" => aggregations, + "status" => 200 + }.compact + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/aggregations_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/aggregations_spec.rb new file mode 100644 index 00000000..722fd823 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/aggregations_spec.rb @@ -0,0 +1,442 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "aggregation_resolver_support" + +module ElasticGraph + class GraphQL + module Aggregation + RSpec.describe Resolvers, "for aggregations (without sub-aggregations)" do + include_context "aggregation resolver support" + + it "resolves an ungrouped aggregation query just requesting the count" do + response = resolve_target_nodes(<<~QUERY, aggs: nil, hit_count: 17) + target: widget_aggregations { + nodes { + count + } + } + QUERY + + expect(response).to eq [{"count" => 17}] + end + + it "resolves a simple grouped aggregation query just requesting the count" do + target_buckets = [ + {"key" => {"name" => "foo"}, "doc_count" => 2}, + {"key" => {"name" => "bar"}, "doc_count" => 7} + ] + + response = resolve_target_nodes(<<~QUERY, target_buckets: target_buckets) + target: widget_aggregations { + nodes { + grouped_by { name } + count + } + } + QUERY + + expect(response).to eq [ + { + "grouped_by" => {"name" => "foo"}, + "count" => 2 + }, + { + "grouped_by" => {"name" => "bar"}, + "count" => 7 + } + ] + end + + it "resolves nested `grouped_by` fields" do + target_buckets = [ + {"key" => {"name" => "foo", "options.color" => "GREEN", "options.size" => "LARGE"}, "doc_count" => 2}, + {"key" => {"name" => "bar", "options.color" => "RED", "options.size" => "SMALL"}, "doc_count" => 7} + ] + + response = resolve_target_nodes(<<~QUERY, target_buckets: target_buckets) + target: widget_aggregations { + nodes { + grouped_by { + name + options { + color + size + } + } + count + } + } + QUERY + + expect(response).to eq [ + { + "grouped_by" => {"name" => "foo", "options" => {"color" => "GREEN", "size" => "LARGE"}}, + "count" => 2 + }, + { + "grouped_by" => {"name" => "bar", "options" => {"color" => "RED", "size" => "SMALL"}}, + "count" => 7 + } + ] + end + + it "uses the GraphQL query field name rather than the `name_in_index` for the aggregation bucket keys" do + target_buckets = [ + {"key" => {"workspace_id2" => "foo", "the_opts.the_sighs" => "SMALL"}, "doc_count" => 2}, + {"key" => {"workspace_id2" => "bar", "the_opts.the_sighs" => "LARGE"}, "doc_count" => 7} + ] + + response = resolve_target_nodes(<<~QUERY, target_buckets: target_buckets) + target: widget_aggregations { + nodes { + grouped_by { + workspace_id2: workspace_id + the_opts: the_options { + the_sighs: the_size + } + } + count + } + } + QUERY + + expect(response).to eq [ + { + "grouped_by" => {"workspace_id2" => "foo", "the_opts" => {"the_sighs" => "SMALL"}}, + "count" => 2 + }, + { + "grouped_by" => {"workspace_id2" => "bar", "the_opts" => {"the_sighs" => "LARGE"}}, + "count" => 7 + } + ] + end + + it "resolves `count`, `aggregated_values` and `grouped_by` fields for a completely empty response that we get querying a rollover index pattern and no concrete indexes yet exist that match the pattern" do + response = resolve_target_nodes(<<~QUERY, aggs: nil) + target: widget_aggregations { + nodes { + count + + aggregated_values { + amount_cents { exact_sum } + cost { + amount_cents { exact_max } + } + } + + grouped_by { + name + options { + color + size + } + } + } + } + QUERY + + expect(response).to eq [{ + "grouped_by" => { + "name" => nil, + "options" => { + "color" => nil, + "size" => nil + } + }, + "count" => 0, + "aggregated_values" => { + "amount_cents" => {"exact_sum" => 0}, + "cost" => {"amount_cents" => {"exact_max" => nil}} + } + }] + end + + it "resolves `count`, `aggregated_values` and `grouped_by` fields for a completely empty response when field aliases are used" do + response = resolve_target_nodes(<<~QUERY, aggs: nil) + target: widget_aggregations { + nodes { + count + + aggregated_values { + ac: amount_cents { exact_sum } + c: cost { + amount_cents { exact_max } + } + } + + grouped_by { + nm: name + opt: options { + c: color + s: size + } + } + } + } + QUERY + + expect(response).to eq [{ + "grouped_by" => { + "nm" => nil, + "opt" => { + "c" => nil, + "s" => nil + } + }, + "count" => 0, + "aggregated_values" => { + "ac" => {"exact_sum" => 0}, + "c" => {"amount_cents" => {"exact_max" => nil}} + } + }] + end + + it "resolves an ungrouped aggregation query containing aggregated values" do + aggs = { + aggregated_value_key_of("amount_cents", "exact_sum") => {"value" => 900.0}, + aggregated_value_key_of("cost", "amount_cents", "exact_max") => {"value" => 400.0}, + aggregated_value_key_of("cost", "amount_cents", "exact_sum") => {"value" => 1400.0}, + aggregated_value_key_of("cost", "currency", "approximate_distinct_value_count") => {"value" => 5.0} + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: widget_aggregations { + nodes { + aggregated_values { + amount_cents { exact_sum } + cost { + amount_cents { + exact_max + exact_sum + } + currency { approximate_distinct_value_count } + } + } + } + } + QUERY + + expect(response).to eq [ + { + "aggregated_values" => { + "amount_cents" => {"exact_sum" => 900}, + "cost" => { + "amount_cents" => { + "exact_max" => 400, + "exact_sum" => 1400 + }, + "currency" => { + "approximate_distinct_value_count" => 5 + } + } + } + } + ] + end + + it "uses GraphQL field aliases when resolving `aggregated_values` subfields" do + aggs = { + aggregated_value_key_of("amount_cents", "exact_sum") => {"value" => 900.0}, + aggregated_value_key_of("cost", "amt_cts", "exact_max") => {"value" => 400.0}, + aggregated_value_key_of("cost", "amt_cts", "exact_sum") => {"value" => 1400.0}, + aggregated_value_key_of("cost", "currency", "approximate_distinct_value_count") => {"value" => 5.0} + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: widget_aggregations { + nodes { + aggregated_values { + amount_cents { exact_sum } + cost { + amt_cts: amount_cents { + exact_max + exact_sum + } + currency { approximate_distinct_value_count } + } + } + } + } + QUERY + + expect(response).to eq [ + { + "aggregated_values" => { + "amount_cents" => {"exact_sum" => 900}, + "cost" => { + "amt_cts" => { + "exact_max" => 400, + "exact_sum" => 1400 + }, + "currency" => { + "approximate_distinct_value_count" => 5 + } + } + } + } + ] + end + + it "resolves aggregated Date/DateTime/LocalTime values" do + aggs = { + aggregated_value_key_of("created_at", "exact_min") => {"value" => 1696854612000.0, "value_as_string" => "2023-10-09T12:30:12.000Z"}, + aggregated_value_key_of("created_at", "exact_max") => {"value" => 1704792612000.0, "value_as_string" => "2024-01-09T09:30:12.000Z"}, + aggregated_value_key_of("created_at", "approximate_avg") => {"value" => 1701039012530.0, "value_as_string" => "2023-11-26T22:50:12.530Z"}, + aggregated_value_key_of("created_on", "exact_min") => {"value" => 1696809600000.0, "value_as_string" => "2023-10-09"}, + aggregated_value_key_of("created_on", "exact_max") => {"value" => 1704758400000.0, "value_as_string" => "2024-01-09"}, + aggregated_value_key_of("created_on", "approximate_avg") => {"value" => 1700985600000.0, "value_as_string" => "2023-11-26"}, + aggregated_value_key_of("created_at_time_of_day", "exact_min") => {"value" => 34212000.0, "value_as_string" => "09:30:12"}, + aggregated_value_key_of("created_at_time_of_day", "exact_max") => {"value" => 81012000.0, "value_as_string" => "22:30:12"}, + aggregated_value_key_of("created_at_time_of_day", "approximate_avg") => {"value" => 53412000.0, "value_as_string" => "14:50:12"}, + aggregated_value_key_of("created_on", "approximate_distinct_value_count") => {"value" => 3.0}, + aggregated_value_key_of("created_at", "approximate_distinct_value_count") => {"value" => 3.0}, + aggregated_value_key_of("created_at_time_of_day", "approximate_distinct_value_count") => {"value" => 3.0} + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: widget_aggregations { + nodes { + aggregated_values { + created_at { exact_min, exact_max, approximate_avg, approximate_distinct_value_count } + created_on { exact_min, exact_max, approximate_avg, approximate_distinct_value_count } + created_at_time_of_day { exact_min, exact_max, approximate_avg, approximate_distinct_value_count } + } + } + } + QUERY + + expect(response).to eq [ + { + "aggregated_values" => { + "created_at" => { + "exact_min" => "2023-10-09T12:30:12.000Z", + "exact_max" => "2024-01-09T09:30:12.000Z", + "approximate_avg" => "2023-11-26T22:50:12.530Z", + "approximate_distinct_value_count" => 3 + }, + "created_on" => { + "exact_min" => "2023-10-09", + "exact_max" => "2024-01-09", + "approximate_avg" => "2023-11-26", + "approximate_distinct_value_count" => 3 + }, + "created_at_time_of_day" => { + "exact_min" => "09:30:12", + "exact_max" => "22:30:12", + "approximate_avg" => "14:50:12", + "approximate_distinct_value_count" => 3 + } + } + } + ] + end + + it "resolves legacy aggregated Date/DateTime/LocalTime values" do + aggs = { + aggregated_value_key_of("created_at_legacy", "exact_min") => {"value" => 1696854612000.0, "value_as_string" => "2023-10-09T12:30:12.000Z"}, + aggregated_value_key_of("created_at_legacy", "exact_max") => {"value" => 1704792612000.0, "value_as_string" => "2024-01-09T09:30:12.000Z"}, + aggregated_value_key_of("created_at_legacy", "approximate_avg") => {"value" => 1701039012530.0, "value_as_string" => "2023-11-26T22:50:12.530Z"}, + aggregated_value_key_of("created_on_legacy", "exact_min") => {"value" => 1696809600000.0, "value_as_string" => "2023-10-09"}, + aggregated_value_key_of("created_on_legacy", "exact_max") => {"value" => 1704758400000.0, "value_as_string" => "2024-01-09"}, + aggregated_value_key_of("created_on_legacy", "approximate_avg") => {"value" => 1700985600000.0, "value_as_string" => "2023-11-26"}, + aggregated_value_key_of("created_at_time_of_day", "exact_min") => {"value" => 34212000.0, "value_as_string" => "09:30:12"}, + aggregated_value_key_of("created_at_time_of_day", "exact_max") => {"value" => 81012000.0, "value_as_string" => "22:30:12"}, + aggregated_value_key_of("created_at_time_of_day", "approximate_avg") => {"value" => 53412000.0, "value_as_string" => "14:50:12"}, + aggregated_value_key_of("created_on_legacy", "approximate_distinct_value_count") => {"value" => 3.0}, + aggregated_value_key_of("created_at_legacy", "approximate_distinct_value_count") => {"value" => 3.0}, + aggregated_value_key_of("created_at_time_of_day", "approximate_distinct_value_count") => {"value" => 3.0} + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: widget_aggregations { + nodes { + aggregated_values { + created_at_legacy { exact_min, exact_max, approximate_avg, approximate_distinct_value_count } + created_on_legacy { exact_min, exact_max, approximate_avg, approximate_distinct_value_count } + created_at_time_of_day { exact_min, exact_max, approximate_avg, approximate_distinct_value_count } + } + } + } + QUERY + + expect(response).to eq [ + { + "aggregated_values" => { + "created_at_legacy" => { + "exact_min" => "2023-10-09T12:30:12.000Z", + "exact_max" => "2024-01-09T09:30:12.000Z", + "approximate_avg" => "2023-11-26T22:50:12.530Z", + "approximate_distinct_value_count" => 3 + }, + "created_on_legacy" => { + "exact_min" => "2023-10-09", + "exact_max" => "2024-01-09", + "approximate_avg" => "2023-11-26", + "approximate_distinct_value_count" => 3 + }, + "created_at_time_of_day" => { + "exact_min" => "09:30:12", + "exact_max" => "22:30:12", + "approximate_avg" => "14:50:12", + "approximate_distinct_value_count" => 3 + } + } + } + ] + end + + it "resolves the `cursor` on an ungrouped aggregation" do + response = resolve_target_nodes(<<~QUERY, aggs: nil, path: ["data", "target", "edges"]) + target: widget_aggregations { + edges { + cursor + } + } + QUERY + + expect(response).to eq [{"cursor" => DecodedCursor::SINGLETON.encode}] + end + + it "resolves the `cursor` on a grouped aggregation" do + target_buckets = [ + {"key" => {"name" => "foo"}}, + {"key" => {"name" => "bar"}} + ] + + response = resolve_target_nodes(<<~QUERY, target_buckets: target_buckets, path: ["data", "target", "edges"]) + target: widget_aggregations { + edges { + cursor + node { + grouped_by { name } + } + } + } + QUERY + + expect(response).to eq [ + { + "cursor" => DecodedCursor.new({"name" => "foo"}).encode, + "node" => {"grouped_by" => {"name" => "foo"}} + }, + { + "cursor" => DecodedCursor.new({"name" => "bar"}).encode, + "node" => {"grouped_by" => {"name" => "bar"}} + } + ] + end + + def aggregated_value_key_of(*field_path, function_name, aggregation_name: "target") + super(*field_path, function_name, aggregation_name: "target").encode + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/relay_connection_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/relay_connection_spec.rb new file mode 100644 index 00000000..cd658050 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/relay_connection_spec.rb @@ -0,0 +1,358 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/graphql/aggregation/resolvers/relay_connection_builder" +require "elastic_graph/support/hash_util" + +module ElasticGraph + class GraphQL + module Aggregation + module Resolvers + RSpec.describe "RelayConnection for aggregations" do + let(:indexed_widget_count) { 10000 } + + attr_accessor :schema_artifacts + + before(:context) do + self.schema_artifacts = generate_schema_artifacts do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "workspace_id", "ID" + t.field "name", "String" + t.index "widgets" + end + end + end + + context "when grouping by a field" do + it "contains the requested number of groupings when the caller passed `first: N`" do + results = execute_widgets_aggregation_query(<<~QUERY) + widget_aggregations(first: 2) { + edges { + node { + grouped_by { workspace_id } + count + } + } + } + QUERY + + expect(results.dig("edges").size).to eq 2 + + results = execute_widgets_aggregation_query(<<~QUERY) + widget_aggregations(first: 3) { + edges { + node { + grouped_by { workspace_id } + count + } + } + } + QUERY + + expect(results.dig("edges").size).to eq 3 + end + + it "contains the default page size number of groupings when the caller does not pass `first: N`" do + results = execute_widgets_aggregation_query(<<~QUERY, default_page_size: 5) + widget_aggregations { + edges { + node { + grouped_by { workspace_id } + count + } + } + } + QUERY + + expect(results.dig("edges").size).to eq 5 + end + + it "returns the available number of buckets when it is less than the `first` arg" do + results = execute_widgets_aggregation_query(<<~QUERY, available_buckets: 3) + widget_aggregations(first: 5) { + edges { + node { + grouped_by { workspace_id } + count + } + } + } + QUERY + + expect(results.dig("edges").size).to eq 3 + end + + it "returns the available number of buckets when it is less than the default page size and no `first` arg was passed" do + results = execute_widgets_aggregation_query(<<~QUERY, available_buckets: 3, default_page_size: 5) + widget_aggregations { + edges { + node { + grouped_by { workspace_id } + count + } + } + } + QUERY + + expect(results.dig("edges").size).to eq 3 + end + + it "exposes a unique cursor for each `node`" do + results = execute_widgets_aggregation_query(<<~QUERY) + widget_aggregations(first: 3) { + edges { + cursor + node { + grouped_by { workspace_id } + } + } + } + QUERY + + expect(results.dig("edges")).to match [ + {"cursor" => /\w+/, "node" => {"grouped_by" => {"workspace_id" => "1"}}}, + {"cursor" => /\w+/, "node" => {"grouped_by" => {"workspace_id" => "2"}}}, + {"cursor" => /\w+/, "node" => {"grouped_by" => {"workspace_id" => "3"}}} + ] + + cursors = cursors_from(results) + expect(cursors.uniq).to match_array(cursors) + end + + it "exposes the cursor of the first and last nodes as the `start_cursor` and `end_cursor`, respectively" do + results = execute_widgets_aggregation_query(<<~QUERY) + widget_aggregations(first: 3) { + page_info { + start_cursor + end_cursor + } + + edges { + cursor + node { + grouped_by { workspace_id } + } + } + } + QUERY + + cursors = cursors_from(results) + + expect(results.dig("page_info")).to eq({ + "start_cursor" => cursors.first, + "end_cursor" => cursors.last + }) + end + + it "returns `null` for page_info cursors if there are no aggregation buckets" do + results = execute_widgets_aggregation_query(<<~QUERY) + widget_aggregations(first: 0) { + page_info { + start_cursor + end_cursor + } + + edges { + cursor + node { + grouped_by { workspace_id } + } + } + } + QUERY + + expect(results.dig("edges")).to be_empty + + expect(results.dig("page_info")).to eq({ + "start_cursor" => nil, + "end_cursor" => nil + }) + end + end + + context "when not grouping by anything" do + it "returns a single bucket when no `first:` arg is passed" do + results = execute_widgets_aggregation_query(<<~QUERY) + widget_aggregations { + edges { + node { + count + } + } + } + QUERY + + expect(results.dig("edges").size).to eq 1 + end + + it "returns an empty collection of buckets when `first: 0` is passed" do + results = execute_widgets_aggregation_query(<<~QUERY) + widget_aggregations(first: 0) { + page_info { + start_cursor + end_cursor + } + + edges { + node { + count + } + } + } + QUERY + + expect(results.dig("edges").size).to eq 0 + + expect(results.dig("page_info")).to eq({ + "start_cursor" => nil, + "end_cursor" => nil + }) + end + + it "still exposes a cursor even though there can be at most one node, to satisfy the Relay spec" do + results = execute_widgets_aggregation_query(<<~QUERY) + widget_aggregations { + page_info { + start_cursor + end_cursor + } + + edges { + cursor + node { + count + } + } + } + QUERY + + expect(results).to eq({ + "page_info" => { + "start_cursor" => SINGLETON_CURSOR, + "end_cursor" => SINGLETON_CURSOR + }, + "edges" => [{ + "cursor" => SINGLETON_CURSOR, + "node" => { + "count" => indexed_widget_count + } + }] + }) + end + end + + it "supports multiple aliased aggregation fields with different groupings" do + results = execute_widgets_aggregation_query(<<~QUERY, path: ["data"]) + by_workspace_id: widget_aggregations(first: 2) { + edges { + node { + grouped_by { workspace_id } + count + } + } + } + + by_name: widget_aggregations(first: 3) { + edges { + node { + grouped_by { name } + count + } + } + } + QUERY + + expect(results.dig("by_workspace_id", "edges").size).to eq 2 + expect(results.dig("by_name", "edges").size).to eq 3 + end + + it "supports `nodes` being used instead of `edges`" do + results = execute_widgets_aggregation_query(<<~QUERY) + widget_aggregations(first: 3) { + nodes { + grouped_by { workspace_id } + } + } + QUERY + + expect(results.dig("nodes")).to match [ + {"grouped_by" => {"workspace_id" => "1"}}, + {"grouped_by" => {"workspace_id" => "2"}}, + {"grouped_by" => {"workspace_id" => "3"}} + ] + end + + def execute_widgets_aggregation_query(inner_query, available_buckets: nil, path: ["data", "widget_aggregations"], **config_overrides) + allow(datastore_client).to receive(:msearch) do |request| + build_datastore_response(request, available_buckets: available_buckets) + end + + graphql = build_graphql(schema_artifacts: schema_artifacts, **config_overrides) + + query = "query { #{inner_query} }" + response = graphql.graphql_query_executor.execute(query) + expect(response["errors"]).to eq([]).or eq(nil) + response.dig(*path) + end + + # Builds a dynamic response for our fake datastore client, based on the request itself, + # and the `available_buckets` (which limits how many groupings are "available" in the datastore) + def build_datastore_response(request, available_buckets:) + # Our query logic generates a payload with a mixture of string and symbol keys + # (it doesn't matter to the datastore client since it serializes in JSON the same). + # Here we do not want to be mix and match (or be coupled to the current key form + # being used) so we normalize to string keys here. + normalized_request = Support::HashUtil.stringify_keys(request) + + responses = normalized_request["body"].each_slice(2).map do |(search_header, search_body)| + expect(search_header).to include("index" => "widgets") + + aggregations = search_body.fetch("aggs", {}).select { |k, v| v.key?("composite") }.to_h do |agg_name, agg_subhash| + composite_agg_request = agg_subhash.fetch("composite") + # We'll return the smaller of the requested count and the available count (defaults to unbounded) + count_to_return = [composite_agg_request.fetch("size"), available_buckets].compact.min + bucket_keys = composite_agg_request.fetch("sources").flat_map(&:keys) + buckets = Array.new(count_to_return) { |i| build_bucket(i, bucket_keys) } + + [agg_name, {"after_key" => {}, "buckets" => buckets}] + end + + datastore_response_payload_with_aggs(aggregations) + end + + {"responses" => responses} + end + + def datastore_response_payload_with_aggs(aggregations) + { + "took" => 25, + "timed_out" => false, + "_shards" => {"total" => 30, "successful" => 30, "skipped" => 0, "failed" => 0}, + "hits" => {"total" => {"value" => indexed_widget_count, "relation" => "eq"}, "max_score" => nil, "hits" => []}, + "aggregations" => aggregations, + "status" => 200 + }.compact + end + + def build_bucket(index, bucket_keys) + { + "key" => bucket_keys.each_with_object({}) { |key, hash| hash[key] = (index + 1).to_s }, + "doc_count" => (index + 1) * 2 + } + end + + def cursors_from(results) + results.fetch("edges").map { |e| e.fetch("cursor") } + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/sub_aggregations_with_composite_adapter_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/sub_aggregations_with_composite_adapter_spec.rb new file mode 100644 index 00000000..b8b8f85e --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/sub_aggregations_with_composite_adapter_spec.rb @@ -0,0 +1,204 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "aggregation_resolver_support" +require_relative "ungrouped_sub_aggregation_shared_examples" +require "support/sub_aggregation_support" + +module ElasticGraph + class GraphQL + module Aggregation + RSpec.describe Resolvers, "for sub-aggregations, when the `CompositeGroupingAdapter` adapter is used" do + include_context "aggregation resolver support" + include_context "sub-aggregation support", Aggregation::CompositeGroupingAdapter + it_behaves_like "ungrouped sub-aggregations" + + context "with grouping" do + it "resolves a sub-aggregation grouping on multiple fields" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested"]}), + "doc_count" => 9, + "seasons_nested" => { + "after_key" => {"seasons_nested.year" => 2022, "seasons_nested.note" => nil}, + "buckets" => [ + { + "key" => {"seasons_nested.year" => 2019, "seasons_nested.note" => "old rules"}, + "doc_count" => 3 + }, + { + "key" => {"seasons_nested.year" => 2020, "seasons_nested.note" => "covid"}, + "doc_count" => 4 + }, + { + "key" => {"seasons_nested.year" => 2020, "seasons_nested.note" => "pandemic"}, + "doc_count" => 2 + } + ] + } + } + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested { + nodes { + grouped_by { year, note } + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq [ + { + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + { + "grouped_by" => {"year" => 2019, "note" => "old rules"}, + "count_detail" => {"approximate_value" => 3} + }, + { + "grouped_by" => {"year" => 2020, "note" => "covid"}, + "count_detail" => {"approximate_value" => 4} + }, + { + "grouped_by" => {"year" => 2020, "note" => "pandemic"}, + "count_detail" => {"approximate_value" => 2} + } + ] + } + } + } + ] + end + + it "resolves all `count_detail` fields with the same value (since we always have the exact count!)" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested"]}), + "doc_count" => 7, + "seasons_nested" => { + "after_key" => {"seasons_nested.year" => 2022, "seasons_nested.note" => nil}, + "buckets" => [ + { + "key" => {"seasons_nested.year" => 2019, "seasons_nested.note" => "old rules"}, + "doc_count" => 3 + }, + { + "key" => {"seasons_nested.year" => 2020, "seasons_nested.note" => "covid"}, + "doc_count" => 4 + } + ] + } + } + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested { + nodes { + grouped_by { year, note } + count_detail { approximate_value, exact_value, upper_bound } + } + } + } + } + } + QUERY + + expect(response).to eq [ + { + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + { + "grouped_by" => {"year" => 2019, "note" => "old rules"}, + "count_detail" => {"approximate_value" => 3, "exact_value" => 3, "upper_bound" => 3} + }, + { + "grouped_by" => {"year" => 2020, "note" => "covid"}, + "count_detail" => {"approximate_value" => 4, "exact_value" => 4, "upper_bound" => 4} + } + ] + } + } + } + ] + end + + it "handles filtering" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested:filtered", "seasons_nested"]}), + "doc_count" => 9, + "seasons_nested:filtered" => { + "doc_count" => 3, + "seasons_nested" => { + "after_key" => {"seasons_nested.year" => 2020}, + "buckets" => [ + { + "key" => {"seasons_nested.year" => 2019}, + "doc_count" => 1 + }, + { + "key" => {"seasons_nested.year" => 2020}, + "doc_count" => 2 + } + ] + } + } + } + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested(filter: {year: {lt: 2021}}) { + nodes { + grouped_by { year } + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq [ + { + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + { + "grouped_by" => {"year" => 2019}, + "count_detail" => {"approximate_value" => 1} + }, + { + "grouped_by" => {"year" => 2020}, + "count_detail" => {"approximate_value" => 2} + } + ] + } + } + } + ] + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/sub_aggregations_with_non_composite_adapter_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/sub_aggregations_with_non_composite_adapter_spec.rb new file mode 100644 index 00000000..89fa6052 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/sub_aggregations_with_non_composite_adapter_spec.rb @@ -0,0 +1,1832 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "aggregation_resolver_support" +require_relative "ungrouped_sub_aggregation_shared_examples" +require "support/sub_aggregation_support" + +module ElasticGraph + class GraphQL + module Aggregation + RSpec.describe Resolvers, "for sub-aggregations, when the `NonCompositeGroupingAdapter` adapter is used" do + using SubAggregationRefinements + include_context "aggregation resolver support" + include_context "sub-aggregation support", Aggregation::NonCompositeGroupingAdapter + it_behaves_like "ungrouped sub-aggregations" + + context "with `count_detail` fields and grouping" do + it "indicates the count is exact when grouping only on date fields" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at.as_date_time"]}), + "doc_count" => 6, + "seasons_nested.started_at.as_date_time" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.started_at.as_date_time"], "key_path" => ["key_as_string"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 3 + } + ] + } + }.with_missing_value_bucket(0) + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested { + nodes { + grouped_by { started_at { as_date_time(truncation_unit: YEAR) }} + count_detail { + approximate_value + exact_value + upper_bound + } + } + } + } + } + } + QUERY + + expect(response.dig(0, "sub_aggregations", "seasons_nested", "nodes", 0, "count_detail")).to be_exactly_equal_to(3) + end + + it "indicates the count is exact when grouping only on legacy date fields" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at_legacy"]}), + "doc_count" => 6, + "seasons_nested.started_at_legacy" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.started_at_legacy"], "key_path" => ["key_as_string"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 3 + } + ] + } + }.with_missing_value_bucket(0) + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested { + nodes { + grouped_by { started_at_legacy(granularity: YEAR) } + count_detail { + approximate_value + exact_value + upper_bound + } + } + } + } + } + } + QUERY + + expect(response.dig(0, "sub_aggregations", "seasons_nested", "nodes", 0, "count_detail")).to be_exactly_equal_to(3) + end + + it "computes `exact_value` and `upper_bound` based on `doc_count_error_upper_bound` when available on a non-date grouping" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}), + "doc_count" => 4, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.year"], "key_path" => ["key"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2020, + "doc_count" => 200, + "doc_count_error_upper_bound" => 7 + }, + { + "key" => 2021, + "doc_count" => 100, + "doc_count_error_upper_bound" => 0 + } + ] + } + }.with_missing_value_bucket(5) + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested { + nodes { + grouped_by { year } + count_detail { + approximate_value + exact_value + upper_bound + } + } + } + } + } + } + QUERY + + expect(response.dig(0, "sub_aggregations", "seasons_nested", "nodes", 0, "count_detail")).to be_approximately_equal_to(200, with_upper_bound: 207) + expect(response.dig(0, "sub_aggregations", "seasons_nested", "nodes", 1, "count_detail")).to be_exactly_equal_to(100) + expect(response.dig(0, "sub_aggregations", "seasons_nested", "nodes", 2, "count_detail")).to be_exactly_equal_to(5) + end + + def be_exactly_equal_to(count) + eq({"approximate_value" => count, "exact_value" => count, "upper_bound" => count}) + end + + def be_approximately_equal_to(count, with_upper_bound:) + eq({"approximate_value" => count, "exact_value" => nil, "upper_bound" => with_upper_bound}) + end + end + + context "with grouping" do + it "resolves a sub-aggregation grouping on one non-date field" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}), + "doc_count" => 4, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.year"], "key_path" => ["key"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2020, + "doc_count" => 2 + }, + { + "key" => 2019, + "doc_count" => 1 + }, + { + "key" => 2022, + "doc_count" => 1 + } + ] + } + }.with_missing_value_bucket(0) + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested { + nodes { + grouped_by { year } + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + {"count_detail" => {"approximate_value" => 2}, "grouped_by" => {"year" => 2020}}, + {"count_detail" => {"approximate_value" => 1}, "grouped_by" => {"year" => 2019}}, + {"count_detail" => {"approximate_value" => 1}, "grouped_by" => {"year" => 2022}} + ] + } + } + }]) + end + + it "includes a node for the missing value bucket when its count is greater than zero" do + response_for_0, response_for_3 = [0, 3].map do |missing_bucket_doc_count| + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}), + "doc_count" => 4, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.year"], "key_path" => ["key"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2020, + "doc_count" => 2 + } + ] + } + }.with_missing_value_bucket(missing_bucket_doc_count) + } + + resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested { + nodes { + grouped_by { year } + count_detail { approximate_value } + } + } + } + } + } + QUERY + end + + expect(response_for_0).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + {"count_detail" => {"approximate_value" => 2}, "grouped_by" => {"year" => 2020}} + ] + } + } + }]) + + expect(response_for_3).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + {"count_detail" => {"approximate_value" => 3}, "grouped_by" => {"year" => nil}}, + {"count_detail" => {"approximate_value" => 2}, "grouped_by" => {"year" => 2020}} + ] + } + } + }]) + end + + it "handles a non-empty missing value bucket when a sub-aggregation filter has been applied" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested:filtered", "seasons_nested.year"]}), + "doc_count" => 10, + "seasons_nested:filtered" => { + "doc_count" => 5, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.year"], "key_path" => ["key"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2020, + "doc_count" => 3 + } + ] + } + }.with_missing_value_bucket(2) + } + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested(filter: {year: {not: {equal_to_any_of: [2021]}}}) { + nodes { + grouped_by { year } + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + {"count_detail" => {"approximate_value" => 3}, "grouped_by" => {"year" => 2020}}, + {"count_detail" => {"approximate_value" => 2}, "grouped_by" => {"year" => nil}} + ] + } + } + }]) + end + + it "resolves a sub-aggregation grouping on multiple non-date fields when it used multiple terms aggregations" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}), + "doc_count" => 4, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"buckets_path" => ["seasons_nested.count"], "grouping_fields" => ["seasons_nested.year"], "key_path" => ["key"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2020, + "doc_count" => 2, + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "seasons_nested.count" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.count"], "key_path" => ["key"]}), + "buckets" => [{"key" => 3, "doc_count" => 2}] + } + }.with_missing_value_bucket(0), + { + "key" => 2019, + "doc_count" => 1, + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "seasons_nested.count" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.count"], "key_path" => ["key"]}), + "buckets" => [{"key" => 4, "doc_count" => 1}] + } + }.with_missing_value_bucket(0), + { + "key" => 2022, + "doc_count" => 1, + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "seasons_nested.count" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.count"], "key_path" => ["key"]}), + "buckets" => [{"key" => 1, "doc_count" => 1}] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0) + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested { + nodes { + grouped_by { year, count } + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + {"count_detail" => {"approximate_value" => 2}, "grouped_by" => {"year" => 2020, "count" => 3}}, + {"count_detail" => {"approximate_value" => 1}, "grouped_by" => {"year" => 2019, "count" => 4}}, + {"count_detail" => {"approximate_value" => 1}, "grouped_by" => {"year" => 2022, "count" => 1}} + ] + } + } + }]) + end + + # Note: we used to use multi-terms aggregation but no longer do. Still, it's useful that our resolver is able to + # handle either kind of response structure. So while it no longer must handle this case, it's useful that it does + # so and given the test was already written we decided to keep it. If that flexibility ever becomes a maintenance + # burden, feel free to remove this test. + it "resolves a sub-aggregation grouping on multiple non-date fields when it used a single multi-terms aggregation" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year_and_count"]}), + "doc_count" => 4, + "seasons_nested.year_and_count" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.year", "seasons_nested.count"], "key_path" => ["key"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => [2020, 3], + "doc_count" => 2 + }, + { + "key" => [2019, 4], + "doc_count" => 1 + }, + { + "key" => [2022, 1], + "doc_count" => 1 + } + ] + } + }.with_missing_value_bucket(0) + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested { + nodes { + grouped_by { year, count } + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + {"count_detail" => {"approximate_value" => 2}, "grouped_by" => {"year" => 2020, "count" => 3}}, + {"count_detail" => {"approximate_value" => 1}, "grouped_by" => {"year" => 2019, "count" => 4}}, + {"count_detail" => {"approximate_value" => 1}, "grouped_by" => {"year" => 2022, "count" => 1}} + ] + } + } + }]) + end + + it "sorts and truncates `terms` buckets on the doc count (descending), with the key (ascending) as a tie-breaker" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}, size: 3), + "doc_count" => 10, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.year"], "key_path" => ["key"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2020, + "doc_count" => 2 + }, + { + "key" => 2019, + "doc_count" => 6 + }, + { + "key" => 2022, + "doc_count" => 2 + }, + { + "key" => 2021, + "doc_count" => 2 + } + ] + } + }.with_missing_value_bucket(0) + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested(first: 3) { + nodes { + grouped_by { year } + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + {"count_detail" => {"approximate_value" => 6}, "grouped_by" => {"year" => 2019}}, + {"count_detail" => {"approximate_value" => 2}, "grouped_by" => {"year" => 2020}}, + {"count_detail" => {"approximate_value" => 2}, "grouped_by" => {"year" => 2021}} + ] + } + } + }]) + end + + it "tolerates comparing null/missing key values against string values when sorting the buckets" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.note"]}, size: 2), + "doc_count" => 6, + "seasons_nested.note" => { + "meta" => inner_terms_meta({"buckets_path" => ["seasons_nested.record.losses"], "grouping_fields" => ["seasons_nested.note"], "key_path" => ["key"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => "pandemic", + "doc_count" => 2, + "doc_count_error_upper_bound" => 0, + "seasons_nested.record.losses" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.record.losses"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + {"key" => 12, "doc_count" => 1, "doc_count_error_upper_bound" => 0}, + {"key" => 22, "doc_count" => 1, "doc_count_error_upper_bound" => 0} + ] + }, + "seasons_nested.record.losses:m" => {"doc_count" => 0} + }, + { + "key" => "covid", + "doc_count" => 1, + "doc_count_error_upper_bound" => 0, + "seasons_nested.record.losses" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.record.losses"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + {"key" => 22, "doc_count" => 1, "doc_count_error_upper_bound" => 0} + ] + }, + "seasons_nested.record.losses:m" => {"doc_count" => 0} + } + ] + }, + "seasons_nested.note:m" => { + "meta" => inner_terms_meta({"buckets_path" => ["seasons_nested.record.losses"], "grouping_fields" => ["seasons_nested.note"], "key_path" => ["key"]}), + "doc_count" => 3, + "seasons_nested.record.losses" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.record.losses"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + {"key" => 15, "doc_count" => 1, "doc_count_error_upper_bound" => 0}, + {"key" => 22, "doc_count" => 1, "doc_count_error_upper_bound" => 0} + ] + }, + "seasons_nested.record.losses:m" => {"doc_count" => 1} + } + } + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested(first: 3) { + nodes { + grouped_by { note, record { losses } } + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + {"count_detail" => {"approximate_value" => 1}, "grouped_by" => {"note" => nil, "record" => {"losses" => nil}}}, + {"count_detail" => {"approximate_value" => 1}, "grouped_by" => {"note" => nil, "record" => {"losses" => 15}}} + ] + } + } + }]) + end + + it "sorts and truncates `terms` buckets on the doc count (descending), with the key values (ascending) as a tie-breaker when a multiple terms aggregations were used" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}, size: 3), + "doc_count" => 21, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"buckets_path" => ["seasons_nested.count"], "grouping_fields" => ["seasons_nested.year"], "key_path" => ["key"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2022, + "doc_count" => 4, + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "seasons_nested.count" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.count"], "key_path" => ["key"]}), + "buckets" => [ + {"key" => 1, "doc_count" => 4} + ] + } + }.with_missing_value_bucket(0), + { + "key" => 2020, + "doc_count" => 8, + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "seasons_nested.count" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.count"], "key_path" => ["key"]}), + "buckets" => [ + {"key" => 1, "doc_count" => 4}, + {"key" => 3, "doc_count" => 4} + ] + } + }.with_missing_value_bucket(0), + { + "key" => 2019, + "doc_count" => 9, + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "seasons_nested.count" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.count"], "key_path" => ["key"]}), + "buckets" => [ + {"key" => 4, "doc_count" => 9} + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0) + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested(first: 3) { + nodes { + grouped_by { year, count } + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + {"count_detail" => {"approximate_value" => 9}, "grouped_by" => {"year" => 2019, "count" => 4}}, + {"count_detail" => {"approximate_value" => 4}, "grouped_by" => {"year" => 2020, "count" => 1}}, + {"count_detail" => {"approximate_value" => 4}, "grouped_by" => {"year" => 2020, "count" => 3}} + ] + } + } + }]) + end + + # Note: we used to use multi-terms aggregation but no longer do. Still, it's useful that our resolver is able to + # handle either kind of response structure. So while it no longer must handle this case, it's useful that it does + # so and given the test was already written we decided to keep it. If that flexibility ever becomes a maintenance + # burden, feel free to remove this test. + it "sorts and truncates `terms` buckets on the doc count (descending), with the key values (ascending) as a tie-breaker when a single multi-terms aggregation was used" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year_and_count"]}, size: 3), + "doc_count" => 21, + "seasons_nested.year_and_count" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.year", "seasons_nested.count"], "key_path" => ["key"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => [2022, 1], + "doc_count" => 4 + }, + { + "key" => [2020, 1], + "doc_count" => 4 + }, + { + "key" => [2019, 4], + "doc_count" => 9 + }, + { + "key" => [2020, 3], + "doc_count" => 4 + } + ] + } + }.with_missing_value_bucket(0) + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested(first: 3) { + nodes { + grouped_by { year, count } + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + {"count_detail" => {"approximate_value" => 9}, "grouped_by" => {"year" => 2019, "count" => 4}}, + {"count_detail" => {"approximate_value" => 4}, "grouped_by" => {"year" => 2020, "count" => 1}}, + {"count_detail" => {"approximate_value" => 4}, "grouped_by" => {"year" => 2020, "count" => 3}} + ] + } + } + }]) + end + + it "resolves a sub-aggregation grouping on a single date field" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at.as_date_time"]}), + "doc_count" => 6, + "seasons_nested.started_at.as_date_time" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.started_at.as_date_time"], "key_path" => ["key_as_string"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 3 + }, + { + "key_as_string" => "2020-01-01T00:00:00.000Z", + "key" => 1577836800000, + "doc_count" => 2 + }, + { + "key_as_string" => "2022-01-01T00:00:00.000Z", + "key" => 1640995200000, + "doc_count" => 1 + } + ] + } + }.with_missing_value_bucket(0) + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested { + nodes { + grouped_by { started_at { as_date_time(truncation_unit: YEAR) }} + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + {"count_detail" => {"approximate_value" => 3}, "grouped_by" => {"started_at" => {"as_date_time" => "2019-01-01T00:00:00.000Z"}}}, + {"count_detail" => {"approximate_value" => 2}, "grouped_by" => {"started_at" => {"as_date_time" => "2020-01-01T00:00:00.000Z"}}}, + {"count_detail" => {"approximate_value" => 1}, "grouped_by" => {"started_at" => {"as_date_time" => "2022-01-01T00:00:00.000Z"}}} + ] + } + } + }]) + end + + it "resolves a sub-aggregation grouping on a single legacy date field" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at_legacy"]}), + "doc_count" => 6, + "seasons_nested.started_at_legacy" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.started_at_legacy"], "key_path" => ["key_as_string"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 3 + }, + { + "key_as_string" => "2020-01-01T00:00:00.000Z", + "key" => 1577836800000, + "doc_count" => 2 + }, + { + "key_as_string" => "2022-01-01T00:00:00.000Z", + "key" => 1640995200000, + "doc_count" => 1 + } + ] + } + }.with_missing_value_bucket(0) + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested { + nodes { + grouped_by { started_at_legacy(granularity: YEAR) } + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + {"count_detail" => {"approximate_value" => 3}, "grouped_by" => {"started_at_legacy" => "2019-01-01T00:00:00.000Z"}}, + {"count_detail" => {"approximate_value" => 2}, "grouped_by" => {"started_at_legacy" => "2020-01-01T00:00:00.000Z"}}, + {"count_detail" => {"approximate_value" => 1}, "grouped_by" => {"started_at_legacy" => "2022-01-01T00:00:00.000Z"}} + ] + } + } + }]) + end + + it "resolves a sub-aggregation grouping on a non-date field and multiple date fields (requiring a 3-level datastore response structure)" do + # This is the response structure we get when we put a `terms` aggregation outside a `date_histogram` aggregation. + terms_outside_date_aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}), + "doc_count" => 18, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"buckets_path" => ["seasons_nested.started_at.as_date_time"], "key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2020, + "doc_count" => 11, + "seasons_nested.started_at.as_date_time" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.won_games_at"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.started_at.as_date_time"]}), + "buckets" => [ + { + "key_as_string" => "2020-01-01T00:00:00.000Z", + "key" => 1577836800000, + "doc_count" => 11, + "seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at.as_date_time"]}), + "buckets" => [ + { + "key_as_string" => "2020-01-01T00:00:00.000Z", + "key" => 1577836800000, + "doc_count" => 6 + }, + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 5 + } + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0), + { + "key" => 2019, + "doc_count" => 7, + "seasons_nested.started_at.as_date_time" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.won_games_at"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.started_at.as_date_time"]}), + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 4, + "seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at.as_date_time"]}), + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 4 + } + ] + } + }.with_missing_value_bucket(0), + { + "key_as_string" => "2021-01-01T00:00:00.000Z", + "key" => 1609459200000, + "doc_count" => 3, + "seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at.as_date_time"]}), + "buckets" => [ + { + "key_as_string" => "2021-01-01T00:00:00.000Z", + "key" => 1609459200000, + "doc_count" => 3 + } + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0) + } + + # This is the response structure we get when we put a `terms` aggregation inside a `date_histogram` aggregation. + terms_inside_date_aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at.as_date_time"]}), + "doc_count" => 18, + "seasons_nested.started_at.as_date_time" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.won_games_at"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.started_at.as_date_time"]}), + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 4, + "seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.year"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at.as_date_time"]}), + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 4, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2019, + "doc_count" => 4 + } + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0), + { + "key_as_string" => "2020-01-01T00:00:00.000Z", + "key" => 1577836800000, + "doc_count" => 2, + "seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.year"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at.as_date_time"]}), + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 5, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2020, + "doc_count" => 5 + } + ] + } + }.with_missing_value_bucket(0), + { + "key_as_string" => "2020-01-01T00:00:00.000Z", + "key" => 1577836800000, + "doc_count" => 6, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2020, + "doc_count" => 6 + } + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0), + { + "key_as_string" => "2021-01-01T00:00:00.000Z", + "key" => 1609459200000, + "doc_count" => 1, + "seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.year"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at.as_date_time"]}), + "buckets" => [ + { + "key_as_string" => "2021-01-01T00:00:00.000Z", + "key" => 1609459200000, + "doc_count" => 3, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2019, + "doc_count" => 3 + } + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0) + } + + terms_inside_date_response, terms_outside_date_response = [terms_inside_date_aggs, terms_outside_date_aggs].map do |aggs| + resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested { + nodes { + grouped_by { + started_at { as_date_time(truncation_unit: YEAR) } + year + won_game_at { as_date_time(truncation_unit: YEAR) } + } + count_detail { approximate_value } + } + } + } + } + } + QUERY + end + + # Our resolver should be able to handle either nesting ordering. On 2023-12-05, we swapped the ordering + # (from `terms { date { ... } }` to `date { terms { ... } }`). We want our resolver to handle either + # order, so here we test both, expecting the same response in either case. + expect(terms_inside_date_response).to eq(terms_outside_date_response) + expect(terms_inside_date_response).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + { + "count_detail" => {"approximate_value" => 6}, + "grouped_by" => { + "year" => 2020, + "started_at" => {"as_date_time" => "2020-01-01T00:00:00.000Z"}, + "won_game_at" => {"as_date_time" => "2020-01-01T00:00:00.000Z"} + } + }, + { + "count_detail" => {"approximate_value" => 5}, + "grouped_by" => { + "year" => 2020, + "started_at" => {"as_date_time" => "2020-01-01T00:00:00.000Z"}, + "won_game_at" => {"as_date_time" => "2019-01-01T00:00:00.000Z"} + } + }, + { + "count_detail" => {"approximate_value" => 4}, + "grouped_by" => { + "year" => 2019, + "started_at" => {"as_date_time" => "2019-01-01T00:00:00.000Z"}, + "won_game_at" => {"as_date_time" => "2019-01-01T00:00:00.000Z"} + } + }, + { + "count_detail" => {"approximate_value" => 3}, + "grouped_by" => { + "year" => 2019, + "started_at" => {"as_date_time" => "2021-01-01T00:00:00.000Z"}, + "won_game_at" => {"as_date_time" => "2021-01-01T00:00:00.000Z"} + } + } + ] + } + } + }]) + end + + it "resolves a sub-aggregation grouping on a non-date field and multiple legacy date fields (requiring a 3-level datastore response structure)" do + # This is the response structure we get when we put a `terms` aggregation outside a `date_histogram` aggregation. + terms_outside_date_aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}), + "doc_count" => 18, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"buckets_path" => ["seasons_nested.started_at_legacy"], "key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2020, + "doc_count" => 11, + "seasons_nested.started_at_legacy" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.won_games_at_legacy"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.started_at_legacy"]}), + "buckets" => [ + { + "key_as_string" => "2020-01-01T00:00:00.000Z", + "key" => 1577836800000, + "doc_count" => 11, + "seasons_nested.won_games_at_legacy" => { + "meta" => inner_date_meta({"key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at_legacy"]}), + "buckets" => [ + { + "key_as_string" => "2020-01-01T00:00:00.000Z", + "key" => 1577836800000, + "doc_count" => 6 + }, + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 5 + } + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0), + { + "key" => 2019, + "doc_count" => 7, + "seasons_nested.started_at_legacy" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.won_games_at_legacy"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.started_at_legacy"]}), + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 4, + "seasons_nested.won_games_at_legacy" => { + "meta" => inner_date_meta({"key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at_legacy"]}), + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 4 + } + ] + } + }.with_missing_value_bucket(0), + { + "key_as_string" => "2021-01-01T00:00:00.000Z", + "key" => 1609459200000, + "doc_count" => 3, + "seasons_nested.won_games_at_legacy" => { + "meta" => inner_date_meta({"key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at_legacy"]}), + "buckets" => [ + { + "key_as_string" => "2021-01-01T00:00:00.000Z", + "key" => 1609459200000, + "doc_count" => 3 + } + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0) + } + + # This is the response structure we get when we put a `terms` aggregation inside a `date_histogram` aggregation. + terms_inside_date_aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at_legacy"]}), + "doc_count" => 18, + "seasons_nested.started_at_legacy" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.won_games_at_legacy"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.started_at_legacy"]}), + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 4, + "seasons_nested.won_games_at_legacy" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.year"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at_legacy"]}), + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 4, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2019, + "doc_count" => 4 + } + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0), + { + "key_as_string" => "2020-01-01T00:00:00.000Z", + "key" => 1577836800000, + "doc_count" => 2, + "seasons_nested.won_games_at_legacy" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.year"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at_legacy"]}), + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 5, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2020, + "doc_count" => 5 + } + ] + } + }.with_missing_value_bucket(0), + { + "key_as_string" => "2020-01-01T00:00:00.000Z", + "key" => 1577836800000, + "doc_count" => 6, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2020, + "doc_count" => 6 + } + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0), + { + "key_as_string" => "2021-01-01T00:00:00.000Z", + "key" => 1609459200000, + "doc_count" => 1, + "seasons_nested.won_games_at_legacy" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.year"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at_legacy"]}), + "buckets" => [ + { + "key_as_string" => "2021-01-01T00:00:00.000Z", + "key" => 1609459200000, + "doc_count" => 3, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2019, + "doc_count" => 3 + } + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0) + } + + terms_inside_date_response, terms_outside_date_response = [terms_inside_date_aggs, terms_outside_date_aggs].map do |aggs| + resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested { + nodes { + grouped_by { + started_at_legacy(granularity: YEAR) + year + won_game_at_legacy(granularity: YEAR) + } + count_detail { approximate_value } + } + } + } + } + } + QUERY + end + + # Our resolver should be able to handle either nesting ordering. On 2023-12-05, we swapped the ordering + # (from `terms { date { ... } }` to `date { terms { ... } }`). We want our resolver to handle either + # order, so here we test both, expecting the same response in either case. + expect(terms_inside_date_response).to eq(terms_outside_date_response) + expect(terms_inside_date_response).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + { + "count_detail" => {"approximate_value" => 6}, + "grouped_by" => { + "year" => 2020, + "started_at_legacy" => "2020-01-01T00:00:00.000Z", + "won_game_at_legacy" => "2020-01-01T00:00:00.000Z" + } + }, + { + "count_detail" => {"approximate_value" => 5}, + "grouped_by" => { + "year" => 2020, + "started_at_legacy" => "2020-01-01T00:00:00.000Z", + "won_game_at_legacy" => "2019-01-01T00:00:00.000Z" + } + }, + { + "count_detail" => {"approximate_value" => 4}, + "grouped_by" => { + "year" => 2019, + "started_at_legacy" => "2019-01-01T00:00:00.000Z", + "won_game_at_legacy" => "2019-01-01T00:00:00.000Z" + } + }, + { + "count_detail" => {"approximate_value" => 3}, + "grouped_by" => { + "year" => 2019, + "started_at_legacy" => "2021-01-01T00:00:00.000Z", + "won_game_at_legacy" => "2021-01-01T00:00:00.000Z" + } + } + ] + } + } + }]) + end + + it "sorts and truncates single-layer `date_histogram` buckets on the doc count (descending), with the key values (ascending) as a tie-breaker" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at.as_date_time"]}, size: 3), + "doc_count" => 18, + "seasons_nested.started_at.as_date_time" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.started_at.as_date_time"], "key_path" => ["key_as_string"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 3 + }, + { + "key_as_string" => "2022-01-01T00:00:00.000Z", + "key" => 1640995200000, + "doc_count" => 3 + }, + { + "key_as_string" => "2021-01-01T00:00:00.000Z", + "key" => 1609459200000, + "doc_count" => 9 + }, + { + "key_as_string" => "2020-01-01T00:00:00.000Z", + "key" => 1577836800000, + "doc_count" => 3 + } + ] + } + }.with_missing_value_bucket(0) + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested(first: 3) { + nodes { + grouped_by { started_at { as_date_time(truncation_unit: YEAR) }} + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + {"count_detail" => {"approximate_value" => 9}, "grouped_by" => {"started_at" => {"as_date_time" => "2021-01-01T00:00:00.000Z"}}}, + {"count_detail" => {"approximate_value" => 3}, "grouped_by" => {"started_at" => {"as_date_time" => "2019-01-01T00:00:00.000Z"}}}, + {"count_detail" => {"approximate_value" => 3}, "grouped_by" => {"started_at" => {"as_date_time" => "2020-01-01T00:00:00.000Z"}}} + ] + } + } + }]) + end + + it "sorts and truncates single-layer legacy `date_histogram` buckets on the doc count (descending), with the key values (ascending) as a tie-breaker" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at_legacy"]}, size: 3), + "doc_count" => 18, + "seasons_nested.started_at_legacy" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.started_at_legacy"], "key_path" => ["key_as_string"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 3 + }, + { + "key_as_string" => "2022-01-01T00:00:00.000Z", + "key" => 1640995200000, + "doc_count" => 3 + }, + { + "key_as_string" => "2021-01-01T00:00:00.000Z", + "key" => 1609459200000, + "doc_count" => 9 + }, + { + "key_as_string" => "2020-01-01T00:00:00.000Z", + "key" => 1577836800000, + "doc_count" => 3 + } + ] + } + }.with_missing_value_bucket(0) + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested(first: 3) { + nodes { + grouped_by { started_at_legacy(granularity: YEAR) } + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + {"count_detail" => {"approximate_value" => 9}, "grouped_by" => {"started_at_legacy" => "2021-01-01T00:00:00.000Z"}}, + {"count_detail" => {"approximate_value" => 3}, "grouped_by" => {"started_at_legacy" => "2019-01-01T00:00:00.000Z"}}, + {"count_detail" => {"approximate_value" => 3}, "grouped_by" => {"started_at_legacy" => "2020-01-01T00:00:00.000Z"}} + ] + } + } + }]) + end + + it "sorts and truncates a complex multi-layer `date_histogram` + `term` buckets on the doc count (descending), with the key values (ascending) as a tie-breaker" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}, size: 5), + "doc_count" => 18, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"buckets_path" => ["seasons_nested.started_at.as_date_time"], "key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2020, + "doc_count" => 18, + "seasons_nested.started_at.as_date_time" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.won_games_at"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.started_at.as_date_time"]}), + "buckets" => [ + { + "key_as_string" => "2020-01-01T00:00:00.000Z", + "key" => 1577836800000, + "doc_count" => 18, + "seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at.as_date_time"]}), + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 3 + }, + { + "key_as_string" => "2022-01-01T00:00:00.000Z", + "key" => 1640995200000, + "doc_count" => 3 + }, + { + "key_as_string" => "2021-01-01T00:00:00.000Z", + "key" => 1609459200000, + "doc_count" => 9 + }, + { + "key_as_string" => "2020-01-01T00:00:00.000Z", + "key" => 1577836800000, + "doc_count" => 3 + } + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0), + { + "key" => 2019, + "doc_count" => 11, + "seasons_nested.started_at.as_date_time" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.won_games_at"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.started_at.as_date_time"]}), + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 11, + "seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at.as_date_time"]}), + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 11 + } + ] + } + }.with_missing_value_bucket(0), + { + "key_as_string" => "2021-01-01T00:00:00.000Z", + "key" => 1609459200000, + "doc_count" => 2, + "seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at.as_date_time"]}), + "buckets" => [ + { + "key_as_string" => "2021-01-01T00:00:00.000Z", + "key" => 1609459200000, + "doc_count" => 2 + } + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0) + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested(first: 5) { + nodes { + grouped_by { + started_at { as_date_time(truncation_unit: YEAR) } + year + won_game_at { as_date_time(truncation_unit: YEAR) } + } + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + { + "count_detail" => {"approximate_value" => 11}, + "grouped_by" => { + "year" => 2019, + "started_at" => {"as_date_time" => "2019-01-01T00:00:00.000Z"}, + "won_game_at" => {"as_date_time" => "2019-01-01T00:00:00.000Z"} + } + }, + { + "count_detail" => {"approximate_value" => 9}, + "grouped_by" => { + "year" => 2020, + "started_at" => {"as_date_time" => "2020-01-01T00:00:00.000Z"}, + "won_game_at" => {"as_date_time" => "2021-01-01T00:00:00.000Z"} + } + }, + { + "count_detail" => {"approximate_value" => 3}, + "grouped_by" => { + "year" => 2020, + "started_at" => {"as_date_time" => "2020-01-01T00:00:00.000Z"}, + "won_game_at" => {"as_date_time" => "2019-01-01T00:00:00.000Z"} + } + }, + { + "count_detail" => {"approximate_value" => 3}, + "grouped_by" => { + "year" => 2020, + "started_at" => {"as_date_time" => "2020-01-01T00:00:00.000Z"}, + "won_game_at" => {"as_date_time" => "2020-01-01T00:00:00.000Z"} + } + }, + { + "count_detail" => {"approximate_value" => 3}, + "grouped_by" => { + "year" => 2020, + "started_at" => {"as_date_time" => "2020-01-01T00:00:00.000Z"}, + "won_game_at" => {"as_date_time" => "2022-01-01T00:00:00.000Z"} + } + } + ] + } + } + }]) + end + + it "sorts and truncates a complex multi-layer legacy `date_histogram` + `term` buckets on the doc count (descending), with the key values (ascending) as a tie-breaker" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}, size: 5), + "doc_count" => 18, + "seasons_nested.year" => { + "meta" => inner_terms_meta({"buckets_path" => ["seasons_nested.started_at_legacy"], "key_path" => ["key"], "grouping_fields" => ["seasons_nested.year"]}), + "doc_count_error_upper_bound" => 0, + "sum_other_doc_count" => 0, + "buckets" => [ + { + "key" => 2020, + "doc_count" => 18, + "seasons_nested.started_at_legacy" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.won_games_at_legacy"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.started_at_legacy"]}), + "buckets" => [ + { + "key_as_string" => "2020-01-01T00:00:00.000Z", + "key" => 1577836800000, + "doc_count" => 18, + "seasons_nested.won_games_at_legacy" => { + "meta" => inner_date_meta({"key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at_legacy"]}), + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 3 + }, + { + "key_as_string" => "2022-01-01T00:00:00.000Z", + "key" => 1640995200000, + "doc_count" => 3 + }, + { + "key_as_string" => "2021-01-01T00:00:00.000Z", + "key" => 1609459200000, + "doc_count" => 9 + }, + { + "key_as_string" => "2020-01-01T00:00:00.000Z", + "key" => 1577836800000, + "doc_count" => 3 + } + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0), + { + "key" => 2019, + "doc_count" => 11, + "seasons_nested.started_at_legacy" => { + "meta" => inner_date_meta({"buckets_path" => ["seasons_nested.won_games_at_legacy"], "key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.started_at_legacy"]}), + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 11, + "seasons_nested.won_games_at_legacy" => { + "meta" => inner_date_meta({"key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at_legacy"]}), + "buckets" => [ + { + "key_as_string" => "2019-01-01T00:00:00.000Z", + "key" => 1546300800000, + "doc_count" => 11 + } + ] + } + }.with_missing_value_bucket(0), + { + "key_as_string" => "2021-01-01T00:00:00.000Z", + "key" => 1609459200000, + "doc_count" => 2, + "seasons_nested.won_games_at_legacy" => { + "meta" => inner_date_meta({"key_path" => ["key_as_string"], "grouping_fields" => ["seasons_nested.won_game_at_legacy"]}), + "buckets" => [ + { + "key_as_string" => "2021-01-01T00:00:00.000Z", + "key" => 1609459200000, + "doc_count" => 2 + } + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0) + ] + } + }.with_missing_value_bucket(0) + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested(first: 5) { + nodes { + grouped_by { + started_at_legacy(granularity: YEAR) + year + won_game_at_legacy(granularity: YEAR) + } + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq([{ + "sub_aggregations" => { + "seasons_nested" => { + "nodes" => [ + { + "count_detail" => {"approximate_value" => 11}, + "grouped_by" => { + "year" => 2019, + "started_at_legacy" => "2019-01-01T00:00:00.000Z", + "won_game_at_legacy" => "2019-01-01T00:00:00.000Z" + } + }, + { + "count_detail" => {"approximate_value" => 9}, + "grouped_by" => { + "year" => 2020, + "started_at_legacy" => "2020-01-01T00:00:00.000Z", + "won_game_at_legacy" => "2021-01-01T00:00:00.000Z" + } + }, + { + "count_detail" => {"approximate_value" => 3}, + "grouped_by" => { + "year" => 2020, + "started_at_legacy" => "2020-01-01T00:00:00.000Z", + "won_game_at_legacy" => "2019-01-01T00:00:00.000Z" + } + }, + { + "count_detail" => {"approximate_value" => 3}, + "grouped_by" => { + "year" => 2020, + "started_at_legacy" => "2020-01-01T00:00:00.000Z", + "won_game_at_legacy" => "2020-01-01T00:00:00.000Z" + } + }, + { + "count_detail" => {"approximate_value" => 3}, + "grouped_by" => { + "year" => 2020, + "started_at_legacy" => "2020-01-01T00:00:00.000Z", + "won_game_at_legacy" => "2022-01-01T00:00:00.000Z" + } + } + ] + } + } + }]) + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/ungrouped_sub_aggregation_shared_examples.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/ungrouped_sub_aggregation_shared_examples.rb new file mode 100644 index 00000000..37099037 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/resolvers/ungrouped_sub_aggregation_shared_examples.rb @@ -0,0 +1,603 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class GraphQL + module Aggregation + # The CompositeAggregationAdapter and NonCompositeAggregationAdapter deal with the same requests and responses + # when no groupign is involved, so we can use shared examples for those cases. + RSpec.shared_examples_for "ungrouped sub-aggregations" do + it "resolves ungrouped sub-aggregations with just a count_detail under an ungrouped aggregation" do + aggs = { + "target:seasons_nested" => {"doc_count" => 423, "meta" => outer_meta}, + "target:current_players_nested" => {"doc_count" => 201, "meta" => outer_meta} + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested { + nodes { + count_detail { approximate_value } + } + } + + current_players_nested { + nodes { + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq [{ + "sub_aggregations" => { + "seasons_nested" => {"nodes" => [{ + "count_detail" => {"approximate_value" => 423} + }]}, + "current_players_nested" => {"nodes" => [{ + "count_detail" => {"approximate_value" => 201} + }]} + } + }] + end + + it "resolves an aliased sub-aggregation" do + aggs = { + "target:inner_target" => {"doc_count" => 423, "meta" => outer_meta} + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + inner_target: seasons_nested { + nodes { + count_detail { approximate_value } + } + } + } + } + } + QUERY + + expect(response).to eq [{ + "sub_aggregations" => { + "inner_target" => {"nodes" => [{ + "count_detail" => {"approximate_value" => 423} + }]} + } + }] + end + + it "resolves an ungrouped sub-aggregation with just a count_detail under a grouped aggregation" do + target_buckets = [ + { + "key" => {"current_name" => "Yankees"}, + "doc_count" => 1, + "target:seasons_nested" => {"doc_count" => 3, "meta" => outer_meta} + }, + { + "key" => {"current_name" => "Dodgers"}, + "doc_count" => 1, + "target:seasons_nested" => {"doc_count" => 9, "meta" => outer_meta} + } + ] + + response = resolve_target_nodes(<<~QUERY, target_buckets: target_buckets) + target: team_aggregations { + nodes { + grouped_by { current_name } + sub_aggregations { + seasons_nested { + nodes { + count_detail { + approximate_value + } + } + } + } + } + } + QUERY + + expect(response).to eq [ + { + "grouped_by" => {"current_name" => "Yankees"}, + "sub_aggregations" => {"seasons_nested" => {"nodes" => [{ + "count_detail" => {"approximate_value" => 3} + }]}} + }, + { + "grouped_by" => {"current_name" => "Dodgers"}, + "sub_aggregations" => {"seasons_nested" => {"nodes" => [{ + "count_detail" => {"approximate_value" => 9} + }]}} + } + ] + end + + it "resolves a sub-aggregation embedded under an extra object layer and respects alternate `name_in_index`" do + aggs = { + "target:nested_fields.seasons" => {"doc_count" => 423, "meta" => outer_meta} + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + nested_fields { # called the_nested_fields in the index + seasons { # called the_seasons in the index + nodes { + count_detail { approximate_value } + } + } + } + } + } + } + QUERY + + expect(response).to eq [{ + "sub_aggregations" => { + "nested_fields" => { + "seasons" => {"nodes" => [{ + "count_detail" => {"approximate_value" => 423} + }]} + } + } + }] + end + + it "resolves sub-aggregations of sub-aggregations of sub-aggregations" do + aggs = { + "target:seasons_nested" => { + "doc_count" => 4, + "meta" => outer_meta, + "target:seasons_nested:seasons_nested.players_nested" => { + "doc_count" => 25, + "meta" => outer_meta, + "target:seasons_nested:players_nested:seasons_nested.players_nested.seasons_nested" => {"doc_count" => 47, "meta" => outer_meta} + } + } + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested { + nodes { + count_detail { approximate_value } + sub_aggregations { + players_nested { + nodes { + count_detail { approximate_value } + sub_aggregations { + seasons_nested { + nodes { + count_detail { approximate_value } + } + } + } + } + } + } + } + } + } + } + } + QUERY + + expect(response).to eq [{ + "sub_aggregations" => { + "seasons_nested" => {"nodes" => [{ + "count_detail" => {"approximate_value" => 4}, + "sub_aggregations" => { + "players_nested" => {"nodes" => [{ + "count_detail" => {"approximate_value" => 25}, + "sub_aggregations" => { + "seasons_nested" => {"nodes" => [{ + "count_detail" => {"approximate_value" => 47} + }]} + } + }]} + } + }]} + } + }] + end + + it "resolves filtered sub-aggregations" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"bucket_path" => ["seasons_nested:filtered"]}), + "doc_count" => 423, + "seasons_nested:filtered" => {"doc_count" => 57} + } + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested(filter: {year: {gt: 2000}}) { + nodes { + count_detail { + approximate_value + } + } + } + } + } + } + QUERY + + expect(response).to eq [{ + "sub_aggregations" => { + "seasons_nested" => {"nodes" => [{ + "count_detail" => { + "approximate_value" => 57 + } + }]} + } + }] + end + + it "ignores an empty filter" do + aggs = { + "target:seasons_nested" => {"doc_count" => 423, "meta" => outer_meta} + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested(filter: {year: {gt: null}}) { + nodes { + count_detail { + approximate_value + } + } + } + } + } + } + QUERY + + expect(response).to eq [{ + "sub_aggregations" => { + "seasons_nested" => {"nodes" => [{ + "count_detail" => { + "approximate_value" => 423 + } + }]} + } + }] + end + + it "resolves a filtered sub-aggregation under a filtered sub-aggregation" do + aggs = { + "target:current_players_nested" => { + "meta" => outer_meta({"bucket_path" => ["current_players_nested:filtered"]}), + "doc_count" => 5, + "current_players_nested:filtered" => { + "doc_count" => 3, + "target:current_players_nested:current_players_nested.seasons_nested" => { + "meta" => outer_meta({"bucket_path" => ["seasons_nested:filtered"]}), + "doc_count" => 2, + "seasons_nested:filtered" => {"doc_count" => 1} + } + } + } + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + current_players_nested(filter: {name: {equal_to_any_of: ["Bob"]}}) { + nodes { + count_detail { + approximate_value + } + + sub_aggregations { + seasons_nested(filter: {year: {gt: 2000}}) { + nodes { + count_detail { + approximate_value + } + } + } + } + } + } + } + } + } + QUERY + + expect(response).to eq [{ + "sub_aggregations" => { + "current_players_nested" => {"nodes" => [{ + "count_detail" => {"approximate_value" => 3}, + "sub_aggregations" => { + "seasons_nested" => {"nodes" => [{ + "count_detail" => {"approximate_value" => 1} + }]} + } + }]} + } + }] + end + + it "resolves a filtered sub-aggregation under an unfiltered sub-aggregation under a filtered sub-aggregation" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"bucket_path" => ["seasons_nested:filtered"]}), + "doc_count" => 20, + "seasons_nested:filtered" => { + "doc_count" => 19, + "target:seasons_nested:seasons_nested.players_nested" => { + "doc_count" => 18, + "meta" => outer_meta, + "target:seasons_nested:players_nested:seasons_nested.players_nested.seasons_nested" => { + "meta" => outer_meta({"bucket_path" => ["seasons_nested:filtered"]}), + "doc_count" => 12, + "seasons_nested:filtered" => {"doc_count" => 8} + } + } + } + } + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested(filter: {year: {gt: 2000}}) { + nodes { + count_detail { approximate_value } + sub_aggregations { + players_nested { + nodes { + count_detail { approximate_value } + sub_aggregations { + seasons_nested(filter: {year: {gt: 2010}}) { + nodes { + count_detail { approximate_value } + } + } + } + } + } + } + } + } + } + } + } + QUERY + + expect(response).to eq [{ + "sub_aggregations" => { + "seasons_nested" => {"nodes" => [{ + "count_detail" => {"approximate_value" => 19}, + "sub_aggregations" => { + "players_nested" => {"nodes" => [{ + "count_detail" => {"approximate_value" => 18}, + "sub_aggregations" => { + "seasons_nested" => {"nodes" => [{ + "count_detail" => {"approximate_value" => 8} + }]} + } + }]} + } + }]} + } + }] + end + + context "with `count_detail` fields" do + it "indicates the count is exact when no filtering or grouping has been applied" do + aggs = { + "target:seasons_nested" => {"doc_count" => 423, "meta" => outer_meta} + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested { + nodes { + count_detail { + approximate_value + exact_value + upper_bound + } + } + } + } + } + } + QUERY + + expect(response.dig(0, "sub_aggregations", "seasons_nested", "nodes", 0, "count_detail")).to be_exactly_equal_to(423) + end + + it "indicates the count is exact when filtering has been applied (but no grouping)" do + aggs = { + "target:seasons_nested" => { + "meta" => outer_meta({"bucket_path" => ["seasons_nested:filtered"]}), + "doc_count" => 423, + "seasons_nested:filtered" => {"doc_count" => 57} + } + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + seasons_nested(filter: {year: {gt: 2000}}) { + nodes { + count_detail { + approximate_value + exact_value + upper_bound + } + } + } + } + } + } + QUERY + + expect(response.dig(0, "sub_aggregations", "seasons_nested", "nodes", 0, "count_detail")).to be_exactly_equal_to(57) + end + + def be_exactly_equal_to(count) + eq({"approximate_value" => count, "exact_value" => count, "upper_bound" => count}) + end + end + + context "with aggregated values" do + it "resolves ungrouped aggregated values" do + aggs = { + "target:current_players_nested" => { + "doc_count" => 5, + "meta" => outer_meta, + "current_players_nested:current_players_nested.name:approximate_distinct_value_count" => { + "value" => 5 + } + } + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + current_players_nested { + nodes { + aggregated_values { name { approximate_distinct_value_count } } + } + } + } + } + } + QUERY + + expect(response).to eq [{ + "sub_aggregations" => { + "current_players_nested" => { + "nodes" => [ + { + "aggregated_values" => { + "name" => {"approximate_distinct_value_count" => 5} + } + } + ] + } + } + }] + end + + it "uses GraphQL field aliases when resolving `aggregated_values` subfields" do + aggs = { + "target:current_players_nested" => { + "doc_count" => 5, + "meta" => outer_meta, + "current_players_nested:current_players_nested.the_name:approximate_distinct_value_count" => { + "value" => 5 + } + } + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + current_players_nested { + nodes { + aggregated_values { the_name: name { approximate_distinct_value_count } } + } + } + } + } + } + QUERY + + expect(response).to eq [{ + "sub_aggregations" => { + "current_players_nested" => { + "nodes" => [ + { + "aggregated_values" => { + "the_name" => {"approximate_distinct_value_count" => 5} + } + } + ] + } + } + }] + end + + context "with a `grouped_by` field which excludes all groupings via an `@include` directive" do + it "returns an empty object for `grouped_by`" do + aggs = { + "target:current_players_nested" => { + "doc_count" => 5, + "meta" => outer_meta, + "current_players_nested:current_players_nested.name:approximate_distinct_value_count" => { + "value" => 3 + } + } + } + + response = resolve_target_nodes(<<~QUERY, aggs: aggs) + target: team_aggregations { + nodes { + sub_aggregations { + current_players_nested { + nodes { + grouped_by { name @include(if: false) } + count_detail { approximate_value } + aggregated_values { name { approximate_distinct_value_count } } + } + } + } + } + } + QUERY + + expect(response).to eq [{ + "sub_aggregations" => { + "current_players_nested" => { + "nodes" => [ + { + "grouped_by" => {}, + "count_detail" => {"approximate_value" => 5}, + "aggregated_values" => { + "name" => {"approximate_distinct_value_count" => 3} + } + } + ] + } + } + }] + end + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/script_term_grouping_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/script_term_grouping_spec.rb new file mode 100644 index 00000000..c751ad26 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/aggregation/script_term_grouping_spec.rb @@ -0,0 +1,102 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/script_term_grouping" +require "elastic_graph/constants" +require "support/aggregations_helpers" + +module ElasticGraph + class GraphQL + module Aggregation + RSpec.describe ScriptTermGrouping do + include AggregationsHelpers + + let(:script_id) { "some_script_id" } + + describe "#key" do + it "returns the encoded field path" do + grouping = script_term_grouping_of("foo", "bar") + + expect(grouping.key).to eq "foo.bar" + end + + it "uses GraphQL query field names when they differ from the name of the field in the index" do + grouping = script_term_grouping_of("foo", "bar", field_names_in_graphql_query: ["oof", "rab"]) + + expect(grouping.key).to eq "oof.rab" + end + end + + describe "#encoded_index_field_path" do + it "returns the encoded field path" do + grouping = script_term_grouping_of("foo", "bar") + + expect(grouping.encoded_index_field_path).to eq "foo.bar" + end + + it "uses the names in the index when they differ from the GraphQL names" do + grouping = script_term_grouping_of("oof", "rab", field_names_in_graphql_query: ["foo", "bar"]) + + expect(grouping.encoded_index_field_path).to eq "oof.rab" + end + + it "allows a `name_in_index` that references a child field" do + grouping = script_term_grouping_of("foo.c", "bar.d", field_names_in_graphql_query: ["foo", "bar"]) + + expect(grouping.encoded_index_field_path).to eq "foo.c.bar.d" + end + end + + describe "#composite_clause" do + it 'builds a datastore aggregation terms clause in the form: {"terms" => {"script" => {"id" => ... , "params" => ...}}}' do + grouping = script_term_grouping_of("foo", "bar") + + expect(grouping.composite_clause).to eq({ + "terms" => { + "script" => { + "id" => script_id, + "params" => { + "field" => "foo.bar" + } + } + } + }) + end + + it "uses the names of the fields in the index rather than the GraphQL query field names when they differ" do + grouping = script_term_grouping_of("foo", "bar", field_names_in_graphql_query: ["oof", "rab"]) + + expect(grouping.composite_clause.dig("terms", "script", "params", "field")).to eq("foo.bar") + end + + it "allows arbitrary params to be set" do + grouping = script_term_grouping_of("foo", "bar", params: {"some_param" => "some_value", "another_param" => "another_value"}) + expect(grouping.composite_clause.dig("terms", "script", "params", "some_param")).to eq("some_value") + expect(grouping.composite_clause.dig("terms", "script", "params", "another_param")).to eq("another_value") + end + end + + describe "#inner_meta" do + it "returns inner meta" do + grouping = script_term_grouping_of("foo", "bar") + expect(grouping.inner_meta).to eq({ + "key_path" => [ + "key" + ], + "merge_into_bucket" => {} + }) + end + end + + def script_term_grouping_of(*field_names_in_index, **args) + super(*field_names_in_index, script_id: script_id, **args) + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/client_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/client_spec.rb new file mode 100644 index 00000000..6e27b44d --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/client_spec.rb @@ -0,0 +1,27 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/client" + +module ElasticGraph + class GraphQL + RSpec.describe Client do + describe "#description" do + it "combines the name and id in a readable way" do + client = Client.new(name: "John", source_description: "42") + + expect(client.description).to eq("John (42)") + end + + it "avoids returning duplicate info when the name and id are the same (such as for `ANONYMOUS`)" do + expect(Client::ANONYMOUS.description).to eq("(anonymous)") + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/config_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/config_spec.rb new file mode 100644 index 00000000..a6f92945 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/config_spec.rb @@ -0,0 +1,223 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/config" +require "yaml" + +module ElasticGraph + class GraphQL + RSpec.describe Config do + it "sets config values from the given parsed YAML" do + config = Config.from_parsed_yaml("graphql" => { + "default_page_size" => 27, + "max_page_size" => 270, + "slow_query_latency_warning_threshold_in_ms" => 3200, + "extension_modules" => [], + "client_resolver" => { + "extension_name" => "ElasticGraph::GraphQL::ClientResolvers::ViaHTTPHeader", + "require_path" => "support/client_resolvers", + "header_name" => "X-Client-Name" + } + }) + + expect(config.default_page_size).to eq 27 + expect(config.max_page_size).to eq 270 + expect(config.slow_query_latency_warning_threshold_in_ms).to eq 3200 + expect(config.extension_modules).to eq [] + expect(config.client_resolver).to eq ClientResolvers::ViaHTTPHeader.new({"header_name" => "X-Client-Name"}) + end + + it "provides reasonable defaults for some optional settings" do + config = Config.from_parsed_yaml("graphql" => { + "default_page_size" => 27, + "max_page_size" => 270 + }) + + expect(config.default_page_size).to eq 27 + expect(config.max_page_size).to eq 270 + expect(config.slow_query_latency_warning_threshold_in_ms).to eq 5000 + expect(config.extension_modules).to eq [] + expect(config.client_resolver).to be_a Client::DefaultResolver + end + + it "raises an error when given an unrecognized config setting" do + expect { + Config.from_parsed_yaml("graphql" => { + "default_page_size" => 27, + "max_page_size" => 270, + "fake_setting" => 23 + }) + }.to raise_error Errors::ConfigError, a_string_including("fake_setting") + end + + describe "#client_resolver" do + it "raises an error when given an invalid require path" do + expect { + Config.from_parsed_yaml("graphql" => { + "default_page_size" => 27, + "max_page_size" => 270, + "client_resolver" => { + "extension_name" => "ElasticGraph::GraphQL::ClientResolvers::ViaHTTPHeader", + "require_path" => "support/client_resolvers_typo", + "header_name" => "X-CLIENT-NAME" + } + }) + }.to raise_error LoadError, a_string_including("support/client_resolvers_typo") + end + + it "raises an error when given an invalid name" do + expect { + Config.from_parsed_yaml("graphql" => { + "default_page_size" => 27, + "max_page_size" => 270, + "client_resolver" => { + "extension_name" => "ElasticGraph::GraphQL::ClientResolvers::ViaHTTPHeaderTypo", + "require_path" => "support/client_resolvers", + "header_name" => "X-CLIENT-NAME" + } + }) + }.to raise_error NameError, a_string_including("uninitialized constant ElasticGraph::GraphQL::ClientResolvers::ViaHTTPHeaderTypo") + end + + it "raises an error when given an extension that does not implement the right interface" do + expect { + Config.from_parsed_yaml("graphql" => { + "default_page_size" => 27, + "max_page_size" => 270, + "client_resolver" => { + "extension_name" => "ElasticGraph::GraphQL::ClientResolvers::Invalid", + "require_path" => "support/client_resolvers", + "header_name" => "X-CLIENT-NAME" + } + }) + }.to raise_error Errors::InvalidExtensionError, a_string_including("Missing instance methods: `resolve`") + end + end + + describe "#extension_settings" do + it "is empty if the config YAML file contains no settings beyond the core ElasticGraph ones" do + config = Config.from_parsed_yaml(parsed_test_settings_yaml) + + expect(config.extension_settings).to eq({}) + end + + it "includes any additional settings that aren't part of ElasticGraph's core configuration" do + parsed_yaml = parsed_test_settings_yaml.merge( + "ext1" => {"a" => 3, "b" => false}, + "ext2" => [12, 24] + ) + + config = Config.from_parsed_yaml(parsed_yaml) + + expect(config.extension_settings).to eq( + "ext1" => {"a" => 3, "b" => false}, + "ext2" => [12, 24] + ) + end + end + + describe "#extension_modules", :in_temp_dir do + it "loads the extension modules from disk" do + File.write("eg_extension_module1.rb", <<~EOS) + module EgExtensionModule1 + end + EOS + + File.write("eg_extension_module2.rb", <<~EOS) + module EgExtensionModule2 + end + EOS + + extension_modules = extension_modules_from(<<~YAML) + extension_modules: + - require_path: ./eg_extension_module1 + extension_name: EgExtensionModule1 + - require_path: ./eg_extension_module2 + extension_name: EgExtensionModule2 + YAML + + expect(extension_modules).to eq([::EgExtensionModule1, ::EgExtensionModule2]) + end + + it "raises a clear error if the extension can't be loaded" do + expect { + extension_modules_from(<<~YAML) + extension_modules: + - require_path: ./not_real + extension_name: NotReal + YAML + }.to raise_error LoadError, a_string_including("not_real") + end + + it "raises a clear error if the config is malformed" do + expect { + extension_modules_from(<<~YAML) + extension_modules: + - require: ./not_real + extension_name: NotReal + YAML + }.to raise_error a_string_including("require_path") + + File.write("eg_extension_module1.rb", <<~EOS) + module EgExtensionModule1 + end + EOS + + expect { + extension_modules_from(<<~YAML) + extension_modules: + - require_path: ./eg_extension_module1 + extension: EgExtensionModule1 + YAML + }.to raise_error a_string_including("extension_name") + end + + it "raises a clear error if the named extension is not a module" do + File.write("eg_extension_class1.rb", <<~EOS) + class EgExtensionClass1 + end + EOS + + expect { + extension_modules_from(<<~YAML) + extension_modules: + - require_path: ./eg_extension_class1 + extension_name: EgExtensionClass1 + YAML + }.to raise_error a_string_including("not a module") + + File.write("eg_extension_object1.rb", <<~EOS) + EgExtensionObject1 = Object.new + EOS + + expect { + extension_modules_from(<<~YAML) + extension_modules: + - require_path: ./eg_extension_object1 + extension_name: EgExtensionObject1 + YAML + }.to raise_error a_string_including("not a class or module") + end + + def load_config_from_yaml(yaml) + yaml = <<~EOS + default_page_size: 27 + max_page_size: 270 + #{yaml} + EOS + + Config.from_parsed_yaml("graphql" => ::YAML.safe_load(yaml)) + end + + def extension_modules_from(yaml) + load_config_from_yaml(yaml).extension_modules + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/aggregations_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/aggregations_spec.rb new file mode 100644 index 00000000..8d667b47 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/aggregations_spec.rb @@ -0,0 +1,237 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_unit_support" +require "support/aggregations_helpers" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "aggregations" do + include AggregationsHelpers + include_context "DatastoreQueryUnitSupport" + + it "excludes `aggs` if `aggregations` is `nil`" do + query = new_query(aggregations: nil) + expect(datastore_body_of(query)).to exclude(:aggs) + end + + it "excludes `aggs` if `aggregations` is not given" do + expect(datastore_body_of(new_query)).to exclude(:aggs) + end + + it "excludes `aggs` if `aggregations` is empty" do + expect(datastore_body_of(new_query(aggregations: {}))).to exclude(:aggs) + end + + it "excludes `aggs` if `aggregations.size` is 0" do + query = new_query(aggregations: [aggregation_query_of( + computations: [computation_of("amountMoney", "amount", :sum)], + groupings: [field_term_grouping_of("options", "size")], + first: 0 + )]) + + expect(datastore_body_of(query)).to exclude(:aggs) + end + + it "populates `aggs` with a metric aggregation when given only computations in `aggregations`" do + query = new_query(aggregations: [aggregation_query_of( + name: "my_aggs", + computations: [ + computation_of("amountMoney", "amount", :sum), + computation_of("amountMoney", "amount", :avg) + ] + )]) + + expect(datastore_body_of(query)).to include_aggs( + aggregated_value_key_of("amountMoney", "amount", "sum", aggregation_name: "my_aggs") => {"sum" => {"field" => "amountMoney.amount"}}, + aggregated_value_key_of("amountMoney", "amount", "avg", aggregation_name: "my_aggs") => {"avg" => {"field" => "amountMoney.amount"}} + ) + end + + it "uses the GraphQL query field names in computation aggregation keys when they differ from the field names in the index" do + query = new_query(aggregations: [aggregation_query_of( + name: "my_aggs", + computations: [ + computation_of("amountMoney", "amount", :sum, field_names_in_graphql_query: ["amtOuter", "amtInner"]), + computation_of("amountMoney", "amount", :avg, field_names_in_graphql_query: ["amtOuter", "amtInner"]) + ] + )]) + + expect(datastore_body_of(query)).to include_aggs( + aggregated_value_key_of("amtOuter", "amtInner", "sum", aggregation_name: "my_aggs") => {"sum" => {"field" => "amountMoney.amount"}}, + aggregated_value_key_of("amtOuter", "amtInner", "avg", aggregation_name: "my_aggs") => {"avg" => {"field" => "amountMoney.amount"}} + ) + end + + it "populates `aggs` with a composite aggregation when given only groupings in `aggregations`" do + query = new_query(aggregations: [aggregation_query_of(name: "my_agg", first: 12, groupings: [ + field_term_grouping_of("options", "size"), + field_term_grouping_of("options", "color"), + date_histogram_grouping_of("created_at", "day", time_zone: "UTC") + ])]) + + expect(datastore_body_of(query)).to include_aggs("my_agg" => {"composite" => { + "size" => 13, # add 1 so that we can detect if there are more results. + "sources" => [ + {"options.size" => {"terms" => {"field" => "options.size", "missing_bucket" => true}}}, + {"options.color" => {"terms" => {"field" => "options.color", "missing_bucket" => true}}}, + {"created_at" => {"date_histogram" => {"field" => "created_at", "missing_bucket" => true, "calendar_interval" => "day", "format" => DATASTORE_DATE_TIME_FORMAT, "time_zone" => "UTC"}}} + ] + }}) + end + + it "populates `aggs` for `as_day_of_week` using a script" do + query = new_query(aggregations: [aggregation_query_of(name: "my_agg", first: 12, groupings: [ + as_day_of_week_grouping_of("created_at", time_zone: "UTC", offset_ms: 10_800_000, script_id: "some_script_id") + ])]) + + expect(datastore_body_of(query)).to include_aggs("my_agg" => {"composite" => { + "size" => 13, # add 1 so that we can detect if there are more results. + "sources" => [ + {"created_at" => {"terms" => {"missing_bucket" => true, "script" => {"id" => "some_script_id", "params" => {"field" => "created_at", "offset_ms" => 10800000, "time_zone" => "UTC"}}}}} + ] + }}) + end + + it "populates `aggs` for `as_time_of_day` using a script" do + query = new_query(aggregations: [aggregation_query_of(name: "my_agg", first: 12, groupings: [ + as_time_of_day_grouping_of("created_at", "hour", time_zone: "UTC", offset_ms: 10_800_000, script_id: "some_script_id") + ])]) + + expect(datastore_body_of(query)).to include_aggs("my_agg" => {"composite" => { + "size" => 13, # add 1 so that we can detect if there are more results. + "sources" => [ + {"created_at" => {"terms" => {"missing_bucket" => true, "script" => {"id" => "some_script_id", "params" => {"field" => "created_at", "offset_ms" => 10800000, "time_zone" => "UTC", "interval" => "hour"}}}}} + ] + }}) + end + + it "uses the GraphQL query field names in composite aggregation keys when they differ from the field names in the index" do + query = new_query(aggregations: [aggregation_query_of(name: "my_agg", first: 12, groupings: [ + field_term_grouping_of("options", "size", field_names_in_graphql_query: ["opts", "the_size"]), + field_term_grouping_of("options", "color", field_names_in_graphql_query: ["opts", "color"]), + date_histogram_grouping_of("created_at", "day", time_zone: "UTC", field_names_in_graphql_query: ["the_created_at"]) + ])]) + + expect(datastore_body_of(query)).to include_aggs("my_agg" => {"composite" => { + "size" => 13, # add 1 so that we can detect if there are more results. + "sources" => [ + {"opts.the_size" => {"terms" => {"field" => "options.size", "missing_bucket" => true}}}, + {"opts.color" => {"terms" => {"field" => "options.color", "missing_bucket" => true}}}, + {"the_created_at" => {"date_histogram" => {"field" => "created_at", "missing_bucket" => true, "calendar_interval" => "day", "format" => DATASTORE_DATE_TIME_FORMAT, "time_zone" => "UTC"}}} + ] + }}) + end + + it "populates `aggs` with a composite aggregation and a metric sub-aggregation when given both groupings and computations in `aggregations`" do + query = new_query(aggregations: [aggregation_query_of( + name: "agg2", + computations: [computation_of("amountMoney", "amount", :sum)], + groupings: [field_term_grouping_of("options", "size")], + first: 17 + )]) + + expect(datastore_body_of(query)).to include_aggs("agg2" => { + "composite" => { + "size" => 18, # add 1 so that we can detect if there are more results. + "sources" => [ + {"options.size" => {"terms" => {"field" => "options.size", "missing_bucket" => true}}} + ] + }, + "aggs" => { + aggregated_value_key_of("amountMoney", "amount", "sum", aggregation_name: "agg2") => {"sum" => {"field" => "amountMoney.amount"}} + } + }) + end + + it "supports multiple aggregations in a single query" do + by_name = aggregation_query_of(name: "by_name", first: 3, computations: [computation_of("amount_cents", :sum)], groupings: [field_term_grouping_of("name")]) + by_month = aggregation_query_of(name: "by_month", first: 4, computations: [computation_of("amount_cents", :sum)], groupings: [date_histogram_grouping_of("created_at", "month", time_zone: "UTC")]) + just_sum = aggregation_query_of(name: "just_sum", computations: [computation_of("amount_cents", :sum)]) + min_and_max = aggregation_query_of(name: "min_and_max", computations: [computation_of("amount_cents", :min), computation_of("amount_cents", :max)]) + just_count = aggregation_query_of(name: "just_count", needs_doc_count: true) + + query = new_query(aggregations: [by_name, by_month, just_sum, min_and_max, just_count]) + + # `track_total_hits: true` is from the `just_count` aggregation. + expect(datastore_body_of(query)).to include(track_total_hits: true).and include_aggs({ + aggregated_value_key_of("amount_cents", "sum", aggregation_name: "just_sum") => { + "sum" => { + "field" => "amount_cents" + } + }, + aggregated_value_key_of("amount_cents", "min", aggregation_name: "min_and_max") => { + "min" => { + "field" => "amount_cents" + } + }, + aggregated_value_key_of("amount_cents", "max", aggregation_name: "min_and_max") => { + "max" => { + "field" => "amount_cents" + } + }, + "by_month" => { + "aggs" => { + aggregated_value_key_of("amount_cents", "sum", aggregation_name: "by_month") => { + "sum" => { + "field" => "amount_cents" + } + } + }, + "composite" => { + "size" => 5, + "sources" => [ + { + "created_at" => { + "date_histogram" => { + "calendar_interval" => "month", + "field" => "created_at", + "format" => "strict_date_time", + "missing_bucket" => true, + "time_zone" => "UTC" + } + } + } + ] + } + }, + "by_name" => { + "aggs" => { + aggregated_value_key_of("amount_cents", "sum", aggregation_name: "by_name") => { + "sum" => { + "field" => "amount_cents" + } + } + }, + "composite" => { + "size" => 4, + "sources" => [ + { + "name" => { + "terms" => { + "field" => "name", + "missing_bucket" => true + } + } + } + ] + } + } + }) + end + + def include_aggs(aggs) + include(aggs: aggs) + end + + def aggregated_value_key_of(...) + super.encode + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/cluster_name_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/cluster_name_spec.rb new file mode 100644 index 00000000..f1387373 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/cluster_name_spec.rb @@ -0,0 +1,53 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_unit_support" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "#cluster_name" do + include_context "DatastoreQueryUnitSupport" + + let(:graphql) do + build_graphql do |datastore_config| + datastore_config.with( + index_definitions: datastore_config.index_definitions.merge( + "components" => config_index_def_of(query_cluster: "other1") + ) + ) + end + end + + let(:widgets_def) { graphql.datastore_core.index_definitions_by_name.fetch("widgets") } + let(:addresses_def) { graphql.datastore_core.index_definitions_by_name.fetch("addresses") } + let(:components_def) { graphql.datastore_core.index_definitions_by_name.fetch("components") } + + before do + expect(widgets_def.cluster_to_query).to eq "main" + expect(addresses_def.cluster_to_query).to eq "main" + expect(components_def.cluster_to_query).to eq "other1" + end + + it "returns the name of the datastore cluster from the search index definitions" do + main_query = new_query(search_index_definitions: [widgets_def, addresses_def]) + expect(main_query.cluster_name).to eq "main" + + other_query = new_query(search_index_definitions: [components_def]) + expect(other_query.cluster_name).to eq "other1" + end + + it "raises an error if the index definitions do not agree about the cluster" do + query = new_query(search_index_definitions: [widgets_def, components_def]) + + expect { + query.cluster_name + }.to raise_error Errors::ConfigError, a_string_including("main", "other1") + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/datastore_query_unit_support.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/datastore_query_unit_support.rb new file mode 100644 index 00000000..f28207de --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/datastore_query_unit_support.rb @@ -0,0 +1,36 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/aggregation/key" +require "elastic_graph/graphql/datastore_query" +require "support/aggregations_helpers" + +module ElasticGraph + class GraphQL + RSpec.shared_context "DatastoreQueryUnitSupport", :capture_logs do + include AggregationsHelpers + let(:graphql) { build_graphql } + let(:builder) { graphql.datastore_query_builder } + + def datastore_body_of(query) + query.send(:to_datastore_body).tap do |datastore_body| + # To ensure that aggregations always satisfy the `QueryOptimizer` requirements, we validate all queries here. + verify_aggregations_satisfy_optimizer_requirements(datastore_body[:aggs], for_query: query) + end + end + + def new_query(aggregations: [], **options) + builder.new_query( + aggregations: aggregations.to_h { |agg| [agg.name, agg] }, + search_index_definitions: graphql.datastore_core.index_definitions_by_graphql_type.fetch("Widget"), + **options + ) + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/filtering_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/filtering_spec.rb new file mode 100644 index 00000000..46d085c6 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/filtering_spec.rb @@ -0,0 +1,1507 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_unit_support" +require "tempfile" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "filtering" do + include_context "DatastoreQueryUnitSupport" + + let(:always_false_condition) do + {bool: {filter: Filtering::BooleanQuery::ALWAYS_FALSE_FILTER.clauses}} + end + + it "builds a `nil` datastore body when given no filters (passed as `nil`)" do + query = new_query(filter: nil) + + expect(datastore_body_of(query)).to not_filter_datastore_at_all + end + + it "builds a `nil` datastore body when given no filters (passed as an empty hash)" do + query = new_query(filter: {}) + + expect(datastore_body_of(query)).to not_filter_datastore_at_all + end + + it "ignores unknown filtering operators and logs a warning" do + query = new_query(filter: {"name" => {"like" => "abc"}}) + + expect { + expect(datastore_body_of(query)).to not_filter_datastore_at_all + }.to log a_string_including('Ignoring unknown filtering operator (like: "abc") on field `name`') + end + + it "ignores malformed filters and logs a warning" do + query = new_query(filter: {"name" => [7]}) + + expect { + expect(datastore_body_of(query)).to not_filter_datastore_at_all + }.to log a_string_including("Ignoring unknown filtering operator (name: [7]) on field ``") + end + + it "takes advantage of ids query when filtering on id" do + query = new_query(filter: {"id" => {"equal_to_any_of" => ["testid"]}}) + + expect(datastore_body_of(query)).to filter_datastore_with(ids: {values: ["testid"]}) + end + + it "deduplicates the `id` to a unique set of values when given duplicates in an `equal_to_any_of: [...]` filter" do + query = new_query(filter: {"id" => {"equal_to_any_of" => ["a", "b", "a"]}}) + + expect(datastore_body_of(query)).to filter_datastore_with(ids: {values: ["a", "b"]}) + end + + it "builds a `terms` condition when given an `equal_to_any_of: [...]` filter" do + query = new_query(filter: {"age" => {"equal_to_any_of" => [25, 30]}}) + + expect(datastore_body_of(query)).to filter_datastore_with(terms: {"age" => [25, 30]}) + end + + it "deduplicates the `terms` to a unique set of values when given duplicates in an `equal_to_any_of: [...]` filter" do + query = new_query(filter: {"age" => {"equal_to_any_of" => [25, 30, 25]}}) + + expect(datastore_body_of(query)).to filter_datastore_with(terms: {"age" => [25, 30]}) + end + + it "builds a `range` condition when given an `gt: scalar` filter" do + query = new_query(filter: {"age" => {"gt" => 25}}) + + expect(datastore_body_of(query)).to filter_datastore_with(range: {"age" => {gt: 25}}) + end + + it "builds a `range` condition when given an `gte: scalar` filter" do + query = new_query(filter: {"age" => {"gte" => 25}}) + + expect(datastore_body_of(query)).to filter_datastore_with(range: {"age" => {gte: 25}}) + end + + it "builds a `range` condition when given an `lt: scalar` filter" do + query = new_query(filter: {"age" => {"lt" => 25}}) + + expect(datastore_body_of(query)).to filter_datastore_with(range: {"age" => {lt: 25}}) + end + + it "builds a `range` condition when given an `lte: scalar` filter" do + query = new_query(filter: {"age" => {"lte" => 25}}) + + expect(datastore_body_of(query)).to filter_datastore_with(range: {"age" => {lte: 25}}) + end + + it "merges multiple `range` clauses that are on the same field" do + query1 = new_query(filter: {"age" => {"gt" => 10, "lte" => 25}}) + expect(datastore_body_of(query1)).to filter_datastore_with(range: {"age" => {gt: 10, lte: 25}}) + + query2 = new_query(filter: {"age" => {"gt" => 10, "lte" => 25, "gte" => 20, "lt" => 50}}) + expect(datastore_body_of(query2)).to filter_datastore_with(range: {"age" => {gt: 10, gte: 20, lt: 50, lte: 25}}) + end + + it "leaves multiple `range` clauses that are on different fields unmerged" do + query = new_query(filter: {"age" => {"gt" => 10}, "height" => {"lte" => 120}}) + + expect(datastore_body_of(query)).to filter_datastore_with( + {range: {"age" => {gt: 10}}}, + {range: {"height" => {lte: 120}}} + ) + end + + it "builds a `match` must condition when given a `matches`: 'string' filter" do + query = new_query(filter: {"name_text" => {"matches" => "foo"}}) + + expect(datastore_body_of(query)).to query_datastore_with(bool: {must: [{match: {"name_text" => "foo"}}]}) + end + + it "builds a `match` must condition when given a `matches_query`: 'MatchesQueryFilterInput' filter" do + query = new_query( + filter: { + "name_text" => { + "matches_query" => { + "query" => "foo", + "allowed_edits_per_term" => enum_value("MatchesQueryAllowedEditsPerTermInput", "DYNAMIC"), + "require_all_terms" => false + } + } + } + ) + + expect(datastore_body_of(query)).to query_datastore_with(bool: {must: [{match: {"name_text" => {query: "foo", fuzziness: "AUTO", operator: "OR"}}}]}) + end + + it "builds a `match` must condition with specified fuzziness when given a `matches_query`: 'MatchesQueryFilterInput' filter" do + query = new_query( + filter: { + "name_text" => { + "matches_query" => { + "query" => "foo", + "allowed_edits_per_term" => enum_value("MatchesQueryAllowedEditsPerTermInput", "NONE"), + "require_all_terms" => false + } + } + } + ) + expect(datastore_body_of(query)).to query_datastore_with(bool: {must: [{match: {"name_text" => {query: "foo", fuzziness: "0", operator: "OR"}}}]}) + + query = new_query( + filter: { + "name_text" => { + "matches_query" => { + "query" => "foo", + "allowed_edits_per_term" => enum_value("MatchesQueryAllowedEditsPerTermInput", "ONE"), + "require_all_terms" => false + } + } + } + ) + expect(datastore_body_of(query)).to query_datastore_with(bool: {must: [{match: {"name_text" => {query: "foo", fuzziness: "1", operator: "OR"}}}]}) + + query = new_query( + filter: { + "name_text" => { + "matches_query" => { + "query" => "foo", + "allowed_edits_per_term" => enum_value("MatchesQueryAllowedEditsPerTermInput", "TWO"), + "require_all_terms" => false + } + } + } + ) + expect(datastore_body_of(query)).to query_datastore_with(bool: {must: [{match: {"name_text" => {query: "foo", fuzziness: "2", operator: "OR"}}}]}) + + query = new_query( + filter: { + "name_text" => { + "matches_query" => { + "query" => "foo", + "allowed_edits_per_term" => enum_value("MatchesQueryAllowedEditsPerTermInput", "DYNAMIC"), + "require_all_terms" => false + } + } + } + ) + expect(datastore_body_of(query)).to query_datastore_with(bool: {must: [{match: {"name_text" => {query: "foo", fuzziness: "AUTO", operator: "OR"}}}]}) + end + + it "builds a `match` must condition with specified operator when given a `matches_query`: 'MatchesQueryFilterInput' filter" do + query = new_query( + filter: { + "name_text" => { + "matches_query" => { + "query" => "foo", + "allowed_edits_per_term" => enum_value("MatchesQueryAllowedEditsPerTermInput", "DYNAMIC"), + "require_all_terms" => true + } + } + } + ) + + expect(datastore_body_of(query)).to query_datastore_with(bool: {must: [{match: {"name_text" => {query: "foo", fuzziness: "AUTO", operator: "AND"}}}]}) + end + + it "builds a `match_phrase_prefix` must condition when given a `matches_phrase`: 'MatchesPhraseFilterInput' filter" do + query = new_query(filter: {"name_text" => {"matches_phrase" => {"phrase" => "foo"}}}) + + expect(datastore_body_of(query)).to query_datastore_with(bool: {must: [{match_phrase_prefix: {"name_text" => {query: "foo"}}}]}) + end + + it "builds a `terms` condition on a nested path when given a deeply nested (3 levels) `equal_to_any_of: [...]` filter" do + query = new_query(filter: {"options" => {"color" => {"red" => {"equal_to_any_of" => [100, 200]}}}}) + + expect(datastore_body_of(query)).to filter_datastore_with(terms: {"options.color.red" => [100, 200]}) + end + + it "builds a `terms` condition on a nested path when given a nested (2 levels) `equal_to_any_of: [...]` filter" do + query = new_query(filter: {"options" => {"size" => {"equal_to_any_of" => [10]}}}) + + expect(datastore_body_of(query)).to filter_datastore_with(terms: {"options.size" => [10]}) + end + + it "supports an `equal_to_any_of` operator on multiple fields, converting them to multiple `terms` conditions" do + query = new_query(filter: {"age" => {"equal_to_any_of" => [25, 30]}, "size" => {"equal_to_any_of" => [10]}}) + + expect(datastore_body_of(query)).to filter_datastore_with( + {terms: {"age" => [25, 30]}}, + {terms: {"size" => [10]}} + ) + end + + describe "`equal_to_any_of` with `[nil]`" do + it "builds a `must_not` `exists` condition when given an `equal_to_any_of: [nil]` filter" do + query = new_query(filter: {"age" => {"equal_to_any_of" => [nil]}}) + + expect(datastore_body_of(query)).to query_datastore_with({ + bool: {must_not: [ + {bool: {filter: [{exists: {"field" => "age"}}]}} + ]} + }) + end + + it "builds a `must_not` `exists` condition when given an `equal_to_any_of: [nil, nil]` filter" do + query = new_query(filter: {"age" => {"equal_to_any_of" => [nil, nil]}}) + + expect(datastore_body_of(query)).to query_datastore_with({ + bool: {must_not: [ + {bool: {filter: [{exists: {"field" => "age"}}]}} + ]} + }) + end + + it "handles `equal_to_any_of: [nil, non_nil_values, ...]` by ORing together multiple conditions (for a field that's not `id`)" do + query = new_query(filter: {"age" => {"equal_to_any_of" => [nil, 25, 40]}}) + + expect(datastore_body_of(query)).to filter_datastore_with({ + bool: {minimum_should_match: 1, should: [ + {bool: {filter: [{terms: {"age" => [25, 40]}}]}}, + {bool: {must_not: [{bool: {filter: [{exists: {"field" => "age"}}]}}]}} + ]} + }) + end + + it "handles `equal_to_any_of: [nil, non_nil_values, ...]` by ORing together multiple conditions (for the `id` field)" do + query = new_query(filter: {"id" => {"equal_to_any_of" => [nil, 25, 40]}}) + + expect(datastore_body_of(query)).to filter_datastore_with({ + bool: {minimum_should_match: 1, should: [ + {bool: {filter: [{ids: {values: [25, 40]}}]}}, + {bool: {must_not: [{bool: {filter: [{exists: {"field" => "id"}}]}}]}} + ]} + }) + end + + it "builds an `exists` condition when given a `not` equal_to_any_of filter" do + query = new_query(filter: {"not" => {"age" => {"equal_to_any_of" => [nil]}}}) + + expect(datastore_body_of(query)).to query_datastore_with({ + bool: {filter: [{exists: {"field" => "age"}}]} + }) + end + + it "builds an `exists` condition when given `equal_to_any_of: [nil]` filter, and does not drop other boolean occurrences" do + query = new_query(filter: { + "age" => {"equal_to_any_of" => [nil]}, + "color" => {"equal_to_any_of" => %w[blue green]} + }) + + expect(datastore_body_of(query)).to query_datastore_with({bool: { + filter: [{terms: {"color" => %w[blue green]}}], + must_not: [{bool: {filter: [{exists: {"field" => "age"}}]}}] + }}) + end + + it "builds an `exists` condition when given `equal_to_any_of: [nil]` filter, and combines must_not occurrences" do + query = new_query(filter: { + "age" => {"equal_to_any_of" => [nil]}, + "color" => {"not" => {"equal_to_any_of" => %w[blue green]}} + }) + + expect(datastore_body_of(query)).to query_datastore_with({bool: { + must_not: [ + {bool: {filter: [{exists: {"field" => "age"}}]}}, + {bool: {filter: [{terms: {"color" => %w[blue green]}}]}} + ] + }}) + end + + it "builds an `exists` condition when given a `not` `equal_to_any_of` filter, and combines it with other boolean occurrences" do + query = new_query(filter: { + "not" => {"age" => {"equal_to_any_of" => [nil]}}, + "color" => {"equal_to_any_of" => %w[blue green]} + }) + + expect(datastore_body_of(query)).to query_datastore_with({bool: { + filter: [ + {exists: {"field" => "age"}}, + {terms: {"color" => %w[blue green]}} + ] + }}) + end + + it "builds an `exists` condition when given a `not` `equal_to_any_of` filter, and does not drop other negated boolean occurrences" do + query = new_query(filter: { + "age" => {"not" => {"equal_to_any_of" => [nil]}}, + "color" => {"not" => {"equal_to_any_of" => %w[blue green]}} + }) + + expect(datastore_body_of(query)).to query_datastore_with({bool: { + filter: [{exists: {"field" => "age"}}], + must_not: [{bool: {filter: [{terms: {"color" => %w[blue green]}}]}}] + }}) + end + + it "builds an `exists` when `equal_to_any_of: [nil]` is the only filter and nested in `any_of`" do + query = new_query(filter: {"any_of" => [ + {"age" => {"equal_to_any_of" => [nil]}} + ]}) + + expect(datastore_body_of(query)).to query_datastore_with({ + bool: { + minimum_should_match: 1, + should: [{bool: {must_not: [{bool: {filter: [{exists: {"field" => "age"}}]}}]}}] + } + }) + end + + it "builds an `exists` when the `equal_to_any_of: [nil]` part is among other filters" do + query = new_query(filter: { + "name_text" => {"matches" => "foo"}, + "age" => {"equal_to_any_of" => [nil]}, + "currency" => {"equal_to_any_of" => ["USD"]} + }) + + expect(datastore_body_of(query)).to query_datastore_with({ + bool: { + filter: [{terms: {"currency" => ["USD"]}}], + must: [{match: {"name_text" => "foo"}}], + must_not: [{bool: {filter: [{exists: {"field" => "age"}}]}}] + } + }) + end + end + + describe "`all_of` operator" do + it "can be used to wrap multiple `any_satisfy` expressions to require multiple sub-filters to be satisfied by a list element" do + query = new_query(filter: { + "tags" => {"all_of" => [ + {"any_satisfy" => {"equal_to_any_of" => ["a", "b"]}}, + {"any_satisfy" => {"equal_to_any_of" => ["c", "d"]}} + ]} + }) + + expect(datastore_body_of(query)).to query_datastore_with({bool: {filter: [ + {terms: {"tags" => ["a", "b"]}}, + {terms: {"tags" => ["c", "d"]}} + ]}}) + end + + it "is ignored when `null` is passed" do + query = new_query(filter: {"tags" => {"all_of" => nil}}) + + expect(datastore_body_of(query)).to not_filter_datastore_at_all + end + + it "is ignored when `[]` is passed" do + query = new_query(filter: {"tags" => {"all_of" => []}}) + + expect(datastore_body_of(query)).to not_filter_datastore_at_all + end + end + + describe "`any_satisfy` operator" do + context "on a list-of-scalars field" do + it "returns the body of the sub-filters because the semantics indicated by `any_satisfy` is what the datastore automatically provides on list fields" do + query1 = new_query(filter: { + "tags" => {"any_satisfy" => {"equal_to_any_of" => ["a", "b"]}}, + "ages" => {"any_satisfy" => {"gt" => 30}} + }) + + query2 = new_query(filter: { + "tags" => {"equal_to_any_of" => ["a", "b"]}, + "ages" => {"gt" => 30} + }) + + expect(datastore_body_of(query1)).to eq(datastore_body_of(query2)) + end + + it "merges multiple range operators into a single clause to force a single value to satisfy all (rather than separate values satisfying each)" do + query = new_query(filter: { + "ages" => {"any_satisfy" => {"gt" => 30, "lt" => 60}} + }) + + expect(datastore_body_of(query)).to query_datastore_with({bool: {filter: [{range: {"ages" => { + gt: 30, + lt: 60 + }}}]}}) + end + + it "can be used within an `all_of` to require multiple sub-filters to be satisfied by a list element" do + query = new_query(filter: { + "tags" => {"all_of" => [ + {"any_satisfy" => {"equal_to_any_of" => ["a", "b"]}}, + {"any_satisfy" => {"equal_to_any_of" => ["c", "d"]}} + ]} + }) + + expect(datastore_body_of(query)).to query_datastore_with({bool: {filter: [ + {terms: {"tags" => ["a", "b"]}}, + {terms: {"tags" => ["c", "d"]}} + ]}}) + end + + context "when using `snake_case` schema names" do + let(:graphql) { build_graphql(schema_element_name_form: :snake_case) } + + it "rejects a query that produces multiple query clauses under `any_satisfy` because the datastore does not require a single value to match them all" do + query = new_query(filter: { + # We don't expect users to send us a filter like this, but if they did, we can't support it. + "ages" => {"any_satisfy" => {"gt" => 30, "equal_to_any_of" => [50]}} + }) + + expect { + datastore_body_of(query) + }.to raise_error ::GraphQL::ExecutionError, a_string_including( + "`any_satisfy: {gt: 30, equal_to_any_of: [50]}` is not supported because it produces multiple filtering clauses under `any_satisfy`" + ) + end + end + + context "when using `camelCase` schema names" do + let(:graphql) { build_graphql(schema_element_name_form: :camelCase) } + + it "rejects a query that produces multiple query clauses under `any_satisfy` because the datastore does not require a single value to match them all" do + query = new_query(filter: { + # We don't expect users to send us a filter like this, but if they did, we can't support it. + "ages" => {"anySatisfy" => {"gt" => 30, "equalToAnyOf" => [50]}} + }) + + expect { + datastore_body_of(query) + }.to raise_error ::GraphQL::ExecutionError, a_string_including( + "`anySatisfy: {gt: 30, equalToAnyOf: [50]}` is not supported because it produces multiple filtering clauses under `anySatisfy`" + ) + end + end + + it "still allows multiple query clauses under `any_satisfy: {any_of: [...]}}` because that has OR semantics" do + query = new_query(filter: { + "ages" => {"any_satisfy" => {"any_of" => [{"gt" => 30}, {"equal_to_any_of" => [50]}]}} + }) + + expect(datastore_body_of(query)).to query_datastore_with({bool: {minimum_should_match: 1, should: [ + {bool: {filter: [{range: {"ages" => {gt: 30}}}]}}, + {bool: {filter: [{terms: {"ages" => [50]}}]}} + ]}}) + end + + it "does not allow `any_of` alongside another filter because that would also produce multiple query clauses that we can't support" do + query = new_query(filter: { + "ages" => {"any_satisfy" => {"any_of" => [{"gt" => 30}], "equal_to_any_of" => [50]}} + }) + + expect { + datastore_body_of(query) + }.to raise_error ::GraphQL::ExecutionError, a_string_including( + "`any_satisfy: {any_of: [{gt: 30}], equal_to_any_of: [50]}` is not supported because it produces multiple filtering clauses under `any_satisfy`" + ) + end + + it "returns the standard always false filter for `any_satisfy: {any_of: []}`" do + query = new_query(filter: { + "ages" => {"any_satisfy" => {"any_of" => []}} + }) + + expect(datastore_body_of(query)).to query_datastore_with(always_false_condition) + end + + it "applies no filtering when given `any_satisfy: {}`" do + query = new_query(filter: { + "ages" => {"any_satisfy" => {}} + }) + + expect(datastore_body_of(query)).to not_filter_datastore_at_all + end + end + + context "on a list-of-nested-objects field" do + it "builds a `nested` filter" do + query = new_query(filter: { + "people" => {"friends" => {"any_satisfy" => {"age" => {"gt" => 30}}}} + }) + + expect(datastore_body_of(query)).to filter_datastore_with({nested: { + path: "people.friends", + query: {bool: {filter: [{ + range: {"people.friends.age" => {gt: 30}} + }]}} + }}) + end + + it "correctly builds a `nested` filter when `any_satisfy: {not: ...}` is used" do + query = new_query(filter: { + "people" => {"friends" => {"any_satisfy" => {"not" => {"age" => {"gt" => 30}}}}} + }) + + expect(datastore_body_of(query)).to filter_datastore_with({nested: { + path: "people.friends", + query: {bool: {must_not: [{bool: {filter: [{ + range: {"people.friends.age" => {gt: 30}} + }]}}]}} + }}) + end + + it "correctly builds a `nested` filter when `any_satisfy: {field: {any_satisfy: ...}}}` is used" do + query = new_query(filter: { + "line_items" => {"any_satisfy" => {"tags" => {"any_satisfy" => {"equal_to_any_of" => ["a"]}}}} + }) + + expect(datastore_body_of(query)).to filter_datastore_with({nested: { + path: "line_items", + query: {bool: {filter: [{ + terms: {"line_items.tags" => ["a"]} + }]}} + }}) + end + + it "correctly builds a `nested` filter when `any_satisfy: {any_of: [{field: ...}]}` is used" do + query = new_query(filter: { + "line_items" => {"any_satisfy" => {"any_of" => [{"name" => {"equal_to_any_of" => ["a"]}}]}} + }) + + expect(datastore_body_of(query)).to filter_datastore_with({nested: { + path: "line_items", + query: {bool: { + minimum_should_match: 1, + should: [{bool: {filter: [{ + terms: {"line_items.name" => ["a"]} + }]}}] + }} + }}) + end + + it "builds an empty filter when given `any_satisfy: {field: {}}`" do + query = new_query(filter: { + "line_items" => {"any_satisfy" => {"name" => {}}} + }) + + expect(datastore_body_of(query)).to not_filter_datastore_at_all + end + + it "builds an empty filter when given `any_satisfy: {field: {predicate: nil}}`" do + query = new_query(filter: { + "line_items" => {"any_satisfy" => {"name" => {"equal_to_any_of" => nil}}} + }) + + expect(datastore_body_of(query)).to not_filter_datastore_at_all + end + end + end + + # Note: a `count` filter gets translated into `__counts` (to distinguish it from a schema field named `count`), + # and that translation happens as the query is being built, so we use `__counts` in our example filters here. + describe "`count` operator on a list" do + it "builds a query on the hidden `#{LIST_COUNTS_FIELD}` field where we have indexed list counts" do + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"gt" => 10}} + }) + + expect(datastore_body_of(query)).to filter_datastore_with(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {gt: 10} + }) + end + + it "correctly references a list field embedded on an object field" do + query = new_query(filter: { + "details" => {"uniform_colors" => {LIST_COUNTS_FIELD => {"gt" => 10}}} + }) + + expect(datastore_body_of(query)).to filter_datastore_with(range: { + "#{LIST_COUNTS_FIELD}.details|uniform_colors" => {gt: 10} + }) + end + + it "correctly references a list field under a list-of-nested-objects field" do + query = new_query(filter: { + "seasons_nested" => {"any_satisfy" => {"notes" => {LIST_COUNTS_FIELD => {"gt" => 10}}}} + }) + + expect(datastore_body_of(query)).to filter_datastore_with({nested: { + path: "seasons_nested", + query: {bool: {filter: [{ + range: {"seasons_nested.#{LIST_COUNTS_FIELD}.notes" => {gt: 10}} + }]}} + }}) + end + + it "treats a filter on a `count` schema field like a filter on any other schema field" do + query = new_query(filter: { + "past_names" => {"count" => {"gt" => 10}} + }) + + expect(datastore_body_of(query)).to filter_datastore_with(range: { + "past_names.count" => {gt: 10} + }) + end + + describe "including an extra must_not exists filter for a predicate that matches zero in order to match documents indexed before the list field got defined" do + it "correctly detects when an `lt` expression could match zero" do + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"lt" => 10}} + }) + expect(datastore_body_of(query)).to filter_datastore_with_must_not_exists_or(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {lt: 10} + }) + + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"lt" => 1}} + }) + expect(datastore_body_of(query)).to filter_datastore_with_must_not_exists_or(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {lt: 1} + }) + + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"lt" => 0}} + }) + expect(datastore_body_of(query)).to filter_datastore_with(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {lt: 0} + }) + + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"lt" => -1}} + }) + expect(datastore_body_of(query)).to filter_datastore_with(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {lt: -1} + }) + end + + it "correctly detects when an `lte` expression could match zero" do + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"lte" => 10}} + }) + expect(datastore_body_of(query)).to filter_datastore_with_must_not_exists_or(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {lte: 10} + }) + + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"lte" => 1}} + }) + expect(datastore_body_of(query)).to filter_datastore_with_must_not_exists_or(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {lte: 1} + }) + + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"lte" => 0}} + }) + expect(datastore_body_of(query)).to filter_datastore_with_must_not_exists_or(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {lte: 0} + }) + + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"lte" => -1}} + }) + expect(datastore_body_of(query)).to filter_datastore_with(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {lte: -1} + }) + end + + it "correctly detects when an `gt` expression could match zero" do + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"gt" => -10}} + }) + expect(datastore_body_of(query)).to filter_datastore_with_must_not_exists_or(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {gt: -10} + }) + + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"gt" => -1}} + }) + expect(datastore_body_of(query)).to filter_datastore_with_must_not_exists_or(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {gt: -1} + }) + + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"gt" => 0}} + }) + expect(datastore_body_of(query)).to filter_datastore_with(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {gt: 0} + }) + + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"gt" => 1}} + }) + expect(datastore_body_of(query)).to filter_datastore_with(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {gt: 1} + }) + end + + it "correctly detects when an `gte` expression could match zero" do + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"gte" => -10}} + }) + expect(datastore_body_of(query)).to filter_datastore_with_must_not_exists_or(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {gte: -10} + }) + + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"gte" => -1}} + }) + expect(datastore_body_of(query)).to filter_datastore_with_must_not_exists_or(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {gte: -1} + }) + + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"gte" => 0}} + }) + expect(datastore_body_of(query)).to filter_datastore_with_must_not_exists_or(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {gte: 0} + }) + + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"gte" => 1}} + }) + expect(datastore_body_of(query)).to filter_datastore_with(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {gte: 1} + }) + end + + it "correctly detects when an `equal_to_any_of` could match zero" do + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"equal_to_any_of" => [3, 0, 5]}} + }) + expect(datastore_body_of(query)).to filter_datastore_with_must_not_exists_or(terms: { + "#{LIST_COUNTS_FIELD}.past_names" => [3, 0, 5] + }) + + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"equal_to_any_of" => [3, 5]}} + }) + expect(datastore_body_of(query)).to filter_datastore_with(terms: { + "#{LIST_COUNTS_FIELD}.past_names" => [3, 5] + }) + end + + it "correct detects when an expression with multiple predicates could match zero" do + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"gte" => 0, "lt" => 10}} + }) + expect(datastore_body_of(query)).to filter_datastore_with_must_not_exists_or(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {gte: 0, lt: 10} + }) + + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"gte" => 1, "lt" => 10}} + }) + expect(datastore_body_of(query)).to filter_datastore_with(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {gte: 1, lt: 10} + }) + + query = new_query(filter: { + "past_names" => {LIST_COUNTS_FIELD => {"gte" => 0, "lt" => 0}} + }) + expect(datastore_body_of(query)).to filter_datastore_with(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {gte: 0, lt: 0} + }) + end + + it "ignores `count` filter predicates that have a `nil` value" do + query = new_query(filter: {"past_names" => {LIST_COUNTS_FIELD => {"gt" => nil}}}) + expect(datastore_body_of(query)).to not_filter_datastore_at_all + + query = new_query(filter: {"past_names" => {LIST_COUNTS_FIELD => {"gte" => nil}}}) + expect(datastore_body_of(query)).to not_filter_datastore_at_all + + query = new_query(filter: {"past_names" => {LIST_COUNTS_FIELD => {"lt" => nil}}}) + expect(datastore_body_of(query)).to not_filter_datastore_at_all + + query = new_query(filter: {"past_names" => {LIST_COUNTS_FIELD => {"lte" => nil}}}) + expect(datastore_body_of(query)).to not_filter_datastore_at_all + + query = new_query(filter: {"past_names" => {LIST_COUNTS_FIELD => {"equal_to_any_of" => nil}}}) + expect(datastore_body_of(query)).to not_filter_datastore_at_all + + query = new_query(filter: {"past_names" => {LIST_COUNTS_FIELD => {"gt" => nil, "lt" => 10}}}) + expect(datastore_body_of(query)).to filter_datastore_with_must_not_exists_or(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {lt: 10} + }) + + query = new_query(filter: {"past_names" => {LIST_COUNTS_FIELD => {"gt" => 10, "lt" => nil}}}) + expect(datastore_body_of(query)).to filter_datastore_with(range: { + "#{LIST_COUNTS_FIELD}.past_names" => {gt: 10} + }) + end + + def filter_datastore_with_must_not_exists_or(other) + filter_datastore_with(bool: { + should: [ + {bool: {filter: [other]}}, + {bool: {must_not: [{bool: {filter: [ + {exists: {"field" => "#{LIST_COUNTS_FIELD}.past_names"}} + ]}}]}} + ], + minimum_should_match: 1 + }) + end + end + end + + describe "`near` operator" do + datastore_abbreviations_by_distance_unit = { + "MILE" => "mi", + "FOOT" => "ft", + "INCH" => "in", + "YARD" => "yd", + "KILOMETER" => "km", + "METER" => "m", + "CENTIMETER" => "cm", + "MILLIMETER" => "mm", + "NAUTICAL_MILE" => "nmi" + } + + shared_examples_for "`near` filtering with all distance units" do |enum_type_name| + let(:distance_unit_enum) { graphql.schema.type_named(enum_type_name) } + + specify "the examples here cover all `#{enum_type_name}` values" do + expect( + graphql.runtime_metadata.enum_types_by_name.fetch(enum_type_name).values_by_name.keys + ).to match_array(datastore_abbreviations_by_distance_unit.keys) + end + + datastore_abbreviations_by_distance_unit.each do |distance_unit, datastore_abbreviation| + it "supports filtering using the `#{distance_unit}` distance unit" do + query = new_query(filter: {"address_location" => {"near" => { + "latitude" => 37.5, + "longitude" => 67.5, + "max_distance" => 500, + "unit" => distance_unit_enum.enum_value_named(distance_unit) + }}}) + + expect(datastore_body_of(query)).to filter_datastore_with( + geo_distance: { + "distance" => "500#{datastore_abbreviation}", + "address_location" => { + "lat" => 37.5, + "lon" => 67.5 + } + } + ) + end + end + end + + context "when using the standard `InputEnum` naming format" do + include_examples "`near` filtering with all distance units", "DistanceUnitInput" + end + + context "when configured to use an alternate `InputEnum` naming format" do + attr_accessor :schema_artifacts + + before(:context) do + self.schema_artifacts = generate_schema_artifacts(derived_type_name_formats: {InputEnum: "%{base}InputAlt"}) + end + + let(:graphql) { build_graphql(schema_artifacts: schema_artifacts) } + include_examples "`near` filtering with all distance units", "DistanceUnitInputAlt" + end + end + + context "when there are no `GeoLocation` fields" do + let(:graphql) do + schema_artifacts = build_datastore_core.schema_artifacts + runtime_meta = schema_artifacts.runtime_metadata.with( + enum_types_by_name: schema_artifacts.runtime_metadata.enum_types_by_name.except("DistanceUnit") + ) + + allow(schema_artifacts).to receive(:runtime_metadata).and_return(runtime_meta) + build_graphql(schema_artifacts: schema_artifacts) + end + + specify "the `near` filter implementation doesn't fail due to a lack of a `DistanceUnit` type" do + expect(graphql.runtime_metadata.enum_types_by_name.keys).to exclude("DistanceUnit") + + query = new_query(filter: {"id" => {"equal_to_any_of" => ["a"]}}) + + expect(datastore_body_of(query)).to filter_datastore_with({ids: {values: ["a"]}}) + end + end + + describe "`time_of_day` operator" do + it "uses a `script` query, passing along the `gte` param" do + query = new_query(filter: {"timestamps" => {"created_at" => {"time_of_day" => {"gte" => "07:00:00"}}}}) + + expect_script_query_with_params(query, { + field: "timestamps.created_at", + gte: "07:00:00" + }) + end + + it "uses a `script` query, passing along the `gt` param" do + query = new_query(filter: {"timestamps" => {"created_at" => {"time_of_day" => {"gt" => "07:00:00"}}}}) + + expect_script_query_with_params(query, { + field: "timestamps.created_at", + gt: "07:00:00" + }) + end + + it "uses a `script` query, passing along the `lte` param" do + query = new_query(filter: {"timestamps" => {"created_at" => {"time_of_day" => {"lte" => "07:00:00"}}}}) + + expect_script_query_with_params(query, { + field: "timestamps.created_at", + lte: "07:00:00" + }) + end + + it "uses a `script` query, passing along the `lt` param" do + query = new_query(filter: {"timestamps" => {"created_at" => {"time_of_day" => {"lt" => "07:00:00"}}}}) + + expect_script_query_with_params(query, { + field: "timestamps.created_at", + lt: "07:00:00" + }) + end + + it "uses a `script` query, passing along the `equal_to_any_of` param" do + query = new_query(filter: {"timestamps" => {"created_at" => {"time_of_day" => {"equal_to_any_of" => ["07:00:00", "08:00:00"]}}}}) + + expect_script_query_with_params(query, { + field: "timestamps.created_at", + equal_to_any_of: ["07:00:00", "08:00:00"] + }) + end + + it "uses a `script` query, passing along the `time_zone` param" do + query = new_query(filter: {"timestamps" => {"created_at" => {"time_of_day" => { + "time_zone" => "America/Los_Angeles", + # We have to include at least one comparison operator so that the filter is included in the datastore body. + "gt" => "08:00:00" + }}}}) + + expect_script_query_with_params(query, { + field: "timestamps.created_at", + time_zone: "America/Los_Angeles", + gt: "08:00:00" + }) + end + + it "omits the filter from the query payload when no operators are provided" do + query = new_query(filter: {"timestamps" => {"created_at" => {"time_of_day" => {"time_zone" => "America/Los_Angeles"}}}}) + + expect(datastore_body_of(query)).to not_filter_datastore_at_all + end + + context "when configured to use different schema element names" do + let(:graphql) do + build_graphql(schema_element_name_overrides: { + gt: "greater", + lt: "lesser", + gte: "greater_or_equal", + lte: "lesser_or_equal", + equal_to_any_of: "in", + time_zone: "tz", + time_of_day: "time_part" + }) + end + + it "recognizes the alternate schema element names but uses our standard names in the script params because the script expects that" do + query = new_query(filter: {"timestamps" => {"created_at" => {"time_part" => { + "tz" => "America/Los_Angeles", + "greater" => "01:00:00", + "lesser" => "02:00:00", + "greater_or_equal" => "03:00:00", + "lesser_or_equal" => "04:00:00", + "in" => ["05:00:00"] + }}}}) + + expect_script_query_with_params(query, { + field: "timestamps.created_at", + time_zone: "America/Los_Angeles", + gt: "01:00:00", + lt: "02:00:00", + gte: "03:00:00", + lte: "04:00:00", + equal_to_any_of: ["05:00:00"] + }) + end + end + + def expect_script_query_with_params(query, expected_params) + body = datastore_body_of(query) + script_args = body.dig(:query, :bool, :filter, 0, :script, :script) + + expect(script_args.keys).to include(:id, :params) + expect(script_args[:id]).to start_with("filter_by_time_of_day_") + + expected_params_in_nanos = expected_params.transform_values do |value| + case value + when /^\d\d:/ + Support::TimeUtil.nano_of_day_from_local_time(value) + when ::Array + value.map { |v| Support::TimeUtil.nano_of_day_from_local_time(v) } + else + value + end + end + + expect(script_args[:params]).to eq(expected_params_in_nanos) + end + end + + it "supports an `any_of` operator" do + query = new_query(filter: { + "any_of" => [ + { + "transaction_type" => {"equal_to_any_of" => ["CARD"]}, + "total_amount" => { + "any_of" => [ + {"amount" => {"gt" => 10000}}, + { + "amount" => {"gt" => 1000}, + "currency" => {"equal_to_any_of" => ["USD"]} + } + ] + } + }, + { + "transaction_type" => {"equal_to_any_of" => ["CASH"]} + } + ] + }) + + expect(datastore_body_of(query)).to query_datastore_with({ + bool: {minimum_should_match: 1, should: [ + { + bool: { + filter: [ + {terms: {"transaction_type" => ["CARD"]}}, + {bool: {minimum_should_match: 1, should: [ + {bool: {filter: [ + {range: {"total_amount.amount" => {gt: 10000}}} + ]}}, + {bool: {filter: [ + {range: {"total_amount.amount" => {gt: 1000}}}, + {terms: {"total_amount.currency" => ["USD"]}} + ]}} + ]}} + ] + } + }, + { + bool: {filter: [ + {terms: {"transaction_type" => ["CASH"]}} + ]} + } + ]} + }) + end + + it "handles `any_of` used at nested cousin nodes correctly" do + query = new_query(filter: { + "cost" => { + "any_of" => [ + {"currency" => {"equal_to_any_of" => ["USD"]}}, + {"amount_cents" => {"gt" => 100}} + ] + }, + "options" => { + "any_of" => [ + {"size" => {"equal_to_any_of" => [size_of("MEDIUM")]}}, + {"color" => {"equal_to_any_of" => [color_of("RED")]}} + ] + } + }) + + expect(datastore_body_of(query)).to query_datastore_with({bool: {filter: [ + { + bool: { + minimum_should_match: 1, should: [ + {bool: {filter: [ + {terms: {"cost.currency" => ["USD"]}} + ]}}, + {bool: {filter: [ + {range: {"cost.amount_cents" => {gt: 100}}} + ]}} + ] + } + }, + { + bool: { + minimum_should_match: 1, should: [ + {bool: {filter: [ + {terms: {"options.size" => ["MEDIUM"]}} + ]}}, + {bool: {filter: [ + {terms: {"options.color" => ["RED"]}} + ]}} + ] + } + } + ]}}) + end + + describe "`not`" do + it "negates the inner filter expression, regardless of where the `not` goes" do + body_for_inner_not = datastore_body_of(new_query(filter: {"age" => {"not" => { + "equal_to_any_of" => [25, 30] + }}})) + + body_for_outer_not = datastore_body_of(new_query(filter: {"not" => {"age" => { + "equal_to_any_of" => [25, 30] + }}})) + + expect(body_for_inner_not).to eq(body_for_outer_not).and query_datastore_with({bool: {must_not: [{bool: {filter: [{terms: {"age" => [25, 30]}}]}}]}}) + end + + it "can negate multiple inner filter predicates" do + body_for_inner_not = datastore_body_of(new_query(filter: {"age" => {"not" => { + "gte" => 30, + "lt" => 25 + }}})) + + body_for_outer_not = datastore_body_of(new_query(filter: {"not" => {"age" => { + "gte" => 30, + "lt" => 25 + }}})) + + expect(body_for_inner_not).to eq(body_for_outer_not).and query_datastore_with({bool: {must_not: [{bool: {filter: [ + {range: {"age" => {gte: 30, lt: 25}}} + ]}}]}}) + end + + it "negates a complex compound inner filter expression" do + query = new_query(filter: {"not" => { + "any_of" => [ + { + "transaction_type" => {"equal_to_any_of" => ["CARD"]}, + "total_amount" => { + "any_of" => [ + {"amount" => {"gt" => 10000}}, + { + "amount" => {"gt" => 1000}, + "currency" => {"equal_to_any_of" => ["USD"]} + } + ] + } + }, + { + "transaction_type" => {"equal_to_any_of" => ["CASH"]} + } + ] + }}) + + expect(datastore_body_of(query)).to query_datastore_with({ + bool: {must_not: [{bool: {minimum_should_match: 1, should: [ + { + bool: { + filter: [ + {terms: {"transaction_type" => ["CARD"]}}, + {bool: {minimum_should_match: 1, should: [ + {bool: {filter: [ + {range: {"total_amount.amount" => {gt: 10000}}} + ]}}, + {bool: {filter: [ + {range: {"total_amount.amount" => {gt: 1000}}}, + {terms: {"total_amount.currency" => ["USD"]}} + ]}} + ]}} + ] + } + }, + { + bool: {filter: [ + {terms: {"transaction_type" => ["CASH"]}} + ]} + } + ]}}]} + }) + end + + it "works correctly when included alongside other filtering operators" do + body_for_inner_not = datastore_body_of(new_query(filter: {"age" => { + "not" => {"equal_to_any_of" => [15, 30]}, + "gte" => 20 + }})) + + body_for_outer_not = datastore_body_of(new_query(filter: { + "not" => { + "age" => {"equal_to_any_of" => [15, 30]} + }, + "age" => { + "gte" => 20 + } + })) + + expect(body_for_inner_not).to eq(body_for_outer_not).and query_datastore_with({bool: { + filter: [{range: {"age" => {gte: 20}}}], + must_not: [{bool: {filter: [{terms: {"age" => [15, 30]}}]}}] + }}) + end + + it "works correctly when included alongside an `any_of`" do + body_for_inner_not = datastore_body_of(new_query(filter: {"age" => { + "not" => {"equal_to_any_of" => [15, 50]}, + "any_of" => [ + {"gt" => 35}, + {"lt" => 20} + ] + }})) + + body_for_outer_not = datastore_body_of(new_query(filter: { + "not" => { + "age" => {"equal_to_any_of" => [15, 50]} + }, + "age" => { + "any_of" => [ + {"gt" => 35}, + {"lt" => 20} + ] + } + })) + + # The location of the `filter: [{bool: }]` wrapper differs between `body_for_inner_not` and `body_for_outer_not` but does not impact behavior. + # These two queries yield the same results - confirmed by the integration test "`not` works correctly when included alongside an `any_of`", + # so even though the behavior is the same, we must assert on the two payloads separately. + expect(body_for_inner_not).to query_datastore_with({bool: {filter: [{bool: { + must_not: [ + {bool: {filter: [{terms: {"age" => [15, 50]}}]}} + ], + should: [ + {bool: {filter: [{range: {"age" => {gt: 35}}}]}}, + {bool: {filter: [{range: {"age" => {lt: 20}}}]}} + ], + minimum_should_match: 1 + }}]}}) + + expect(body_for_outer_not).to query_datastore_with({bool: { + must_not: [ + {bool: {filter: [{terms: {"age" => [15, 50]}}]}} + ], + filter: [ + {bool: { + should: [ + {bool: {filter: [{range: {"age" => {gt: 35}}}]}}, + {bool: {filter: [{range: {"age" => {lt: 20}}}]}} + ], + minimum_should_match: 1 + }} + ] + }}) + end + + it "is ignored when set to nil" do + body_for_inner_not = datastore_body_of(new_query(filter: {"age" => {"not" => nil}})) + body_for_outer_not = datastore_body_of(new_query(filter: {"not" => {"age" => nil}})) + + expect(body_for_inner_not).to eq(body_for_outer_not).and not_filter_datastore_at_all + end + + it "is ignored when set to nil when alongside other filters" do + body_for_inner_not = datastore_body_of(new_query(filter: {"age" => { + "not" => nil, + "gt" => 25 + }})) + + body_for_outer_not = datastore_body_of(new_query(filter: { + "not" => nil, + "age" => { + "gt" => 25 + } + })) + + expect(body_for_inner_not).to eq(body_for_outer_not).and query_datastore_with({bool: {filter: [{range: {"age" => {gt: 25}}}]}}) + end + + it "is ignored when the inner filter is also ignored" do + body_for_inner_not = datastore_body_of(new_query(filter: {"age" => {"not" => {"equal_to_any_of" => nil}}})) + body_for_outer_not = datastore_body_of(new_query(filter: {"not" => {"age" => {"equal_to_any_of" => nil}}})) + + expect(body_for_inner_not).to eq(body_for_outer_not).and not_filter_datastore_at_all + end + end + + describe "behavior of empty/null filter values" do + it "prunes out filtering predicates that are empty no-ops on root fields" do + query = new_query(filter: { + "age" => {"gt" => nil}, + "name" => {"equal_to_any_of" => ["Jane"]}, + "height" => {"gte" => nil, "lt" => nil}, + "id" => {} + }) + + expect(datastore_body_of(query)).to filter_datastore_with(terms: {"name" => ["Jane"]}) + end + + it "prunes out filtering predicates that are empty no-ops on subfields" do + query = new_query(filter: { + "bio" => { + "age" => {"gt" => nil}, + "name" => {"equal_to_any_of" => ["Jane"]}, + "height" => {"gte" => nil, "lt" => nil}, + "id" => {} + } + }) + + expect(datastore_body_of(query)).to filter_datastore_with(terms: {"bio.name" => ["Jane"]}) + end + + it "does not filter at all when all predicate are empty/null on root fields" do + query = new_query(filter: { + "age" => {"gt" => nil}, + "name" => {"equal_to_any_of" => nil}, + "height" => {"gte" => nil, "lt" => nil}, + "id" => {} + }) + + expect(datastore_body_of(query)).to not_filter_datastore_at_all + end + + it "does not filter at all when all predicate are empty/null on subfields" do + query = new_query(filter: { + "bio" => { + "age" => {"gt" => nil}, + "name" => {"equal_to_any_of" => nil}, + "height" => {"gte" => nil, "lt" => nil}, + "id" => {} + } + }) + + expect(datastore_body_of(query)).to not_filter_datastore_at_all + end + + it "does not prune out `equal_to_any_of: []` to be consistent with `equal_to_any_of` being like an `IN` in SQL and `where field IN ()` should match nothing" do + query = new_query(filter: { + "name" => {"equal_to_any_of" => []} + }) + + expect(datastore_body_of(query)).to filter_datastore_with(terms: {"name" => []}) + end + + it "does not prune out `any_of: []` to be consistent with `equal_to_any_of: []`, instead providing an 'always false' condition to achieve the same behavior" do + query = new_query(filter: { + "age" => {"gt" => 18}, + "name" => {"any_of" => []} + }) + + expect(datastore_body_of(query)).to query_datastore_with(bool: {filter: [ + {range: {"age" => {gt: 18}}}, + always_false_condition + ]}) + end + + it "reduces an `any_of` composed entirely of empty predicates to a false condition" do + query = new_query(filter: { + "age" => {"any_of" => [{"gt" => nil}, {"lt" => nil}]} + }) + + expect(datastore_body_of(query)).to filter_datastore_with(always_false_condition) + end + + it "does not filter at all when given only `any_of: nil` on a root field" do + query = new_query(filter: { + "age" => {"any_of" => nil} + }) + + expect(datastore_body_of(query)).to not_filter_datastore_at_all + end + + it "does not filter at all when given only `any_of: nil` on a subfield" do + query = new_query(filter: { + "bio" => {"age" => {"any_of" => nil}} + }) + + expect(datastore_body_of(query)).to not_filter_datastore_at_all + end + + # Note: the GraphQL schema does not allow `any_of: {}` (`any_of` is a list field). However, we're testing + # it here for completeness--as a defense-in-depth measure, it's good for the filter interpreter to handle + # whatever is thrown at it. Including these tests allows us to exercise an edge case in the code that + # can't otherwise be exercised. + describe "`any_of: {}` (which the GraphQL schema does not allow)" do + it "does not filter at all when given on a root field" do + query = new_query(filter: { + "age" => {"any_of" => {}} + }) + + expect(datastore_body_of(query)).to not_filter_datastore_at_all + end + + it "does not filter at all when given on a subfield" do + query = new_query(filter: { + "bio" => {"age" => {"any_of" => {}}} + }) + + expect(datastore_body_of(query)).to not_filter_datastore_at_all + end + end + end + + def not_filter_datastore_at_all + exclude(:query) + end + + def filter_datastore_with(*filters) + # `filter` uses the datastore's filtering context + query_datastore_with({bool: {filter: filters}}) + end + + def enum_value(type_name, value_name) + graphql.schema.type_named(type_name).enum_value_named(value_name) + end + + def size_of(value_name) + enum_value("SizeInput", value_name) + end + + def color_of(value_name) + enum_value("ColorInput", value_name) + end + + # Custom matcher that provides nicer, more detailed failure output than what built-in RSpec matchers provide. + matcher :query_datastore_with do |expected_query| + match do |datastore_body| + @datastore_body = datastore_body + @expected_query = expected_query + expect(datastore_body[:query]).to eq(expected_query) + end + + # :nocov: -- only covered when an expectation fails + def expected_json + @expected_json ||= ::JSON.pretty_generate(normalize(@expected_query)) + end + + def actual_json + @actual_json ||= ::JSON.pretty_generate(normalize(@datastore_body[:query])) + end + + failure_message do + <<~EOS + Expected `query` in datastore body[^1] to use a specific query[^2]. + + [^1] Actual: + #{actual_json} + + + [^2] Expected: + #{expected_json} + + + Diff: #{generate_git_diff} + EOS + end + + # RSpec's differ doesn't ignore whitespace, but we want our diffs here to do so to make it easier to read the output. + def generate_git_diff + ::Tempfile.create do |expected_file| + expected_file.write(expected_json + "\n") + expected_file.fsync + + ::Tempfile.create do |actual_file| + actual_file.write(actual_json + "\n") + actual_file.fsync + + `git diff --no-index #{actual_file.path} #{expected_file.path} --ignore-all-space #{" --color" unless ENV["CI"]}` + .gsub(expected_file.path, "/expected") + .gsub(actual_file.path, "/actual") + end + end + end + + # We sort hashes by their key so that ordering differences don't show up in the textual diff. + def normalize(value) + case value + when ::Hash + value.sort_by { |k, v| k }.to_h { |k, v| [k, normalize(v)] } + when ::Array + value.map { |v| normalize(v) } + else + value + end + end + # :nocov: + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/individual_docs_needed_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/individual_docs_needed_spec.rb new file mode 100644 index 00000000..ef8f566f --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/individual_docs_needed_spec.rb @@ -0,0 +1,36 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_unit_support" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "#individual_docs_needed" do + include_context "DatastoreQueryUnitSupport" + + it "defaults `individual_docs_needed` to `false` if there are no requested fields" do + query = new_query(requested_fields: []) + + expect(query.individual_docs_needed).to be false + end + + it "allows `individual_docs_needed` to be forced to `true` by the caller" do + query = new_query(requested_fields: [], individual_docs_needed: true) + expect(query.individual_docs_needed).to be true + end + + it "forces `individual_docs_needed` to `true` if there are requested field, because we will not get back the requested fields if we do not fetch documents" do + query = new_query(requested_fields: ["id"]) + expect(query.individual_docs_needed).to be true + + query = new_query(requested_fields: ["id"], individual_docs_needed: false) + expect(query.individual_docs_needed).to be true + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/merge_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/merge_spec.rb new file mode 100644 index 00000000..0db38324 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/merge_spec.rb @@ -0,0 +1,328 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_unit_support" +require "support/sort" +require "support/aggregations_helpers" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "#merge" do + include SortSupport, AggregationsHelpers + include_context "DatastoreQueryUnitSupport" + + before(:context) do + # These are derived from app state and don't vary in two different queries for the same app, + # so we don't have to deal with merging them. + app_level_attributes = %i[ + logger filter_interpreter routing_picker index_expression_builder + default_page_size max_page_size schema_element_names + ] + + @attributes_needing_merge_test_coverage = (DatastoreQuery.members - app_level_attributes).to_set + end + + before(:example) do |ex| + Array(ex.metadata[:covers]).each do |attribute| + @attributes_needing_merge_test_coverage.delete(attribute) + end + end + + after(:context) do + expect(@attributes_needing_merge_test_coverage).to be_empty, "`#merge` tests are expected to cover all attributes, " \ + "but the following do not appear to have coverage: #{@attributes_needing_merge_test_coverage}" + end + + it "throws exception if attempting to merge two queries with different `search_index_definitions` values", covers: :search_index_definitions do + widgets_def = graphql.datastore_core.index_definitions_by_name.fetch("widgets") + components_def = graphql.datastore_core.index_definitions_by_name.fetch("components") + + query1 = new_query(search_index_definitions: [widgets_def]) + query2 = new_query(search_index_definitions: [components_def]) + + expect { + merge(query1, query2) + }.to raise_error(ElasticGraph::Errors::InvalidMergeError, a_string_including("search_index_definitions", "widgets", "components")) + end + + it "can merge `equal_to_any_of` conditions from two separate queries that are on separate fields", covers: :filters do + query1 = new_query(filter: {"age" => {"equal_to_any_of" => [25, 30]}}) + query2 = new_query(filter: {"size" => {"equal_to_any_of" => [10]}}) + + merged = merge(query1, query2) + + expect(datastore_body_of(merged)).to filter_datastore_with( + {terms: {"age" => [25, 30]}}, + {terms: {"size" => [10]}} + ) + end + + it "can merge `equal_to_any_of` conditions from two separate queries that are on the same field", covers: :filters do + query1 = new_query(filter: {"age" => {"equal_to_any_of" => [25, 30]}}) + query2 = new_query(filter: {"age" => {"equal_to_any_of" => [35, 30]}}) + + merged = merge(query1, query2) + + expect(datastore_body_of(merged)).to filter_datastore_with( + {terms: {"age" => [25, 30]}}, + {terms: {"age" => [35, 30]}} + ) + end + + it "can merge using `merge_with(**query_options)` as well", covers: :filters do + query1 = new_query(filter: {"age" => {"equal_to_any_of" => [25, 30]}}) + + merged = nil + + expect { + merged = query1.merge_with(filter: {"age" => {"equal_to_any_of" => [35, 30]}, "size" => {"equal_to_any_of" => [10]}}) + }.to maintain { query1 }.and maintain { query1.filters } + + expect(datastore_body_of(merged)).to filter_datastore_with( + {terms: {"age" => [25, 30]}}, + {terms: {"age" => [35, 30]}}, + {terms: {"size" => [10]}} + ) + end + + it "de-duplicates filters that are present in both queries", covers: :filters do + query1 = new_query(filter: {"age" => {"equal_to_any_of" => [25, 30]}}) + + merged = merge(query1, query1) + + expect(merged.filters).to contain_exactly({"age" => {"equal_to_any_of" => [25, 30]}}) + end + + it "uses only the tiebreaking sort clauses when merging two queries that have a `nil` sort", covers: :sort do + query1 = new_query(sort: nil, individual_docs_needed: true) + query2 = new_query(sort: nil, individual_docs_needed: true) + + expect(datastore_body_of(merge(query1, query2))).to include_sort_with_tiebreaker + end + + it "does not use tiebreaking sort clauses when any of the two queries already specifies them", covers: :sort do + # tiebreaker uses `asc` instead of `desc` + query1 = new_query(sort: [{"id" => {"order" => "desc"}}], individual_docs_needed: true) + query2 = new_query(sort: nil, individual_docs_needed: true) + + expect(datastore_body_of(merge(query1, query2))).to include(sort: [{"id" => {"order" => "desc", "missing" => "_last"}}]) + end + + it "uses the `sort` value from either query when only one of them has a value", covers: :sort do + query1 = new_query(sort: [{created_at: {"order" => "asc"}}], individual_docs_needed: true) + query2 = new_query(sort: nil, individual_docs_needed: true) + + merged = merge(query1, query2) + merged_reverse = merge(query2, query1) + + expect(datastore_body_of(merged)).to include_sort_with_tiebreaker(created_at: {"order" => "asc"}) + expect(datastore_body_of(merged_reverse)).to eq(datastore_body_of(merged)) + end + + it "uses the `sort` value from the `query` argument when both queries have a `sort` value and logs a warning", covers: :sort do + query1 = new_query(sort: [{created_at: {"order" => "asc"}}], individual_docs_needed: true) + query2 = new_query(sort: [{created_at: {"order" => "desc"}}], individual_docs_needed: true) + + expect { + merged = merge(query1, query2) + expect(datastore_body_of(merged)).to include_sort_with_tiebreaker(created_at: {"order" => "desc"}) + }.to log a_string_including("Tried to merge two queries that both define `sort`") + end + + it "uses one of the `sort` values when `sort` values are the same and does not log a warning", covers: :sort do + query1 = new_query(sort: [{created_at: {"order" => "asc"}}], individual_docs_needed: true) + query2 = new_query(sort: [{created_at: {"order" => "asc"}}], individual_docs_needed: true) + + expect { + merged = merge(query1, query2) + expect(datastore_body_of(merged)).to include_sort_with_tiebreaker(created_at: {"order" => "asc"}) + }.to avoid_logging_warnings + end + + it "maintains a `document_pagination` value of `nil` when merging two queries that have a `nil` `document_pagination`", covers: :document_pagination do + query1 = new_query(document_pagination: nil) + query2 = new_query(document_pagination: nil) + + merged = merge(query1, query2) + expect(merged.document_pagination).to eq({}) + end + + it "uses the `document_pagination` value from either query when only one of them has a value", covers: :document_pagination do + query1 = new_query(document_pagination: {first: 2}) + query2 = new_query(document_pagination: nil) + + merged = merge(query1, query2) + merged_reverse = merge(query2, query1) + expect(merged.document_pagination).to eq({first: 2}) + expect(merged_reverse.document_pagination).to eq(merged.document_pagination) + end + + it "uses the `document_pagination` value from the `query` argument when both queries have a `document_pagination` value and logs a warning", covers: :document_pagination do + query1 = new_query(document_pagination: {first: 2}) + query2 = new_query(document_pagination: {first: 5}) + + expect { + merged = merge(query1, query2) + expect(merged.document_pagination).to eq({first: 5}) + }.to log a_string_including("Tried to merge two queries that both define `document_pagination`") + end + + it "uses one of the `document_pagination` values when `document_pagination` values are the same and does not log a warning", covers: :document_pagination do + query1 = new_query(document_pagination: {first: 10}) + query2 = new_query(document_pagination: {first: 10}) + + expect { + merged = merge(query1, query2) + expect(merged.document_pagination).to eq({first: 10}) + }.to avoid_logging_warnings + end + + it "merges `aggregations` by merging the hashes", covers: :aggregations do + agg1 = aggregation_query_of(name: "a1", groupings: [ + field_term_grouping_of("foo1", "bar1"), + field_term_grouping_of("foo2", "bar2") + ]) + + agg2 = aggregation_query_of(name: "a2", groupings: [ + field_term_grouping_of("foo1", "bar1"), + field_term_grouping_of("foo3", "bar3") + ]) + + agg3 = aggregation_query_of(name: "a3", groupings: [ + field_term_grouping_of("foo1", "bar1") + ]) + + query1 = new_query(aggregations: [agg1, agg3]) + query2 = new_query(aggregations: [agg2, agg3]) + + merged1 = query1.merge(query2).aggregations + merged2 = query2.merge(query1).aggregations + + expect(merged1).to eq(merged2) + expect(merged1).to eq({ + "a1" => agg1, + "a2" => agg2, + "a3" => agg3 + }) + end + + it "correctly merges requested fields from multiple queries by concatenating and de-duplicating them", covers: :requested_fields do + query1 = new_query(requested_fields: ["a", "b"]) + query2 = new_query(requested_fields: ["b", "c"]) + + expect { + expect(merge(query1, query2).requested_fields).to contain_exactly("a", "b", "c") + }.to avoid_logging_warnings + end + + it "sets `individual_docs_needed` to `true` if it is set on either query", covers: :individual_docs_needed do + query1 = new_query(individual_docs_needed: true) + query2 = new_query(individual_docs_needed: false) + + expect(query1.merge(query2).individual_docs_needed).to be true + expect(query2.merge(query1).individual_docs_needed).to be true + end + + it "sets `individual_docs_needed` to `false` if it is set to `false` on both queries", covers: :individual_docs_needed do + query1 = new_query(individual_docs_needed: false) + query2 = new_query(individual_docs_needed: false) + + expect(query1.merge(query2).individual_docs_needed).to be false + expect(query2.merge(query1).individual_docs_needed).to be false + end + + it "sets `total_document_count_needed` to `true` if it is set on either query", covers: :total_document_count_needed do + query1 = new_query(total_document_count_needed: true) + query2 = new_query(total_document_count_needed: false) + + expect(query1.merge(query2).total_document_count_needed).to be true + expect(query2.merge(query1).total_document_count_needed).to be true + end + + it "sets `total_document_count_needed` to `false` if it is set to `false` on both queries", covers: :total_document_count_needed do + query1 = new_query(total_document_count_needed: false) + query2 = new_query(total_document_count_needed: false) + + expect(query1.merge(query2).total_document_count_needed).to be false + expect(query2.merge(query1).total_document_count_needed).to be false + end + + it "forces `total_document_count_needed` to `true` if either query has an aggregation query that requires it", covers: :total_document_count_needed do + query1 = new_query(total_document_count_needed: false, aggregations: [aggregation_query_of(needs_doc_count: true)]) + query2 = new_query(total_document_count_needed: false) + + expect(query1.merge(query2).total_document_count_needed).to be true + expect(query2.merge(query1).total_document_count_needed).to be true + end + + it "does not force `total_document_count_needed` to `true` if the aggregations query has groupings", covers: :total_document_count_needed do + query1 = new_query(total_document_count_needed: false, aggregations: [aggregation_query_of( + needs_doc_count: true, + groupings: [field_term_grouping_of("age")] + )]) + query2 = new_query(total_document_count_needed: false) + + expect(query1.merge(query2).total_document_count_needed).to be false + expect(query2.merge(query1).total_document_count_needed).to be false + end + + specify "#merge_with can merge in an empty filter", covers: :filters do + query1 = new_query(filter: {"age" => {"equal_to_any_of" => [25, 30]}}) + + expect(query1.merge_with).to eq query1 + expect(query1.merge_with(filter: nil)).to eq query1 + expect(query1.merge_with(filter: {})).to eq query1 + end + + it "prefers a set `monotonic_clock_deadline` value to an unset one", covers: :monotonic_clock_deadline do + query1 = new_query(monotonic_clock_deadline: 5000) + query2 = new_query(monotonic_clock_deadline: nil) + + expect(query1.merge(query2).monotonic_clock_deadline).to eq 5000 + expect(query2.merge(query1).monotonic_clock_deadline).to eq 5000 + end + + it "prefers the shorter `monotonic_clock_deadline` value so that we can default to an application config setting, " \ + "and override it with a shorter deadline", covers: :monotonic_clock_deadline do + query1 = new_query(monotonic_clock_deadline: 3000) + query2 = new_query(monotonic_clock_deadline: 6000) + + expect(query1.merge(query2).monotonic_clock_deadline).to eq 3000 + expect(query2.merge(query1).monotonic_clock_deadline).to eq 3000 + end + + it "leaves `monotonic_clock_deadline` unset if unset on both source queries", covers: :monotonic_clock_deadline do + query1 = new_query(monotonic_clock_deadline: nil) + query2 = new_query(monotonic_clock_deadline: nil) + + expect(query1.merge(query2).monotonic_clock_deadline).to eq nil + expect(query2.merge(query1).monotonic_clock_deadline).to eq nil + end + + def filter_datastore_with(*filters) + # `filter` uses the datastore's filtering context + include(query: {bool: {filter: filters}}) + end + + def include_sort_with_tiebreaker(*sort_clauses) + include(sort: sort_list_with_missing_option_for(*sort_clauses)) + end + + def merge(query1, query2) + merged = nil + + # merging should not mutate either query, so we assert that here + expect { + merged = query1.merge(query2) + }.to maintain { query1 }.and maintain { query2 } + + merged + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/misc_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/misc_spec.rb new file mode 100644 index 00000000..086c4fd4 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/misc_spec.rb @@ -0,0 +1,31 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_unit_support" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "misc" do + include_context "DatastoreQueryUnitSupport" + let(:default_page_size) { 73 } + let(:graphql) { build_graphql(default_page_size: default_page_size) } + + it "raises an error if instantiated with an empty collection of `search_index_definitions`" do + expect { + new_query(search_index_definitions: []) + }.to raise_error Errors::SearchFailedError, a_string_including("search_index_definitions") + end + + it "inspects nicely, but redacts filters since they could contain PII" do + expect(new_query(filter: {"ssn" => {"equal_to_any_of" => ["123-45-6789"]}}, individual_docs_needed: true).inspect).to eq(<<~EOS.strip) + #{"order"=>"asc", "missing"=>"_first"}}] track_total_hits=false query= _source=false> + EOS + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/pagination_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/pagination_spec.rb new file mode 100644 index 00000000..b1c7d33e --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/pagination_spec.rb @@ -0,0 +1,51 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_unit_support" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "pagination" do + include_context "DatastoreQueryUnitSupport" + let(:default_page_size) { 73 } + let(:max_page_size) { 200 } + let(:graphql) { build_graphql(default_page_size: default_page_size, max_page_size: max_page_size) } + + it "excludes `search_after` when document_pagination is nil" do + query = new_query(document_pagination: nil) + expect(datastore_body_of(query).keys).to_not include(:search_after) + end + + it "uses the configured default page size when not overridden by a document_pagination option" do + query = new_query(individual_docs_needed: true) + # we allow for `default_page_size + 1` so if we need to fetch an additional document + # to see if there's another page, we can. + expect(datastore_body_of(query)).to include(size: a_value_within(1).of(default_page_size)) + end + + it "limits the page size to the configured max page size" do + query = new_query(individual_docs_needed: true, document_pagination: {first: max_page_size + 10}) + # we allow for `default_page_size + 1` so if we need to fetch an additional document + # to see if there's another page, we can. + expect(datastore_body_of(query)).to include(size: a_value_within(1).of(max_page_size)) + + query = new_query(individual_docs_needed: true, document_pagination: {last: max_page_size + 10}) + # we allow for `default_page_size + 1` so if we need to fetch an additional document + # to see if there's another page, we can. + expect(datastore_body_of(query)).to include(size: a_value_within(1).of(max_page_size)) + end + + it "queries the datastore with a page size of 0 if `individual_docs_needed` is false" do + query = new_query(requested_fields: []) + expect(query.individual_docs_needed).to be false + + expect(datastore_body_of(query)).to include(size: 0) + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/perform_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/perform_spec.rb new file mode 100644 index 00000000..bf898d45 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/perform_spec.rb @@ -0,0 +1,138 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_unit_support" +require "support/aggregations_helpers" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, ".perform" do + include AggregationsHelpers + include_context "DatastoreQueryUnitSupport" + + let(:graphql) do + build_graphql do |datastore_config| + datastore_config.with( + index_definitions: datastore_config.index_definitions.merge( + "components" => config_index_def_of(query_cluster: "other1") + ) + ) + end + end + + let(:raw_doc) do + { + "_index" => "widgets", + "_type" => "_doc", + "_id" => "zwbfffaijhkljtfmcuwv", + "_score" => nil, + "_source" => {}, + "sort" => [300] + } + end + + specify "performs multiple queries, wrapping each response in an `DatastoreResponse::SearchResponse`" do + widgets_def = graphql.datastore_core.index_definitions_by_name.fetch("widgets") + components_def = graphql.datastore_core.index_definitions_by_name.fetch("components") + + expect(widgets_def.cluster_to_query).to eq "main" + expect(components_def.cluster_to_query).to eq "other1" + + query0 = new_query(search_index_definitions: [widgets_def], filter: {"age" => {"equal_to_any_of" => [0]}}, requested_fields: ["name"]) + query1 = new_query(search_index_definitions: [widgets_def], filter: {"age" => {"equal_to_any_of" => [10]}}, requested_fields: ["name"]) + query2 = new_query(search_index_definitions: [components_def], filter: {"age" => {"equal_to_any_of" => [20]}}, requested_fields: ["name"]) + + yielded_header_body_tuples_by_query = nil + + responses = DatastoreQuery.perform([query0, query1, query2]) do |header_body_tuples_by_query| + yielded_header_body_tuples_by_query = header_body_tuples_by_query + + { + query0 => raw_response_with_docs(raw_doc, raw_doc), + query1 => raw_response_with_docs(raw_doc), + query2 => raw_response_with_docs(raw_doc, raw_doc) + } + end + + expect(yielded_header_body_tuples_by_query).to match({ + query0 => [{index: "widgets_rollover__*"}, a_hash_including(query: {bool: {filter: [{terms: {"age" => [0]}}]}})], + query1 => [{index: "widgets_rollover__*"}, a_hash_including(query: {bool: {filter: [{terms: {"age" => [10]}}]}})], + query2 => [{index: "components"}, a_hash_including(query: {bool: {filter: [{terms: {"age" => [20]}}]}})] + }) + + expect(responses.values).to all be_a DatastoreResponse::SearchResponse + expect(responses.size).to eq 3 + expect(responses.values.map(&:size)).to eq [2, 1, 2] + end + + it "avoids yielding empty queries, providing a default empty search response to the caller" do + empty_query = new_minimal_query + with_fields = new_minimal_query(requested_fields: ["name"]) + with_total_hits = new_minimal_query(total_document_count_needed: true) + with_aggs = new_minimal_query(aggregations: [agg = aggregation_query_of(computations: [computation_of("amountMoney", "amount", :sum)])]) + + yielded_header_body_tuples_by_query = nil + query_response = raw_response_with_docs(raw_doc) + + responses = DatastoreQuery.perform([empty_query, with_fields, with_total_hits, with_aggs]) do |header_body_tuples_by_query| + yielded_header_body_tuples_by_query = header_body_tuples_by_query + header_body_tuples_by_query.transform_values { query_response } + end + + # Notably, `empty_query` shouldn't be yielded to the block... + expect(yielded_header_body_tuples_by_query.keys).to contain_exactly( + with_fields, with_total_hits, + with_aggs.with(aggregations: {agg.name => agg}) + ) + + # ...but we should still get a response for it. + expect(responses.transform_values { |response| response.size }).to eq( + empty_query => 0, + with_fields => 1, + with_total_hits => 1, + with_aggs => 1 + ) + end + + it "raises an error if the logic has failed to return a response for a query" do + widgets_def = graphql.datastore_core.index_definitions_by_name.fetch("widgets") + components_def = graphql.datastore_core.index_definitions_by_name.fetch("components") + + expect(widgets_def.cluster_to_query).to eq "main" + expect(components_def.cluster_to_query).to eq "other1" + + query0 = new_query(search_index_definitions: [widgets_def], filter: {"age" => {"equal_to_any_of" => [0]}}, requested_fields: ["name"]) + query1 = new_query(search_index_definitions: [widgets_def], filter: {"age" => {"equal_to_any_of" => [10]}}, requested_fields: ["name"]) + query2 = new_query(search_index_definitions: [components_def], filter: {"age" => {"equal_to_any_of" => [20]}}, requested_fields: ["name"]) + + expect { + DatastoreQuery.perform([query0, query1, query2]) do |header_body_tuples_by_query| + { + query0 => raw_response_with_docs(raw_doc, raw_doc), + # Here we omit `query1` to simulate a response missing. + # query1 => raw_response_with_docs(raw_doc), + query2 => raw_response_with_docs(raw_doc, raw_doc) + } + end + }.to raise_error Errors::SearchFailedError, a_string_including("does not have the expected set of queries", query1.inspect) + end + + def new_minimal_query(requested_fields: [], total_document_count_needed: false, aggregations: [], **options) + new_query(requested_fields: requested_fields, total_document_count_needed: total_document_count_needed, aggregations: aggregations, **options) + end + + def raw_response_with_docs(*raw_docs) + # deep copy RAW_EMPTY so our updates don't impact the original + Marshal.load(Marshal.dump(DatastoreResponse::SearchResponse::RAW_EMPTY)).tap do |response| + response["hits"]["hits"] = raw_docs + response["hits"]["total"]["value"] = raw_docs.count + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/requested_fields_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/requested_fields_spec.rb new file mode 100644 index 00000000..8bbbfd6e --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/requested_fields_spec.rb @@ -0,0 +1,41 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_unit_support" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "#requested_fields" do + include_context "DatastoreQueryUnitSupport" + + it "requests only non-id fields from the datastore when building the request body" do + query = new_query(requested_fields: ["name", "id"]) + + expect(datastore_body_of(query)[:_source][:includes]).to contain_exactly("name") + end + + it "requests all fields in requested_fields when requested_fields does not include id" do + query = new_query(requested_fields: ["name", "age"]) + + expect(datastore_body_of(query)[:_source][:includes]).to contain_exactly("name", "age") + end + + it "does not request _source when id is the only requested field" do + query = new_query(requested_fields: ["id"]) + + expect(datastore_body_of(query)[:_source]).to eq(false) + end + + it "does not request _source when no fields are requested" do + query = new_query(requested_fields: []) + + expect(datastore_body_of(query)[:_source]).to eq(false) + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/route_with_field_paths_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/route_with_field_paths_spec.rb new file mode 100644 index 00000000..8514ee90 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/route_with_field_paths_spec.rb @@ -0,0 +1,39 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_unit_support" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "#route_with_field_paths" do + include_context "DatastoreQueryUnitSupport" + + let(:widgets_def) { graphql.datastore_core.index_definitions_by_name.fetch("widgets") } + let(:addresses_def) { graphql.datastore_core.index_definitions_by_name.fetch("addresses") } + let(:manufacturers_def) { graphql.datastore_core.index_definitions_by_name.fetch("manufacturers") } + + before do + expect(widgets_def.route_with).to eq "workspace_id2" + expect(addresses_def.route_with).to eq "id" + expect(manufacturers_def.route_with).to eq "id" + end + + it "returns a list of `route_with` values from the `search_index_definitions`" do + query = new_query(search_index_definitions: [widgets_def, addresses_def]) + + expect(query.route_with_field_paths).to contain_exactly("workspace_id2", "id") + end + + it "deduplicates values when multiple search index definitions have the same `route_with` value" do + query = new_query(search_index_definitions: [manufacturers_def, addresses_def]) + + expect(query.route_with_field_paths).to contain_exactly("id") + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/search_index_expression_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/search_index_expression_spec.rb new file mode 100644 index 00000000..c742995b --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/search_index_expression_spec.rb @@ -0,0 +1,543 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_unit_support" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "#search_index_expression" do + include_context "DatastoreQueryUnitSupport" + + shared_examples_for "search index expression logic" do |timestamp_field_type| + attr_accessor :schema_artifacts + + before(:context) do + self.schema_artifacts = generate_schema_artifacts + end + + it "joins the search expressions from the individual index definitions into the `index` of the search headers" do + widgets_def = graphql.datastore_core.index_definitions_by_name.fetch("widgets") + components_def = graphql.datastore_core.index_definitions_by_name.fetch("components") + + expect(widgets_def.index_expression_for_search).to eq "widgets_rollover__*" + expect(components_def.index_expression_for_search).to eq "components" + + query = new_query(search_index_definitions: [components_def, widgets_def]) + + expect(query.search_index_expression).to eq "components,widgets_rollover__*" + end + + context "when a rollover timestamp field is being filtered on" do + it "avoids searching indices that come on or before a `gt` lower bound" do + parts = search_index_expression_parts_for({"created_at" => {"gt" => "2021-04-15T12:30:00Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("01", "02", "03") + end + + it "avoids searching indices that come before a `gte` lower bound" do + parts = search_index_expression_parts_for({"created_at" => {"gte" => "2021-04-15T12:30:00Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("01", "02", "03") + + parts = search_index_expression_parts_for({"created_at" => {"gte" => "2021-02-28T23:59:58Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("01") + + parts = search_index_expression_parts_for({"created_at" => {"gte" => "2021-02-28T23:59:59.999Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("01") + + parts = search_index_expression_parts_for({"created_at" => {"gte" => "2021-03-01T00:00:00Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("01", "02") + end + + it "uses the higher of `gt` and `gte` when both are set, because the filter predicates are ANDed together" do + parts = search_index_expression_parts_for({"created_at" => {"gte" => "2021-02-15T12:30:00Z", "gt" => "2021-04-01T12:30:00Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("01", "02", "03") + + parts = search_index_expression_parts_for({"created_at" => {"gt" => "2021-02-15T12:30:00Z", "gte" => "2021-04-01T12:30:00Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("01", "02", "03") + end + + it "avoids searching indices that come on or after a `lt` upper bound" do + parts = search_index_expression_parts_for({"created_at" => {"lt" => "2021-09-15T12:30:00Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("10", "11", "12") + end + + it "avoids searching indices that come after a `lte` upper bound" do + parts = search_index_expression_parts_for({"created_at" => {"lte" => "2021-09-15T12:30:00Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("10", "11", "12") + + parts = search_index_expression_parts_for({"created_at" => {"lte" => "2021-11-01T00:00:01Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("12") + + parts = search_index_expression_parts_for({"created_at" => {"lte" => "2021-11-01T00:00:00Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("12") + + parts = search_index_expression_parts_for({"created_at" => {"lte" => "2021-10-30T23:59:59.999Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("11", "12") + end + + it "uses the lower of `lt` and `lte` when both are set, because the filter predicates are ANDed together" do + parts = search_index_expression_parts_for({"created_at" => {"lte" => "2021-10-01T12:30:00Z", "lt" => "2021-08-15T12:30:00Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("09", "10", "11", "12") + + parts = search_index_expression_parts_for({"created_at" => {"lt" => "2021-10-01T12:30:00Z", "lte" => "2021-08-15T12:30:00Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("09", "10", "11", "12") + end + + it "excludes indices not targeted by a filter that has both an upper and lower bound" do + parts = search_index_expression_parts_for({"created_at" => {"gt" => "2021-03-01T00:00:00Z", "lt" => "2021-08-15T12:30:00Z"}}) + + expect(parts).to target_widget_indices_excluding_2021_months("01", "02", "09", "10", "11", "12") + end + + it "puts the excluded indices after the included ones because the datastore returns errors if exclusions are listed first" do + parts = search_index_expression_parts_for({"created_at" => {"gt" => "2021-03-15T12:30:00Z"}}) + + expect(parts).to eq(["widgets_rollover__*", "-widgets_rollover__2021-01", "-widgets_rollover__2021-02"]) + # Note: we don't care about the order of the excluded indices so we use an `or` here to allow them in either order. + .or eq(["widgets_rollover__*", "-widgets_rollover__2021-02", "-widgets_rollover__2021-01"]) + end + + it "ANDs together multiple filter hashes when determining what indices to exclude" do + parts = search_index_expression_parts_for([ + {"created_at" => {"gt" => "2021-03-01T00:00:00Z"}}, + {"created_at" => {"lt" => "2021-10-01T00:00:00Z"}} + ]) + + expect(parts).to target_widget_indices_excluding_2021_months("01", "02", "10", "11", "12") + end + + it "ignores filter operators it does not understand" do + # greater_than isn't an understood filter operator. While this shouldn't get in to this logic, + # if it did we should just ignore it. + parts = search_index_expression_parts_for({"created_at" => {"greater_than" => "2021-03-01T12:30:00Z"}}) + + expect(parts).to target_all_widget_indices + end + + it "searches no indices when the time range filter excludes all timestamps" do + parts = search_index_expression_parts_for({"created_at" => {"gt" => "2021-10-01T00:00:00Z", "lt" => "2021-01-01T00:00:00Z"}}) + + expect(parts).to eq [] + end + + it "supports nested timestamp fields" do + self.schema_artifacts = generate_schema_artifacts(timestamp_field: "foo.bar.created_at") + parts = search_index_expression_parts_for({"foo" => {"bar" => {"created_at" => {"gt" => "2021-04-15T12:30:00Z"}}}}) + + expect(parts).to target_widget_indices_excluding_2021_months("01", "02", "03") + end + + context "when `equal_to_any_of` is used" do + it "excludes all indices that the `equal_to_any_of` timestamps do not fall in" do + parts = search_index_expression_parts_for({"created_at" => {"equal_to_any_of" => [ + "2021-01-15T12:30:00Z", + "2021-03-31T23:59:59.999Z", + "2021-05-15T12:30:00Z", + "2021-05-30T12:30:00Z", + "2021-07-15T12:30:00Z", + "2021-09-15T12:30:00Z", + "2021-11-01T00:00:00Z" + ]}}) + + expect(parts).to target_widget_indices_excluding_2021_months("02", "04", "06", "08", "10", "12") + end + + it "is ANDed together with other predicates when selecting which indices to exclude" do + parts = search_index_expression_parts_for({"created_at" => { + "gte" => "2021-03-01T00:00:00Z", "gt" => "2021-04-01T00:00:00Z", # reduces to > 2021-04-01 + "lt" => "2021-08-01T00:00:00Z", "lte" => "2021-09-15T00:00:00Z", # reduces to < 2021-08-01 + "equal_to_any_of" => [ + "2021-01-15T12:30:00Z", + "2021-03-31T23:59:59.999Z", + "2021-05-15T12:30:00Z", + "2021-05-30T12:30:00Z", + "2021-07-15T12:30:00Z", + "2021-09-15T12:30:00Z", + "2021-11-01T00:00:00Z" + ] + }}) + + expect(parts).to target_widget_indices_excluding_all_2021_months_except("05", "07") + end + + it "searches no indices when `equal_to_any_of` ONLY contains `[nil]`" do + parts = search_index_expression_parts_for({"created_at" => {"equal_to_any_of" => [nil]}}) + + expect(parts).to eq [] + end + + it "ignores `nil` when `equal_to_any_of` includes `nil` with other timestamps" do + parts = search_index_expression_parts_for({"created_at" => {"equal_to_any_of" => [ + "2021-01-15T12:30:00Z", + nil, + "2021-03-31T23:59:59.999Z" + ]}}) + + expect(parts).to target_widget_indices_excluding_all_2021_months_except("01", "03") + end + end + + context "when `not` is used" do + it "excludes indices targeted by a filter without both an upper and lower bound" do + parts = search_index_expression_parts_for({"created_at" => {"not" => { + "gte" => "2021-03-01T00:00:00Z", + "lt" => "2021-06-01T00:00:00Z" + }}}) + + expect(parts).to target_widget_indices_excluding_2021_months("03", "04", "05") + end + + it "excludes the index containing the month the time range start in when at the beginning of the month" do + parts = search_index_expression_parts_for({"created_at" => {"not" => { + "gte" => "2021-03-01T00:00:00Z" + }}}) + + expect(parts).to target_widget_indices_excluding_all_2021_months_except("01", "02") + end + + it "includes the index containing the month the time range starts when after the beginning of the month" do + parts = search_index_expression_parts_for({"created_at" => {"not" => { + "gt" => "2021-03-15T23:59:58Z" + }}}) + + expect(parts).to target_widget_indices_excluding_all_2021_months_except("01", "02", "03") + end + + it "correctly excludes months outside of the given time ranges when using `any_of`" do + parts = search_index_expression_parts_for({"created_at" => {"not" => { + "any_of" => [ + {"lt" => "2021-01-01T00:00:00Z"}, + {"gte" => "2021-03-01T00:00:00Z"} + ] + }}}) + + expect(parts).to target_widget_indices_excluding_all_2021_months_except("01", "02") + end + + it "returns all indices when `equal_to_any_of` are individual timestamps" do + parts = search_index_expression_parts_for({"created_at" => {"not" => { + "equal_to_any_of" => [ + "2021-01-15T12:30:00Z", + "2021-03-31T23:59:59.999Z", + "2021-05-15T12:30:00Z", + "2021-07-15T12:30:00Z" + ] + }}}) + + expect(parts).to target_all_widget_indices + end + + it "returns all indices when `equal_to_any_of` includes `nil` with individual timestamps" do + parts = search_index_expression_parts_for({"created_at" => {"not" => { + "equal_to_any_of" => [ + "2021-01-15T12:30:00Z", + "2021-03-31T23:59:59.999Z", + nil, + "2021-05-15T12:30:00Z", + "2021-07-15T12:30:00Z" + ] + }}}) + + expect(parts).to target_all_widget_indices + end + + it "returns all indices when `equal_to_any_of` is `[nil]`" do + parts = search_index_expression_parts_for({"created_at" => {"not" => {"equal_to_any_of" => [nil]}}}) + + expect(parts).to target_all_widget_indices + end + + it "is ANDed together with other predicates when selecting which indices to exclude" do + parts = search_index_expression_parts_for({"created_at" => { + "gte" => "2021-03-01T00:00:00Z", + "lt" => "2021-08-01T00:00:00Z", + "not" => { + "any_of" => [ + {"lt" => "2021-04-01T00:00:00Z"}, + {"gte" => "2021-05-01T00:00:00Z"} + ] + } + }}) + + expect(parts).to target_widget_indices_excluding_all_2021_months_except("04") + end + + it "can handle nested `not`s" do + nested_once = search_index_expression_parts_for({"created_at" => {"not" => {"not" => { + "gte" => "2021-03-01T00:00:00Z" + }}}}) + + nested_twice = search_index_expression_parts_for({"created_at" => {"not" => {"not" => {"not" => { + "gte" => "2021-03-01T00:00:00Z" + }}}}}) + + expect(nested_once).to target_widget_indices_excluding_2021_months("01", "02") + expect(nested_twice).to target_widget_indices_excluding_all_2021_months_except("01", "02") + end + + it "can handle `any_of` between nested `not`s" do + parts = search_index_expression_parts_for({"created_at" => {"not" => { + "any_of" => [ + { + "not" => { + "gte" => "2021-03-01T00:00:00Z", + "lt" => "2021-08-01T00:00:00Z" + } + }, + { + "gte" => "2021-07-01T00:00:00Z" + } + ] + }}}) + + expect(parts).to target_widget_indices_excluding_all_2021_months_except("03", "04", "05", "06") + end + + it "searches all indices when the time range filter matches all timestamps" do + parts = search_index_expression_parts_for({"created_at" => {"not" => { + "lt" => "2021-03-01T00:00:00Z", + "gte" => "2021-06-01T00:00:00Z" + }}}) + + expect(parts).to target_all_widget_indices + end + + it "searches no indices when the time range filter excludes all timestamps" do + parts = search_index_expression_parts_for({"created_at" => {"not" => { + "any_of" => [ + {"gte" => "2021-01-01T00:00:00Z"}, + {"lt" => "2021-01-01T00:00:00Z"} + ] + }}}) + + expect(parts).to eq [] + end + + it "searches all indices when set to nil" do + parts = search_index_expression_parts_for({"created_at" => {"not" => nil}}) + + expect(parts).to target_all_widget_indices + end + end + + %w[equal_to_any_of gt gte lt lte any_of].each do |operator| + it "ignores a `nil` value for a `#{operator}` filter" do + parts = search_index_expression_parts_for({"created_at" => {operator => nil}}) + + expect(parts).to target_all_widget_indices + end + end + + context "when `any_of` is used" do + it "ORs together the date filter criteria from the `any_of` subfilters when determining what indices to exclude" do + parts1 = search_index_expression_parts_for({"any_of" => [ + {"created_at" => {"lt" => "2021-03-01T00:00:00Z"}}, + {"created_at" => {"gt" => "2021-08-15T12:30:00Z"}} + ]}) + + # Order of subfilters shouldn't matter + parts2 = search_index_expression_parts_for({"any_of" => [ + {"created_at" => {"gt" => "2021-08-15T12:30:00Z"}}, + {"created_at" => {"lt" => "2021-03-01T00:00:00Z"}} + ]}) + + # It shouldn't matter if the any_of is nested or on the outside. + parts3 = search_index_expression_parts_for({"created_at" => {"any_of" => [ + {"lt" => "2021-03-01T00:00:00Z"}, + {"gt" => "2021-08-15T12:30:00Z"} + ]}}) + + expect(parts1).to eq(parts2).and eq(parts3).and target_widget_indices_excluding_2021_months("03", "04", "05", "06", "07") + end + + it "excludes no indices when one of the `any_of` subfilters does not filter on the timestamp field at all" do + parts = search_index_expression_parts_for({"any_of" => [ + {"created_at" => {"lt" => "2021-03-01T00:00:00Z"}}, + {"created_at" => {"gt" => "2021-08-15T12:30:00Z"}}, + {"id" => {"equal_to_any_of" => "some-id"}} + ]}) + + expect(parts).to target_all_widget_indices + end + + it "excludes no indices when we have an `any_of: []` filter because that will match all results" do + parts = search_index_expression_parts_for({"any_of" => []}) + + expect(parts).to target_all_widget_indices + end + end + + context "for a query that includes aggregations" do + it "filters out indices just like a non-aggregations query" do + parts = search_index_expression_parts_for({"created_at" => {"gte" => "2021-04-15T12:30:00Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("01", "02", "03") + + parts = search_index_expression_parts_for({"created_at" => {"lte" => "2021-09-15T12:30:00Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("10", "11", "12") + end + + it "searches exactly one index when the time range filter excludes all documents, to ensure a consistent aggregations response" do + parts = search_index_expression_parts_for({"created_at" => {"equal_to_any_of" => []}}) + + expect(parts).to contain_exactly("widgets_rollover__2021-01") + end + + it "excludes all but one index when the time range filter excludes all known indices, to ensure a consistent aggregations response" do + parts = search_index_expression_parts_for({"created_at" => {"gte" => "2022-04-15T12:30:00Z"}}) + + expect(parts).to target_widget_indices_excluding_2021_months( + "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12" + ) + end + + def search_index_expression_parts_for(filter_or_filters) + aggregations = {"aggs" => aggregation_query_of( + name: "aggs", + groupings: [date_histogram_grouping_of("created_at", "month")] + )} + + super(filter_or_filters, aggregations: aggregations) + end + end + end + + def search_index_expression_parts_for(filter_or_filters, aggregations: {}) + filter_or_filters = coerce_date_times_in(filter_or_filters) + + allow(datastore_client).to receive(:list_indices_matching).with("widgets_rollover__*").and_return( + # Materialize monthly indices for all of 2021. + ("01".."12").map { |month| "widgets_rollover__2021-#{month}" } + ) + + graphql = build_graphql( + clients_by_name: {"main" => datastore_client}, + schema_artifacts: schema_artifacts, + # Clear our normal index settings for `widgets` since that defines indices based + # on yearly rollover but in these tests we use monthly rollover to have a bit finer + # granularity to work with. + index_definitions: {"widgets" => config_index_def_of} + ) + + options = if filter_or_filters.is_a?(Array) + {filters: filter_or_filters} + else + {filter: filter_or_filters} + end + + index_def = graphql.datastore_core.index_definitions_by_name.fetch("widgets") + builder.new_query(search_index_definitions: [index_def], aggregations: aggregations, **options).search_index_expression.split(",") + end + + def target_all_widget_indices + contain_exactly("widgets_rollover__*") + end + + def target_widget_indices_excluding_2021_months(*month_num_strings) + indices_to_exclude = month_num_strings.map { |month| "-widgets_rollover__2021-#{month}" } + contain_exactly("widgets_rollover__*", *indices_to_exclude) + end + + def target_widget_indices_excluding_all_2021_months_except(*month_num_strings) + exclusion_months = ("01".."12").to_set - month_num_strings + target_widget_indices_excluding_2021_months(*exclusion_months) + end + + define_method :generate_schema_artifacts do |timestamp_field: "created_at"| + super() do |schema| + schema.object_type "Foo" do |t| + t.field "bar", "Bar" + end + + schema.object_type "Bar" do |t| + t.field "created_at", timestamp_field_type + end + + schema.object_type "Widget" do |t| + t.field "id", "ID!" + + if timestamp_field.start_with?("foo.") + t.field "foo", "Foo" # for the nested case + else + t.field timestamp_field, timestamp_field_type + end + + t.index "widgets" do |i| + i.rollover :monthly, timestamp_field + end + end + end + end + end + + context "when the timestamp field is a `DateTime`" do + include_examples "search index expression logic", "DateTime" do + it "avoids searching an index when filtering on a timestamp `gt` the last ms of the index" do + parts = search_index_expression_parts_for({"created_at" => {"gt" => "2021-02-28T23:59:59.999Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("01", "02") + + parts = search_index_expression_parts_for({"created_at" => {"gt" => "2021-02-28T23:59:59.998Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("01") + end + + it "avoids searching an index when filtering on a timestamp `lt` the first ms of the index" do + parts = search_index_expression_parts_for({"created_at" => {"lt" => "2021-11-01T00:00:00.001Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("12") + + parts = search_index_expression_parts_for({"created_at" => {"lt" => "2021-11-01T00:00:00Z"}}) + expect(parts).to target_widget_indices_excluding_2021_months("11", "12") + end + + it "supports non-UTC timestamps" do + parts = search_index_expression_parts_for({"created_at" => {"gte" => "2021-02-28T15:59:59.999-08:00"}}) + expect(parts).to target_widget_indices_excluding_2021_months("01") + + parts = search_index_expression_parts_for({"created_at" => {"gte" => "2021-02-28T16:00:00-08:00"}}) + expect(parts).to target_widget_indices_excluding_2021_months("01", "02") + end + + def coerce_date_times_in(object) + # No coercion necessary for DateTime objects. + object + end + end + end + + context "when the timestamp field is a `Date`" do + include_examples "search index expression logic", "Date" do + it "avoids searching an index when filtering on a date `gt` the last day of the index" do + parts = search_index_expression_parts_for({"created_at" => {"gt" => "2021-02-28"}}) + expect(parts).to target_widget_indices_excluding_2021_months("01", "02") + + parts = search_index_expression_parts_for({"created_at" => {"gt" => "2021-02-27"}}) + expect(parts).to target_widget_indices_excluding_2021_months("01") + end + + it "avoids searching an index when filtering on a date `lt` the first day of the index" do + parts = search_index_expression_parts_for({"created_at" => {"lt" => "2021-11-02"}}) + expect(parts).to target_widget_indices_excluding_2021_months("12") + + parts = search_index_expression_parts_for({"created_at" => {"lt" => "2021-11-01"}}) + expect(parts).to target_widget_indices_excluding_2021_months("11", "12") + end + + def coerce_date_times_in(object) + case object + when ::Hash + object.transform_values { |value| coerce_date_times_in(value) } + when ::Array + object.map { |item| coerce_date_times_in(item) } + when /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/ + object.split("T").first + else + object + end + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/shard_routing_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/shard_routing_spec.rb new file mode 100644 index 00000000..31eaa5a1 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/shard_routing_spec.rb @@ -0,0 +1,527 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_unit_support" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "shard routing" do + include_context "DatastoreQueryUnitSupport" + + attr_accessor :schema_artifacts_by_route_with_field_paths + + before(:context) do + self.schema_artifacts_by_route_with_field_paths = ::Hash.new do |hash, route_with_field_paths| + hash[route_with_field_paths] = generate_schema_artifacts do |schema| + schema.object_type "Foo" do |t| + t.field "bar", "Bar" + end + + schema.object_type "Bar" do |t| + t.field "name", "String" + end + + route_with_field_paths.each_with_index do |path, number| + schema.object_type "Type#{number}" do |t| + t.field "id", "ID" + + if path.start_with?("foo.") + t.field "foo", "Foo" # for the nested case + else + t.field path, "ID!" + end + + t.index "index#{number}" do |i| + i.route_with path + end + end + end + end + end + end + + it "searches all shards when the query does not filter on a single `route_with_field_paths` field" do + expect(shard_routing_for(["name"], {})).to search_all_shards + expect(shard_routing_for(["name"], { + "id" => {"equal_to_any_of" => ["abc"]} + })).to search_all_shards + end + + it "searches the shards identified by the values in an `equal_to_any_of` filter for a single `route_with_field_paths` field" do + expect(shard_routing_for(["name"], { + "name" => {"equal_to_any_of" => ["abc", "def"]} + })).to search_shards_identified_by "abc", "def" + end + + it "ignores `nil` among other values in `equal_to_any_of` filter for a single `route_with_field_paths` field" do + expect(shard_routing_for(["name"], { + "name" => {"equal_to_any_of" => ["abc", nil, "def"]} + })).to search_shards_identified_by "abc", "def" + end + + it "searches all shards when the query filters on single `route_with_field_paths` field using an inexact operator" do + expect(shard_routing_for(["name"], {"name" => {"gt" => "abc"}})).to search_all_shards + expect(shard_routing_for(["name"], {"name" => {"gte" => "abc"}})).to search_all_shards + expect(shard_routing_for(["name"], {"name" => {"lt" => "abc"}})).to search_all_shards + expect(shard_routing_for(["name"], {"name" => {"lte" => "abc"}})).to search_all_shards + expect(shard_routing_for(["name"], {"name" => {"matches" => "abc"}})).to search_all_shards + expect(shard_routing_for(["name"], {"name" => {"matches_query" => {"query" => "abc"}}})).to search_all_shards + expect(shard_routing_for(["name"], {"name" => {"matches_phrase" => {"phrase" => "abc"}}})).to search_all_shards + end + + it "ignores inequality operators on a single `route_with_field_paths` field when that field also has an exact equality operator" do + # the fact that we are filtering `> abc` can be ignored because we are only looking for `def` based on `equal_to_any_of`. + expect(shard_routing_for(["name"], {"name" => { + "gt" => "abc", + "equal_to_any_of" => ["def"] + }})).to search_shards_identified_by "def" + + # ordering of operators shouldn't matter... + expect(shard_routing_for(["name"], {"name" => { + "equal_to_any_of" => ["def"], + "gt" => "abc" + }})).to search_shards_identified_by "def" + end + + it "ignores filters on other fields so long as they are not in an `any_of` clause (for multiple filters in one hash)" do + expect(shard_routing_for(["name"], { + "name" => {"equal_to_any_of" => ["abc", "def"]}, + "cost" => {"gt" => 10} + })).to search_shards_identified_by "abc", "def" + end + + it "ignores filters on other fields so long as they are not in an `any_of` clause (for multiple filters in an array of hashes)" do + expect(shard_routing_for(["name"], [ + {"name" => {"equal_to_any_of" => ["abc", "def"]}}, + {"cost" => {"gt" => 10}} + ])).to search_shards_identified_by "abc", "def" + + # order should not matter... + expect(shard_routing_for(["name"], [ + {"cost" => {"gt" => 10}}, + {"name" => {"equal_to_any_of" => ["abc", "def"]}} + ])).to search_shards_identified_by "abc", "def" + end + + it "searches the shards identified by the set intersection of filter values when we have multiple `equal_to_any_of` filters on the same `route_with_field_paths` field" do + expect(shard_routing_for(["name"], [ + {"name" => {"equal_to_any_of" => ["abc", "def"]}}, + {"name" => {"equal_to_any_of" => ["def", "ghi"]}} + ])).to search_shards_identified_by "def" + end + + context "on a query with no aggregations" do + it "searches no shards when the result of the set intersection of filter values has no values, because no documents can match the filter" do + expect(shard_routing_for(["name"], [ + {"name" => {"equal_to_any_of" => ["abc", "def"]}}, + {"name" => {"equal_to_any_of" => ["ghi", "jkl"]}} + ])).to search_no_shards + end + + it "searches no shards when the query filters with `equal_to_any_of: []` on a single `route_with_field_paths` field, because no documents can match the filter" do + expect(shard_routing_for(["name"], { + "name" => {"equal_to_any_of" => []} + })).to search_no_shards + end + + it "searches no shards when the query filters with `equal_to_any_of: [nil]` on a single `route_with_field_paths` field, because no documents can match the filter" do + expect(shard_routing_for(["name"], { + "name" => {"equal_to_any_of" => [nil]} + })).to search_no_shards + end + end + + context "on a query with aggregations" do + it "searches the fallback shard when the result of the set intersection of filter values has no values, because we must search a shard to get a response with the expected structure" do + expect(shard_routing_for(["name"], [ + {"name" => {"equal_to_any_of" => ["abc", "def"]}}, + {"name" => {"equal_to_any_of" => ["ghi", "jkl"]}} + ])).to search_the_fallback_shard + end + + it "searches the fallback shard when the query filters with `equal_to_any_of: []` on a single `route_with_field_paths` field, because we must search a shard to get a response with the expected structure" do + expect(shard_routing_for(["name"], { + "name" => {"equal_to_any_of" => []} + })).to search_the_fallback_shard + end + + it "searches the fallback shard when the query filters with `equal_to_any_of: [nil]` on a single `route_with_field_paths` field, because we must search a shard to get a response with the expected structure" do + expect(shard_routing_for(["name"], { + "name" => {"equal_to_any_of" => [nil]} + })).to search_the_fallback_shard + end + + def shard_routing_for(route_with_field_paths, filter_or_filters) + aggregations = [aggregation_query_of(computations: [computation_of("amountMoney", "amount", :sum)])] + super(route_with_field_paths, filter_or_filters, aggregations: aggregations) + end + end + + it "searches all shards when the query filters with `equal_to_any_of: nil` on a single `route_with_field_paths` field because our current filter logic ignores that filter and we must search all shards" do + expect(shard_routing_for(["name"], { + "name" => {"equal_to_any_of" => nil} + })).to search_all_shards + end + + it "searches all shards when the query filters with `equal_to_any_on` on a single `route_with_field_paths` field only contains ignored routing values" do + expect(shard_routing_for( + ["name"], + {"name" => {"equal_to_any_of" => ["ignored_value"]}}, + ignored_routing_values: ["ignored_value"] + )).to search_all_shards + end + + it "searches all shards when the query filters with `equal_to_any_on` on a single `route_with_field_paths` field contains both an ignored routing value and non-ignored routing values" do + expect(shard_routing_for( + ["name"], + {"name" => {"equal_to_any_of" => ["ignored_value", "not_ignored_value"]}}, + ignored_routing_values: ["ignored_value"] + )).to search_all_shards + end + + it "supports nested field paths for routing fields" do + filters = {"foo" => {"bar" => {"name" => {"equal_to_any_of" => ["abc", "def"]}}}} + + routing = shard_routing_for(["foo.bar.name"], filters) + + expect(routing).to search_shards_identified_by "abc", "def" + end + + it "supports nested field paths for routing fields with `equal_to_any_of` containing `nil`" do + filters = {"foo" => {"bar" => {"name" => {"equal_to_any_of" => ["abc", nil, "def"]}}}} + + routing = shard_routing_for(["foo.bar.name"], filters) + + expect(routing).to search_shards_identified_by "abc", "def" + end + + it "searches all shards when there is an `any_of` filter clause that could match documents not covered by an exact value filter" do + expect(shard_routing_for(["name"], {"any_of" => [ + {"name" => {"equal_to_any_of" => ["abc", "def"]}}, + {"cost" => {"gt" => 10}} + ]})).to search_all_shards + + # nil value shouldn't matter + expect(shard_routing_for(["name"], {"any_of" => [ + {"name" => {"equal_to_any_of" => ["abc", nil, "def"]}}, + {"cost" => {"gt" => 10}} + ]})).to search_all_shards + + # order shouldn't matter... + expect(shard_routing_for(["name"], {"any_of" => [ + {"cost" => {"gt" => 10}}, + {"name" => {"equal_to_any_of" => ["abc", "def"]}} + ]})).to search_all_shards + end + + it "searches the shards identified by the set union of filter values when all `any_of` clauses filter on the same `route_with_field_paths` field" do + expect(shard_routing_for(["name"], {"any_of" => [ + {"name" => {"equal_to_any_of" => ["abc", "def"]}}, + {"name" => {"equal_to_any_of" => ["def", "ghi"]}} + ]})).to search_shards_identified_by "abc", "def", "ghi" + + # nil value shouldn't matter + expect(shard_routing_for(["name"], {"any_of" => [ + {"name" => {"equal_to_any_of" => ["abc", nil, "def"]}}, + {"name" => {"equal_to_any_of" => ["def", "ghi", nil]}} + ]})).to search_shards_identified_by "abc", "def", "ghi" + end + + it "searches all shards when one branch of an `any_of` could match documents on any shard" do + expect(shard_routing_for(["name"], {"any_of" => [ + {"name" => {"equal_to_any_of" => ["abc", "def"]}}, + {"name" => {"equal_to_any_of" => ["def", "ghi"]}}, + {"name" => {"gt" => "xyz"}} + ]})).to search_all_shards + + # order shouldn't matter... + expect(shard_routing_for(["name"], {"any_of" => [ + {"name" => {"equal_to_any_of" => ["abc", "def"]}}, + {"name" => {"gt" => "xyz"}}, + {"name" => {"equal_to_any_of" => ["def", "ghi"]}} + ]})).to search_all_shards + + expect(shard_routing_for(["name"], {"any_of" => [ + {"name" => {"gt" => "xyz"}}, + {"name" => {"equal_to_any_of" => ["abc", "def"]}}, + {"name" => {"equal_to_any_of" => ["def", "ghi"]}} + ]})).to search_all_shards + end + + it "searches all shards when we have an `any_of: []` filter because that will match all results" do + expect(shard_routing_for(["name"], { + "any_of" => [] + })).to search_all_shards + end + + describe "not" do + it "searches all shards when there are values in an `equal_to_any_of` filter" do + expect(shard_routing_for(["name"], + {"name" => {"not" => {"equal_to_any_of" => ["abc", "def"]}}})).to search_all_shards + end + + it "searches the shards identified by the values in an `equal_to_any_of` filter alongside `not`" do + expect(shard_routing_for(["name"], {"name" => { + "not" => {"equal_to_any_of" => ["abc"]}, + "equal_to_any_of" => ["abc", "def"] + }})).to search_shards_identified_by "def" + + # nil value shouldn't matter + expect(shard_routing_for(["name"], {"name" => { + "not" => {"equal_to_any_of" => ["abc", nil]}, + "equal_to_any_of" => ["abc", "def"] + }})).to search_shards_identified_by "def" + end + + it "searches all shards when `any_of` is an empty set" do + expect(shard_routing_for(["name"], { + "not" => {"any_of" => []} + })).to search_all_shards + end + + it "searches all shards when the query filters with `equal_to_any_of: []`" do + expect(shard_routing_for(["name"], { + "name" => { + "not" => {"equal_to_any_of" => []} + } + })).to search_all_shards + end + + it "searches all shards when the query filters with `equal_to_any_of: [nil]`" do + expect(shard_routing_for(["name"], { + "name" => { + "not" => {"equal_to_any_of" => [nil]} + } + })).to search_all_shards + end + + it "searches all shards when the query filters with `equal_to_any_of: nil`" do + expect(shard_routing_for(["name"], { + "name" => { + "not" => {"equal_to_any_of" => nil} + } + })).to search_all_shards + end + + it "searches all shards when set to nil`" do + expect(shard_routing_for(["name"], { + "name" => {"not" => nil} + })).to search_all_shards + end + + it "searches all shards when the query does not filter on a single `route_with_field_paths` field" do + expect(shard_routing_for(["name"], { + "id" => {"not" => {"equal_to_any_of" => ["abc"]}} + })).to search_all_shards + end + + it "searches all shards when the query filters on single `route_with_field_paths` field using an inexact operator" do + expect(shard_routing_for(["name"], {"name" => {"not" => {"gt" => "abc"}}})).to search_all_shards + expect(shard_routing_for(["name"], {"name" => {"not" => {"gte" => "abc"}}})).to search_all_shards + expect(shard_routing_for(["name"], {"name" => {"not" => {"lt" => "abc"}}})).to search_all_shards + expect(shard_routing_for(["name"], {"name" => {"not" => {"lte" => "abc"}}})).to search_all_shards + expect(shard_routing_for(["name"], {"name" => {"not" => {"matches" => "abc"}}})).to search_all_shards + expect(shard_routing_for(["name"], {"name" => {"not" => {"matches_query" => {"query" => "abc"}}}})).to search_all_shards + expect(shard_routing_for(["name"], {"name" => {"not" => {"matches_phrase" => {"phrase" => "abc"}}}})).to search_all_shards + end + + it "ignores inequality operators on a single `route_with_field_paths` field when that field also has an exact equality operator" do + # the fact that we are filtering `!(> xyz)` can be ignored because we are only looking for `def` based on `equal_to_any_of`. + expect(shard_routing_for(["name"], {"name" => { + "not" => {"gt" => "xyz"}, + "equal_to_any_of" => ["def"] + }})).to search_shards_identified_by "def" + + # ordering of operators shouldn't matter... + expect(shard_routing_for(["name"], {"name" => { + "equal_to_any_of" => ["def"], + "not" => {"gt" => "xyz"} + }})).to search_shards_identified_by "def" + end + + it "ignores filters on other fields so long as they are not in an `any_of` clause (for multiple filters in one hash)" do + expect(shard_routing_for(["name"], { + "name" => {"not" => {"equal_to_any_of" => ["abc", "def"]}}, + "cost" => {"gt" => 10} + })).to search_all_shards + end + + it "ignores filters on other fields so long as they are not in an `any_of` clause (for multiple filters in an array of hashes)" do + expect(shard_routing_for(["name"], [ + {"name" => {"not" => {"equal_to_any_of" => ["abc", "def"]}}}, + {"cost" => {"gt" => 10}} + ])).to search_all_shards + + expect(shard_routing_for(["name"], [ + {"name" => {"not" => {"equal_to_any_of" => ["abc", nil, "def"]}}}, + {"cost" => {"gt" => 10}} + ])).to search_all_shards + + expect(shard_routing_for(["name"], [ + {"name" => {"not" => {"equal_to_any_of" => []}}}, + {"cost" => {"gt" => 10}} + ])).to search_all_shards + + # order should not matter... + expect(shard_routing_for(["name"], [ + {"cost" => {"gt" => 10}}, + {"name" => {"not" => {"equal_to_any_of" => ["abc", "def"]}}} + ])).to search_all_shards + + expect(shard_routing_for(["name"], [ + {"cost" => {"gt" => 10}}, + {"name" => {"not" => {"equal_to_any_of" => ["abc", nil, "def"]}}} + ])).to search_all_shards + + expect(shard_routing_for(["name"], [ + {"cost" => {"gt" => 10}}, + {"name" => {"not" => {"equal_to_any_of" => []}}} + ])).to search_all_shards + end + + it "searches the shards identified by the set intersection of filter values when we have multiple `equal_to_any_of` filters on the same `route_with_field_paths` field" do + expect(shard_routing_for(["name"], [ + {"name" => {"equal_to_any_of" => ["abc", "def"]}}, + {"name" => {"not" => {"equal_to_any_of" => ["def", "ghi"]}}} + ])).to search_shards_identified_by "abc" + + expect(shard_routing_for(["name"], [ + {"name" => {"equal_to_any_of" => ["abc", "def"]}}, + {"name" => {"not" => {"equal_to_any_of" => ["def", nil, "ghi"]}}} + ])).to search_shards_identified_by "abc" + + expect(shard_routing_for(["name"], [ + {"name" => {"equal_to_any_of" => ["abc", nil, "def"]}}, + {"name" => {"not" => {"equal_to_any_of" => ["def", "ghi"]}}} + ])).to search_shards_identified_by "abc" + + expect(shard_routing_for(["name"], [ + {"name" => {"equal_to_any_of" => ["abc", "def"]}}, + {"name" => {"not" => {"equal_to_any_of" => []}}} + ])).to search_shards_identified_by "abc", "def" + end + + it "supports nested field paths for routing fields" do + filters = {"foo" => {"bar" => {"name" => {"not" => {"equal_to_any_of" => ["abc", "def"]}}}}} + + routing = shard_routing_for(["foo.bar.name"], filters) + + expect(routing).to search_all_shards + end + + it "can handle nested `not`s" do + expect(shard_routing_for( + ["name"], + {"name" => {"equal_to_any_of" => ["abc", "def"]}} + )).to search_shards_identified_by "abc", "def" + + expect(shard_routing_for( + ["name"], + {"name" => {"not" => {"equal_to_any_of" => ["abc", "def"]}}} + )).to search_all_shards + + expect(shard_routing_for( + ["name"], + {"not" => {"name" => {"not" => {"equal_to_any_of" => ["abc", "def"]}}}} + )).to search_shards_identified_by "abc", "def" + + expect(shard_routing_for( + ["name"], + {"not" => {"not" => {"name" => {"not" => {"equal_to_any_of" => ["abc", "def"]}}}}} + )).to search_all_shards + end + end + + context "when there are multiple fields in `route_with_field_paths`" do + it "searches all shards when the query does not filter on any of the `route_with_field_paths`" do + expect(shard_routing_for(["name", "user_id"], {})).to search_all_shards + expect(shard_routing_for(["name", "user_id"], { + "id" => {"equal_to_any_of" => ["abc"]} + })).to search_all_shards + end + + it "searches the shards identified by the set union of filter values, provided we are filtering on all routing fields, to ensure we search all shards that may contain matching documents" do + expect(shard_routing_for(["name", "user_id"], { + "name" => {"equal_to_any_of" => ["abc", "def"]}, + "user_id" => {"equal_to_any_of" => ["123", "456"]} + })).to search_shards_identified_by "abc", "def", "123", "456" + end + + it "searches all shards when one or more of the `route_with_field_paths` is not filtered on at all, to ensure we can find documents in the index with that routing field" do + expect(shard_routing_for(["name", "user_id"], { + "name" => {"equal_to_any_of" => ["abc", "def"]} + })).to search_all_shards + + expect(shard_routing_for(["name", "user_id"], { + "user_id" => {"equal_to_any_of" => ["123", "456"]} + })).to search_all_shards + end + + it "works correctly when `any_of` is used by multiple cousin nodes" do + expect(shard_routing_for(["name", "user_id"], { + "name" => {"any_of" => [{"equal_to_any_of" => ["abc", "def"]}]}, + "user_id" => {"any_of" => [{"equal_to_any_of" => ["123", "456"]}]} + })).to search_shards_identified_by "abc", "def", "123", "456" + end + end + + def shard_routing_for(route_with_field_paths, filter_or_filters, ignored_routing_values: [], aggregations: nil) + options = if filter_or_filters.is_a?(Array) + {filters: filter_or_filters} + else + {filter: filter_or_filters} + end + + search_index_definitions = search_index_definitions_for(route_with_field_paths, ignored_routing_values) + + query = new_query(search_index_definitions: search_index_definitions, aggregations: aggregations, **options) + + datastore_msearch_header_of(query)[:routing]&.split(",").tap do |used_routing_values| + expect(used_routing_values).to eq(query.shard_routing_values) + end + end + + def search_all_shards + eq(nil) # when no routing value is provided, the datastore will search all shards + end + + def search_shards_identified_by(*routing_values) + contain_exactly(*routing_values) + end + + def search_no_shards + eq [] # when an empty set of routing values are provided, the datastore will search no shards + end + + def search_the_fallback_shard + eq ["fallback_shard_routing_value"] + end + + def search_index_definitions_for(route_with_field_paths, ignored_routing_values) + index_definitions = route_with_field_paths.length.times.map do |i| + ["index#{i}", config_index_def_of(ignore_routing_values: ignored_routing_values)] + end.to_h + + graphql = build_graphql( + index_definitions: index_definitions, + schema_artifacts: schema_artifacts_by_route_with_field_paths[route_with_field_paths] + ) + + route_with_field_paths.map.with_index do |path, number| + graphql.datastore_core.index_definitions_by_name.fetch("index#{number}").tap do |index_def| + expect(index_def.route_with).to eq path + end + end + end + + def datastore_msearch_header_of(query) + query.send(:to_datastore_msearch_header) + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/sorting_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/sorting_spec.rb new file mode 100644 index 00000000..8f17f448 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/sorting_spec.rb @@ -0,0 +1,52 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_unit_support" +require "support/sort" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "sorting" do + include_context "DatastoreQueryUnitSupport" + include SortSupport + + it "uses only the tiebreaker sort clauses when given an empty `sort`" do + query = new_query(sort: [], individual_docs_needed: true) + expect(datastore_body_of(query)).to include_sort_with_tiebreaker + end + + it "uses only the tiebreaker sort clauses when given a nil `sort`" do + query = new_query(sort: nil, individual_docs_needed: true) + expect(datastore_body_of(query)).to include_sort_with_tiebreaker + end + + it "ignores duplicate sort fields, preferring whichever direction comes first" do + query = new_query(sort: [{"foo" => {"order" => "asc"}}, {"foo" => {"order" => "desc"}}], individual_docs_needed: true) + expect(datastore_body_of(query)).to include(sort: [{"foo" => {"order" => "asc", "missing" => "_first"}}, {"id" => {"order" => "asc", "missing" => "_first"}}]) + + query = new_query(sort: [{"foo" => {"order" => "desc"}}, {"foo" => {"order" => "asc"}}], individual_docs_needed: true) + expect(datastore_body_of(query)).to include(sort: [{"foo" => {"order" => "desc", "missing" => "_last"}}, {"id" => {"order" => "asc", "missing" => "_first"}}]) + end + + it "sets `sort:` when given a non-nil `sort`" do + query = new_query(sort: [{created_at: {"order" => "asc"}}], individual_docs_needed: true) + expect(datastore_body_of(query)).to include_sort_with_tiebreaker(created_at: {"order" => "asc"}) + end + + it "omits `sort` when `individual_docs_needed` is `false` since there will be no documents to sort" do + query = new_query(sort: [{created_at: {"order" => "asc"}}], individual_docs_needed: false) + + expect(datastore_body_of(query)).to exclude("sort", :sort) + end + + def include_sort_with_tiebreaker(*sort_clauses) + include(sort: sort_list_with_missing_option_for(*sort_clauses)) + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/sub_aggregations_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/sub_aggregations_spec.rb new file mode 100644 index 00000000..bd093b57 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/sub_aggregations_spec.rb @@ -0,0 +1,999 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_unit_support" +require "support/aggregations_helpers" +require "support/sub_aggregation_support" + +module ElasticGraph + class GraphQL + module SubAggregationQueryRefinements + refine ::Hash do + # Helper method that can be used to add a missing value bucket aggregation to an + # existing aggregation hash. Defined as a refinement to support a chainable syntax + # in order to minimize churn in our specs at the point we added missing value buckets. + def with_missing_value_agg + grouped_field = SubAggregationQueryRefinements.grouped_field_from(self) + grouped_sub_hash = fetch(grouped_field) + copied_entries = grouped_sub_hash.key?("aggs") ? grouped_sub_hash.slice("meta", "aggs") : {} + missing_agg_hash = copied_entries.merge({"missing" => {"field" => grouped_field}}) + + merge({Aggregation::Key.missing_value_bucket_key(grouped_field) => missing_agg_hash}) + end + end + + extend ::RSpec::Matchers + + def self.grouped_field_from(agg_hash) + grouped_field_candidates = agg_hash.except("aggs", "meta").keys + + # We expect only one candidate; here we use an expectation that will show them all if there are more. + expect(grouped_field_candidates).to eq([grouped_field_candidates.first]) + grouped_field_candidates.first + end + end + + RSpec.describe DatastoreQuery, "sub-aggregations" do + using SubAggregationQueryRefinements + include_context "DatastoreQueryUnitSupport" + include_context "sub-aggregation support", Aggregation::NonCompositeGroupingAdapter + + it "excludes a sub-aggregation requesting an empty page from the generated query body" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["current_players_nested"], query: sub_aggregation_query_of(name: "current_players_nested", first: 0)) + ])]) + + expect(datastore_body_of(query)).to exclude_aggs + end + + it "builds a `nested` aggregation query for a nested sub-aggregation" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["current_players_nested"], query: sub_aggregation_query_of(name: "current_players_nested", first: 12)), + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", first: 9)) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:current_players_nested" => {"nested" => {"path" => "current_players_nested"}, "meta" => outer_meta(size: 12)}, + "teams:seasons_nested" => {"nested" => {"path" => "seasons_nested"}, "meta" => outer_meta(size: 9)} + }) + end + + it "builds sub-aggregations of sub-aggregations of sub-aggregations when the query has that structure" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + first: 14, + sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested", "players_nested"], query: sub_aggregation_query_of( + name: "players_nested", + first: 15, + sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested", "players_nested", "seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + first: 16 + )) + ] + )) + ] + )) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "nested" => {"path" => "seasons_nested"}, + "meta" => outer_meta(size: 14), + "aggs" => { + "teams:seasons_nested:seasons_nested.players_nested" => { + "nested" => {"path" => "seasons_nested.players_nested"}, + "meta" => outer_meta(size: 15), + "aggs" => { + "teams:seasons_nested:players_nested:seasons_nested.players_nested.seasons_nested" => { + "nested" => {"path" => "seasons_nested.players_nested.seasons_nested"}, + "meta" => outer_meta(size: 16) + } + } + } + } + } + }) + end + + it "supports nesting sub-aggreations under an extra object layer" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["the_nested_fields", "current_players"], query: sub_aggregation_query_of(name: "current_players")) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:the_nested_fields.current_players" => { + "nested" => {"path" => "the_nested_fields.current_players"}, + "meta" => outer_meta + } + }) + end + + it "can handle sub-aggregation fields of the same name under parents of a different name" do + query = new_query(aggregations: [aggregation_query_of(name: "team_aggregations", sub_aggregations: [ + nested_sub_aggregation_of( + path_in_index: ["nested_fields", "seasons"], + query: sub_aggregation_query_of( + name: "seasons", + computations: [computation_of("nested_fields", "seasons", "year", :min, computed_field_name: "exact_min")] + ) + ), + nested_sub_aggregation_of( + path_in_index: ["nested_fields2", "seasons"], + query: sub_aggregation_query_of( + name: "seasons", + computations: [computation_of("nested_fields2", "seasons", "year", :min, computed_field_name: "exact_min")] + ) + ) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "team_aggregations:nested_fields.seasons" => { + "aggs" => { + "seasons:nested_fields.seasons.year:exact_min" => { + "min" => {"field" => "nested_fields.seasons.year"} + } + }, + "meta" => outer_meta, + "nested" => {"path" => "nested_fields.seasons"} + }, + "team_aggregations:nested_fields2.seasons" => { + "aggs" => { + "seasons:nested_fields2.seasons.year:exact_min" => { + "min" => {"field" => "nested_fields2.seasons.year"} + } + }, + "meta" => outer_meta, + "nested" => {"path" => "nested_fields2.seasons"} + } + }) + end + + it "supports filtered sub-aggregations" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["current_players_nested"], query: sub_aggregation_query_of(name: "current_players_nested", filter: { + "name" => {"equal_to_any_of" => %w[Dan Ted Bob]} + })) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:current_players_nested" => { + "meta" => outer_meta({"bucket_path" => ["current_players_nested:filtered"]}), + "nested" => {"path" => "current_players_nested"}, + "aggs" => { + "current_players_nested:filtered" => { + "filter" => { + bool: {filter: [{terms: {"current_players_nested.name" => ["Dan", "Ted", "Bob"]}}]} + } + } + } + } + }) + end + + it "ignores empty filters" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["current_players_nested"], query: sub_aggregation_query_of(name: "current_players_nested", filter: { + "name" => {"equal_to_any_of" => nil} + })) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:current_players_nested" => { + "nested" => {"path" => "current_players_nested"}, + "meta" => outer_meta + } + }) + end + + it "supports sub-aggregations under a filtered sub-aggregation" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["current_players_nested"], query: sub_aggregation_query_of( + name: "current_players_nested", + filter: {"name" => {"equal_to_any_of" => %w[Dan Ted Bob]}}, + sub_aggregations: [nested_sub_aggregation_of( + path_in_index: ["current_players_nested", "seasons_nested"], + query: sub_aggregation_query_of(name: "seasons_nested") + )] + )) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:current_players_nested" => { + "meta" => outer_meta({"bucket_path" => ["current_players_nested:filtered"]}), + "nested" => {"path" => "current_players_nested"}, + "aggs" => { + "current_players_nested:filtered" => { + "filter" => { + bool: {filter: [{terms: {"current_players_nested.name" => ["Dan", "Ted", "Bob"]}}]} + }, + "aggs" => { + "teams:current_players_nested:current_players_nested.seasons_nested" => { + "nested" => {"path" => "current_players_nested.seasons_nested"}, + "meta" => outer_meta + } + } + } + } + } + }) + end + + it "supports filtered sub-aggregations under a filtered sub-aggregation" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["current_players_nested"], query: sub_aggregation_query_of( + name: "current_players_nested", + filter: {"name" => {"equal_to_any_of" => %w[Dan Ted Bob]}}, + sub_aggregations: [nested_sub_aggregation_of( + path_in_index: ["current_players_nested", "seasons_nested"], + query: sub_aggregation_query_of(name: "seasons_nested", filter: {"year" => {"gt" => 2020}}) + )] + )) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:current_players_nested" => { + "meta" => outer_meta({"bucket_path" => ["current_players_nested:filtered"]}), + "nested" => {"path" => "current_players_nested"}, + "aggs" => { + "current_players_nested:filtered" => { + "filter" => { + bool: {filter: [{terms: {"current_players_nested.name" => ["Dan", "Ted", "Bob"]}}]} + }, + "aggs" => { + "teams:current_players_nested:current_players_nested.seasons_nested" => { + "meta" => outer_meta({"bucket_path" => ["seasons_nested:filtered"]}), + "nested" => {"path" => "current_players_nested.seasons_nested"}, + "aggs" => { + "seasons_nested:filtered" => { + "filter" => { + bool: {filter: [{range: {"current_players_nested.seasons_nested.year" => {gt: 2020}}}]} + } + } + } + } + } + } + } + } + }) + end + + context "with computations" do + it "supports ungrouped aggregated values" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", computations: [ + computation_of("seasons_nested", "year", :min, computed_field_name: "exact_min"), + computation_of("seasons_nested", "the_record", "win_count", :max, computed_field_name: "exact_max"), + computation_of("seasons_nested", "the_record", "win_count", :avg, computed_field_name: "approximate_avg") + ])) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "nested" => {"path" => "seasons_nested"}, + "meta" => outer_meta, + "aggs" => { + "seasons_nested:seasons_nested.the_record.win_count:approximate_avg" => { + "avg" => {"field" => "seasons_nested.the_record.win_count"} + }, + "seasons_nested:seasons_nested.the_record.win_count:exact_max" => { + "max" => {"field" => "seasons_nested.the_record.win_count"} + }, + "seasons_nested:seasons_nested.year:exact_min" => { + "min" => {"field" => "seasons_nested.year"} + } + } + } + }) + end + + it "uses the GraphQL query field names in the aggregation key when they differ from the field names in the index" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", computations: [ + computation_of("seasons_nested", "year", :min, computed_field_name: "exact_min", field_names_in_graphql_query: ["sea_nest", "the_year"]), + computation_of("seasons_nested", "the_record", "win_count", :avg, computed_field_name: "approximate_avg", field_names_in_graphql_query: ["sea_nest", "rec", "wins"]) + ])) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "nested" => {"path" => "seasons_nested"}, + "meta" => outer_meta, + "aggs" => { + "seasons_nested:sea_nest.rec.wins:approximate_avg" => { + "avg" => {"field" => "seasons_nested.the_record.win_count"} + }, + "seasons_nested:sea_nest.the_year:exact_min" => { + "min" => {"field" => "seasons_nested.year"} + } + } + } + }) + end + + it "supports filtered aggregated values" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + filter: {"year" => {"gt" => 2021}}, + computations: [ + computation_of("seasons_nested", "year", :min, computed_field_name: "exact_min") + ] + )) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "nested" => {"path" => "seasons_nested"}, + "aggs" => { + "seasons_nested:filtered" => { + "filter" => {bool: {filter: [{range: {"seasons_nested.year" => {gt: 2021}}}]}}, + "aggs" => { + "seasons_nested:seasons_nested.year:exact_min" => { + "min" => {"field" => "seasons_nested.year"} + } + } + } + }, + "meta" => outer_meta({"bucket_path" => ["seasons_nested:filtered"]}) + } + }) + end + + it "supports grouped aggregated values" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + computations: [ + computation_of("seasons_nested", "year", :min, computed_field_name: "exact_min"), + computation_of("seasons_nested", "the_record", "win_count", :max, computed_field_name: "exact_max"), + computation_of("seasons_nested", "the_record", "win_count", :avg, computed_field_name: "approximate_avg") + ], + groupings: [ + field_term_grouping_of("seasons_nested", "notes") + ] + )) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "nested" => {"path" => "seasons_nested"}, + "meta" => outer_meta({"buckets_path" => ["seasons_nested.notes"]}), + "aggs" => { + "seasons_nested.notes" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.notes"], "key_path" => ["key"]}), + "terms" => terms({"field" => "seasons_nested.notes", "collect_mode" => "depth_first"}), + "aggs" => { + "seasons_nested:seasons_nested.year:exact_min" => { + "min" => {"field" => "seasons_nested.year"} + }, + "seasons_nested:seasons_nested.the_record.win_count:exact_max" => { + "max" => {"field" => "seasons_nested.the_record.win_count"} + }, + "seasons_nested:seasons_nested.the_record.win_count:approximate_avg" => { + "avg" => {"field" => "seasons_nested.the_record.win_count"} + } + } + } + }.with_missing_value_agg + } + }) + end + + it "uses the GraphQL query field name (instead of the `name_in_index`) for `grouping_fields` meta on terms groupings" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + computations: [ + computation_of("seasons_nested", "year", :min, computed_field_name: "exact_min") + ], + groupings: [ + field_term_grouping_of("seasons_nested", "notes", field_names_in_graphql_query: ["sea_nest", "the_notes"]) + ] + )) + ])]) + + expect(datastore_body_of(query).dig(:aggs, "teams:seasons_nested", "aggs", "sea_nest.the_notes", "meta")).to eq( + inner_terms_meta({"grouping_fields" => ["sea_nest.the_notes"], "key_path" => ["key"]}) + ) + end + + it "supports aggregated values on multiple levels of sub-aggregations" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + computations: [computation_of("seasons_nested", "year", :min)], + sub_aggregations: [nested_sub_aggregation_of(path_in_index: ["seasons_nested", "players_nested"], query: sub_aggregation_query_of( + name: "players_nested", + computations: [computation_of("seasons_nested", "players_nested", "name", :cardinality)], + sub_aggregations: [nested_sub_aggregation_of(path_in_index: ["seasons_nested", "players_nested", "seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + computations: [computation_of("seasons_nested", "players_nested", "seasons_nested", "year", :max)] + ))] + ))] + )) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "nested" => {"path" => "seasons_nested"}, + "meta" => outer_meta, + "aggs" => { + "seasons_nested:seasons_nested.year:min" => { + "min" => {"field" => "seasons_nested.year"} + }, + "teams:seasons_nested:seasons_nested.players_nested" => { + "nested" => {"path" => "seasons_nested.players_nested"}, + "meta" => outer_meta, + "aggs" => { + "players_nested:seasons_nested.players_nested.name:cardinality" => { + "cardinality" => {"field" => "seasons_nested.players_nested.name"} + }, + "teams:seasons_nested:players_nested:seasons_nested.players_nested.seasons_nested" => { + "nested" => {"path" => "seasons_nested.players_nested.seasons_nested"}, + "meta" => outer_meta, + "aggs" => { + "seasons_nested:seasons_nested.players_nested.seasons_nested.year:max" => { + "max" => {"field" => "seasons_nested.players_nested.seasons_nested.year"} + } + } + } + } + } + } + } + }) + end + end + + context "with groupings (using the `NonCompositeGroupingAdapter`)" do + include_context "sub-aggregation support", Aggregation::NonCompositeGroupingAdapter + + it "can group sub-aggregations on a single non-date field" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", groupings: [ + field_term_grouping_of("seasons_nested", "year") + ])) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "nested" => {"path" => "seasons_nested"}, + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}), + "aggs" => { + "seasons_nested.year" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.year"], "key_path" => ["key"]}), + "terms" => terms({"field" => "seasons_nested.year", "collect_mode" => "depth_first"}) + } + }.with_missing_value_agg + } + }) + end + + it "can group sub-aggregations on multiple non-date fields" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", groupings: [ + field_term_grouping_of("seasons_nested", "year"), + field_term_grouping_of("seasons_nested", "notes") + ])) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "nested" => {"path" => "seasons_nested"}, + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}), + "aggs" => { + "seasons_nested.year" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.year"], "key_path" => ["key"], "buckets_path" => ["seasons_nested.notes"]}), + "terms" => terms({"field" => "seasons_nested.year", "collect_mode" => "depth_first"}), + "aggs" => { + "seasons_nested.notes" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.notes"], "key_path" => ["key"]}), + "terms" => terms({"field" => "seasons_nested.notes", "collect_mode" => "depth_first"}) + } + }.with_missing_value_agg + } + }.with_missing_value_agg + } + }) + end + + it "limits `terms` aggregations based on the query `size`" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + first: 14, + groupings: [field_term_grouping_of("seasons_nested", "year")] + )) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "nested" => {"path" => "seasons_nested"}, + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}, size: 14), + "aggs" => { + "seasons_nested.year" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.year"], "key_path" => ["key"]}), + "terms" => terms({"field" => "seasons_nested.year", "collect_mode" => "depth_first"}, size: 14) + } + }.with_missing_value_agg + } + }) + end + + it "limits `terms` aggregations based on the query `size`" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + first: 17, + groupings: [ + field_term_grouping_of("seasons_nested", "year"), + field_term_grouping_of("seasons_nested", "notes") + ] + )) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "nested" => {"path" => "seasons_nested"}, + "meta" => outer_meta({"buckets_path" => ["seasons_nested.year"]}, size: 17), + "aggs" => { + "seasons_nested.year" => { + "meta" => inner_terms_meta({"buckets_path" => ["seasons_nested.notes"], "grouping_fields" => ["seasons_nested.year"], "key_path" => ["key"]}), + "terms" => terms({"field" => "seasons_nested.year", "collect_mode" => "depth_first"}, size: 17), + "aggs" => { + "seasons_nested.notes" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.notes"], "key_path" => ["key"]}), + "terms" => terms({"field" => "seasons_nested.notes", "collect_mode" => "depth_first"}, size: 17) + } + }.with_missing_value_agg + } + }.with_missing_value_agg + } + }) + end + + it "can group sub-aggregations on a single date field" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", groupings: [ + date_histogram_grouping_of("seasons_nested", "started_at", "year") + ])) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "nested" => {"path" => "seasons_nested"}, + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at"]}), + "aggs" => { + "seasons_nested.started_at" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.started_at"], "key_path" => ["key_as_string"]}), + "date_histogram" => { + "calendar_interval" => "year", + "field" => "seasons_nested.started_at", + "format" => "strict_date_time", + "time_zone" => "UTC", + "min_doc_count" => 1 + } + } + }.with_missing_value_agg + } + }) + end + + it "uses the GraphQL query field name (instead of the `name_in_index`) for `grouping_fields` meta on date groupings" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", groupings: [ + date_histogram_grouping_of("seasons_nested", "started_at", "year", field_names_in_graphql_query: ["sea_nest", "started"]) + ])) + ])]) + + expect(datastore_body_of(query).dig(:aggs, "teams:seasons_nested", "aggs", "sea_nest.started", "meta")).to eq( + inner_date_meta({"grouping_fields" => ["sea_nest.started"], "key_path" => ["key_as_string"]}) + ) + end + + it "uses the GraphQL query field name (instead of the `name_in_index`) for `grouping_fields` meta on date groupings" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", groupings: [ + date_histogram_grouping_of("seasons_nested", "started_at", "year") + ])) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "nested" => {"path" => "seasons_nested"}, + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at"]}), + "aggs" => { + "seasons_nested.started_at" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.started_at"], "key_path" => ["key_as_string"]}), + "date_histogram" => { + "calendar_interval" => "year", + "field" => "seasons_nested.started_at", + "format" => "strict_date_time", + "time_zone" => "UTC", + "min_doc_count" => 1 + } + } + }.with_missing_value_agg + } + }) + end + + it "can group sub-aggregations on a multiple date fields" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", groupings: [ + date_histogram_grouping_of("seasons_nested", "started_at", "year"), + date_histogram_grouping_of("seasons_nested", "won_games_at", "year") + ])) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "nested" => {"path" => "seasons_nested"}, + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at"]}), + "aggs" => { + "seasons_nested.started_at" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.started_at"], "buckets_path" => ["seasons_nested.won_games_at"], "key_path" => ["key_as_string"]}), + "date_histogram" => { + "calendar_interval" => "year", + "field" => "seasons_nested.started_at", + "format" => "strict_date_time", + "time_zone" => "UTC", + "min_doc_count" => 1 + }, + "aggs" => { + "seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.won_games_at"], "key_path" => ["key_as_string"]}), + "date_histogram" => { + "calendar_interval" => "year", + "field" => "seasons_nested.won_games_at", + "format" => "strict_date_time", + "time_zone" => "UTC", + "min_doc_count" => 1 + } + } + }.with_missing_value_agg + } + }.with_missing_value_agg + } + }) + end + + it "can group sub-aggregations on a single non-date field and a single date field" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", groupings: [ + date_histogram_grouping_of("seasons_nested", "started_at", "year"), + field_term_grouping_of("seasons_nested", "notes") + ])) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "nested" => {"path" => "seasons_nested"}, + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at"]}), + "aggs" => { + "seasons_nested.started_at" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.started_at"], "buckets_path" => ["seasons_nested.notes"], "key_path" => ["key_as_string"]}), + "date_histogram" => { + "calendar_interval" => "year", + "field" => "seasons_nested.started_at", + "format" => "strict_date_time", + "time_zone" => "UTC", + "min_doc_count" => 1 + }, + "aggs" => { + "seasons_nested.notes" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.notes"], "key_path" => ["key"]}), + "terms" => terms({"field" => "seasons_nested.notes", "collect_mode" => "depth_first"}) + } + }.with_missing_value_agg + } + }.with_missing_value_agg + } + }) + end + + it "can group sub-aggregations on multiple non-date fields and a single date field" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", groupings: [ + date_histogram_grouping_of("seasons_nested", "started_at", "year"), + field_term_grouping_of("seasons_nested", "year"), + field_term_grouping_of("seasons_nested", "notes") + ])) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "nested" => {"path" => "seasons_nested"}, + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at"]}), + "aggs" => { + "seasons_nested.started_at" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.started_at"], "buckets_path" => ["seasons_nested.year"], "key_path" => ["key_as_string"]}), + "date_histogram" => { + "calendar_interval" => "year", + "field" => "seasons_nested.started_at", + "format" => "strict_date_time", + "time_zone" => "UTC", + "min_doc_count" => 1 + }, + "aggs" => { + "seasons_nested.year" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.year"], "buckets_path" => ["seasons_nested.notes"], "key_path" => ["key"]}), + "terms" => terms({"field" => "seasons_nested.year", "collect_mode" => "depth_first"}), + "aggs" => { + "seasons_nested.notes" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.notes"], "key_path" => ["key"]}), + "terms" => terms({"field" => "seasons_nested.notes", "collect_mode" => "depth_first"}) + } + }.with_missing_value_agg + } + }.with_missing_value_agg + } + }.with_missing_value_agg + } + }) + end + + it "can group sub-aggregations on multiple non-date fields and multiple date fields" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of(name: "seasons_nested", groupings: [ + date_histogram_grouping_of("seasons_nested", "started_at", "year"), + date_histogram_grouping_of("seasons_nested", "won_games_at", "year"), + field_term_grouping_of("seasons_nested", "year"), + field_term_grouping_of("seasons_nested", "notes") + ])) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "nested" => {"path" => "seasons_nested"}, + "meta" => outer_meta({"buckets_path" => ["seasons_nested.started_at"]}), + "aggs" => { + "seasons_nested.started_at" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.started_at"], "buckets_path" => ["seasons_nested.won_games_at"], "key_path" => ["key_as_string"]}), + "date_histogram" => { + "calendar_interval" => "year", + "field" => "seasons_nested.started_at", + "format" => "strict_date_time", + "time_zone" => "UTC", + "min_doc_count" => 1 + }, + "aggs" => { + "seasons_nested.won_games_at" => { + "meta" => inner_date_meta({"grouping_fields" => ["seasons_nested.won_games_at"], "buckets_path" => ["seasons_nested.year"], "key_path" => ["key_as_string"]}), + "date_histogram" => { + "calendar_interval" => "year", + "field" => "seasons_nested.won_games_at", + "format" => "strict_date_time", + "time_zone" => "UTC", + "min_doc_count" => 1 + }, + "aggs" => { + "seasons_nested.year" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.year"], "buckets_path" => ["seasons_nested.notes"], "key_path" => ["key"]}), + "terms" => terms({"field" => "seasons_nested.year", "collect_mode" => "depth_first"}), + "aggs" => { + "seasons_nested.notes" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.notes"], "key_path" => ["key"]}), + "terms" => terms({"field" => "seasons_nested.notes", "collect_mode" => "depth_first"}) + } + }.with_missing_value_agg + } + }.with_missing_value_agg + } + }.with_missing_value_agg + } + }.with_missing_value_agg + } + }) + end + + it "accounts for an extra filtering layer in the `buckets_path` meta" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + groupings: [field_term_grouping_of("seasons_nested", "year")], + filter: {"year" => {"gt" => 2020}} + )) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "nested" => {"path" => "seasons_nested"}, + "meta" => outer_meta({"buckets_path" => ["seasons_nested:filtered", "seasons_nested.year"]}), + "aggs" => { + "seasons_nested:filtered" => { + "filter" => { + bool: {filter: [{range: {"seasons_nested.year" => {gt: 2020}}}]} + }, + "aggs" => { + "seasons_nested.year" => { + "meta" => inner_terms_meta({"grouping_fields" => ["seasons_nested.year"], "key_path" => ["key"]}), + "terms" => terms({"field" => "seasons_nested.year", "collect_mode" => "depth_first"}) + } + }.with_missing_value_agg + } + } + } + }) + end + + it "encodes embedded grouping fields in the meta" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + groupings: [ + field_term_grouping_of("seasons_nested", "parent1", "parent2", "year"), + date_histogram_grouping_of("seasons_nested", "parent1", "parent2", "started_at", "year") + ] + )) + ])]) + + date_groupings = datastore_body_of(query).dig(:aggs, "teams:seasons_nested", "aggs", "seasons_nested.parent1.parent2.started_at") + + expect(date_groupings.dig("meta", "grouping_fields")).to eq ["seasons_nested.parent1.parent2.started_at"] + expect(date_groupings.dig("aggs", "seasons_nested.parent1.parent2.year", "meta", "grouping_fields")).to eq ["seasons_nested.parent1.parent2.year"] + end + + it "sets `show_term_doc_count_error` based on the `needs_doc_count_error` query flag" do + term_options_for_true, term_options_for_false = [true, false].map do |needs_doc_count_error| + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + needs_doc_count_error: needs_doc_count_error, + groupings: [field_term_grouping_of("seasons_nested", "year")] + )) + ])]) + + datastore_body_of(query).dig(:aggs, "teams:seasons_nested", "aggs", "seasons_nested.year", "terms") + end + + expect(term_options_for_true).to include("show_term_doc_count_error" => true) + expect(term_options_for_false).to include("show_term_doc_count_error" => false) + end + end + + context "with groupings (using the `CompositeGroupingAdapter`)" do + include_context "sub-aggregation support", Aggregation::CompositeGroupingAdapter + + it "builds a `composite` sub-aggregation query" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + needs_doc_count: true, + computations: [ + computation_of("seasons_nested", "year", :min, computed_field_name: "exact_min"), + computation_of("seasons_nested", "the_record", "win_count", :max, computed_field_name: "exact_max"), + computation_of("seasons_nested", "the_record", "win_count", :avg, computed_field_name: "approximate_avg") + ], + groupings: [ + date_histogram_grouping_of("seasons_nested", "started_at", "year"), + field_term_grouping_of("seasons_nested", "year"), + field_term_grouping_of("seasons_nested", "notes") + ] + )) + ])]) + + expect(datastore_body_of(query)).to include_aggs({ + "teams:seasons_nested" => { + "meta" => outer_meta({"buckets_path" => ["seasons_nested"]}), + "nested" => {"path" => "seasons_nested"}, + "aggs" => { + "seasons_nested" => { + "composite" => { + "size" => 51, + "sources" => [ + { + "seasons_nested.started_at" => { + "date_histogram" => { + "calendar_interval" => "year", + "missing_bucket" => true, + "field" => "seasons_nested.started_at", + "format" => "strict_date_time", + "time_zone" => "UTC" + } + } + }, + { + "seasons_nested.year" => { + "terms" => { + "field" => "seasons_nested.year", + "missing_bucket" => true + } + } + }, + { + "seasons_nested.notes" => { + "terms" => { + "field" => "seasons_nested.notes", + "missing_bucket" => true + } + } + } + ] + }, + "aggs" => { + "seasons_nested:seasons_nested.year:exact_min" => { + "min" => {"field" => "seasons_nested.year"} + }, + "seasons_nested:seasons_nested.the_record.win_count:exact_max" => { + "max" => {"field" => "seasons_nested.the_record.win_count"} + }, + "seasons_nested:seasons_nested.the_record.win_count:approximate_avg" => { + "avg" => {"field" => "seasons_nested.the_record.win_count"} + } + } + } + } + } + }) + end + + it "uses the GraphQL query field names in composite aggregation keys when they differ from the field names in the index" do + query = new_query(aggregations: [aggregation_query_of(name: "teams", sub_aggregations: [ + nested_sub_aggregation_of(path_in_index: ["seasons_nested"], query: sub_aggregation_query_of( + name: "seasons_nested", + needs_doc_count: true, + groupings: [ + date_histogram_grouping_of("seasons_nested", "started_at", "year", field_names_in_graphql_query: ["sea_nest", "started"]), + field_term_grouping_of("seasons_nested", "year", field_names_in_graphql_query: ["sea_nest", "the_year"]), + field_term_grouping_of("seasons_nested", "notes", field_names_in_graphql_query: ["sea_nest", "note"]) + ] + )) + ])]) + + sources = datastore_body_of(query).dig(:aggs, "teams:seasons_nested", "aggs", "seasons_nested", "composite", "sources") + expect(sources).to eq [ + { + "sea_nest.started" => { + "date_histogram" => { + "calendar_interval" => "year", + "missing_bucket" => true, + "field" => "seasons_nested.started_at", + "format" => "strict_date_time", + "time_zone" => "UTC" + } + } + }, + { + "sea_nest.the_year" => { + "terms" => { + "field" => "seasons_nested.year", + "missing_bucket" => true + } + } + }, + { + "sea_nest.note" => { + "terms" => { + "field" => "seasons_nested.notes", + "missing_bucket" => true + } + } + } + ] + end + end + + def include_aggs(aggs) + include(aggs: aggs) + end + + def exclude_aggs + exclude(:aggs) + end + + def terms(terms_hash, size: 50, show_term_doc_count_error: false) + terms_hash.merge({"size" => size, "show_term_doc_count_error" => show_term_doc_count_error}) + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/track_total_hits_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/track_total_hits_spec.rb new file mode 100644 index 00000000..488e94e7 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/track_total_hits_spec.rb @@ -0,0 +1,25 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "datastore_query_unit_support" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreQuery, "#track_total_hits" do + include_context "DatastoreQueryUnitSupport" + + it "sets `track_total_hits` to false when `total_document_count_needed = false`" do + expect(datastore_body_of(new_query(total_document_count_needed: false))).to include(track_total_hits: false) + end + + it "sets `track_total_hits` to true when `total_document_count_needed = true`" do + expect(datastore_body_of(new_query(total_document_count_needed: true))).to include(track_total_hits: true) + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_response/document_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_response/document_spec.rb new file mode 100644 index 00000000..9868dd0f --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_response/document_spec.rb @@ -0,0 +1,149 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/graphql/decoded_cursor" +require "elastic_graph/graphql/datastore_response/document" +require "json" + +module ElasticGraph + class GraphQL + module DatastoreResponse + RSpec.describe Document do + let(:decoded_cursor_factory) { DecodedCursor::Factory.new(%w[amount_cents name]) } + + let(:raw_data) do + { + "_index" => "widgets", + "_type" => "_doc", + "_id" => "qwbfffaijhkljtfmcuwv", + "_score" => 50.23, + "_source" => { + "id" => "qwbfffaijhkljtfmcuwv", + "version" => 10, + "amount_cents" => 300, + "name" => "HuaweiP Smart", + "created_at" => "2019-06-03T22 =>46 =>01Z", + "options" => { + "size" => "MEDIUM", + "color" => "GREEN" + }, + "component_ids" => [] + }, + "sort" => [ + 300, + "HuaweiP Smart" + ] + } + end + + let(:document) { build_doc(raw_data) } + + it "exposes `raw_data`" do + expect(document.raw_data).to eq raw_data + end + + it "exposes `index_name`" do + expect(document.index_name).to eq "widgets" + end + + it "exposes `index_definition_name` (which is the same as the `index_name` on a non-rollover index)" do + expect(document.index_definition_name).to eq "widgets" + end + + it "exposes `index_definition_name` (which is the name of the parent index definition on a rollover index)" do + document = build_doc(raw_data.merge("_index" => "components#{ROLLOVER_INDEX_INFIX_MARKER}2021-02")) + expect(document.index_definition_name).to eq "components" + end + + it "exposes `id`" do + expect(document.id).to eq "qwbfffaijhkljtfmcuwv" + end + + it "exposes `id` in payload even when there's no `_source` field" do + raw_data = { + "_index" => "widgets", + "_type" => "_doc", + "_id" => "qwbfffaijhkljtfmcuwv", + "_score" => 50.23, + "sort" => [ + 300, + "HuaweiP Smart" + ] + } + document = build_doc(raw_data) + expect(document.payload["id"]).to eq "qwbfffaijhkljtfmcuwv" + end + + it "exposes `version`" do + expect(document.version).to eq 10 + end + + it "returns `nil` if `version` field is missing" do + document = build_doc(raw_data.merge("_source" => {})) + expect(document.version).to eq nil + end + + it "exposes `payload`" do + expect(document.payload).to eq raw_data.fetch("_source") + end + + it "exposes its datastore path" do + expect(document.datastore_path).to eq "/widgets/_doc/qwbfffaijhkljtfmcuwv" + end + + it "exposes a `cursor` encoded using the `DecodedCursor::Factory` passed to `build`" do + expect(document.cursor).to eq decoded_cursor_factory.build(raw_data.fetch("sort")) + end + + it "memoizes the `cursor`" do + decoded_cursor_factory = instance_spy(DecodedCursor::Factory, build: "cursor") + document = Document.build(raw_data, decoded_cursor_factory: decoded_cursor_factory) + 3.times { document.cursor } + + expect(decoded_cursor_factory).to have_received(:build).once + end + + it "builds a valid cursor even if built without a cursor encoder" do + document = Document.build(raw_data) + expect(document.cursor).to be_a DecodedCursor + end + + it "does not mutate the passed data" do + expect { + build_doc(raw_data) + }.not_to change { JSON.generate(raw_data) } + end + + it "inspects nicely" do + expect(document.to_s).to eq "#" + expect(document.inspect).to eq document.to_s + end + + it "allows fields to be accessed using `#[]` hash syntax" do + expect(document["amount_cents"]).to eq 300 + end + + it "allows fields to be accessed using `#fetch` like on a hash" do + expect(document.fetch("amount_cents")).to eq 300 + expect { document.fetch("foo") }.to raise_error(KeyError) + end + + it "supports easy construction without the full raw data payload (such as for tests)" do + doc = Document.with_payload({"foo" => 12}) + expect(doc.to_s).to eq "#" + expect(doc["foo"]).to eq 12 + end + + def build_doc(data) + Document.build(data, decoded_cursor_factory: decoded_cursor_factory) + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_response/search_response_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_response/search_response_spec.rb new file mode 100644 index 00000000..bf647bf0 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_response/search_response_spec.rb @@ -0,0 +1,212 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/graphql/decoded_cursor" +require "elastic_graph/graphql/datastore_response/search_response" +require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" +require "json" + +module ElasticGraph + class GraphQL + module DatastoreResponse + RSpec.describe SearchResponse do + let(:decoded_cursor_factory) { DecodedCursor::Factory.new(["amount_cents"]) } + let(:raw_data) do + { + "took" => 50, + "timed_out" => false, + "_shards" => { + "total" => 5, + "successful" => 5, + "skipped" => 0, + "failed" => 0 + }, + "hits" => { + "total" => { + "value" => 17, + "relation" => "eq" + }, + "max_score" => nil, + "hits" => [ + { + "_index" => "widgets", + "_type" => "_doc", + "_id" => "qwbfffaijhkljtfmcuwv", + "_score" => nil, + "_source" => { + "id" => "qwbfffaijhkljtfmcuwv", + "version" => 10, + "amount_cents" => 300, + "name" => "HuaweiP Smart", + "created_at" => "2019-06-03T22 =>46 =>01Z", + "options" => { + "size" => "MEDIUM", + "color" => "GREEN" + }, + "component_ids" => [] + }, + "sort" => [ + 300 + ] + }, + { + "_index" => "widgets", + "_type" => "_doc", + "_id" => "zwbfffaijhkljtfmcuwv", + "_score" => nil, + "_source" => { + "id" => "zwbfffaijhkljtfmcuwv", + "version" => 10, + "amount_cents" => 300, + "name" => "HuaweiP Smart", + "created_at" => "2019-06-03T22 =>46 =>01Z", + "options" => { + "size" => "MEDIUM", + "color" => "GREEN" + }, + "component_ids" => [] + }, + "sort" => [ + 300 + ] + }, + { + "_index" => "widgets", + "_type" => "_doc", + "_id" => "dubsponikrrgasvwbthh", + "_score" => nil, + "_source" => { + "id" => "dubsponikrrgasvwbthh", + "version" => 7, + "amount_cents" => 200, + "name" => "Samsung Galaxy S9", + "created_at" => "2019-06-18T04 =>01 =>51Z", + "options" => { + "size" => "LARGE", + "color" => "BLUE" + }, + "component_ids" => [] + }, + "sort" => [ + 200 + ] + } + ] + } + } + end + + let(:response) { build_response(raw_data) } + + it "builds from a raw datastore JSON response" do + expect(response.raw_data).to eq raw_data + expect(response.documents.size).to eq 3 + end + + it "exposes `metadata` containing everything but the documents themselves" do + expect(response.metadata).to eq( + "took" => 50, + "timed_out" => false, + "_shards" => { + "total" => 5, + "successful" => 5, + "skipped" => 0, + "failed" => 0 + }, + "hits" => { + "total" => { + "value" => 17, + "relation" => "eq" + }, + "max_score" => nil + } + ) + end + + it "exposes a `total_document_count` based on `hits.total`" do + expect(response.total_document_count).to eq 17 + end + + it "avoids mutating the raw data used to build the object" do + expect { + build_response(raw_data) + }.not_to change { JSON.generate(raw_data) } + end + + it "converts the documents to `DatastoreResponse::Document` objects" do + expect(response.documents).to all be_a DatastoreResponse::Document + expect(response.documents.map(&:id)).to eq %w[qwbfffaijhkljtfmcuwv zwbfffaijhkljtfmcuwv dubsponikrrgasvwbthh] + end + + it "passes along the decoded cursor factory so that the documents can expose a cursor" do + expect(response.documents.map { |doc| doc.cursor.encode }).to all match(/\w+/) + end + + it "can be treated as a collection of documents" do + expect(response.to_a).to eq response.documents + expect(response.map(&:id)).to eq %w[qwbfffaijhkljtfmcuwv zwbfffaijhkljtfmcuwv dubsponikrrgasvwbthh] + expect(response.size).to eq 3 + expect(response.empty?).to eq false + end + + it "inspects nicely for when there are no documents" do + response = build_response(raw_data_with_docs(0)) + + expect(response.to_s).to eq "#" + expect(response.inspect).to eq response.to_s + end + + it "inspects nicely for when there is one document" do + response = build_response(raw_data_with_docs(1)) + + expect(response.to_s).to eq "#]>" + expect(response.inspect).to eq response.to_s + end + + it "inspects nicely for when there are two documents" do + response = build_response(raw_data_with_docs(2)) + + expect(response.to_s).to eq "#, " \ + "#]>" + expect(response.inspect).to eq response.to_s + end + + it "inspects nicely for when there are 3 or more documents" do + response = build_response(raw_data_with_docs(3)) + + expect(response.to_s).to eq "#, " \ + "..., " \ + "#]>" + expect(response.inspect).to eq response.to_s + end + + it "exposes an empty response" do + response = SearchResponse::EMPTY + + expect(response).to be_empty + expect(response.to_a).to eq([]) + expect(response.metadata).to eq("hits" => {"total" => {"value" => 0}}) + expect(response.total_document_count).to eq 0 + end + + def raw_data_with_docs(count) + documents = raw_data.fetch("hits").fetch("hits").first(count) + raw_data.merge("hits" => raw_data.fetch("hits").merge("hits" => documents)) + end + + def build_response(data) + SearchResponse.build(data, decoded_cursor_factory: decoded_cursor_factory) + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_search_router_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_search_router_spec.rb new file mode 100644 index 00000000..490358e8 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_search_router_spec.rb @@ -0,0 +1,275 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/elasticsearch/client" +require "elastic_graph/graphql/datastore_query" +require "elastic_graph/graphql/datastore_search_router" +require "elastic_graph/graphql/datastore_response/search_response" +require "elastic_graph/support/monotonic_clock" +require "support/sort" + +module ElasticGraph + class GraphQL + RSpec.describe DatastoreSearchRouter, :capture_logs do + include SortSupport + + attr_accessor :schema_artifacts + + before(:context) do + self.schema_artifacts = generate_schema_artifacts do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "currency", "String" + t.field "name", "String" + t.field "some_field", "String!" + t.index "widgets" + end + end + end + + let(:empty_response) do + DatastoreResponse::SearchResponse::RAW_EMPTY.merge( + "took" => 5, "_shards" => {"total" => 3, "successful" => 1, "skipped" => 0, "failed" => 0}, "status" => 200 + ) + end + + let(:main_datastore_client) { instance_spy(Elasticsearch::Client, cluster_name: "main") } + let(:other_datastore_client) { instance_spy(Elasticsearch::Client, cluster_name: "other") } + let(:graphql) { build_graphql } + let(:router) { graphql.datastore_search_router } + let(:datastore_query_builder) { graphql.datastore_query_builder } + let(:now_when_msearch_is_called) { 100_000 } + let(:monotonic_clock) { instance_double(Support::MonotonicClock, now_in_ms: now_when_msearch_is_called) } + + describe "#msearch" do + before do + allow(main_datastore_client).to receive(:msearch).and_return("took" => 10, "responses" => [ + empty_response.merge("r" => 1), + empty_response.merge("r" => 2), + empty_response.merge("r" => 3) + ]) + end + + let(:sort_list) { [{"foo" => {"order" => "asc"}}] } + let(:query1) { new_widgets_query(default_page_size: 10, individual_docs_needed: true) } + let(:query2) { new_widgets_query(default_page_size: 3, individual_docs_needed: true) } + + it "passes a set of empty headers along with each search body to the datastore, since it requires that for msearch" do + router.msearch([query1, query2]) + + expect(main_datastore_client).to have_received(:msearch).with(a_hash_including(body: [ + {index: "widgets"}, + a_hash_including(sort: sort_list_with_missing_option_for(sort_list), size: a_value_within(1).of(10)), + {index: "widgets"}, + a_hash_including(sort: sort_list_with_missing_option_for(sort_list), size: a_value_within(1).of(3)) + ])) + end + + it "performs the request with a client-side timeout configured from the query deadlines" do + router.msearch([query1.merge_with(monotonic_clock_deadline: now_when_msearch_is_called + 117)]) + + expect(main_datastore_client).to have_received(:msearch).with(a_hash_including(headers: { + TIMEOUT_MS_HEADER => "117" + })) + end + + it "passes no timeout when no queries have a deadline" do + expect(query1.monotonic_clock_deadline).to be nil + expect(query2.monotonic_clock_deadline).to be nil + + router.msearch([query1, query2]) + expect(main_datastore_client).to have_received(:msearch).with(a_hash_including(headers: {})) + end + + it "picks the numerical (not lexicographical) minimum timeout when multiple queries have a deadline" do + router.msearch([ + query1.merge_with(monotonic_clock_deadline: now_when_msearch_is_called + 117), + query2.merge_with(monotonic_clock_deadline: now_when_msearch_is_called + 9) + ]) + + expect(main_datastore_client).to have_received(:msearch).with(a_hash_including(headers: { + TIMEOUT_MS_HEADER => "9" + })) + end + + it "ignores queries that have no `monotonic_clock_deadline` when picking the overall timeout" do + router.msearch([ + query1.merge_with(monotonic_clock_deadline: now_when_msearch_is_called + 117), + query1.merge_with(monotonic_clock_deadline: nil), + query2.merge_with(monotonic_clock_deadline: now_when_msearch_is_called + 9) + ]) + + expect(main_datastore_client).to have_received(:msearch).with(a_hash_including(headers: { + TIMEOUT_MS_HEADER => "9" + })) + end + + it "raises `Errors::RequestExceededDeadlineError` if a query has a deadline in the past" do + queries = [ + query1.merge_with(monotonic_clock_deadline: now_when_msearch_is_called + -3), + query2.merge_with(monotonic_clock_deadline: now_when_msearch_is_called + 9) + ] + + expect { + router.msearch(queries) + }.to raise_error Errors::RequestExceededDeadlineError, /already \d+ ms past/ + end + + it "raises `Errors::RequestExceededDeadlineError` if a query has a deadline at the exact current monotonic clock time" do + queries = [ + query1.merge_with(monotonic_clock_deadline: now_when_msearch_is_called), + query2.merge_with(monotonic_clock_deadline: now_when_msearch_is_called + 9) + ] + + expect { + router.msearch(queries) + }.to raise_error Errors::RequestExceededDeadlineError, /already \d+ ms past/ + end + + it "returns an `DatastoreResponse::SearchResponse` for each response from the datastore in a hash" do + responses = router.msearch([query1, query2]) + + expect(responses.values).to all be_a DatastoreResponse::SearchResponse + expect(responses.keys).to eq [query1, query2] + expect(responses.values.map(&:metadata)).to match [ + a_hash_including("r" => 1), + a_hash_including("r" => 2) + ] + end + + it "raises `Errors::SearchFailedError` if a search fails for any reason" do + allow(main_datastore_client).to receive(:msearch).and_return("took" => 10, "responses" => [ + empty_response, + {"took" => 5, "error" => {"bad stuff" => "happened"}, "status" => 400} + ]) + + expect { + router.msearch([query1, query2]) + }.to raise_error(Errors::SearchFailedError, a_string_including( + "2) ", '{"index":"widgets"}', '{"bad stuff"=>"happened"}' + ).and(excluding( + # These are parts of the body of the request, which we don't want included because it could contain PII!. + "track_total_hits", "size" + ))) + end + + it "logs warning if a query has failed shards" do + shard_failure_bits = { + "_shards" => { + "total" => 640, + "successful" => 620, + "skipped" => 0, + "failed" => 20, + "failures" => [ + { + "shard" => 15, + "index" => "widgets", + "node" => "uMUNaPy6TBa6j9fzRFpv0w", + "reason" => { + "type" => "illegal_argument_exception", + "reason" => "numHits must be > 0; TotalHitCountCollector can be used for the total hit count" + } + } + ] + } + } + + allow(main_datastore_client).to receive(:msearch).and_return("took" => 12, "responses" => [ + empty_response, + empty_response.merge(shard_failure_bits) + ]) + + expect { + router.msearch([query1, query2]) + }.to log(a_string_including( + "The following queries have failed shards", + "Query 2", + "against index `widgets`", + "illegal_argument_exception", + "numHits must be > 0; TotalHitCountCollector can be used for the total hit count" + )) + end + + it "avoids the I/O cost of querying the datastore when given an empty list of queries" do + results = router.msearch([]) + + expect(results).to eq({}) + expect(main_datastore_client).not_to have_received(:msearch) + end + + it "does not assume `Query.perform` yields all queries" do + empty_query = new_widgets_query(requested_fields: [], total_document_count_needed: false) + expect(empty_query).to be_empty + + results = router.msearch([query1, empty_query]) + expect(results.size).to eq(2) + end + + it "records how long the queries took from the client and server's perspective" do + allow(main_datastore_client).to receive(:msearch).and_return( + {"took" => 47, "responses" => [empty_response.merge("r" => 1)]}, + {"took" => 12, "responses" => [empty_response.merge("r" => 2)]} + ) + + allow(monotonic_clock).to receive(:now_in_ms).and_return(100, 340, 500, 530) + query_tracker = QueryDetailsTracker.empty + + expect { + router.msearch([query1], query_tracker: query_tracker) + }.to change { query_tracker.datastore_query_server_duration_ms }.from(0).to(47) + .and change { query_tracker.datastore_query_client_duration_ms }.from(0).to(240) + + expect { + router.msearch([query2], query_tracker: query_tracker) + }.to change { query_tracker.datastore_query_server_duration_ms }.from(47).to(59) + .and change { query_tracker.datastore_query_client_duration_ms }.from(240).to(270) + end + + it "tolerates the datastore server response not indicating how long it took" do + allow(main_datastore_client).to receive(:msearch).and_return("responses" => [ + empty_response.merge("r" => 1) + ]) + + query_tracker = QueryDetailsTracker.empty + + expect { + router.msearch([query1], query_tracker: query_tracker) + }.not_to change { query_tracker.datastore_query_server_duration_ms }.from(0) + end + + it "prints via `puts` the datastore query and response only when `DEBUG_QUERY` is set" do + formatted_messages_pattern = /QUERY:\n.*"index": "widgets".*\nRESPONSE:\n.*"responses"/m + with_env("DEBUG_QUERY" => "1") do + expect { + router.msearch([query1, query2]) + }.to output(formatted_messages_pattern).to_stdout + end + + with_env("DEBUG_QUERY" => nil) do + expect { + router.msearch([query1, query2]) + }.not_to output(formatted_messages_pattern).to_stdout + end + end + + def new_widgets_query(**args) + options = { + search_index_definitions: [graphql.datastore_core.index_definitions_by_name.fetch("widgets")], + sort: sort_list + }.merge(args) + datastore_query_builder.new_query(**options) + end + end + + def build_graphql + super(clients_by_name: {"main" => main_datastore_client, "other" => other_datastore_client}, monotonic_clock: monotonic_clock, schema_artifacts: schema_artifacts) + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/decoded_cursor_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/decoded_cursor_spec.rb new file mode 100644 index 00000000..d0c26707 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/decoded_cursor_spec.rb @@ -0,0 +1,196 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "base64" +require "elastic_graph/graphql/decoded_cursor" +require "faker" + +module ElasticGraph + class GraphQL + RSpec.describe DecodedCursor do + let(:sort_values) do + { + "created_at" => "2019-06-12T12:33:30Z", + "amount" => ::Faker::Number.between(from: 100, to: 900), + "id" => ::Faker::Alphanumeric.alpha(number: 20) + } + end + + describe "#encode" do + it "encodes the provided sort fields and values to a URL-safe string" do + cursor = DecodedCursor.new(sort_values).encode + + expect(cursor).to match(/\A[a-zA-z0-9_-]{32,128}\z/) + end + + it "returns the special singleton cursor string when called on the singleton cursor" do + cursor = DecodedCursor::SINGLETON.encode + + expect(cursor).to eq(SINGLETON_CURSOR) + end + end + + describe "#sort_values" do + it "returns the decoded sort values" do + decoded = DecodedCursor.new(sort_values).sort_values + + expect(decoded).to eq(sort_values) + end + + it "returns an empty hash when called on the singleton cursor, even after parsing it" do + values = DecodedCursor::SINGLETON.sort_values + expect(values).to eq({}) + + values = DecodedCursor.decode!(SINGLETON_CURSOR).sort_values + expect(values).to eq({}) + + values = DecodedCursor.try_decode(SINGLETON_CURSOR).sort_values + expect(values).to eq({}) + end + end + + describe ".decode!" do + it "returns a decoded cursor value" do + cursor = DecodedCursor.new(sort_values).encode + decoded = DecodedCursor.decode!(cursor) + + expect(decoded.sort_values).to eq sort_values + end + + it "returns the special `SINGLETON` value when given the `SINGLETON_CURSOR` string" do + cursor = DecodedCursor.decode!(SINGLETON_CURSOR) + + expect(cursor).to be DecodedCursor::SINGLETON + end + + it "raises a clear error when decoding an invalid base64 string" do + bad_cursor = DecodedCursor.new(sort_values).encode + ' $1!!@#(#@' + + expect { + DecodedCursor.decode!(bad_cursor) + }.to raise_error(Errors::InvalidCursorError, a_string_including(bad_cursor)) + end + + it "raises a clear error when decoding a valid base64 string encoding an invalid JSON string" do + bad_cursor = ::Base64.urlsafe_encode64("[12, 23h", padding: false) + + expect { + DecodedCursor.decode!(bad_cursor) + }.to raise_error(Errors::InvalidCursorError, a_string_including(bad_cursor)) + end + end + + describe ".try_decode" do + it "returns a decoded cursor value" do + cursor = DecodedCursor.new(sort_values).encode + decoded = DecodedCursor.try_decode(cursor) + + expect(decoded.sort_values).to eq sort_values + end + + it "returns the special `SINGLETON` value when given the `SINGLETON_CURSOR` string" do + cursor = DecodedCursor.try_decode(SINGLETON_CURSOR) + + expect(cursor).to be DecodedCursor::SINGLETON + end + + it "raises a clear error when decoding an invalid base64 string" do + bad_cursor = DecodedCursor.new(sort_values).encode + ' $1!!@#(#@' + + expect(DecodedCursor.try_decode(bad_cursor)).to eq nil + end + + it "raises a clear error when decoding a valid base64 string encoding an invalid JSON string" do + bad_cursor = ::Base64.urlsafe_encode64("[12, 23h", padding: false) + + expect(DecodedCursor.try_decode(bad_cursor)).to eq nil + end + end + + describe ".factory_from_sort_list" do + let(:amount) { ::Faker::Number.between(from: 100, to: 900) } + let(:sort_fields) { %w[created_at amount id] } + let(:sort_list) do + [ + {"created_at" => {"order" => "asc"}}, + {"amount" => {"order" => "desc"}}, + {"id" => {"order" => "asc"}} + ] + end + let(:sort_values) { ["2019-06-12T12:33:30Z", amount, ::Faker::Alphanumeric.alpha(number: 20)] } + + it "can be built from a list of `{field => {'order' => direction}}` hashes" do + factory1 = factory_for_sort_fields(sort_fields) + factory2 = DecodedCursor::Factory.from_sort_list(sort_list) + + expect(factory2).to eq factory1 + end + + it "raises when attempting to build from an invalid sort list" do + invalid = {"a" => "asc", "b" => "asc"} + + expect { + DecodedCursor::Factory.from_sort_list(sort_list + [invalid]) + }.to raise_error(Errors::InvalidSortFieldsError, a_string_including(invalid.inspect)) + end + + it "raises when the same field is in the sort list twice, since the encoded JSON cannot represent that (and the extra usage of the field accomplishes nothing...)" do + invalid = [ + {"foo" => {"order" => "asc"}}, + {"bar" => {"order" => "desc"}}, + {"foo" => {"order" => "desc"}} + ] + + expect { + DecodedCursor::Factory.from_sort_list(invalid) + }.to raise_error(Errors::InvalidSortFieldsError, a_string_including(invalid.inspect)) + end + + it "requires sorts on nested fields to be flattened in advance by the caller" do + invalid = {"amount_money" => {"amount" => {"order" => "asc"}}} + valid = {"amount_money.amount" => {"order" => "asc"}} + + expect { + DecodedCursor::Factory.from_sort_list(sort_list + [invalid]) + }.to raise_error(Errors::InvalidSortFieldsError, a_string_including(invalid.inspect)) + + expect(DecodedCursor::Factory.from_sort_list(sort_list + [valid])).to be_a DecodedCursor::Factory + end + + it "inspects well" do + factory = factory_for_sort_fields(sort_fields) + + expect(factory.inspect).to eq "#" + expect(factory.to_s).to eq factory.inspect + end + + it "raises a clear error when the list of values to encode has less than the number of sort fields" do + factory = factory_for_sort_fields(sort_fields) + values = sort_values - [amount] + + expect { + factory.build(values) + }.to raise_error(Errors::CursorEncodingError, a_string_including(values.inspect, sort_fields.inspect)) + end + + it "raises a clear error when the list of values to encode has more than the number of sort fields" do + factory = factory_for_sort_fields(sort_fields) + values = sort_values + ["foo"] + + expect { + factory.build(values) + }.to raise_error(Errors::CursorEncodingError, a_string_including(values.inspect, sort_fields.inspect)) + end + + def factory_for_sort_fields(sort_fields) + DecodedCursor::Factory.new(sort_fields) + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/filtering/filter_interpreter_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/filtering/filter_interpreter_spec.rb new file mode 100644 index 00000000..d11bbfd4 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/filtering/filter_interpreter_spec.rb @@ -0,0 +1,42 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/filtering/filter_interpreter" + +module ElasticGraph + class GraphQL + module Filtering + # Note: most `FilterInterpreter` logic is driven via the `DatastoreQuery` interface. Here we have only + # a couple tests that focus on details that don't impact `DatastoreQuery` behavior. + RSpec.describe FilterInterpreter do + let(:graphql) { build_graphql } + + it "inspects nicely" do + fi = graphql.filter_interpreter + + expect(fi.inspect.length).to be < 200 + expect(fi.to_s.length).to be < 200 + end + + specify "two instances are equal when instantiated with the same args" do + fi1 = FilterInterpreter.new( + filter_node_interpreter: graphql.filter_node_interpreter, + logger: graphql.logger + ) + + fi2 = FilterInterpreter.new( + filter_node_interpreter: graphql.filter_node_interpreter, + logger: graphql.logger + ) + + expect(fi1).to eq(fi2) + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/http_endpoint_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/http_endpoint_spec.rb new file mode 100644 index 00000000..7736b63a --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/http_endpoint_spec.rb @@ -0,0 +1,333 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/datastore_response/search_response" +require "elastic_graph/graphql/http_endpoint" +require "support/client_resolvers" +require "uri" + +module ElasticGraph + class GraphQL + RSpec.describe HTTPEndpoint do + let(:router) { instance_double("ElasticGraph::GraphQL::DatastoreSearchRouter") } + let(:monotonic_clock) { instance_double("ElasticGraph::Support::MonotonicClock", now_in_ms: 0) } + let(:graphql) do + build_graphql( + datastore_search_router: router, + monotonic_clock: monotonic_clock, + client_resolver: ClientResolvers::ViaHTTPHeader.new({"header_name" => "X-CLIENT-NAME"}) + ) + end + let(:expected_query_time) { 100 } + let(:datastore_queries) { [] } + let(:default_query) { "query { addresses { total_edge_count } }" } + + before do + allow(router).to receive(:msearch) do |queries| + if queries.any? { |q| q.monotonic_clock_deadline&.<(expected_query_time) } + raise Errors::RequestExceededDeadlineError, "took too long" + end + + datastore_queries.concat(queries) + queries.to_h { |query| [query, DatastoreResponse::SearchResponse::EMPTY] } + end + end + + shared_examples_for "HTTP processing" do |only_query_string: false| + it "can process a request given no `variables` or `operationName`" do + response_body = process_graphql_expecting(200, query: "query { widgets { __typename } }") + + expect(response_body).to eq("data" => {"widgets" => {"__typename" => "WidgetConnection"}}) + end + + it "passes along the resolved client" do + client = Client.new(name: "Bob", source_description: "X-CLIENT-NAME") + + expect(submitted_value_for(:client, extra_headers: {"X-CLIENT-NAME" => "Bob"})).to eq(client) + end + + it "responds with the HTTP response returned by the client resolver if it returns one instead of a client so it can halt processing" do + process_graphql_expecting(401, extra_headers: {"X-CLIENT-RESOLVER-RESPOND-WITH" => "401"}) + end + + context "when an graphql extension hooks into the HTTPEndpoint" do + let(:graphql) do + adapter = Class.new do + def call(query:, context:, **) + user_name = context.fetch(:http_request).normalized_headers["USER-NAME"] + query.merge_with(filter: {"user_name" => {"equal_to_any_of" => [user_name]}}) + end + end.new + + extension = Module.new do + def graphql_http_endpoint + @graphql_http_endpoint ||= super.tap do |endpoint| + endpoint.extend(Module.new do + def with_context(request) + super do |context| + yield context.merge(color: request.normalized_headers["COLOR"]) + end + end + end) + end + end + + define_method :datastore_query_adapters do + super() + [adapter] + end + end + + build_graphql( + datastore_search_router: router, + monotonic_clock: monotonic_clock, + extension_modules: [extension] + ) + end + + it "can customize the `context` passed down into the GraphQL resolvers" do + expect(submitted_value_for(:context)).to include(color: nil) + expect(submitted_value_for(:context, extra_headers: {"COLOR" => "red"})).to include(color: "red") + end + + it "provides access to the HTTP request in the datastore query adapters, allowing the extension to change the query based on header values" do + process_graphql_expecting(200, extra_headers: {"USER-NAME" => "yoda"}) + + expect(datastore_queries.size).to eq 1 + expect(datastore_queries.first.filters).to contain_exactly( + {"user_name" => {"equal_to_any_of" => ["yoda"]}} + ) + end + end + + it "respects the passed `#{TIMEOUT_MS_HEADER} header, regardless of its casing" do + process_expecting_success_with_timeout_header_named = ->(header_name) do + process_graphql_expecting(200, extra_headers: {header_name => expected_query_time + 1}) + end + + expect(TIMEOUT_MS_HEADER).to eq("ElasticGraph-Request-Timeout-Ms") + process_expecting_success_with_timeout_header_named.call("ElasticGraph-Request-Timeout-Ms") + process_expecting_success_with_timeout_header_named.call("elasticgraph-request-timeout-ms") + process_expecting_success_with_timeout_header_named.call("ELASTICGRAPH-REQUEST-TIMEOUT-MS") + process_expecting_success_with_timeout_header_named.call("ElasticGraph_Request_Timeout_Ms") + process_expecting_success_with_timeout_header_named.call("elasticgraph_request_timeout_ms") + process_expecting_success_with_timeout_header_named.call("ELASTICGRAPH_REQUEST_TIMEOUT_MS") + + expect(datastore_queries.size).to eq(6) + expect(datastore_queries.map(&:monotonic_clock_deadline)).to all eq(expected_query_time + 1) + end + + it "returns a 400 if the `#{TIMEOUT_MS_HEADER}` header value is invalid" do + response_body = process_graphql_expecting(400, extra_headers: {TIMEOUT_MS_HEADER => "twelve"}) + + expect(response_body).to eq error_with("`#{TIMEOUT_MS_HEADER}` header value of \"twelve\" is invalid") + end + + it "returns a 504 Gateway Timeout when a datastore query times out" do + response_body = process_graphql_expecting(504, extra_headers: {TIMEOUT_MS_HEADER => (expected_query_time - 1).to_s}) + + expect(response_body).to eq error_with("Search exceeded requested timeout.") + end + + context "when `max_timeout_in_ms` is passed" do + it "uses that as the timeout if no `#{TIMEOUT_MS_HEADER}` header is passed" do + process_graphql_expecting(200, max_timeout_in_ms: 12000) + + expect(datastore_queries.map(&:monotonic_clock_deadline)).to eq [12000] + end + + it "uses the passed `#{TIMEOUT_MS_HEADER}` header value if it is less than the max" do + process_graphql_expecting(200, max_timeout_in_ms: 12000, extra_headers: {TIMEOUT_MS_HEADER => "11999"}) + + expect(datastore_queries.map(&:monotonic_clock_deadline)).to eq [11999] + end + + it "uses that as the timeout if the passed `#{TIMEOUT_MS_HEADER}` header value exceeds the max" do + process_graphql_expecting(200, max_timeout_in_ms: 12000, extra_headers: {TIMEOUT_MS_HEADER => "12001"}) + + expect(datastore_queries.map(&:monotonic_clock_deadline)).to eq [12000] + end + end + + unless only_query_string + it "ignores `operationName` if set to an empty string" do + response_body = process_graphql_expecting(200, query: "query { widgets { __typename } }", operation_name: "") + + expect(response_body).to eq("data" => {"widgets" => {"__typename" => "WidgetConnection"}}) + end + + it "can select an operation to run using `operationName` when given a multi-operation query" do + query = <<~EOS + query Widgets { widgets { __typename } } + query Components { components { __typename } } + EOS + + response_body = process_graphql_expecting(200, query: query, operation_name: "Widgets") + expect(response_body).to eq("data" => {"widgets" => {"__typename" => "WidgetConnection"}}) + + response_body = process_graphql_expecting(200, query: query, operation_name: "Components") + expect(response_body).to eq("data" => {"components" => {"__typename" => "ComponentConnection"}}) + end + + it "supports variables" do + query = <<~EOS + query Count($filter: WidgetFilterInput) { + widgets(filter: $filter) { + total_edge_count + } + } + EOS + + filter = {"id" => {"equal_to_any_of" => ["1"]}} + + response_body = process_graphql_expecting(200, query: query, variables: {"filter" => filter}) + expect(response_body).to eq("data" => {"widgets" => {"total_edge_count" => 0}}) + expect(datastore_queries.size).to eq(1) + expect(datastore_queries.first.filters.to_a).to eq [filter] + end + end + + def submitted_value_for(option_name, ...) + submitted_value = nil + + query_executor = graphql.graphql_query_executor + allow(query_executor).to receive(:execute).and_wrap_original do |original, query_string, **options| + submitted_value = options[option_name] + original.call(query_string, **options) + end + + process_graphql_expecting(200, ...) + submitted_value + end + end + + context "when given an application/json POST request" do + include_examples "HTTP processing" + + it "returns a 400 response when the body is not parsable JSON" do + response = process_expecting(400, body: "not json") + + expect(response).to eq error_with("Request body is invalid JSON.") + end + + it "tolerates the Content-Type being in different forms (e.g. upper vs lower case)" do + r1 = process_graphql_expecting(200, query: "query { widgets { __typename } }", headers: {"Content-Type" => "application/json"}) + r2 = process_graphql_expecting(200, query: "query { widgets { __typename } }", headers: {"content-type" => "application/json"}) + r3 = process_graphql_expecting(200, query: "query { widgets { __typename } }", headers: {"content_type" => "application/json"}) + r4 = process_graphql_expecting(200, query: "query { widgets { __typename } }", headers: {"CONTENT_TYPE" => "application/json"}) + r5 = process_graphql_expecting(200, query: "query { widgets { __typename } }", headers: {"CONTENT-TYPE" => "application/json"}) + + expect([r1, r2, r3, r4, r5]).to all eq("data" => {"widgets" => {"__typename" => "WidgetConnection"}}) + end + + def process_graphql_expecting(status_code, query: default_query, variables: nil, operation_name: nil, **options) + body = ::JSON.generate({ + "query" => query, + "variables" => variables, + "operationName" => operation_name + }.compact) + + process_expecting(status_code, body: body, **options) + end + + def process(headers: {"Content-Type" => "application/json"}, **options) + super(http_method: :post, headers: headers, **options) + end + end + + context "when given an application/graphql POST request" do + include_examples "HTTP processing", only_query_string: true + + def process_graphql_expecting(status_code, query: default_query, **options) + process_expecting(status_code, body: query, **options) + end + + def process(headers: {"Content-Type" => "application/graphql"}, **options) + super(http_method: :post, headers: headers, **options) + end + end + + context "when given a GET request with query params" do + include_examples "HTTP processing" + + it "returns a 400 response when the variables are not parsable JSON" do + query_params = { + "query" => "query Multiply($operands: Operands!) { multiply(operands: $operands) }", + "variables" => "not a json string", + "operationName" => "Multiply" + } + + response = process_expecting(400, query_params: query_params) + + expect(response).to eq error_with("Variables are invalid JSON.") + end + + it "handles a request with no query params" do + response = process_expecting(200) + + expect(response).to eq error_with("No query string was present") + end + + def process_graphql_expecting(status_code, query: default_query, variables: nil, operation_name: nil, headers: {}, **options) + query_params = { + "query" => query, + "variables" => variables&.then { |v| ::JSON.generate(v) }, + "operationName" => operation_name + }.compact + + process_expecting(status_code, query_params: query_params, http_method: :get, headers: headers, **options) + end + + def process(query_params: {}, **options) + query = ::URI.encode_www_form(query_params) + super(http_method: :get, url: "http://foo.com/bar?#{query}", **options) + end + end + + it "returns a 415 when the request is a POST with an unsupported content type" do + body = ::JSON.generate("query" => "query { widgets { __typename } }") + response = process_expecting(415, http_method: :post, body: body, headers: {"Content-Type" => "text/json"}) + + expect(response).to eq error_with("`text/json` is not a supported content type. Only `application/json` and `application/graphql` are supported.") + end + + it "returns a 405 when the request is not a GET or POST" do + r1 = process_expecting(405, http_method: :delete) + r2 = process_expecting(405, http_method: :put) + r3 = process_expecting(405, http_method: :options) + r4 = process_expecting(405, http_method: :patch) + r5 = process_expecting(405, http_method: :head) + + expect([r1, r2, r3, r4, r5]).to all eq error_with("GraphQL only supports GET and POST requests.") + end + + it "returns an error when given a GET request with no query params" do + response = process_expecting(200, http_method: :get, url: "http://foo.test/no/query/params") + + expect(response).to eq error_with("No query string was present") + end + + def process_expecting(status_code, ...) + response = process(...) + + expect(response.status_code).to eq(status_code) + expect(response.headers).to include("Content-Type" => "application/json") + + ::JSON.parse(response.body) + end + + def process(http_method:, url: "http://foo.test/bar", body: nil, headers: {}, extra_headers: {}, **options) + request = HTTPRequest.new(url: url, http_method: http_method, body: body, headers: headers.merge(extra_headers)) + graphql.graphql_http_endpoint.process(request, **options) + end + + def error_with(message) + {"errors" => [{"message" => message}]} + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/monkey_patches/schema_object_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/monkey_patches/schema_object_spec.rb new file mode 100644 index 00000000..d21c9d44 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/monkey_patches/schema_object_spec.rb @@ -0,0 +1,29 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/monkey_patches/schema_object" + +module ElasticGraph + class GraphQL + module MonkeyPatches + RSpec.describe SchemaObjectVisibilityDecorator do + it "does not interfere with the ability to parse and re-dump an SDL string" do + schema_string = <<~EOS + type Query { + foo: Int + } + EOS + + dumped = ::GraphQL::Schema.from_definition(schema_string).to_definition + + expect(dumped).to eq(schema_string) + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/query_adapter/filters_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/query_adapter/filters_spec.rb new file mode 100644 index 00000000..6e75b556 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/query_adapter/filters_spec.rb @@ -0,0 +1,1073 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/graphql/query_adapter/filters" + +module ElasticGraph + class GraphQL + class QueryAdapter + RSpec.describe Filters, :query_adapter do + attr_accessor :schema_artifacts + + before(:context) do + self.schema_artifacts = generate_schema_artifacts do |schema| + schema.enum_type "Size" do |t| + t.value "LARGE" + t.value "MEDIUM" + t.value "SMALL" + end + + schema.object_type "Options" do |t| + t.field "color", "String", name_in_index: "rgb_color" + t.field "size", "String" + end + + schema.object_type "Person" do |t| + t.field "name", "String", name_in_index: "name_es" + t.field "nationality", "String", name_in_index: "nationality_es" + end + + schema.object_type "Company" do |t| + t.field "name", "String", name_in_index: "name_es" + t.field "stock_ticker", "String" + end + + schema.union_type "Inventor" do |t| + t.subtypes "Company", "Person" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "name", "String!" + t.field "count", "Int" + t.field "tags", "[String]" + t.field "description", "String!", name_in_index: "description_in_es" + t.field "options", "Options", name_in_index: "widget_opts" + t.field "nested_options", "[Options!]" do |f| + f.mapping type: "nested" + end + + t.field "inventor", "Inventor" + t.field "size", "Size" + t.field "component_id", "ID" + t.index "widgets" + end + + schema.object_type "Component" do |t| + t.field "id", "ID" + t.field "name", "String" + t.field "cost", "Int" + t.field "options", "Options" + t.relates_to_one "widget", "Widget", via: "component_id", dir: :in + + t.field "widget_name", "String" do |f| + f.sourced_from "widget", "name" + end + + t.field "nested_options", "[Options!]" do |f| + f.mapping type: "nested" + f.sourced_from "widget", "nested_options" + end + + t.index "components" + end + + schema.union_type "WidgetOrComponent" do |t| + t.root_query_fields plural: "widgets_or_components" + t.subtypes "Widget", "Component" + end + + schema.object_type "Unfilterable" do |t| + t.root_query_fields plural: "unfilterables" + t.field "id", "ID", filterable: false + t.index "unfilterables" + end + end + end + + it "translates GraphQL filtering options to datastore filters, converting field names as needed" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets(filter: {name: {equal_to_any_of: ["abc"]}, description: {equal_to_any_of: ["def"]}}) { + edges { + node { + id + } + } + } + } + QUERY + + expect(query.filters.first).to eq({ + "name" => {"equal_to_any_of" => ["abc"]}, + "description_in_es" => {"equal_to_any_of" => ["def"]} + }) + end + + it "does field name translations at all levels of the filter" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets(filter: {options: {color: {equal_to_any_of: ["abc"]}, size: {equal_to_any_of: ["def"]}}}) { + edges { + node { + id + } + } + } + } + QUERY + + expect(query.filters.first).to eq({"widget_opts" => { + "rgb_color" => {"equal_to_any_of" => ["abc"]}, + "size" => {"equal_to_any_of" => ["def"]} + }}) + end + + it "translates enum value strings to enum value objects so that the `runtime_metadata` of the enum value is available to our `FilterInterpreter`" do + graphql, queries_by_field = graphql_and_datastore_queries_by_field_for(<<~QUERY) + query { + widgets(filter: {size: {equal_to_any_of: [LARGE, SMALL, null]}}) { + nodes { + id + } + } + } + QUERY + + size_input_type = graphql.schema.type_named("SizeInput") + + expect(queries_by_field.fetch("Query.widgets").first.filters.first).to eq({ + "size" => {"equal_to_any_of" => [ + size_input_type.enum_value_named("LARGE"), + size_input_type.enum_value_named("SMALL"), + nil + ]} + }) + end + + it "translates the sub-expressions of an `any_of`" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets(filter: {any_of: [ + {name: {equal_to_any_of: ["bob"]}, description: {equal_to_any_of: ["foo"]}}, + {options: {any_of: [ + {color: {equal_to_any_of: ["red"]}}, + {size: {equal_to_any_of: ["large"]}} + ]}} + ]}) { + edges { + node { + id + } + } + } + } + QUERY + + expect(query.filters.first).to eq({ + "any_of" => [ + { + "name" => {"equal_to_any_of" => ["bob"]}, + "description_in_es" => {"equal_to_any_of" => ["foo"]} + }, + { + "widget_opts" => { + "any_of" => [ + {"rgb_color" => {"equal_to_any_of" => ["red"]}}, + {"size" => {"equal_to_any_of" => ["large"]}} + ] + } + } + ] + }) + end + + it "can translate filter fields that exist on multiple union subtypes on the indexed types" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets(filter: {inventor: {name: {equal_to_any_of: ["abc"]}}}) { + edges { + node { + id + } + } + } + } + QUERY + + expect(query.filters.first).to eq({ + "inventor" => {"name_es" => {"equal_to_any_of" => ["abc"]}} + }) + end + + it "can translate filter fields that exist on only one of a type union's subtypes on the indexed types" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets(filter: {inventor: {stock_ticker: {equal_to_any_of: ["abc"]}, nationality: {equal_to_any_of: ["def"]}}}) { + edges { + node { + id + } + } + } + } + QUERY + + expect(query.filters.first).to eq({"inventor" => { + "stock_ticker" => {"equal_to_any_of" => ["abc"]}, + "nationality_es" => {"equal_to_any_of" => ["def"]} + }}) + end + + it "translates `count` on a list field to `#{LIST_COUNTS_FIELD}` while leaving a `count` schema field unchanged" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets(filter: {count: {gt: 1}, tags: {count: {gt: 1}}}) { + nodes { id } + } + } + QUERY + + expect(query.filters.first).to eq({ + "count" => {"gt" => 1}, + "tags" => {LIST_COUNTS_FIELD => {"gt" => 1}} + }) + end + + context "on a type that has never had any `sourced_from` fields" do + it "sets no `filters` on the datastore query when the GraphQL query has no filters (but the query field supports arguments)" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + edges { + node { + id + } + } + } + } + QUERY + + expect(query.filters).to be_empty + end + + it "sets no `filters` on the datastore query when the GraphQL query field does not have a `filter` argument" do + graphql, queries_by_field = graphql_and_datastore_queries_by_field_for(<<~QUERY, schema_artifacts: schema_artifacts) + query { + unfilterables { + edges { + node { + id + } + } + } + } + QUERY + + # Verify it does not support a `filter` argument. + expect(graphql.schema.field_named("Query", "unfilterables").graphql_field.arguments.keys).to contain_exactly( + "order_by", + "first", "after", + "last", "before" + ) + + expect(queries_by_field.keys).to contain_exactly("Query.unfilterables") + expect(queries_by_field.fetch("Query.unfilterables").size).to eq(1) + expect(queries_by_field.fetch("Query.unfilterables").first.filters).to be_empty + end + end + + context "on a type that has (or has had) `sourced_from` fields" do + let(:exclude_incomplete_docs_filter) do + {"__sources" => {"equal_to_any_of" => [SELF_RELATIONSHIP_NAME]}} + end + + it "excludes incomplete documents as the only filter when the client specifies no filters" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly(exclude_incomplete_docs_filter) + end + + it "excludes incomplete documents as an additional automatic filter on top of any user-specified filters" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {widget_name: {equal_to_any_of: ["thingy"]}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + exclude_incomplete_docs_filter, + {"widget_name" => {"equal_to_any_of" => ["thingy"]}} + ) + end + + it "omits the incomplete doc exclusion filter when the specified query filters cannot match an incomplete doc due to requiring a value coming from the `#{SELF_RELATIONSHIP_NAME}` source" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {name: {equal_to_any_of: ["thingy"]}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + {"name" => {"equal_to_any_of" => ["thingy"]}} + ) + end + + it "understands that a filter with `equal_to_any_of: [null]` may still match incomplete documents" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {name: {equal_to_any_of: [null]}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + exclude_incomplete_docs_filter, + {"name" => {"equal_to_any_of" => [nil]}} + ) + end + + it "understands that a filter with `equal_to_any_of: [null, null]` may still match incomplete documents" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {name: {equal_to_any_of: [null, null]}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + exclude_incomplete_docs_filter, + {"name" => {"equal_to_any_of" => [nil, nil]}} + ) + end + + it "understands that a filter with `equal_to_any_of: [null, something_else]` may still match incomplete documents" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {name: {equal_to_any_of: [null, "thingy"]}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + exclude_incomplete_docs_filter, + {"name" => {"equal_to_any_of" => [nil, "thingy"]}} + ) + end + + it "understands that a filter with `equal_to_any_of: []` cannot match incomplete documents" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {name: {equal_to_any_of: []}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + {"name" => {"equal_to_any_of" => []}} + ) + end + + context "when multiple fields are filtered on" do + it "omits the incomplete doc exclusion filter when none of the field filters could match incomplete documents" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: { + name: {equal_to_any_of: ["thingy"]} + cost: {equal_to_any_of: [7] + }}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + { + "name" => {"equal_to_any_of" => ["thingy"]}, + "cost" => {"equal_to_any_of" => [7]} + } + ) + end + + it "omits the incomplete doc exclusion filter when some (but not all) of the field filters could match incomplete documents" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: { + name: {equal_to_any_of: [null]} + cost: {equal_to_any_of: [7] + }}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + { + "name" => {"equal_to_any_of" => [nil]}, + "cost" => {"equal_to_any_of" => [7]} + } + ) + end + + it "includes the incomplete doc exclusion filter when all of the field filters could match incomplete documents" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: { + name: {equal_to_any_of: [null]} + cost: {equal_to_any_of: [null] + }}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + exclude_incomplete_docs_filter, + { + "name" => {"equal_to_any_of" => [nil]}, + "cost" => {"equal_to_any_of" => [nil]} + } + ) + end + end + + it "correctly handles subfields" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {options: {size: {equal_to_any_of: ["LARGE"]}}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + {"options" => {"size" => {"equal_to_any_of" => ["LARGE"]}}} + ) + + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {options: {size: {equal_to_any_of: [null]}}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + exclude_incomplete_docs_filter, + {"options" => {"size" => {"equal_to_any_of" => [nil]}}} + ) + end + + it "understands that a range filter on a self-sourced field cannot match incomplete docs, even if mixed with an operator that can match incomplete docs" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {cost: {gt: 7}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + {"cost" => {"gt" => 7}} + ) + + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {cost: {gt: 7, equal_to_any_of: [null]}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + {"cost" => {"gt" => 7, "equal_to_any_of" => [nil]}} + ) + end + + describe "`null` leaves" do + it "ignores `filter: null`" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: null) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly(exclude_incomplete_docs_filter) + end + + it "ignores a `field: null` filter since it will get pruned" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {cost: null}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + exclude_incomplete_docs_filter, + {"cost" => nil} + ) + + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: { + cost: null + name: {equal_to_any_of: ["thingy"]} + }) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly({"cost" => nil, "name" => {"equal_to_any_of" => ["thingy"]}}) + end + + it "ignores a `field: {predicate: null}` filter since it will get pruned" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {cost: {equal_to_any_of: null}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + exclude_incomplete_docs_filter, + {"cost" => {"equal_to_any_of" => nil}} + ) + + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: { + cost: {equal_to_any_of: null} + name: {equal_to_any_of: ["thingy"]} + }) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly({ + "cost" => {"equal_to_any_of" => nil}, + "name" => {"equal_to_any_of" => ["thingy"]} + }) + end + + it "ignores a `null` filter since it will get pruned" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {options: null}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + exclude_incomplete_docs_filter, + {"options" => nil} + ) + + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: { + options: null + name: {equal_to_any_of: ["thingy"]} + }) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly({ + "options" => nil, + "name" => {"equal_to_any_of" => ["thingy"]} + }) + end + + it "ignores a `parent_field: {child_field: null}` filter since it will get pruned" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {options: {size: null}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + {"options" => {"size" => nil}}, + exclude_incomplete_docs_filter + ) + + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: { + options: {size: null} + name: {equal_to_any_of: ["thingy"]} + }) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly({ + "options" => {"size" => nil}, + "name" => {"equal_to_any_of" => ["thingy"]} + }) + end + + it "ignores a `parent_field: {child_field: {predicate: null}}` filter since it will get pruned" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {options: {size: {equal_to_any_of: null}}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + {"options" => {"size" => {"equal_to_any_of" => nil}}}, + exclude_incomplete_docs_filter + ) + + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: { + options: {size: {equal_to_any_of: null}} + name: {equal_to_any_of: ["thingy"]} + }) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly({ + "options" => {"size" => {"equal_to_any_of" => nil}}, + "name" => {"equal_to_any_of" => ["thingy"]} + }) + end + end + + context "when `not` is used" do + it "still includes the incomplete doc exclusion filter when `not` is applied to a field with an alternate source" do + # not on the outside... + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {not: {widget_name: {equal_to_any_of: ["thingy"]}}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + exclude_incomplete_docs_filter, + {"not" => {"widget_name" => {"equal_to_any_of" => ["thingy"]}}} + ) + + # not on the inside... + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {widget_name: {not: {equal_to_any_of: ["thingy"]}}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + exclude_incomplete_docs_filter, + {"widget_name" => {"not" => {"equal_to_any_of" => ["thingy"]}}} + ) + end + + it "still includes the incomplete doc exclusion filter when `not` is applied to self-sourced field with a non-nil filter value" do + # not on the outside... + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {not: {name: {equal_to_any_of: ["thingy"]}}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + exclude_incomplete_docs_filter, + {"not" => {"name" => {"equal_to_any_of" => ["thingy"]}}} + ) + + # not on the inside... + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {name: {not: {equal_to_any_of: ["thingy"]}}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + exclude_incomplete_docs_filter, + {"name" => {"not" => {"equal_to_any_of" => ["thingy"]}}} + ) + end + + it "omits the incomplete doc exclusion filter when `not` is applied to self-sourced field with a nil and a non-nil filter value" do + # not on the outside... + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {not: {name: {equal_to_any_of: ["thingy", null]}}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + {"not" => {"name" => {"equal_to_any_of" => ["thingy", nil]}}} + ) + + # not on the inside... + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {name: {not: {equal_to_any_of: [null, "thingy"]}}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + {"name" => {"not" => {"equal_to_any_of" => [nil, "thingy"]}}} + ) + end + + it "omits the incomplete doc exclusion filter when `not` is applied to self-sourced field with a single nil filter value" do + # not on the outside... + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {not: {name: {equal_to_any_of: [null]}}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + {"not" => {"name" => {"equal_to_any_of" => [nil]}}} + ) + + # not on the inside... + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {name: {not: {equal_to_any_of: [null]}}}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + {"name" => {"not" => {"equal_to_any_of" => [nil]}}} + ) + end + end + + context "when `any_of` is used" do + it "includes the incomplete doc exclusion filter when there are multiple sub-clauses on different fields, because it's hard to optimize and it's safest to include it" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {any_of: [ + {name: {equal_to_any_of: ["abc"]}}, + {cost: {equal_to_any_of: [7]}} + ]}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + exclude_incomplete_docs_filter, + {"any_of" => [{"name" => {"equal_to_any_of" => ["abc"]}}, {"cost" => {"equal_to_any_of" => [7]}}]} + ) + end + + it "omits the incomplete doc exclusion filter when there are multiple sub-clause on the same field, that both can't match incomplete docs" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {any_of: [ + {cost: {lt: 100}}, + {cost: {gt: 200}} + ]}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + {"any_of" => [{"cost" => {"lt" => 100}}, {"cost" => {"gt" => 200}}]} + ) + end + + it "includes the incomplete doc exclusion filter when there are multiple sub-clause on the same field, one of which can match incomplete docs" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {any_of: [ + {cost: {lt: 100}}, + {cost: {equal_to_any_of: [null]}} + ]}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + exclude_incomplete_docs_filter, + {"any_of" => [{"cost" => {"lt" => 100}}, {"cost" => {"equal_to_any_of" => [nil]}}]} + ) + end + + it "omits the incomplete doc exclusion filter when there is one sub-clause, that can't match incomplete docs" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {any_of: [ + {name: {equal_to_any_of: ["abc"]}} + ]}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + {"any_of" => [{"name" => {"equal_to_any_of" => ["abc"]}}]} + ) + end + + it "includes the incomplete doc exclusion filter when there are no sub-clauses, because the filter is ignored" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components(filter: {any_of: []}) { + nodes { + id + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + exclude_incomplete_docs_filter, + {"any_of" => []} + ) + end + end + + context "when querying a type backed by multiple index definitions" do + it "includes the incomplete doc exclusion filter if incomplete docs could be hit by the search on at least one of the filters" do + query = datastore_query_for(:Query, :widgets_or_components, <<~QUERY) + query { + widgets_or_components(filter: { + name: {equal_to_any_of: [null]} + }) { + nodes { + __typename + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + exclude_incomplete_docs_filter, + {"name" => {"equal_to_any_of" => [nil]}} + ) + end + + it "omits the incomplete doc exclusion filter if incomplete docs could not be hit by the search on either of the filters" do + query = datastore_query_for(:Query, :widgets_or_components, <<~QUERY) + query { + widgets_or_components(filter: { + name: {equal_to_any_of: ["thingy"]} + }) { + nodes { + __typename + } + } + } + QUERY + + expect(query.filters).to contain_exactly( + {"name" => {"equal_to_any_of" => ["thingy"]}} + ) + end + end + end + + shared_examples_for "filtering on a `nested` field" do |root_field| + it "translates the sub-expressions of an `all_of`" do + query = datastore_query_for(:Query, root_field, <<~QUERY) + query { + #{root_field}(filter: {nested_options: {all_of: [ + {any_satisfy: {color: {equal_to_any_of: ["red"]}}}, + {any_satisfy: {color: {equal_to_any_of: ["green"]}}} + ]}}) { + total_edge_count + } + } + QUERY + + expect(query.filters.first).to eq({"nested_options" => {"all_of" => [ + # the `name_in_index` of `color` is `rgb_color`. + {"any_satisfy" => {"rgb_color" => {"equal_to_any_of" => ["red"]}}}, + {"any_satisfy" => {"rgb_color" => {"equal_to_any_of" => ["green"]}}} + ]}}) + end + + it "translates an `any_satisfy` filter" do + query = datastore_query_for(:Query, root_field, <<~QUERY) + query { + #{root_field}(filter: { + nested_options: { + any_satisfy: { + color: {equal_to_any_of: ["red"]} + } + } + }) { + total_edge_count + } + } + QUERY + + expect(query.filters.first).to eq({ + "nested_options" => {"any_satisfy" => {"rgb_color" => {"equal_to_any_of" => ["red"]}}} + }) + end + + it "translates a `count` filter" do + query = datastore_query_for(:Query, root_field, <<~QUERY) + query { + #{root_field}(filter: { + nested_options: {count: {equal_to_any_of: [0]}} + }) { + total_edge_count + } + } + QUERY + + expect(query.filters.first).to eq({ + "nested_options" => {"__counts" => {"equal_to_any_of" => [0]}} + }) + end + end + + context "on a `nested` field which is directly indexed" do + include_examples "filtering on a `nested` field", :widgets + end + + context "on a `nested` field which is sourced from another type" do + include_examples "filtering on a `nested` field", :components + end + + def graphql_and_datastore_queries_by_field_for(graphql_query, **graphql_opts) + super(graphql_query, schema_artifacts: schema_artifacts, **graphql_opts) + end + + def datastore_query_for(type, field, graphql_query) + super( + schema_artifacts: schema_artifacts, + graphql_query: graphql_query, + type: type, + field: field, + ) + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/query_adapter/requested_fields_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/query_adapter/requested_fields_spec.rb new file mode 100644 index 00000000..6fd4d85b --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/query_adapter/requested_fields_spec.rb @@ -0,0 +1,791 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/query_adapter/requested_fields" + +module ElasticGraph + class GraphQL + class QueryAdapter + RSpec.describe RequestedFields, :query_adapter do + attr_accessor :schema_artifacts + + before(:context) do + self.schema_artifacts = generate_schema_artifacts do |schema| + schema.object_type "WidgetOptions" do |t| + t.field "size", "String" + t.field "color", "String" + end + + schema.object_type "Person" do |t| + t.implements "NamedInventor" + t.field "name", "String" + t.field "nationality", "String" + end + + schema.object_type "Company" do |t| + t.implements "NamedInventor" + t.field "name", "String" + t.field "stock_ticker", "String" + end + + schema.union_type "Inventor" do |t| + t.subtypes "Person", "Company" + end + + # Embedded interface type. + schema.interface_type "NamedInventor" do |t| + t.field "name", "String" + end + + # Indexed interfae type. + schema.interface_type "NamedEntity" do |t| + t.root_query_fields plural: "named_entities" + t.field "id", "ID" + t.field "name", "String" + end + + schema.object_type "Widget" do |t| + t.implements "NamedEntity" + t.field "id", "ID" + t.field "name", "String" + t.field "created_at", "DateTime" + t.field "last_name", "String" + t.field "amount_cents", "Int" + t.field "options", "WidgetOptions" + t.field "inventor", "Inventor" + t.field "named_inventor", "NamedInventor" + t.relates_to_many "components", "Component", via: "component_ids", dir: :out, singular: "component" + t.relates_to_many "child_widgets", "Widget", via: "parent_id", dir: :in, singular: "child_widget" + t.relates_to_one "parent_widget", "Widget", via: "parent_id", dir: :out + t.index "widgets" + end + + schema.object_type "MechanicalPart" do |t| + t.implements "NamedEntity" + t.field "id", "ID" + t.field "name", "String" + t.field "created_at", "DateTime" + t.index "mechanical_parts" + end + + schema.object_type "ElectricalPart" do |t| + t.implements "NamedEntity" + t.field "id", "ID" + t.field "name", "String" + t.field "created_at", "DateTime" + t.field "voltage", "Int" + t.index "electrical_parts" + end + + schema.union_type "Part" do |t| + t.subtypes "MechanicalPart", "ElectricalPart" + end + + schema.object_type "Component" do |t| + t.implements "NamedEntity" + t.field "id", "ID" + t.field "name", "String" + t.field "last_name", "String" + t.relates_to_one "widget", "Widget", via: "component_ids", dir: :in + t.relates_to_many "parts", "Part", via: "part_ids", dir: :out, singular: "part" + t.index "components" + end + end + end + + it "can request the fields under `edges.node` for a relay connection" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + edges { + node { + id + name + } + } + } + } + QUERY + + expect(query.requested_fields).to contain_exactly("id", "name") + expect(query.individual_docs_needed).to be true + expect(query.total_document_count_needed).to be false + end + + it "can request the fields under `nodes` for a relay connection" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + nodes { + id + name + } + } + } + QUERY + + expect(query.requested_fields).to contain_exactly("id", "name") + expect(query.individual_docs_needed).to be true + expect(query.total_document_count_needed).to be false + end + + it "does not request the fields under `edges.node` for an aggregations query" do + query = datastore_query_for(:Query, :widget_aggregations, <<~QUERY) + query { + widget_aggregations { + edges { + node { + grouped_by { + name + } + + aggregated_values { + amount_cents { + exact_sum + } + } + } + } + } + } + QUERY + + expect(query.requested_fields).to be_empty + expect(query.individual_docs_needed).to be false + expect(query.total_document_count_needed).to be false + end + + it "includes a path prefix when requesting a field under an embedded object" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + edges { + node { + id + name + options { + size + } + } + } + } + } + QUERY + + expect(query.requested_fields).to contain_exactly("id", "name", "options.size") + expect(query.individual_docs_needed).to be true + expect(query.total_document_count_needed).to be false + end + + it "requests __typename when the client asks for __typename on a union or interface (but not when they ask for __typename on an object)" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + edges { + node { + id + + options { + __typename + } + + inventor { + __typename + } + + named_inventor { + __typename + } + } + } + } + } + QUERY + + expect(query.requested_fields).to contain_exactly("id", "inventor.__typename", "named_inventor.__typename") + expect(query.individual_docs_needed).to be true + expect(query.total_document_count_needed).to be false + end + + it "requests the set union of fields from union subtypes" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + edges { + node { + inventor { + __typename + + ... on Person { + name + nationality + } + + ... on Company { + name + stock_ticker + } + } + } + } + } + } + QUERY + + expect(query.requested_fields).to contain_exactly("inventor.__typename", "inventor.name", "inventor.nationality", "inventor.stock_ticker") + expect(query.individual_docs_needed).to be true + expect(query.total_document_count_needed).to be false + end + + it "requests the set union of fields from interface subtypes" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + edges { + node { + named_inventor { + __typename + name + + ... on Person { + nationality + } + + ... on Company { + stock_ticker + } + } + } + } + } + } + QUERY + + expect(query.requested_fields).to contain_exactly("named_inventor.__typename", "named_inventor.name", "named_inventor.nationality", "named_inventor.stock_ticker") + expect(query.individual_docs_needed).to be true + expect(query.total_document_count_needed).to be false + end + + it "always requests __typename for union fields even if the GraphQL query is not asking for it so that the ElasticGraph framework can determine the subtype" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + edges { + node { + inventor { + ... on Person { + nationality + } + + ... on Company { + stock_ticker + } + } + } + } + } + } + QUERY + + expect(query.requested_fields).to contain_exactly("inventor.__typename", "inventor.nationality", "inventor.stock_ticker") + expect(query.individual_docs_needed).to be true + expect(query.total_document_count_needed).to be false + end + + it "always requests __typename for interface fields even if the GraphQL query is not asking for it so that the ElasticGraph framework can determine the subtype" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + edges { + node { + named_inventor { + ... on Person { + nationality + } + + ... on Company { + stock_ticker + } + } + } + } + } + } + QUERY + + expect(query.requested_fields).to contain_exactly("named_inventor.__typename", "named_inventor.nationality", "named_inventor.stock_ticker") + expect(query.individual_docs_needed).to be true + expect(query.total_document_count_needed).to be false + end + + it "ignores relay connection sub-fields that are not directly under `edges.node` (e.g. `page_info`)" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + page_info { + has_next_page + } + edges { + node { + id + name + } + cursor + } + } + } + QUERY + + expect(query.requested_fields).to contain_exactly("id", "name") + expect(query.individual_docs_needed).to be true + expect(query.total_document_count_needed).to be false + end + + it "requests no fields when only fetching page_info on a relay connection" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + page_info { + has_next_page + } + edges { + cursor + } + } + } + QUERY + + expect(query.requested_fields).to be_empty + expect(query.total_document_count_needed).to be false + end + + describe "individual_docs_needed" do + it "sets `individual_docs_needed = false` when no fields are requested, no cursor is requested and page info is not requested" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + __typename + } + } + QUERY + + expect(query.individual_docs_needed).to be false + end + + it "sets `individual_docs_needed = true` when a node field is requested" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + edges { + node { + id + } + } + } + } + QUERY + + expect(query.individual_docs_needed).to be true + end + + it "sets `individual_docs_needed = true` when an edge cursor is requested" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + edges { + cursor + } + } + } + QUERY + + expect(query.individual_docs_needed).to be true + end + + %w[start_cursor end_cursor has_next_page has_previous_page].each do |page_info_field| + it "sets `individual_docs_needed = true` when page info field #{page_info_field} is requested" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + page_info { + #{page_info_field} + } + } + } + QUERY + + expect(query.individual_docs_needed).to be true + end + end + end + + describe "total_document_count_needed" do + it "is set to false when `total_edge_count` is not requested" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + edges { + node { + id + name + } + } + } + } + QUERY + + expect(query.total_document_count_needed).to be false + end + + it "is set to true when `total_edge_count` is requested" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + total_edge_count + edges { + node { + id + name + } + } + } + } + QUERY + + expect(query.total_document_count_needed).to be true + end + + it "is set to false when `total_edge_count` is requested in a nested object" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + edges { + node { + components { + total_edge_count + } + } + } + } + } + QUERY + + expect(query.total_document_count_needed).to be false + end + end + + context "for a nested relation field with an outbound foreign key" do + it "includes the foreign key field" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + edges { + node { + name + + components { + edges { + node { + last_name + } + } + } + } + } + } + } + QUERY + + expect(query.requested_fields).to contain_exactly("name", "component_ids") + expect(query.individual_docs_needed).to be true + expect(query.total_document_count_needed).to be false + end + + it "also includes `id` if it is a self-referential relation" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + edges { + node { + name + + parent_widget { + amount_cents + } + } + } + } + } + QUERY + + expect(query.requested_fields).to contain_exactly("name", "id", "parent_id") + expect(query.individual_docs_needed).to be true + expect(query.total_document_count_needed).to be false + end + end + + context "for a nested relation field with an inbound foreign key" do + it "includes the `id` field" do + query = datastore_query_for(:Query, :components, <<~QUERY) + query { + components { + edges { + node { + last_name + + widget { + name + } + } + } + } + } + QUERY + + expect(query.requested_fields).to contain_exactly("last_name", "id") + expect(query.individual_docs_needed).to be true + expect(query.total_document_count_needed).to be false + end + + it "also includes foreign key field if it is a self-referential relation" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + edges { + node { + name + + child_widgets { + edges { + node { + amount_cents + } + } + } + } + } + } + } + QUERY + + expect(query.requested_fields).to contain_exactly("name", "id", "parent_id") + expect(query.individual_docs_needed).to be true + expect(query.total_document_count_needed).to be false + end + end + + context "when building a query for a nested relation" do + it "requests the fields required based on its sub-fields" do + query = datastore_query_for(:Widget, :components, <<~QUERY) + query { + widgets { + edges { + node { + id + name + components { + edges { + node { + last_name + + widget { + amount_cents + } + } + } + } + } + } + } + } + QUERY + + expect(query.requested_fields).to contain_exactly("last_name", "id") + expect(query.individual_docs_needed).to be true + expect(query.total_document_count_needed).to be false + end + end + + context "when building a query for a nested relation with a nested type union relay connection relation" do + it "requests the foreign key field rather than the sub-fields from the fragments" do + query = datastore_query_for(:Widget, :components, <<~QUERY) + query { + widgets { + edges { + node { + components { + edges { + node { + last_name + + parts { + edges { + node { + ... on MechanicalPart { + name + } + + ... on ElectricalPart { + name + voltage + } + } + } + } + } + } + } + } + } + } + } + QUERY + + expect(query.requested_fields).to contain_exactly("last_name", "part_ids") + expect(query.individual_docs_needed).to be true + expect(query.total_document_count_needed).to be false + end + end + + context "when building a query for a a nested type union relay connection relation" do + it "requests the set union of fields from each type fragment" do + query = datastore_query_for(:Component, :parts, <<~QUERY) + query { + widgets { + edges { + node { + components { + edges { + node { + last_name + + parts { + edges { + node { + ... on MechanicalPart { + name + } + + ... on ElectricalPart { + name + voltage + } + } + } + } + } + } + } + } + } + } + } + QUERY + + expect(query.requested_fields).to contain_exactly("name", "voltage", "__typename") + expect(query.individual_docs_needed).to be true + expect(query.total_document_count_needed).to be false + end + end + + it "ignores built-in introspection fields as they never exist in the datastore" do + query = datastore_query_for(:Query, :widgets, <<~QUERY) + query { + widgets { + edges { + node { + id + name + __typename + } + } + } + } + QUERY + + expect(query.requested_fields).to contain_exactly("id", "name") + expect(query.individual_docs_needed).to be true + expect(query.total_document_count_needed).to be false + end + + it "only identifies requested fields for query nodes of indexed types" do + graphql_fields = graphql_fields_with_request_fields_for(<<~QUERY) + query { + components { + edges { + node { + id + } + } + } + + widgets { + page_info { + has_next_page + } + + edges { + node { + id + + components { + edges { + node { + last_name + + parts { + edges { + node { + ... on MechanicalPart { + name + } + + ... on ElectricalPart { + name + voltage + } + } + } + } + } + } + } + } + } + } + } + QUERY + + expect(graphql_fields).to contain_exactly( + "Query.widgets", + "Query.components", + "Widget.components", + "Component.parts" + ) + end + + def datastore_query_for(type, field, graphql_query) + super( + schema_artifacts: schema_artifacts, + graphql_query: graphql_query, + type: type, + field: field, + ) + end + + def graphql_fields_with_request_fields_for(graphql_query) + queries_by_graphql_field = datastore_queries_by_field_for(graphql_query, schema_artifacts: schema_artifacts) + + queries_by_graphql_field.reject do |field, queries| + queries.all? { |q| q.requested_fields.empty? } + end.keys + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/query_executor_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/query_executor_spec.rb new file mode 100644 index 00000000..8c60d771 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/query_executor_spec.rb @@ -0,0 +1,503 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/query_executor" +require "elastic_graph/graphql/schema" +require "elastic_graph/support/monotonic_clock" + +module ElasticGraph + class GraphQL + RSpec.describe QueryExecutor do + describe "#execute", :capture_logs do + attr_accessor :schema_artifacts + + before(:context) do + self.schema_artifacts = generate_schema_artifacts do |s| + s.object_type "Color" do |t| + t.field "id", "ID" + t.field "red", "Int" + t.field "green", "Int" + t.field "blue", "Int" + t.index "colors" + end + + s.object_type "Color2" do |t| + t.field "id", "ID" + t.field "red", "Int" + t.field "green", "Int" + t.field "blue", "Int" + t.index "alt_colors" + end + + # The tests below require some custom GraphQL schema elements that we need to define by + # hand in order for them to be available, so we use `raw_sdl` here for that. + s.raw_sdl <<~EOS + input ColorArgs { + red: Int + } + + type Query { + colors(args: ColorArgs): [Color!]! + colors2(args: ColorArgs): [Color2!]! + } + EOS + end + end + + let(:monotonic_clock) { instance_double(Support::MonotonicClock, now_in_ms: monontonic_now_time) } + let(:monontonic_now_time) { 100_000 } + let(:slow_query_threshold_ms) { 4000 } + let(:datastore_query_client_duration_ms) { 75 } + let(:datastore_query_server_duration_ms) { 50 } + let(:query_executor) { define_query_executor } + + it "executes the provided query and logs how long it took, including the query fingerprint, and some datastore query details" do + allow(monotonic_clock).to receive(:now_in_ms).and_return(100, 340) + + # Here we query two different indexes so that we have the same routing values and index expressions, + # so that we can demonstrate below that it logs the unique values. + data = execute_expecting_no_errors(<<-QUERY, client: Client.new(name: "client-name", source_description: "client-description"), operation_name: "GetColors") + query GetColors { + red: colors(args: {red: 12}) { + red + } + + green: colors(args: {red: 13}) { + green + } + + blue: colors2(args: {red: 6}) { + blue + } + } + QUERY + + expect(data).to eq("red" => [], "green" => [], "blue" => []) + + expect(logged_duration_message).to include( + "client" => "client-name", + "query_name" => "GetColors", + "duration_ms" => 240, + "datastore_server_duration_ms" => datastore_query_server_duration_ms, + "elasticgraph_overhead_ms" => 240 - datastore_query_client_duration_ms, + "unique_shard_routing_values" => "routing_value_1, routing_value_2", + "unique_shard_routing_value_count" => 2, + "unique_search_index_expressions" => "alt_colors, colors", + "datastore_query_count" => 3, + "datastore_request_count" => 1, + "over_slow_threshold" => "false", + "query_fingerprint" => a_string_starting_with("GetColors/") + ) + end + + it "does not log a duration message when loading dependencies eagerly to avoid skewing metrics derived from the logged messages" do + define_graphql.load_dependencies_eagerly + + expect(logged_duration_message).to be nil + end + + it "includes `slo_result: 'good'` in the logged duration message if the query took less than the `@egLatencySlo` directive's value" do + slo_result = logged_slo_result_for(<<~QUERY, duration_in_ms: 2999) + query GetColors @eg_latency_slo(ms: 3000) { + colors(args: {red: 12}) { + red + } + } + QUERY + + expect(slo_result).to eq("good") + end + + it "includes `slo_result: 'good'` in the logged duration message if the query took exactly the `@egLatencySlo` directive's value" do + slo_result = logged_slo_result_for(<<~QUERY, duration_in_ms: 3000) + query GetColors @eg_latency_slo(ms: 3000) { + colors(args: {red: 12}) { + red + } + } + QUERY + + expect(slo_result).to eq("good") + end + + it "includes `slo_result: false` in the logged duration message if the query took more than the `@egLatencySlo` directive's value" do + slo_result = logged_slo_result_for(<<~QUERY, duration_in_ms: 3001) + query GetColors @eg_latency_slo(ms: 3000) { + colors(args: {red: 12}) { + red + } + } + QUERY + + expect(slo_result).to eq("bad") + end + + it "includes `slo_result: nil` in the logged duration message if the query lacks an `@egLatencySlo` directive" do + slo_result = logged_slo_result_for(<<~QUERY, duration_in_ms: 3001) + query GetColors { + colors(args: {red: 12}) { + red + } + } + QUERY + + expect(slo_result).to eq(nil) + end + + it "logs the full sanitized query if it took longer than our configured slow query threshold" do + allow(monotonic_clock).to receive(:now_in_ms).and_return(100, 101 + slow_query_threshold_ms) + + expect { + data = execute_expecting_no_errors(<<-QUERY, operation_name: "GetColorName") + query GetColorName { + __type(name: "Color") { + name + } + } + QUERY + + expect(data).to eq("__type" => {"name" => "Color"}) + }.to log a_string_including("longer (4001 ms) than the configured slow query threshold (4000 ms)", <<~EOS.strip) + query GetColorName { + __type(name: "") { + name + } + } + EOS + end + + it "logs the full sanitized query with exception details if executing the query triggers an exception" do + self.schema_artifacts = generate_schema_artifacts do |schema| + schema.raw_sdl <<~EOS + type Query { + foo: Int + } + EOS + end + + query_string = <<~EOS + query Foo { + foo + } + EOS + + expect { + execute_expecting_no_errors(query_string) + }.to raise_error(a_string_including("No resolver yet implemented for this case")) + .and log a_string_including( + "Query Foo[1] for client (anonymous) failed with an exception[2]", + query_string.to_s, + "RuntimeError: No resolver yet implemented for this case" + ) + end + + it "logs the query with error details if the query results in errors in the response" do + query_string = <<-QUERY + query GetColorName { + __type { + name + } + } + QUERY + + expected_error_snippet = "'__type' is missing required arguments" + + expect { + result = query_executor.execute(query_string, operation_name: "GetColorName") + + expect(result.dig("errors", 0, "message")).to include(expected_error_snippet) + }.to log a_string_including("GetColorName", "resulted in errors", expected_error_snippet) + end + + it "supports named operations and variables" do + query = <<-QUERY + query GetTypeName($typeName: String!) { + __type(name: $typeName) { + name + } + } + + query GetTypeFields($typeName: String!) { + __type(name: $typeName) { + fields { + name + } + } + } + QUERY + + data1 = execute_expecting_no_errors(query, operation_name: "GetTypeName", variables: {typeName: "Query"}) + expect(data1).to eq("__type" => {"name" => "Query"}) + + data2 = execute_expecting_no_errors(query, operation_name: "GetTypeFields", variables: {typeName: "Color"}) + expect(data2).to eq("__type" => {"fields" => [{"name" => "blue"}, {"name" => "green"}, {"name" => "id"}, {"name" => "red"}]}) + end + + it "ignores unknown variables" do + query = <<-QUERY + query GetTypeName($typeName: String!) { + __type(name: $typeName) { + name + } + } + QUERY + + data1 = execute_expecting_no_errors(query, operation_name: "GetTypeName", variables: {typeName: "Query", unknownVar: 3}) + expect(data1).to eq("__type" => {"name" => "Query"}) + end + + it "fails when variables reference undefined schema elements" do + query = <<~QUERY + query GetColors($colorArgs: ColorArgs) { + colors(args: $colorArgs) { + red + } + } + QUERY + + expect { + result = query_executor.execute(query, operation_name: "GetColors", variables: {colorArgs: {red: 3, orange: 12}}) + expect(result.dig("errors", 0, "message")).to include("$colorArgs", "ColorArgs", "orange") + }.to log(a_string_including("resulted in errors")) + end + + it "treats variable fields with `null` values as being unmentioned, to help static language clients avoid errors as the schema evolves" do + query = <<~QUERY + query GetColors($colorArgs: ColorArgs) { + colors(args: $colorArgs) { + red + } + } + QUERY + + execute_expecting_no_errors(query, operation_name: "GetColors", variables: {colorArgs: {red: 3, orange: nil, brown: nil}}) + end + + it "does not ignore `null` values on unknown arguments in the query itself" do + query = <<~QUERY + query { + colors(args: {red: 3, orange: nil, blue: nil}) { + red + } + } + QUERY + + expect { + result = query_executor.execute(query) + expect(result["errors"].to_s).to include("orange", "blue") + }.to log(a_string_including("resulted in errors")) + end + + it "calculates the `monotonic_clock_deadline` from a provided `timeout_in_ms`, and passes it along in the query context" do + query = <<~QUERY + query { + __type(name: "Color") { + name + } + } + QUERY + + expect(submitted_query_context_for(query, timeout_in_ms: 500)).to include( + monotonic_clock_deadline: monontonic_now_time + 500 + ) + end + + it "passes no `monotonic_clock_deadline` in `context` when no `timeout_in_ms` is provided" do + query = <<~QUERY + query { + __type(name: "Color") { + name + } + } + QUERY + + expect(submitted_query_context_for(query)).not_to include(:monotonic_clock_deadline) + end + + it "allows full introspection on all built-in schema types" do + # "Touch" all the built-in types and fields; at one point, the + # act of doing this caused a later failure when those types were + # fetched in a GraphQL query. + GraphQL::Schema::BUILT_IN_TYPE_NAMES.each do |type_name| + query_executor.schema.type_named(type_name).fields_by_name.values + end + + query = <<~QUERY + query { + __schema { + types { + name + fields { + name + } + } + } + } + QUERY + + field_names_by_type_name = execute_expecting_no_errors(query) + .fetch("__schema") + .fetch("types") + .each_with_object({}) do |type, hash| + hash[type.fetch("name")] = type.fetch("fields")&.map { |f| f.fetch("name") } + end + + expect(field_names_by_type_name).to include("__Field" => ["args", "deprecationReason", "description", "isDeprecated", "name", "type"]) + end + + it "responds reasonably when `query_string` is `nil`" do + expect { + result = query_executor.execute(nil) + + expect(result["errors"].to_s).to include("No query string") + }.to log(a_string_including("resulted in errors")) + end + + context "when the schema has been customized (as in an extension like elasticgraph-apollo)" do + before(:context) do + enumerator_extension = Module.new do + def root_query_type + super.tap do |type| + type.field "multiply", "Int" do |f| + f.argument("operands", "Operands!") + end + end + end + end + + self.schema_artifacts = generate_schema_artifacts do |schema| + schema.factory.extend(Module.new { + define_method :new_graphql_sdl_enumerator do |all_types_except_root_query_type| + super(all_types_except_root_query_type).tap do |enum| + enum.extend enumerator_extension + end + end + }) + + schema.scalar_type "Operands" do |t| + t.mapping type: nil + t.json_schema type: "null" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.index "widgets" + end + end + end + + let(:graphql) do + multiply_resolver = Class.new do + def can_resolve?(field:, object:) + field.name == :multiply + end + + def resolve(field:, object:, args:, context:, lookahead:) + [ + args.dig("operands", "x"), + args.dig("operands", "y"), + context[:additional_operand] + ].compact.reduce(:*) + end + end.new + + build_graphql(schema_artifacts: schema_artifacts, extension_modules: [ + Module.new do + define_method :graphql_resolvers do + @graphql_resolvers ||= [multiply_resolver] + super() + end + end + ]) + end + + it "allows an injected resolver to resolve the custom field" do + query = <<~EOS + query { + multiply(operands: {x: 3, y: 7}) + } + EOS + + result = graphql.graphql_query_executor.execute(query) + + expect(result.to_h).to eq({"data" => {"multiply" => 21}}) + end + + it "passes custom `context` values down to the custom resolver so it can use it in its logic" do + query = <<~EOS + query { + multiply(operands: {x: 3, y: 7}) + } + EOS + + result = graphql.graphql_query_executor.execute(query, context: {additional_operand: 10}) + + expect(result.to_h).to eq({"data" => {"multiply" => 210}}) + end + end + + def define_graphql + router = instance_double("ElasticGraph::GraphQL::DatastoreSearchRouter") + allow(router).to receive(:msearch) do |queries, query_tracker:| + query_tracker.record_datastore_query_duration_ms( + client: datastore_query_client_duration_ms, + server: datastore_query_server_duration_ms + ) + + queries.each_with_object({}) do |query, hash| + allow(query).to receive(:shard_routing_values).and_return(["routing_value_1", "routing_value_2"]) + hash[query] = {} + end + end + + build_graphql( + schema_artifacts: schema_artifacts, + datastore_search_router: router, + monotonic_clock: monotonic_clock, + slow_query_latency_warning_threshold_in_ms: slow_query_threshold_ms + ) + end + + def define_query_executor + define_graphql.graphql_query_executor + end + + def submitted_query_context_for(...) + submitted_context = nil + + allow(::GraphQL::Execution::Interpreter).to receive(:run_all).and_wrap_original do |original, schema, queries, context:| + submitted_context = context + original.call(schema, queries, context: context) + end + + query_executor.execute(...) + + submitted_context + end + + def execute_expecting_no_errors(query, query_executor: self.query_executor, **options) + response = query_executor.execute(query, **options) + expect(response["errors"]).to be nil + response.fetch("data") + end + + def logged_duration_message + logged_jsons = logged_jsons_of_type("ElasticGraphQueryExecutorQueryDuration") + expect(logged_jsons.size).to be < 2 + logged_jsons.first + end + + def logged_slo_result_for(query, duration_in_ms:) + allow(monotonic_clock).to receive(:now_in_ms).and_return(100, 100 + duration_in_ms) + execute_expecting_no_errors(query, client: Client.new(name: "client-name", source_description: "client-description")) + + logged_duration_message.fetch("slo_result") + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/get_record_field_value_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/get_record_field_value_spec.rb new file mode 100644 index 00000000..fbfbd76b --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/get_record_field_value_spec.rb @@ -0,0 +1,126 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/datastore_response/document" +require "elastic_graph/graphql/resolvers/get_record_field_value" + +module ElasticGraph + class GraphQL + module Resolvers + RSpec.describe GetRecordFieldValue, :capture_logs, :resolver do + attr_accessor :schema_artifacts + + before(:context) do + self.schema_artifacts = generate_schema_artifacts do |schema| + schema.scalar_type "MyInt" do |t| + t.mapping type: "integer" + t.json_schema type: "integer" + end + + schema.object_type "PersonIdentifiers" do |t| + t.field "ssn", "String" + end + + schema.object_type "Person" do |t| + t.field "name", "String" + t.field "identifiers", "PersonIdentifiers" + t.field "ssn", "String", name_in_index: "identifiers.ssn", graphql_only: true + t.field "alt_name1", "String", name_in_index: "name", graphql_only: true + t.field "alt_name2", "String", name_in_index: "name", graphql_only: true + t.field "converted_type", "MyInt" + t.field "nicknames", "[String!]" + t.field "alt_nicknames", "[String!]", name_in_index: "nicknames", graphql_only: true + t.field "doc_count", "Int" + end + end + end + + let(:graphql) { build_graphql(schema_artifacts: schema_artifacts) } + + context "for a field without customizations" do + it "fetches a requested scalar field from the document" do + value = resolve(:Person, :name, {"id" => 1, "name" => "Napoleon"}) + + expect(value).to eq "Napoleon" + end + + it "works with an `DatastoreResponse::Document`" do + doc = DatastoreResponse::Document.with_payload("id" => 1, "name" => "Napoleon") + value = resolve(:Person, :name, doc) + + expect(value).to eq "Napoleon" + end + + it "fetches a requested list field from the document" do + value = resolve(:Person, :nicknames, {"id" => 1, "nicknames" => %w[Napo Leon]}) + + expect(value).to eq %w[Napo Leon] + end + + it "returns `nil` when a scalar field is missing" do + value = resolve(:Person, :name, {"id" => 1}) + + expect(value).to eq nil + end + + it "returns a blank list when a list field is missing" do + value = resolve(:Person, :nicknames, {"id" => 2}) + + expect(value).to eq [] + end + end + + context "for a field with customizations" do + it "resolves to the field named via the `name_in_index` option instead of the schema field name" do + value = resolve(:Person, :alt_name1, {"id" => 1, "name" => "Napoleon"}) + + expect(value).to eq "Napoleon" + end + + it "can resolve to a child `name_in_index`" do + value = resolve(:Person, :ssn, {"id" => 1, "identifiers" => {"ssn" => "123-456-7890"}}) + + expect(value).to eq "123-456-7890" + end + + it "can resolve a list field" do + value = resolve(:Person, :alt_nicknames, {"id" => 1, "nicknames" => %w[Napo Leon]}) + + expect(value).to eq %w[Napo Leon] + end + + it "allows the `name` arg value to be a string or symbol" do + value1 = resolve(:Person, :alt_name1, {"id" => 1, "name" => "Napoleon"}) + value2 = resolve(:Person, :alt_name2, {"id" => 1, "name" => "Napoleon"}) + + expect(value1).to eq "Napoleon" + expect(value2).to eq "Napoleon" + end + + it "works with an `DatastoreResponse::Document`" do + doc = DatastoreResponse::Document.with_payload("id" => 1, "name" => "Napoleon") + value = resolve(:Person, :alt_name1, doc) + + expect(value).to eq "Napoleon" + end + + it "returns `nil` for a scalar when the directive field name is missing" do + value = resolve(:Person, :alt_name1, {"id" => 1}) + expect(value).to eq nil + end + + it "returns a blank list for a list field when the directive field name is missing" do + value = resolve(:Person, :alt_nicknames, {"id" => 2}) + + expect(value).to eq [] + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/graphql_adapter_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/graphql_adapter_spec.rb new file mode 100644 index 00000000..0207766f --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/graphql_adapter_spec.rb @@ -0,0 +1,40 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/resolvers/graphql_adapter" + +module ElasticGraph + class GraphQL + module Resolvers + RSpec.describe GraphQLAdapter do + let(:graphql) { build_graphql } + let(:schema) { graphql.schema } + + it "raises a clear error when no resolver can be found" do + adapter = GraphQLAdapter.new( + schema: schema, + datastore_query_builder: graphql.datastore_query_builder, + datastore_query_adapters: graphql.datastore_query_adapters, + runtime_metadata: graphql.runtime_metadata, + resolvers: graphql.graphql_resolvers + ) + + expect { + adapter.call( + schema.type_named(:Widget).graphql_type, + schema.field_named(:Widget, :id).instance_variable_get(:@graphql_field), + nil, + {}, + {} + ) + }.to raise_error(a_string_including("No resolver", "Widget.id")) + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/query_adapter_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/query_adapter_spec.rb new file mode 100644 index 00000000..fdda471b --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/query_adapter_spec.rb @@ -0,0 +1,228 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/resolvers/query_adapter" + +module ElasticGraph + class GraphQL + module Resolvers + RSpec.describe QueryAdapter, :query_adapter do + attr_accessor :schema_artifacts + + before(:context) do + self.schema_artifacts = generate_schema_artifacts do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_at", "DateTime" + t.field "workspace_id", "ID" + t.field "name", "String!" + t.relates_to_many "child_widgets", "Widget", via: "parent_id", dir: :in, singular: "child_widget" + + t.index "widgets" do |i| + i.route_with "workspace_id" + i.default_sort "name", :asc, "created_at", :desc + end + end + + schema.object_type "Component" do |t| + t.field "id", "ID" + t.field "price", "Int" + + t.index "components" do |i| + i.default_sort "price", :asc + end + end + + schema.union_type "WidgetOrComponent" do |t| + t.subtypes "Widget", "Component" + t.root_query_fields plural: "widgets_or_components" + end + + schema.union_type "ComponentOrWidget" do |t| + t.subtypes "Component", "Widget" + t.root_query_fields plural: "components_or_widgets" + end + end + end + + let(:graphql) { build_graphql(schema_artifacts: schema_artifacts) } + let(:field) { graphql.schema.field_named(:Query, :widgets) } + let(:query_adapter) do + QueryAdapter.new( + datastore_query_builder: graphql.datastore_query_builder, + datastore_query_adapters: graphql.datastore_query_adapters + ) + end + + describe "#build_query_from" do + it "returns an `DatastoreQuery`" do + datastore_query = build_query_from({}) + expect(datastore_query).to be_a(DatastoreQuery) + end + + it "sets the `search_index_definitions` based on the field type" do + datastore_query = build_query_from({}) + expect(datastore_query.search_index_definitions).to eq(graphql.schema.type_named(:Widget).search_index_definitions) + end + + describe "sort" do + it "sets it based on the `order_by` argument" do + datastore_query = build_query_from({order_by: "created_at_ASC"}) + expect(datastore_query.sort).to eq(["created_at" => {"order" => "asc"}]) + end + + it "defaults the sort order based on the index sort order if `order_by` is not provided" do + datastore_query = build_query_from({}) + + expect(datastore_query.sort).to eq [ + {"name" => {"order" => "asc"}}, + {"created_at" => {"order" => "desc"}} + ] + end + + context "on a union type where the subtypes have different default sorts" do + it "consistently uses the default sort from the alphabetically first index definition" do + field1 = graphql.schema.field_named(:Query, :widgets_or_components) + field2 = graphql.schema.field_named(:Query, :components_or_widgets) + + sort1 = build_query_from({}, field: field1).sort + sort2 = build_query_from({}, field: field2).sort + + expect(sort1).to eq(sort2).and eq [ + {"price" => {"order" => "asc"}} + ] + end + end + end + + it "supports `filter`" do + datastore_query = build_query_from({filter: {name: {equal_to_any_of: ["ben"]}}}) + expect(datastore_query.filters).to contain_exactly({"name" => {"equal_to_any_of" => ["ben"]}}) + end + + describe "document_pagination" do + context "on an indexed document field" do + let(:field) { graphql.schema.field_named(:Query, :widgets) } + + it "extracts `after`" do + datastore_query = build_query_from({after: "ABC"}) + expect(datastore_query.document_pagination).to include(after: "ABC") + end + + it "extracts `before`" do + datastore_query = build_query_from({before: "CBA"}) + expect(datastore_query.document_pagination).to include(before: "CBA") + end + + it "extracts `first`" do + datastore_query = build_query_from({first: 11}) + expect(datastore_query.document_pagination).to include(first: 11) + end + + it "extracts `last`" do + datastore_query = build_query_from({last: 11}) + expect(datastore_query.document_pagination).to include(last: 11) + end + end + + context "on an indexed aggregation field" do + let(:field) { graphql.schema.field_named(:Query, :widget_aggregations) } + + it "ignores the pagination arguments since the aggregation adapter handles them for aggregations" do + datastore_query = build_query_from({ + first: 3, after: "ABC", + last: 2, before: "DEF" + }) + + expect(datastore_query.document_pagination).to eq({}) + end + end + end + + it "passes along the `monotonic_clock_deadline` from the `context` to the query" do + datastore_query = build_query_from({}, context: {monotonic_clock_deadline: 1234}) + expect(datastore_query.monotonic_clock_deadline).to eq 1234 + end + + it "caches the `DatastoreQuery` objects it builds to avoid performing the same expensive work multiple times" do + datastore_queries = datastore_queries_by_field_for(<<~QUERY, schema_artifacts: schema_artifacts, list_item_count: 10) + query { + widgets { + edges { + node { + child_widgets { + edges { + node { + id + } + } + } + } + } + } + } + QUERY + + child_widget_queries = datastore_queries.fetch("Widget.child_widgets") + + expect(child_widget_queries.size).to eq 10 + expect(child_widget_queries.map(&:__id__).uniq.size).to eq 1 + end + + it "builds different `DatastoreQuery` objects for the same field at different levels" do + datastore_queries = datastore_queries_by_field_for(<<~QUERY, schema_artifacts: schema_artifacts, list_item_count: 10) + query { + widgets { + edges { + node { + child_widgets { + edges { + node { + id + + child_widgets { + edges { + node { + id + } + } + } + } + } + } + } + } + } + } + QUERY + + child_widget_queries = datastore_queries.fetch("Widget.child_widgets") + + expect(child_widget_queries.size).to eq 110 # 10 + (10 * 10) + expect(child_widget_queries.map(&:__id__).uniq.size).to eq 2 + end + + def build_query_from(args, field: self.field, context: {}) + context = ::GraphQL::Query::Context.new( + query: nil, + schema: graphql.schema.graphql_schema, + values: context + ) + + query_adapter.build_query_from( + field: field, + args: field.args_to_schema_form(args), + lookahead: ::GraphQL::Execution::Lookahead::NULL_LOOKAHEAD, + context: context + ) + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/relay_connection/array_adapter_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/relay_connection/array_adapter_spec.rb new file mode 100644 index 00000000..7f01af8e --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/relay_connection/array_adapter_spec.rb @@ -0,0 +1,172 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/resolvers/get_record_field_value" + +module ElasticGraph + class GraphQL + module Resolvers + module RelayConnection + # Note: while this file is `array_adapter_spec.rb`, the describe block below + # must use `GetRecordFieldValue` so we can leverage the handy `resolver` support + # (since `GetRecordFieldValue` is a resolver, but `ArrayAdapter` is not). + # It still primarily exercises the `ArrayAdapter` class defined in `array_adapter.rb`, + # but does so via the `GetRecordFieldValue` resolver, which has the added benefit + # of also verifying that `GetRecordFieldValue` builds `ArrayAdapter` properly. + RSpec.describe GetRecordFieldValue, "on a paginated collection field", :resolver do + attr_accessor :schema_artifacts + + before(:context) do + self.schema_artifacts = generate_schema_artifacts + end + + let(:graphql) { build_graphql } + + it "supports forward pagination" do + response = resolve_nums(11, first: 3) + expect(response.has_previous_page).to eq false + expect(response.has_next_page).to eq true + expect(nodes_of(response)).to eq([1, 2, 3]) + expect(response.nodes).to eq([1, 2, 3]) + + response = resolve_nums(11, first: 3, after: response.end_cursor) + expect(response.has_previous_page).to eq true + expect(response.has_next_page).to eq true + expect(nodes_of(response)).to eq([4, 5, 6]) + expect(response.nodes).to eq([4, 5, 6]) + + response = resolve_nums(11, first: 8, after: response.end_cursor) + expect(response.has_previous_page).to eq true + expect(response.has_next_page).to eq false + expect(nodes_of(response)).to eq([7, 8, 9, 10, 11]) + expect(response.nodes).to eq([7, 8, 9, 10, 11]) + end + + it "supports backwards pagination" do + response = resolve_nums(11, last: 3) + expect(response.has_previous_page).to eq true + expect(response.has_next_page).to eq false + expect(nodes_of(response)).to eq([9, 10, 11]) + expect(response.nodes).to eq([9, 10, 11]) + + response = resolve_nums(11, last: 3, before: response.start_cursor) + expect(response.has_previous_page).to eq true + expect(response.has_next_page).to eq true + expect(nodes_of(response)).to eq([6, 7, 8]) + expect(response.nodes).to eq([6, 7, 8]) + + response = resolve_nums(11, last: 8, before: response.start_cursor) + expect(response.has_previous_page).to eq false + expect(response.has_next_page).to eq true + expect(nodes_of(response)).to eq([1, 2, 3, 4, 5]) + expect(response.nodes).to eq([1, 2, 3, 4, 5]) + end + + it "exposes a unique cursor from each edge" do + response = resolve_nums(11) + cursors = response.edges.map(&:cursor) + + expect(cursors).to all match(/\w+/) + expect(cursors.uniq).to eq cursors + expect(response.start_cursor).to eq(cursors.first) + expect(response.end_cursor).to eq(cursors.last) + end + + it "exposes a `total_edge_count`" do + response = resolve_nums(11) + + expect(response.total_edge_count).to eq(11) + end + + it "behaves reasonably when given an empty array" do + response = resolve_nums(0) + + expect(response.total_edge_count).to eq(0) + expect(response.page_info.start_cursor).to eq(nil) + expect(response.page_info.end_cursor).to eq(nil) + expect(response.has_previous_page).to eq(false) + expect(response.has_next_page).to eq(false) + expect(response.edges).to eq([]) + expect(response.nodes).to eq([]) + end + + it "behaves reasonably when given `nil`" do + response = resolve_nums(nil) + + expect(response.total_edge_count).to eq(0) + expect(response.page_info.start_cursor).to eq(nil) + expect(response.page_info.end_cursor).to eq(nil) + expect(response.has_previous_page).to eq(false) + expect(response.has_next_page).to eq(false) + expect(response.edges).to eq([]) + expect(response.nodes).to eq([]) + end + + context "with `max_page_size` configured" do + let(:graphql) { build_graphql(max_page_size: 10) } + + it "ignores it, allowing the entire array to be returned because we've already paid the cost of fetching the entire array from the datastore, and limiting the page size here doesn't really help us" do + response = resolve_nums(18) + + expect(nodes_of(response)).to eq([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]) + expect(response.has_next_page).to eq(false) + expect(response.has_previous_page).to eq(false) + expect(response.total_edge_count).to eq(18) + end + end + + context "with custom schema element names configured" do + before(:context) do + self.schema_artifacts = generate_schema_artifacts(schema_element_name_overrides: { + "first" => "frst", + "last" => "lst", + "before" => "bfr", + "after" => "aftr" + }) + end + + it "honors those overrides" do + response = resolve_nums(11, frst: 3) + expect(nodes_of(response)).to eq([1, 2, 3]) + + response = resolve_nums(11, frst: 3, aftr: response.end_cursor) + expect(nodes_of(response)).to eq([4, 5, 6]) + + response = resolve_nums(11, lst: 3) + expect(nodes_of(response)).to eq([9, 10, 11]) + + response = resolve_nums(11, lst: 3, bfr: response.start_cursor) + expect(nodes_of(response)).to eq([6, 7, 8]) + end + end + + def resolve_nums(count, **args) + natural_numbers = count.is_a?(::Integer) ? 1.upto(count).to_a : count + resolve("Widget", "natural_numbers", {"natural_numbers" => natural_numbers}, **args) + end + + def nodes_of(response) + response.edges.map(&:node) + end + + def build_graphql(**options) + super(schema_artifacts: schema_artifacts, **options) + end + + def generate_schema_artifacts(**options) + super(**options) do |schema| + schema.object_type "Widget" do |t| + t.paginated_collection_field "natural_numbers", "Int" + end + end + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/relay_connection/search_response_adapter_builder_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/relay_connection/search_response_adapter_builder_spec.rb new file mode 100644 index 00000000..3e8b9918 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/relay_connection/search_response_adapter_builder_spec.rb @@ -0,0 +1,243 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/resolvers/relay_connection/search_response_adapter_builder" +require "support/sort" + +module ElasticGraph + class GraphQL + module Resolvers + module RelayConnection + RSpec.describe SearchResponseAdapterBuilder do + include SortSupport + + let(:decoded_cursor_factory) { decoded_cursor_factory_for("amount_cents") } + + attr_accessor :schema_artifacts + + before(:context) do + self.schema_artifacts = generate_schema_artifacts do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "amount_cents", "Int!" + t.index "widgets" do |i| + i.default_sort "amount_cents", :asc + end + end + end + end + + it "adapts the documents in a datastore search response to the relay connections edges interface" do + results = execute_query(query: <<~QUERY, hits: [hit_for(300, "w1"), hit_for(400, "w2"), hit_for(500, "w3")]) + query { + widgets { + edges { + cursor + node { + amount_cents + } + } + } + } + QUERY + + expect(results).to match( + "data" => { + "widgets" => { + "edges" => [ + edge_for(300, "w1"), + edge_for(400, "w2"), + edge_for(500, "w3") + ] + } + } + ) + end + + it "exposes all of the page info fields defined in the relay 5.0 spec" do + results = execute_query(query: <<~QUERY, hits: [hit_for(300, "w1"), hit_for(400, "w2"), hit_for(500, "w3")]) + query { + widgets { + page_info { + has_previous_page + has_next_page + start_cursor + end_cursor + } + } + } + QUERY + + expect(results).to match( + "data" => { + "widgets" => { + "page_info" => { + "has_previous_page" => a_boolean, + "has_next_page" => a_boolean, + "start_cursor" => decoded_cursor_factory.build([300, "w1"]).encode, + "end_cursor" => decoded_cursor_factory.build([500, "w3"]).encode + } + } + } + ) + end + + it "supports `nodes` being used instead of `edges`" do + results = execute_query(query: <<~QUERY, hits: [hit_for(300, "w1"), hit_for(400, "w2"), hit_for(500, "w3")]) + query { + widgets { + nodes { + amount_cents + } + } + } + QUERY + + expect(results).to match( + "data" => { + "widgets" => { + "nodes" => [ + {"amount_cents" => 300}, + {"amount_cents" => 400}, + {"amount_cents" => 500} + ] + } + } + ) + end + + # Note: the Relay 5.0 spec states `start_cursor`/`end_cursor` MUST be non-nullable: + # https://github.com/facebook/relay/blob/v5.0.0/website/spec/Connections.md#fields-2 + # However, in practice, they must be null when `edges` is empty, and relay itself + # implements this: + # https://github.com/facebook/relay/commit/a17b462b3ff7355df4858a42ddda75f58c161302 + # Hopefully this PR will get merged fixing the spec: + # https://github.com/facebook/relay/pull/2655 + it "exposes `null` for `start_cursor`/`end_cursor` when there are no hits" do + results = execute_query(query: <<~QUERY, hits: []) + query { + widgets { + page_info { + start_cursor + end_cursor + } + } + } + QUERY + + expect(results).to eq( + "data" => { + "widgets" => { + "page_info" => { + "start_cursor" => nil, + "end_cursor" => nil + } + } + } + ) + end + + it "exposes the same cursor value for `start_cursor`/`end_cursor` when there is only one hit" do + results = execute_query(query: <<~QUERY, hits: [hit_for(300, "w1")]) + query { + widgets { + page_info { + start_cursor + end_cursor + } + } + } + QUERY + + expected_cursor = decoded_cursor_factory.build([300, "w1"]).encode + + expect(results).to eq( + "data" => { + "widgets" => { + "page_info" => { + "start_cursor" => expected_cursor, + "end_cursor" => expected_cursor + } + } + } + ) + end + + it "also exposes `total_edge_count` off of the connection even though it is not part of the relay connections spec" do + results = execute_query(query: <<~QUERY, hits: []) + query { + widgets { + total_edge_count + } + } + QUERY + + expect(results).to eq( + "data" => { + "widgets" => { + "total_edge_count" => 5 + } + } + ) + end + + def build_response_hash(hits) + { + "took" => 50, + "timed_out" => false, + "_shards" => { + "total" => 5, + "successful" => 5, + "skipped" => 0, + "failed" => 0 + }, + "hits" => { + "total" => { + "value" => 5, + "relation" => "eq" + }, + "max_score" => nil, + "hits" => hits + } + } + end + + def execute_query(query:, hits:) + raw_data = build_response_hash(hits) + datastore_client = stubbed_datastore_client(msearch: {"responses" => [raw_data]}) + graphql = build_graphql(schema_artifacts: schema_artifacts, clients_by_name: {"main" => datastore_client}) + + graphql.graphql_query_executor.execute(query).to_h + end + + def edge_for(amount_cents, id) + { + "cursor" => decoded_cursor_factory.build([amount_cents, id]).encode, + "node" => {"amount_cents" => amount_cents} + } + end + + def hit_for(amount_cents, id) + { + "_index" => "widgets", + "_type" => "_doc", + "_id" => id, + "_score" => nil, + "_source" => { + "id" => id, + "version" => 10, + "amount_cents" => amount_cents + }, + "sort" => [amount_cents, id] + } + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/relay_connection_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/relay_connection_spec.rb new file mode 100644 index 00000000..fa237265 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/relay_connection_spec.rb @@ -0,0 +1,131 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/resolvers/relay_connection" +require "support/aggregations_helpers" + +module ElasticGraph + class GraphQL + module Resolvers + RSpec.describe RelayConnection do + include AggregationsHelpers + + describe ".maybe_wrap" do + attr_accessor :schema_artifacts + + before(:context) do + self.schema_artifacts = generate_schema_artifacts do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "amount_cents", "Int!" + t.index "widgets" do |i| + i.default_sort "amount_cents", :asc + end + end + + # One test relies on `widgets_non_relay`, which isn't defined by default on `Query` so we define it here. + schema.raw_sdl <<~EOS + type Query { + widgets: WidgetConnection! + widget_aggregations: WidgetAggregationConnection! + widgets_non_relay: [Widget!]! + } + EOS + end + end + + it "uses `SearchResponseAdapterBuilder` to build an adapter when the field is a document relay connection type" do + allow(Resolvers::RelayConnection::SearchResponseAdapterBuilder).to receive(:build_from).and_call_original + + wrapped = maybe_wrap("widgets") + expect(wrapped).to be_a Resolvers::RelayConnection::GenericAdapter + expect(Resolvers::RelayConnection::SearchResponseAdapterBuilder).to have_received(:build_from) + end + + it "uses `Aggregation::Resolvers::RelayConnectionBuilder` to build an adapter when the field is an aggregation relay connection type" do + allow(Aggregation::Resolvers::RelayConnectionBuilder).to receive(:build_from_search_response).and_call_original + + wrapped = maybe_wrap("widget_aggregations") + expect(wrapped).to be_a Resolvers::RelayConnection::GenericAdapter + expect(Aggregation::Resolvers::RelayConnectionBuilder).to have_received(:build_from_search_response) + end + + it "does not wrap a datastore response when the field is a non-relay collection" do + expect(maybe_wrap("widgets_non_relay")).to be_a DatastoreResponse::SearchResponse + end + + def maybe_wrap(field_name) + response = DatastoreResponse::SearchResponse.build(build_response_hash([hit_for(300, "w1")])) + + graphql = build_graphql(schema_artifacts: schema_artifacts) + field = graphql.schema.field_named(:Query, field_name) + context = {schema_element_names: graphql.runtime_metadata.schema_element_names} + + lookahead = ::GraphQL::Execution::Lookahead.new( + query: nil, + field: field.graphql_field, + ast_nodes: [] + ) + + query = instance_double( + "ElasticGraph::GraphQL::DatastoreQuery", + aggregations: {"widget_aggregations" => aggregation_query_of(name: "widget_aggregations")}, + document_paginator: instance_double( + "ElasticGraph::GraphQL::DatastoreQuery::DocumentPaginator", + paginator: instance_double( + "ElasticGraph::GraphQL::DatastoreQuery::Paginator" + ) + ) + ) + + RelayConnection.maybe_wrap(response, field: field, context: context, lookahead: lookahead, query: query) + end + + def build_response_hash(hits) + { + "took" => 50, + "timed_out" => false, + "_shards" => { + "total" => 5, + "successful" => 5, + "skipped" => 0, + "failed" => 0 + }, + "hits" => { + "total" => { + "value" => 5, + "relation" => "eq" + }, + "max_score" => nil, + "hits" => hits + }, + "aggregations" => { + "widget_aggregations" => {"buckets" => []} + } + } + end + + def hit_for(amount_cents, id) + { + "_index" => "widgets", + "_type" => "_doc", + "_id" => id, + "_score" => nil, + "_source" => { + "id" => id, + "version" => 10, + "amount_cents" => amount_cents + }, + "sort" => [amount_cents, id] + } + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/resolvable_value_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/resolvable_value_spec.rb new file mode 100644 index 00000000..d16f7075 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/resolvers/resolvable_value_spec.rb @@ -0,0 +1,219 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/resolvers/resolvable_value" +require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" + +module ElasticGraph + class GraphQL + module Resolvers + PersonFieldNames = SchemaArtifacts::RuntimeMetadata::SchemaElementNamesDefinition.new( + :name, :first_name, :birth_date, :age, + :favorite_quote, :favorite_quote2, :truncate_to, + *SchemaArtifacts::RuntimeMetadata::SchemaElementNames::ELEMENT_NAMES + ) + + RSpec.describe ResolvableValue do + attr_accessor :schema_artifacts_by_first_name + + before(:context) do + self.schema_artifacts_by_first_name = ::Hash.new do |hash, first_name| + hash[first_name] = generate_schema_artifacts do |schema| + schema.raw_sdl <<~EOS + type Person { + name(truncate_to: Int): String + last_name: String + age: Int + #{first_name}: String + birth_date: String + birthDate: String + favorite_quote(truncate_to: Int, foo_bar_bazz: Int): String + favorite_quote2(trunc_to: Int): String + } + EOS + end + end + end + + shared_examples_for ResolvableValue do |person_class| + describe "#resolve" do + it "delegates to a method of the same name on the object" do + value = resolve(name: "Kristin", field: :name) + + expect(value).to eq "Kristin" + end + + it "coerces the field name to its canonical form before calling the method" do + value = resolve(name_form: :camelCase, birth_date: "2001-03-01", field: :birthDate) + + expect(value).to eq "2001-03-01" + end + + it "evaluates the block passed to `ResolvableValue.new` just like `Data.define` does" do + value = resolve(name: "John Doe", field: :first_name) + expect(value).to eq "John" + + value = resolve(name_form: :camelCase, name: "John Doe", field: :firstName) + expect(value).to eq "John" + end + + it "raises a clear error if the field being resolved was not defined in the schema element names" do + expect { + resolve(name: "John Doe", field: :last_name) + }.to raise_error(Errors::SchemaError, /last_name/) + end + + it "raises an error if the field name is not in the form defined by the schema elements" do + expect { + resolve(name_form: :snake_case, birth_date: "2001-03-01", field: :birthDate) + }.to raise_error(Errors::SchemaError, /birthDate/) + end + + it "raises a `NoMethodError` if the field being resolved is not a defined method" do + expect { + resolve(name: "John Doe", field: :age) + }.to raise_error(NoMethodError, /age/) + end + + it "passes query field arguments to the resolver method" do + value = resolve( + name: "John Does", + quote: "To be, or not to be", + field: :favorite_quote, + args: {truncate_to: 5} + ) + + expect(value).to eq "To be" + end + + it "coerces arguments to their canonical form before calling the resolver method" do + value = resolve( + overrides: {truncate_to: "trunc_to"}, + name: "John Does", + quote: "To be, or not to be", + field: :favorite_quote2, + args: {trunc_to: 5} + ) + + expect(value).to eq "To be" + end + + it "raises a clear error if an argument is provided that has no canonical form definition" do + expect { + resolve( + name: "John Does", + quote: "To be, or not to be", + field: :favorite_quote, + args: {truncate_to: 5, foo_bar_bazz: 23} + ) + }.to raise_error Errors::SchemaError, a_string_including("foo_bar_bazz") + end + + it "raises a clear error if arguments are provided to a resolver method that does not expect them" do + expect { + resolve( + name: "John Does", + quote: "To be, or not to be", + field: :name, + args: {truncate_to: 5} + ) + }.to raise_error ArgumentError + end + + def resolve(args: {}, **options) + person, field = person_object_and_schema_field(**options) + args = field.args_to_schema_form(args) + lookahead = instance_double("GraphQL::Execution::Lookahead") + person.resolve(field: field, object: person, context: {}, args: args, lookahead: lookahead) + end + end + + describe "#can_resolve?" do + it "returns `false` if the field name is not defined in the schema elements" do + expect(can_resolve?(field: :last_name)).to be false + end + + it "returns `false` if no method matching the field name is defined on the object" do + expect(can_resolve?(field: :age)).to be false + end + + it "returns `true` if the field name is a defined schema element and the object has a matching method" do + expect(can_resolve?(field: :name)).to be true + expect(can_resolve?(field: :first_name)).to be true + end + + it "considers if the method is defined using the canonical form of the field name" do + expect(can_resolve?(name_form: :camelCase, field: :firstName)).to be true + end + + def can_resolve?(**options) + person, field = person_object_and_schema_field(**options) + person.can_resolve?(field: field, object: person) + end + end + + define_method :person_object_and_schema_field do | + field:, name_form: :snake_case, overrides: {}, + name: "John", birth_date: "2002-09-01", quote: "To be, or not to be, that is the question." + | + element_names = PersonFieldNames.new(form: name_form, overrides: overrides) + + person = person_class.new( + schema_element_names: element_names, + name: name, + quote: quote, + birth_date: birth_date + ) + + [person, build_graphql(element_names).schema.field_named(:Person, field)] + end + + def build_graphql(element_names) + super(schema_artifacts: schema_artifacts_by_first_name[element_names.first_name]) + end + end + + context "with a `ResolvableValue` with methods defined in a passed block" do + block_person = ResolvableValue.new(:name, :birth_date, :quote) do + def first_name + name.split(" ").first + end + + def favorite_quote(truncate_to:) + quote[0...truncate_to] + end + + def favorite_quote2(truncate_to:) + quote[0...truncate_to] + end + end + + include_examples ResolvableValue, block_person + end + + context "with a `ResolvableValue` with methods defined in a subclass" do + subclass_person = ::Class.new(ResolvableValue.new(:name, :birth_date, :quote)) do + def first_name + name.split(" ").first + end + + def favorite_quote(truncate_to:) + quote[0...truncate_to] + end + + def favorite_quote2(truncate_to:) + quote[0...truncate_to] + end + end + + include_examples ResolvableValue, subclass_person + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/cursor_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/cursor_spec.rb new file mode 100644 index 00000000..6c59d837 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/cursor_spec.rb @@ -0,0 +1,96 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "support/scalar_coercion_adapter" + +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + RSpec.describe "Cursor" do + include_context "scalar coercion adapter support", "Cursor" + + context "input coercion" do + it "accepts a properly encoded string cursor" do + cursor = DecodedCursor.new({"a" => 1, "b" => "foo"}) + expect_input_value_to_be_accepted(cursor.encode, as: cursor) + end + + it "accepts an already decoded cursor" do + cursor = DecodedCursor.new({"a" => 1, "b" => "foo"}) + expect_input_value_to_be_accepted(cursor, only_test_variable: true) + end + + it "accepts the special singleton cursor string value" do + expect_input_value_to_be_accepted(DecodedCursor::SINGLETON.encode, as: DecodedCursor::SINGLETON) + end + + it "accepts the special singleton cursor value" do + expect_input_value_to_be_accepted(DecodedCursor::SINGLETON, only_test_variable: true) + end + + it "accepts a `nil` value as-is" do + expect_input_value_to_be_accepted(nil) + end + + it "rejects values that are not strings" do + expect_input_value_to_be_rejected(3) + expect_input_value_to_be_rejected(3.7) + expect_input_value_to_be_rejected(false) + expect_input_value_to_be_rejected([1, 2, 3]) + expect_input_value_to_be_rejected(["a", "b"]) + expect_input_value_to_be_rejected({"a" => 1, "b" => "foo"}) + end + + it "rejects broken string cursors" do + cursor = DecodedCursor.new({"a" => 1, "b" => "foo"}).encode + expect_input_value_to_be_rejected(cursor + "-broken") + end + end + + context "result coercion" do + it "returns the encoded form of a decoded string cursor" do + cursor = DecodedCursor.new({"a" => 1, "b" => "foo"}) + expect_result_to_be_returned(cursor, as: cursor.encode) + end + + it "returns a properly encoded cursor as-is" do + cursor = DecodedCursor.new({"a" => 1, "b" => "foo"}) + expect_result_to_be_returned(cursor.encode, as: cursor.encode) + end + + it "returns the encoded form of the special singleton cursor" do + cursor = DecodedCursor::SINGLETON + expect_result_to_be_returned(cursor, as: cursor.encode) + end + + it "returns the encoded form of the special singleton cursor as-is when given in its string form" do + cursor = DecodedCursor::SINGLETON + expect_result_to_be_returned(cursor.encode, as: cursor.encode) + end + + it "returns `nil` as is" do + expect_result_to_be_returned(nil) + end + + it "returns `nil` for non-string values" do + expect_result_to_be_replaced_with_nil(3) + expect_result_to_be_replaced_with_nil(3.7) + expect_result_to_be_replaced_with_nil(false) + expect_result_to_be_replaced_with_nil([1, 2, 3]) + expect_result_to_be_replaced_with_nil(["a", "b"]) + expect_result_to_be_replaced_with_nil({"a" => 1, "b" => "foo"}) + end + + it "returns `nil` for strings that are not properly encoded cursors" do + expect_result_to_be_replaced_with_nil("not a cursor") + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/date_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/date_spec.rb new file mode 100644 index 00000000..870ce416 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/date_spec.rb @@ -0,0 +1,116 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "support/scalar_coercion_adapter" + +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + RSpec.describe "Date" do + include_context "scalar coercion adapter support", "Date" + + let(:other_format_string) do + "2021/03/27".tap do |string| + ::Date.parse(string) # prove it is parseable + expect { ::Date.iso8601(string) }.to raise_error(ArgumentError) + end + end + + context "input coercion" do + it "accepts an ISO8601 formatted date string with ms precision" do + expect_input_value_to_be_accepted("2021-11-05") + end + + it "rejects a year that is more than 4 digits because the datastore strict format we use requires 4 digits" do + expect_input_value_to_be_rejected("20021-11-05") + expect_input_value_to_be_rejected("200021-11-05") + end + + it "accepts a `nil` value as-is" do + expect_input_value_to_be_accepted(nil) + end + + it "rejects other date formats" do + expect_input_value_to_be_rejected(other_format_string, "ISO8601") + end + + it "rejects an ISO8601-formatted DateTime" do + expect_input_value_to_be_rejected("2021-11-05T12:30:00Z", "ISO8601") + end + + it "rejects a string that is not a date string" do + expect_input_value_to_be_rejected("not a date") + end + + it "rejects numbers" do + expect_input_value_to_be_rejected(1231232131) + expect_input_value_to_be_rejected(34.5) + end + + it "rejects booleans" do + expect_input_value_to_be_rejected(true) + expect_input_value_to_be_rejected(false) + end + + it "rejects a list of ISO8601 date strings" do + expect_input_value_to_be_rejected([::Date.today.iso8601]) + end + end + + context "result coercion" do + it "returns a Date result in ISO8601 format" do + string, date = string_date_pair_from("2021-11-05") + + expect_result_to_be_returned(date, as: string) + end + + it "returns a string already formatted in ISO8061 format as-is" do + string, _date = string_date_pair_from("2021-11-05") + + expect_result_to_be_returned(string, as: string) + end + + it "returns `nil` as is" do + expect_result_to_be_returned(nil) + end + + it "returns `nil` in place of a number" do + expect_result_to_be_replaced_with_nil(1231232131) + expect_result_to_be_replaced_with_nil(34.5) + end + + it "returns `nil` in place of a boolean" do + expect_result_to_be_replaced_with_nil(false) + expect_result_to_be_replaced_with_nil(true) + end + + it "returns `nil` in place of a full timestamp" do + expect_result_to_be_replaced_with_nil(::Time.new(2021, 5, 12, 12, 30, 30)) + end + + it "returns `nil` in place of a String that is not in ISO8601 format" do + expect_result_to_be_replaced_with_nil(other_format_string) + end + + it "returns `nil` in place of a String that is not a date string" do + expect_result_to_be_replaced_with_nil("not a date") + end + + it "returns `nil` in place of a list of valid date values" do + expect_result_to_be_replaced_with_nil([::Date.today.iso8601]) + expect_result_to_be_replaced_with_nil([::Date.today]) + end + end + + def string_date_pair_from(iso8601_string) + [iso8601_string, ::Date.iso8601(iso8601_string)] + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/date_time_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/date_time_spec.rb new file mode 100644 index 00000000..d4c2e5cb --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/date_time_spec.rb @@ -0,0 +1,148 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "support/scalar_coercion_adapter" + +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + RSpec.describe "DateTime" do + include_context "scalar coercion adapter support", "DateTime" + + let(:other_format_string) do + ::Time.iso8601("2021-11-05T12:30:00Z").to_s.tap do |string| + ::Time.parse(string) # prove it is parseable + expect { ::Time.iso8601(string) }.to raise_error(ArgumentError) + end + end + + context "input coercion" do + it "accepts an ISO8601 formatted timestamp string with ms precision" do + expect_input_value_to_be_accepted("2021-11-05T12:30:00.123Z") + end + + it "accepts an ISO8601 formatted timestamp string with s precision" do + string_time = "2021-11-05T12:30:05Z" + + expect_input_value_to_be_accepted(string_time, as: string_time.sub("05Z", "05.000Z")) + end + + it "supports time zones besides UTC" do + expect_input_value_to_be_accepted("2021-11-05T12:30:00.123-07:00") + end + + it "accepts a `nil` value as-is" do + expect_input_value_to_be_accepted(nil) + end + + it "rejects other time formats" do + expect_input_value_to_be_rejected(other_format_string, "ISO8601") + end + + it "rejects an ISO8601-formatted Date" do + expect_input_value_to_be_rejected("2021-11-05", "ISO8601") + end + + it "rejects a string that is not a timestamp string" do + expect_input_value_to_be_rejected("not a timestamp") + end + + it "rejects an ISO8601 formatted timestamp with more than 4 digits for the year because the datastore strict format we use requires 4 digits" do + expect_input_value_to_be_rejected("20021-11-05T12:30:00Z") + expect_input_value_to_be_rejected("200021-11-05T12:30:00Z") + end + + it "allows timestamps before the year 1000" do + expect_input_value_to_be_accepted("0001-01-01T00:00:00.000Z") + end + + it "rejects numbers" do + expect_input_value_to_be_rejected(1231232131) + expect_input_value_to_be_rejected(34.5) + end + + it "rejects booleans" do + expect_input_value_to_be_rejected(true) + expect_input_value_to_be_rejected(false) + end + + it "rejects a list of ISO8601 timestamp strings" do + expect_input_value_to_be_rejected([::Time.now.iso8601]) + end + end + + context "result coercion" do + it "returns a Time result in ISO8601 format" do + string, time = string_time_pair_from("2021-11-05T12:30:00.123Z") + + expect_result_to_be_returned(time, as: string) + end + + it "formats a Time result with ms precision even if the time value only really has second precision" do + string, time = string_time_pair_from("2021-11-05T12:30:00Z") + + expect_result_to_be_returned(time, as: string.sub("00Z", "00.000Z")) + end + + it "returns a string already formatted in ISO8061 format as-is" do + string, _time = string_time_pair_from("2021-11-05T12:30:00.123Z") + + expect_result_to_be_returned(string, as: string) + end + + it "reformats a second-precision ISO8601 string to be ms-precision" do + string, _time = string_time_pair_from("2021-11-05T12:30:00Z") + + expect_result_to_be_returned(string, as: string.sub("00Z", "00.000Z")) + end + + it "supports time zones besides UTC" do + string, time = string_time_pair_from("2021-11-05T12:30:00.123-07:00") + + expect_result_to_be_returned(time, as: string) + end + + it "returns `nil` as is" do + expect_result_to_be_returned(nil) + end + + it "returns `nil` in place of a number" do + expect_result_to_be_replaced_with_nil(1231232131) + expect_result_to_be_replaced_with_nil(34.5) + end + + it "returns `nil` in place of a boolean" do + expect_result_to_be_replaced_with_nil(false) + expect_result_to_be_replaced_with_nil(true) + end + + it "returns `nil` in place of a Date" do + expect_result_to_be_replaced_with_nil(::Date.new(2021, 5, 12)) + end + + it "returns `nil` in place of a String that is not in ISO8601 format" do + expect_result_to_be_replaced_with_nil(other_format_string) + end + + it "returns `nil` in place of a String that is not a timestamp string" do + expect_result_to_be_replaced_with_nil("not a timestamp") + end + + it "returns `nil` in place of a list of valid timestamp values" do + expect_result_to_be_replaced_with_nil([::Time.now.iso8601]) + expect_result_to_be_replaced_with_nil([::Time.now]) + end + end + + def string_time_pair_from(iso8601_string) + [iso8601_string, ::Time.iso8601(iso8601_string)] + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/local_time_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/local_time_spec.rb new file mode 100644 index 00000000..41ee3ead --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/local_time_spec.rb @@ -0,0 +1,214 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "support/scalar_coercion_adapter" + +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + RSpec.describe "LocalTime" do + include_context "scalar coercion adapter support", "LocalTime" + + context "input coercion" do + it "accepts a well-formatted time with hour 00 to 23" do + 0.upto(9) do |hour| + expect_input_value_to_be_accepted("0#{hour}:00:00") + end + + 10.upto(23) do |hour| + expect_input_value_to_be_accepted("#{hour}:00:00") + end + end + + it "rejects hours greater than 23" do + 24.upto(100) do |hour| + expect_input_value_to_be_rejected("#{hour}:00:00") + end + end + + it "accepts a well-formatted time with minute 00 to 59" do + 0.upto(9) do |minute| + expect_input_value_to_be_accepted("00:0#{minute}:00") + end + + 10.upto(59) do |minute| + expect_input_value_to_be_accepted("00:#{minute}:00") + end + end + + it "rejects minutes greater than 59" do + 60.upto(100) do |minute| + expect_input_value_to_be_rejected("00:#{minute}0:00") + end + end + + it "accepts a well-formatted time with second 00 to 59" do + 0.upto(9) do |second| + expect_input_value_to_be_accepted("00:00:0#{second}") + end + + 10.upto(59) do |second| + expect_input_value_to_be_accepted("00:00:#{second}") + end + end + + it "rejects seconds greater than 59" do + 60.upto(100) do |second| + expect_input_value_to_be_rejected("00:00:#{second}") + end + end + + it "rejects single-digit hour, minute, or second" do + expect_input_value_to_be_rejected("7:00:00") + expect_input_value_to_be_rejected("00:7:00") + expect_input_value_to_be_rejected("00:00:7") + end + + it "accepts up to 3 decimal subsecond digits" do + expect_input_value_to_be_accepted("00:00:00.1") + expect_input_value_to_be_accepted("00:00:00.12") + expect_input_value_to_be_accepted("00:00:00.123") + end + + it "rejects malformed or too many decimal digits" do + expect_input_value_to_be_rejected("00:00:00.1a") + expect_input_value_to_be_rejected("00:00:00.") + expect_input_value_to_be_rejected("00:00:00.1234") + end + + it "rejects non-string values" do + expect_input_value_to_be_rejected(3) + expect_input_value_to_be_rejected(3.7) + expect_input_value_to_be_rejected(true) + expect_input_value_to_be_rejected(false) + expect_input_value_to_be_rejected(["00:00:00"]) + expect_input_value_to_be_rejected([]) + end + + it "rejects strings that are not formatted like a time at all" do + expect_input_value_to_be_rejected("not a time") + end + + it "rejects strings that are not quite formatted correctly" do + expect_input_value_to_be_rejected("00:00") # no second part + expect_input_value_to_be_rejected("000000") # no colons + expect_input_value_to_be_rejected("07:00:00am") # am/pm + expect_input_value_to_be_rejected("07:00:00AM") # am/pm + expect_input_value_to_be_rejected("07:00:00 am") # am/pm + expect_input_value_to_be_rejected("07:00:00 AM") # am/pm + expect_input_value_to_be_rejected("07:00:00Z") # time zone + expect_input_value_to_be_rejected("07:00:00+03:00") # time zone offset + end + + it "rejects strings that use another alphanumeric character in place of the dot (ensuring the dot in the pattern is not treated as a regex wildcard)" do + expect_input_value_to_be_rejected("00:00:00a123") + expect_input_value_to_be_rejected("00:00:003123") + end + + it "rejects strings that have extra lines before or after the `LocalTime` value (ensuring we are correctly matching against the entire string, not just one line)" do + expect_input_value_to_be_rejected("a line\n00:00:00") + expect_input_value_to_be_rejected("00:00:00\n a line") + expect_input_value_to_be_rejected("a line\n00:00:00\n a line") + end + end + + context "result coercion" do + it "returns a well-formatted time with hour 00 to 23" do + 0.upto(9) do |hour| + expect_result_to_be_returned("0#{hour}:00:00") + end + + 10.upto(23) do |hour| + expect_result_to_be_returned("#{hour}:00:00") + end + end + + it "returns `nil` in place of a time string with hours greater than 23" do + 24.upto(100) do |hour| + expect_result_to_be_replaced_with_nil("#{hour}:00:00") + end + end + + it "returns a well-formatted time with minute 00 to 59" do + 0.upto(9) do |minute| + expect_result_to_be_returned("00:0#{minute}:00") + end + + 10.upto(59) do |minute| + expect_result_to_be_returned("00:#{minute}:00") + end + end + + it "returns `nil` in place of a time string with minutes greater than 59" do + 60.upto(100) do |minute| + expect_result_to_be_replaced_with_nil("00:#{minute}0:00") + end + end + + it "returns a well-formatted time with second 00 to 59" do + 0.upto(9) do |second| + expect_result_to_be_returned("00:00:0#{second}") + end + + 10.upto(59) do |second| + expect_result_to_be_returned("00:00:#{second}") + end + end + + it "returns `nil` in place of a time string with seconds greater than 59" do + 60.upto(100) do |second| + expect_result_to_be_replaced_with_nil("00:00:#{second}") + end + end + + it "returns `nil` in place of a time string with single-digit hour, minute, or second" do + expect_result_to_be_replaced_with_nil("7:00:00") + expect_result_to_be_replaced_with_nil("00:7:00") + expect_result_to_be_replaced_with_nil("00:00:7") + end + + it "returns up to 3 decimal subsecond digits" do + expect_result_to_be_returned("00:00:00.1") + expect_result_to_be_returned("00:00:00.12") + expect_result_to_be_returned("00:00:00.123") + end + + it "returns `nil` in place of a time string with malformed or too many decimal digits" do + expect_result_to_be_replaced_with_nil("00:00:00.1a") + expect_result_to_be_replaced_with_nil("00:00:00.") + expect_result_to_be_replaced_with_nil("00:00:00.1234") + end + + it "returns `nil` in place of non-string values" do + expect_result_to_be_replaced_with_nil(3) + expect_result_to_be_replaced_with_nil(3.7) + expect_result_to_be_replaced_with_nil(true) + expect_result_to_be_replaced_with_nil(false) + expect_result_to_be_replaced_with_nil(["00:00:00"]) + expect_result_to_be_replaced_with_nil([]) + end + + it "returns `nil` in place of strings that are not formatted like a time at all" do + expect_result_to_be_replaced_with_nil("not a time") + end + + it "returns `nil` in place of strings that are not quite formatted correctly" do + expect_result_to_be_replaced_with_nil("00:00") # no second part + expect_result_to_be_replaced_with_nil("000000") # no colons + expect_result_to_be_replaced_with_nil("07:00:00am") # am/pm + expect_result_to_be_replaced_with_nil("07:00:00AM") # am/pm + expect_result_to_be_replaced_with_nil("07:00:00 am") # am/pm + expect_result_to_be_replaced_with_nil("07:00:00 AM") # am/pm + expect_result_to_be_replaced_with_nil("07:00:00Z") # time zone + expect_result_to_be_replaced_with_nil("07:00:00+03:00") # time zone offset + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/longs_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/longs_spec.rb new file mode 100644 index 00000000..0c38c103 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/longs_spec.rb @@ -0,0 +1,183 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "support/scalar_coercion_adapter" + +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + RSpec.describe "JsonSafeLong" do + include_context "scalar coercion adapter support", "JsonSafeLong" + + context "input coercion" do + it "accepts an input of the minimum valid JsonSafeLong value" do + expect_input_value_to_be_accepted(JSON_SAFE_LONG_MIN) + end + + it "accepts an input of the maximum valid JsonSafeLong value" do + expect_input_value_to_be_accepted(JSON_SAFE_LONG_MAX) + end + + it "accepts an input of the JsonSafeLong values in the middle of the range" do + expect_input_value_to_be_accepted(0) + end + + it "accepts a valid JsonSafeLong when passed as a string" do + expect_input_value_to_be_accepted("10012312", as: 10012312) + end + + it "accepts a `nil` value as-is" do + expect_input_value_to_be_accepted(nil) + end + + it "rejects an input that is an unconvertible type" do + expect_input_value_to_be_rejected(false) + end + + it "rejects an input that is a convertible type with an invalid value" do + expect_input_value_to_be_rejected("fourteen") + end + + it "rejects an input of one less than the minimum valid JsonSafeLong value" do + expect_input_value_to_be_rejected(JSON_SAFE_LONG_MIN - 1) + end + + it "rejects an input of one more than the maximum valid JsonSafeLong value" do + expect_input_value_to_be_rejected(JSON_SAFE_LONG_MAX + 1) + end + end + + context "result coercion" do + it "returns a result of the minimum valid JsonSafeLong value" do + expect_result_to_be_returned(JSON_SAFE_LONG_MIN) + end + + it "returns a result of the maximum valid JsonSafeLong value" do + expect_result_to_be_returned(JSON_SAFE_LONG_MAX) + end + + it "returns a result of the JsonSafeLong values in the middle of the range" do + expect_result_to_be_returned(0) + end + + it "returns a valid JsonSafeLong when it starts as a string" do + expect_result_to_be_returned("10012312", as: 10012312) + end + + it "returns `nil` as is" do + expect_result_to_be_returned(nil) + end + + it "returns `nil` in place of an unconvertible type" do + expect_result_to_be_replaced_with_nil(false) + end + + it "returns `nil` in place of a convertible type with an invalid value" do + expect_result_to_be_replaced_with_nil("fourteen") + end + + it "returns `nil` in place of one less than the minimum valid JsonSafeLong value" do + expect_result_to_be_replaced_with_nil(JSON_SAFE_LONG_MIN - 1) + end + + it "returns `nil` in place of one more than the maximum valid JsonSafeLong value" do + expect_result_to_be_replaced_with_nil(JSON_SAFE_LONG_MAX + 1) + end + end + end + + RSpec.describe "LongString" do + include_context "scalar coercion adapter support", "LongString" + + context "input coercion" do + it "accepts an input of the minimum valid LongString value as a string" do + expect_input_value_to_be_accepted_as_a_string(LONG_STRING_MIN) + end + + it "accepts an input of the maximum valid LongString value as a string" do + expect_input_value_to_be_accepted_as_a_string(LONG_STRING_MAX) + end + + it "accepts a `nil` value as-is" do + expect_input_value_to_be_accepted(nil) + end + + it "accepts an input of the LongString values in the middle of the range as a string" do + expect_input_value_to_be_accepted_as_a_string(0) + end + + it "rejects a valid LongString when passed as a number instead of a string, to guard against the client potentially already having rounded it" do + expect_input_value_to_be_rejected(0) + end + + it "rejects an input that is an unconvertible type" do + expect_input_value_to_be_rejected(false) + end + + it "rejects an input that is a convertible type with an invalid value" do + expect_input_value_to_be_rejected("fourteen") + end + + it "rejects an input of one less than the minimum valid LongString value as a string" do + expect_input_value_to_be_rejected((LONG_STRING_MIN - 1).to_s) + end + + it "rejects an input of one more than the maximum valid LongString value as a string" do + expect_input_value_to_be_rejected((LONG_STRING_MAX + 1).to_s) + end + + def expect_input_value_to_be_accepted_as_a_string(value) + expect_input_value_to_be_accepted(value.to_s, as: value) + end + end + + context "result coercion" do + it "returns a result of the minimum valid LongString value" do + expect_result_to_be_returned_as_a_string(LONG_STRING_MIN) + end + + it "returns a result of the maximum valid LongString value" do + expect_result_to_be_returned_as_a_string(LONG_STRING_MAX) + end + + it "returns a result of the LongString values in the middle of the range" do + expect_result_to_be_returned_as_a_string(0) + end + + it "returns a valid LongString when it starts as a string" do + expect_result_to_be_returned("10012312") + end + + it "returns `nil` as is" do + expect_result_to_be_returned(nil) + end + + it "returns `nil` in place of an unconvertible type" do + expect_result_to_be_replaced_with_nil(false) + end + + it "returns `nil` in place of a convertible type with an invalid value" do + expect_result_to_be_replaced_with_nil("fourteen") + end + + it "returns `nil` in place of one less than the minimum valid LongString value" do + expect_result_to_be_replaced_with_nil(LONG_STRING_MIN - 1) + end + + it "returns `nil` in place of one more than the maximum valid LongString value" do + expect_result_to_be_replaced_with_nil(LONG_STRING_MAX + 1) + end + + def expect_result_to_be_returned_as_a_string(value) + expect_result_to_be_returned(value, as: value.to_s) + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/no_op_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/no_op_spec.rb new file mode 100644 index 00000000..0973399f --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/no_op_spec.rb @@ -0,0 +1,56 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "support/scalar_coercion_adapter" + +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + RSpec.describe "NoOp" do + include_context("scalar coercion adapter support", "SomeCustomScalar", schema_definition: ->(schema) do + schema.scalar_type "SomeCustomScalar" do |t| + t.json_schema type: "null" + t.mapping type: nil + end + + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "scalar", "SomeCustomScalar" + t.index "widgets" + end + end) + + context "input coercion" do + it "leaves values of any type unmodified" do + expect_input_value_to_be_accepted(nil) + expect_input_value_to_be_accepted(3) + expect_input_value_to_be_accepted(3.7) + expect_input_value_to_be_accepted(false) + expect_input_value_to_be_accepted("foo") + expect_input_value_to_be_accepted([1, 2, 3]) + expect_input_value_to_be_accepted(["a", "b"]) + expect_input_value_to_be_accepted({"a" => 1, "b" => "foo"}) + end + end + + context "result coercion" do + it "leaves values of any type unmodified" do + expect_result_to_be_returned(nil) + expect_result_to_be_returned(3) + expect_result_to_be_returned(3.7) + expect_result_to_be_returned(false) + expect_result_to_be_returned("foo") + expect_result_to_be_returned([1, 2, 3]) + expect_result_to_be_returned(["a", "b"]) + expect_result_to_be_returned({"a" => 1, "b" => "foo"}) + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/time_zone_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/time_zone_spec.rb new file mode 100644 index 00000000..9a6db8be --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/time_zone_spec.rb @@ -0,0 +1,102 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "support/scalar_coercion_adapter" + +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + RSpec.describe "TimeZone" do + include_context "scalar coercion adapter support", "TimeZone" + + context "input coercion" do + it "accepts valid time zone ids" do + expect_input_value_to_be_accepted("America/Los_Angeles") + expect_input_value_to_be_accepted("UTC") + end + + it "accepts a `nil` value as-is" do + expect_input_value_to_be_accepted(nil) + end + + it "rejects values that are not strings" do + expect_input_value_to_be_rejected(3) + expect_input_value_to_be_rejected(3.7) + expect_input_value_to_be_rejected(false) + expect_input_value_to_be_rejected([1, 2, 3]) + expect_input_value_to_be_rejected(["a", "b"]) + end + + it "rejects empty strings" do + expect_input_value_to_be_rejected("") + end + + it "rejects unknown time zone ids" do + expect_input_value_to_be_rejected("America/Seattle") + expect_input_value_to_be_rejected("Who knows?") + end + + it "suggests the corrected time zone if given one with a mistake" do + expect_input_value_to_be_rejected("Japan/Tokyo", 'Possible alternative: "Asia/Tokyo".') + end + + it "can offer two suggestions (X or Y)" do + expect_input_value_to_be_rejected("Asia/Kathmando", 'Possible alternatives: "Asia/Kathmandu" or "Asia/Katmandu".') + end + + it "can offer 3 or more suggestions (X, Y, or Z)" do + # Note: we don't assert on the exact order of suggestions here because we found that it's different locally on Mac OS X vs on CI. + expect_input_value_to_be_rejected("SystemV/BST3", "Possible alternatives:", '"SystemV/CST6", ', '"SystemV/YST9", ', '"SystemV/EST5",', ', or "SystemV/') + end + + it "does not suggest anything if it cannot identify any suggestions" do + expect_input_value_to_be_rejected("No Idea", expect_error_to_lack: ["Possible alternatives"]) + end + end + + context "result coercion" do + it "returns valid time zone ids" do + expect_result_to_be_returned("America/Los_Angeles") + expect_result_to_be_returned("UTC") + end + + it "returns `nil` as is" do + expect_result_to_be_returned(nil) + end + + it "returns `nil` for non-string values" do + expect_result_to_be_replaced_with_nil(3) + expect_result_to_be_replaced_with_nil(3.7) + expect_result_to_be_replaced_with_nil(false) + expect_result_to_be_replaced_with_nil([1, 2, 3]) + expect_result_to_be_replaced_with_nil(["a", "b"]) + end + + it "returns `nil` for the empty string" do + expect_result_to_be_replaced_with_nil("") + end + + it "returns `nil` for unknown time zone ids" do + expect_result_to_be_replaced_with_nil("America/Seattle") + expect_result_to_be_replaced_with_nil("Who knows?") + end + end + + describe "VALID_TIME_ZONES" do + it "is up-to-date with the list of time zones available on the JVM (since that's the list that Elasticsearch/OpenSearch use)" do + expected_time_zones_file = `#{SPEC_ROOT}/../script/dump_time_zones --print` + actual_time_zones_file = ::File.read(::File.join(SPEC_ROOT, "..", "lib", "elastic_graph", "graphql", "scalar_coercion_adapters", "valid_time_zones.rb")) + + # Verify the file is up to date. If out of date, run `script/dump_time_zones` to fix it. + expect(actual_time_zones_file).to eq(expected_time_zones_file) + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/untyped_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/untyped_spec.rb new file mode 100644 index 00000000..72143423 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/scalar_coercion_adapters/untyped_spec.rb @@ -0,0 +1,100 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "support/scalar_coercion_adapter" +require "time" + +module ElasticGraph + class GraphQL + module ScalarCoercionAdapters + RSpec.describe "Untyped" do + include_context "scalar coercion adapter support", "Untyped" + + context "input coercion" do + it "accepts integers" do + expect_input_value_to_be_accepted(12, as: "12") + end + + it "accepts floating point numbers" do + expect_input_value_to_be_accepted(-3.75, as: "-3.75") + end + + it "accepts strings" do + expect_input_value_to_be_accepted("foo", as: "\"foo\"") + end + + it "accepts booleans" do + expect_input_value_to_be_accepted(true, as: "true") + expect_input_value_to_be_accepted(false, as: "false") + end + + it "accepts nil" do + expect_input_value_to_be_accepted(nil) + end + + it "accepts arrays of JSON primitives" do + expect_input_value_to_be_accepted([true, "abc", 75, nil], as: "[true,\"abc\",75,null]") + end + + it "accepts a JSON object" do + expect_input_value_to_be_accepted({"name" => "John"}, as: "{\"name\":\"John\"}") + end + + it "rejects types that are not valid in JSON" do + expect_input_value_to_be_rejected(::Time.iso8601("2022-12-01T00:00:00Z"), only_test_variable: true) + end + + it "orders object keys alphabetically when dumping it to normalize to a canonical form" do + expect_input_value_to_be_accepted({"b" => 1, "a" => 2, "c" => 3}, as: '{"a":2,"b":1,"c":3}') + end + end + + context "result coercion" do + it "returns an integer when given an integer as a JSON string" do + expect_result_to_be_returned("12", as: 12) + end + + it "returns a float when given a float as a JSON string" do + expect_result_to_be_returned("-3.75", as: -3.75) + end + + it "returns a string when given a string as a JSON string" do + expect_result_to_be_returned("\"foo\"", as: "foo") + end + + it "returns a boolean when given a boolean as a JSON string" do + expect_result_to_be_returned("true", as: true) + expect_result_to_be_returned("false", as: false) + end + + it "returns nil as-is" do + expect_result_to_be_returned(nil) + end + + it "returns an array of JSON primitives when given that as a JSON string" do + expect_result_to_be_returned("[true,\"abc\",75,null]", as: [true, "abc", 75, nil]) + end + + it "returns a hash when given that as a JSON string" do + expect_result_to_be_returned("{\"name\":\"John\"}", as: {"name" => "John"}) + end + + it "rejects values that are not JSON strings (or nil)" do + expect_result_to_be_replaced_with_nil(3) + expect_result_to_be_replaced_with_nil(true) + expect_result_to_be_replaced_with_nil(false) + expect_result_to_be_replaced_with_nil(3.7) + expect_result_to_be_replaced_with_nil(Time.now) + expect_result_to_be_replaced_with_nil(["abc"]) + expect_result_to_be_replaced_with_nil({"name" => "John"}) + end + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/schema/enum_value_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/schema/enum_value_spec.rb new file mode 100644 index 00000000..4a560dca --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/schema/enum_value_spec.rb @@ -0,0 +1,44 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/graphql/schema/enum_value" + +module ElasticGraph + class GraphQL + class Schema + RSpec.describe EnumValue do + it "inspects well" do + enum_value = define_schema do |s| + s.enum_type "ColorSpace" do |t| + t.value "rgb" + end + end.enum_value_named(:ColorSpace, :rgb) + + expect(enum_value.inspect).to eq "#" + end + + describe "#name" do + it "returns the name as a symbol" do + enum_value = define_schema do |s| + s.enum_type "ColorSpace" do |t| + t.value "rgb" + end + end.enum_value_named(:ColorSpace, :rgb) + + expect(enum_value.name).to eq :rgb + end + end + + def define_schema(&schema_definition) + build_graphql(schema_definition: schema_definition).schema + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/schema/field_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/schema/field_spec.rb new file mode 100644 index 00000000..26d2808e --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/schema/field_spec.rb @@ -0,0 +1,528 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/schema/field" +require "support/aggregations_helpers" + +module ElasticGraph + class GraphQL + class Schema + RSpec.describe Field do + it "exposes the name as a lowercase symbol" do + field = define_schema do |schema| + schema.object_type "Color" do |t| + t.field "red", "Int!" + end + end.field_named(:Color, :red) + + expect(field.name).to eq :red + end + + it "exposes the parent type" do + schema = define_schema do |s| + s.object_type "Color" do |t| + t.field "red", "Int!" + end + end + + field = schema.field_named(:Color, :red) + expect(field.parent_type).to be schema.type_named(:Color) + end + + it "inspects well" do + field = define_schema do |schema| + schema.object_type "Color" do |t| + t.field "red", "Int!" + end + end.field_named(:Color, :red) + + expect(field.inspect).to eq "#" + end + + describe "#type" do + it "returns the type of the field" do + schema = define_schema do |s| + s.object_type "Color" do |t| + t.field "red", "Int" + end + end + + field = schema.field_named(:Color, :red) + + expect(field.type.name).to eq :Int + expect(field.type).to be schema.type_named(:Int) + end + + it "supports wrapped types" do + schema = define_schema do |s| + s.object_type "Color" do |t| + t.field "red", "[Int!]!" + end + end + + field = schema.field_named(:Color, :red) + + expect(field.type.name).to eq :"[Int!]!" + end + end + + describe "#relation_join" do + it "returns a memoized `RelationJoin` if the field is a relation" do + field = define_schema do |s| + s.object_type "Color" do |t| + t.field "id", "ID!" + t.field "photo_id", "ID" + t.index "colors" + end + + s.object_type "Photo" do |t| + t.field "id", "ID!" + t.relates_to_many "colors", "Color", via: "photo_id", dir: :in, singular: "color" + t.index "photos" + end + end.field_named(:Photo, :colors) + + expect(field.relation_join).to be_a(RelationJoin).and be(field.relation_join) + end + + it "returns (and memoizes) nil if the field is not a relation" do + allow(RelationJoin).to receive(:from).and_call_original + + field = define_schema do |s| + s.object_type "Color" do |t| + t.field "red", "Int" + end + end.field_named(:Color, :red) + + expect(field.relation_join).to be(nil).and be(field.relation_join) + expect(RelationJoin).to have_received(:from).once + end + end + + describe "#sort_clauses_for" do + it "raises a clear error if called on a field that lacks an `order_by` argument" do + field = define_schema do |s| + s.object_type "Widget" do |t| + t.field "count", "Int" + end + end.field_named(:Widget, :count) + + expect { + field.sort_clauses_for(:enum_value) + }.to raise_error Errors::SchemaError, a_string_including("order_by", "Widget.count") + end + + let(:schema) do + define_schema do |s| + s.object_type "Photo" do |t| + t.field "id", "ID!" + t.field "pixel_count", "Int!" + t.field "created_at_ms", "Int!" + t.index "photos" do |i| + i.default_sort "created_at_ms", :asc + end + end + end + end + + it "returns a list of datastore sort clauses when passed an array" do + field = schema.field_named(:Query, :photos) + sort_clauses = field.sort_clauses_for([:pixel_count_DESC, :created_at_ms_DESC]) + + expect(sort_clauses).to eq([ + {"pixel_count" => {"order" => "desc"}}, + {"created_at_ms" => {"order" => "desc"}} + ]) + end + + it "returns a list of a single datastore sort clause when passed a scalar" do + field = schema.field_named(:Query, :photos) + sort_clauses = field.sort_clauses_for(:pixel_count_DESC) + + expect(sort_clauses).to eq([{"pixel_count" => {"order" => "desc"}}]) + end + + it "raises an error if a sort value is undefined" do + field = schema.field_named(:Query, :photos) + + expect { + field.sort_clauses_for(:bogus_DESC) + }.to raise_error(Errors::NotFoundError, a_string_including("No enum value named bogus_DESC")) + end + + it "raises an error if a sort enum value lacks `sort_field` in the runtime metadata" do + schema = define_schema do |s| + s.object_type "Photo" do |t| + t.field "id", "ID!" + t.index "photos" + end + + s.raw_sdl <<~EOS + enum PhotoSort { + invalid_photo_sort_DESC + } + + type Query { + photos(order_by: [PhotoSort!]): [Photo!]! + } + EOS + end + + field = schema.field_named(:Query, :photos) + + expect { + field.sort_clauses_for(:invalid_photo_sort_DESC) + }.to raise_error(Errors::SchemaError, a_string_including("sort_field", "invalid_photo_sort_DESC")) + end + + it "returns an empty array when given nil or []" do + field = schema.field_named(:Query, :photos) + + expect(field.sort_clauses_for(nil)).to eq [] + expect(field.sort_clauses_for([])).to eq [] + end + end + + describe "#computation_detail" do + it "returns the aggregation function from an aggregated values field" do + field = define_schema do |s| + s.object_type "Photo" do |t| + t.field "id", "ID!" + t.field "some_field", "Int" + t.index "photos" + end + end.field_named(:IntAggregatedValues, :exact_sum) + + expect(field.computation_detail).to eq( + SchemaArtifacts::RuntimeMetadata::ComputationDetail.new( + function: :sum, + empty_bucket_value: 0 + ) + ) + end + end + + describe "#aggregated" do + it "returns true if the field's type is `FloatAggregatedValues`" do + field = field_of_type("FloatAggregatedValues") + + expect(field.aggregated?).to be true + end + + it "returns true if the field's type is `FloatAggregatedValues!`" do + field = field_of_type("FloatAggregatedValues!") + + expect(field.aggregated?).to be true + end + + it "returns false if the field's type is a list of `FloatAggregatedValues" do + field = field_of_type("[FloatAggregatedValues]") + expect(field.aggregated?).to be false + + field = field_of_type("[FloatAggregatedValues!]") + expect(field.aggregated?).to be false + + field = field_of_type("[FloatAggregatedValues]!") + expect(field.aggregated?).to be false + + field = field_of_type("[FloatAggregatedValues!]!") + expect(field.aggregated?).to be false + end + + it "returns false if the field is another type" do + field = field_of_type("Int") + + expect(field.aggregated?).to be false + end + + def field_of_type(type_name) + define_schema do |schema| + schema.object_type "Photo" do |t| + t.field "some_field", type_name, filterable: false, aggregatable: false, groupable: false + end + end.field_named(:Photo, :some_field) + end + end + + describe "#name_in_index" do + let(:schema) do + define_schema do |s| + s.object_type "Person" do |t| + t.field "first_name", "String" + t.field "alt_name", "String", name_in_index: "name" + end + end + end + + context "when a schema field is defined with a `name_in_index`" do + it "returns the `name_in_index` value" do + field = schema.field_named(:Person, :alt_name) + + expect(field.name_in_index).to eq(:name) + end + end + + context "when a schema field does not have a `name_in_index`" do + it "returns the field name" do + field = schema.field_named(:Person, :first_name) + + expect(field.name_in_index).to eq(:first_name) + end + end + end + + describe "#args_to_schema_form" do + let(:field) do + define_schema do |s| + s.raw_sdl <<~EOS + input Nested { + fooBar_bazzDazz: Int + nested: Nested + } + + type Query { + foo(camelCaseField: Int, maybe_set_to_null: String, nested: Nested): Int + } + EOS + end.field_named(:Query, :foo) + end + + it "converts an args hash from the keyword args style provided by graphql gem to their form in the schema" do + schema_args = field.args_to_schema_form({ + camel_case_field: 3, + nested: {foo_bar_bazz_dazz: 12, nested: {foo_bar_bazz_dazz: nil}} + }) + + expect(schema_args).to eq({ + "camelCaseField" => 3, + "nested" => {"fooBar_bazzDazz" => 12, "nested" => {"fooBar_bazzDazz" => nil}} + }) + end + + it "throws a clear error when it cannot find a matching argument definition" do + expect { + field.args_to_schema_form({some_other_field: 17}) + }.to raise_error Errors::SchemaError, a_string_including("foo", "some_other_field") + end + end + + describe "#index_field_names_for_resolution" do + it "returns the field name for a scalar field" do + fields = index_field_names_for(:field, "foo", "Int") + + expect(fields).to contain_exactly("foo") + end + + it "returns the overridden field name for a scalar field with `name_in_index` set" do + fields = index_field_names_for(:field, "foo", "Int", name_in_index: "bar") + + expect(fields).to contain_exactly("bar") + end + + it "returns an empty list for an embedded object field, because we do not need any fields to resolve it (but subfields will be needed to resolve them)" do + fields = index_field_names_for(:field, "options", "WidgetOptions!") do |schema| + schema.object_type "WidgetOptions" do |t| + t.field "size", "Int" + end + end + + expect(fields).to be_empty + end + + it "returns the foreign key field for a relation with an outbound foreign key" do + fields = index_field_names_for(:relates_to_one, "car", "Car", via: "car_id", dir: :out) do |schema| + define_indexed_car_type_on(schema) + end + + expect(fields).to contain_exactly("car_id") + end + + it "also returns the `id` field for a self-referential relation with an outbound foreign key" do + fields = index_field_names_for(:relates_to_many, "children_widgets", "Widget", via: "children_ids", dir: :out, singular: "child_widget") + + expect(fields).to contain_exactly("id", "children_ids") + end + + it "returns the `id` field for a relation with an inbound foreign key" do + fields = index_field_names_for(:relates_to_one, "car", "Car", via: "widget_id", dir: :in) do |schema| + define_indexed_car_type_on(schema) + end + + expect(fields).to contain_exactly("id") + end + + it "also returns the foreign key field for a self-referential relation with an inbound foreign key" do + fields = index_field_names_for(:relates_to_one, "parent_widget", "Widget", via: "parent_id", dir: :in) + + expect(fields).to contain_exactly("id", "parent_id") + end + + it "understands the semantics of relay connection edges/nodes" do + graphql = build_graphql(schema_definition: lambda do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.index "widgets" + end + end) + + schema = graphql.schema + + expect(schema.field_named(:WidgetConnection, :edges).index_field_names_for_resolution).to eq [] + expect(schema.field_named(:WidgetConnection, :page_info).index_field_names_for_resolution).to eq [] + expect(schema.field_named(:WidgetEdge, :node).index_field_names_for_resolution).to eq [] + expect(schema.field_named(:WidgetEdge, :cursor).index_field_names_for_resolution).to eq [] + end + + def index_field_names_for(field_method, field_name, field_type, **field_args) + graphql = build_graphql(schema_definition: lambda do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.public_send(field_method, field_name, field_type, **field_args) + t.index "widgets" + end + + yield s if block_given? + end) + + graphql.schema.field_named(:Widget, field_name).index_field_names_for_resolution + end + end + + describe "#hidden_from_queries?" do + it "returns `false` on a field whose type that has no backing indexed types" do + schema = define_schema do |s| + s.object_type "Color" do |t| + t.field "name", "String" + end + end + + field = schema.field_named("Color", "name") + + expect(field.hidden_from_queries?).to be false + end + + it "returns `false` on a field whose type has all index definitions accessible from queries on its backing indexed types" do + schema = define_schema(index_definitions: { + "colors" => config_index_def_of(query_cluster: "main") + }) do |s| + s.object_type "Color" do |t| + t.field "id", "ID!" + t.index "colors" + end + end + + field = schema.field_named("Query", "colors") + + expect(field.hidden_from_queries?).to be false + end + + it "returns `true` on a field whose type has all index definitions inaccessible from queries on its backing indexed types" do + schema = define_schema(index_definitions: { + "colors" => config_index_def_of(query_cluster: nil) + }) do |s| + s.object_type "Color" do |t| + t.field "id", "ID!" + t.index "colors" + end + end + + field = schema.field_named("Query", "colors") + + expect(field.hidden_from_queries?).to be true + end + + it "returns `false` on a field whose type has a mixture of accessible and inaccessible index definitions on its backing indexed types" do + schema = define_schema(index_definitions: { + "colors" => config_index_def_of(query_cluster: nil), + "sizes" => config_index_def_of(query_cluster: "main") + }) do |s| + s.object_type "Color" do |t| + t.field "id", "ID!" + t.index "colors" + end + + s.object_type "Size" do |t| + t.field "id", "ID!" + t.index "sizes" + end + + s.union_type "ColorOrSize" do |t| + t.subtypes "Color", "Size" + end + end + + colors = schema.field_named("Query", "colors") + sizes = schema.field_named("Query", "sizes") + colors_or_sizes = schema.field_named("Query", "color_or_sizes") + + expect(colors.hidden_from_queries?).to be true + expect(sizes.hidden_from_queries?).to be false + expect(colors_or_sizes.hidden_from_queries?).to be false + end + end + + describe "#coerce_result" do + it "echoes back most results as-is" do + schema = define_schema do |s| + s.object_type "Color" do |t| + t.field "name", "String" + end + end + + field = schema.field_named("Color", "name") + + expect(field.coerce_result("red")).to eq "red" + end + + context "when an enum value name has been overridden" do + let(:schema) do + define_schema(enum_value_overrides_by_type: { + "DayOfWeek" => { + "MONDAY" => "LUNDI", + "TUESDAY" => "WEDNESDAY", + "WEDNESDAY" => "TUESDAY" + } + }) do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_at", "DateTime" + t.field "created_at_day_of_week", "DayOfWeek" + t.index "widgets" + end + end + end + + it "coerces to the overridden name for a field on a graphql-only return type since ElasticGraph internal logic uses the original name" do + field = schema.field_named("DateTimeGroupedBy", "as_day_of_week") + + expect(field.coerce_result("MONDAY")).to eq "LUNDI" + end + + it "leaves the result as-is for an indexed field, since the overridden names are validated at indexing time and the value should already be in overridden form" do + field = schema.field_named("Widget", "created_at_day_of_week") + + expect(field.coerce_result("TUESDAY")).to eq "TUESDAY" + end + end + end + + def define_indexed_car_type_on(schema) + schema.object_type "Car" do |t| + t.field "id", "ID!" + t.field "widget_id", "ID" + t.index "cars" + end + end + + def define_schema(index_definitions: nil, **options, &schema_def) + build_graphql(schema_definition: schema_def, index_definitions: index_definitions, **options).schema + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/schema/type_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/schema/type_spec.rb new file mode 100644 index 00000000..95e72c27 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/schema/type_spec.rb @@ -0,0 +1,843 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/schema/type" +require "elastic_graph/graphql/schema" + +module ElasticGraph + class GraphQL + class Schema + RSpec.describe Type do + it "exposes the name as a capitalized symbol" do + type = define_schema do |schema| + schema.object_type "Color" + end.type_named("Color") + + expect(type.name).to eq :Color + end + + it "inspects well" do + type = define_schema do |schema| + schema.object_type "Color" + end.type_named("Color") + + expect(type.inspect).to eq "#" + end + + describe "predicates and type wrapping" do + attr_reader :schema + + before(:context) do + # The examples in this example group all depend on this schema, so build it once, + # and reuse it in each example. Schemas are immutable so this is no danger of having + # a subtle ordering dependency. + @schema = define_schema(clients_by_name: {}) do |schema| + schema.object_type "Color" do |t| + t.field "id", "ID" + t.implements "EmbeddedInterface" + t.field "name", "String" + end + + schema.object_type "Velocity" do |t| + t.field "id", "ID" + t.implements "DirectlyIndexedInterface" + t.field "name", "String" + end + + schema.union_type "Attribute" do |t| + t.subtypes "Color", "Velocity" + end + + schema.union_type "IndexedAttribute" do |t| + t.subtypes "Color", "Velocity" + t.index "attributes" + end + + schema.interface_type "EmbeddedInterface" do |t| + t.field "name", "String" + end + + schema.interface_type "DirectlyIndexedInterface" do |t| + t.field "id", "ID" + t.field "name", "String" + t.index "direct_index" + end + + schema.interface_type "IndirectlyIndexedInterface" do |t| + t.field "name", "String" + end + + schema.object_type "Person" do |t| + t.implements "IndirectlyIndexedInterface" + t.field "id", "ID!" + t.field "name", "String" + t.index "people" + end + + schema.object_type "Photo" do |t| + t.field "id", "ID!" + t.index "photos" + end + + schema.union_type "Entity" do |t| + t.subtypes "Person", "Photo" + end + + schema.enum_type "Size" do |t| + t.values "small", "medium", "large" + end + + schema.enum_type "Length" do |t| + t.value "long" + t.value "short" + end + + schema.object_type "ColorEdge" do |t| + t.field "cursor", "String" + t.field "node", "Color!" + end + + # This must be raw SDL because our schema definition API provides no way to define custom `input` types--it + # generates them based on our indexed types. Using `raw_sdl` lets us define what the test expects. We may want + # to update what the test expects to use generated filter types in the future so we don't have to use `raw_sdl` + # here. + %w[Some PersonEdge PersonConnection].each do |type| + schema.raw_sdl "input #{type}FilterInput { foo: Int }" + schema.raw_sdl "input #{type}ListFilterInput { foo: Int }" + end + + schema.object_type "WrappedTypes" do |t| + t.field "int", "Int" + t.field "non_null_int", "Int!" + t.field "list_of_int", "[Int]" + t.field "list_of_non_null_int", "[Int!]" + t.field "non_null_list_of_int", "[Int]!" + t.field "non_null_list_of_non_null_int", "[Int!]!" + t.field "relay_connection", "PersonConnection", filterable: false, groupable: false + t.field "non_null_relay_connection", "PersonConnection!", filterable: false, groupable: false + t.field "relay_edge", "PersonEdge", filterable: false, groupable: false + t.field "non_null_relay_edge", "PersonEdge!", filterable: false, groupable: false + t.field "color", "Color" + t.field "person", "Person" + t.field "size", "Size" + t.field "non_null_size", "Size!" + t.field "list_of_size", "[Size]" + t.field "non_null_person", "Person!" do |f| + f.mapping type: "object" + end + t.field "person_list", "[Person]" do |f| + f.mapping type: "object" + end + t.field "non_null_color", "Color!" + t.field "attribute", "Attribute" + t.field "indexed_attribute", "IndexedAttribute" + t.field "non_null_attribute", "Attribute!" + t.field "entity", "Entity" + t.field "non_null_entity", "Entity!" + t.field "indexed_aggregation", "PersonAggregation", filterable: false, groupable: false + t.field "non_null_indexed_aggregation", "PersonAggregation!", filterable: false, groupable: false + t.field "list_of_indexed_aggregation", "[PersonAggregation]", filterable: false, groupable: false do |f| + f.mapping type: "object" + end + t.field "non_null_list_of_indexed_aggregation", "[PersonAggregation]!", filterable: false, groupable: false do |f| + f.mapping type: "object" + end + end + end + end + + it "can model a scalar" do + type = type_for(:int) + + expect(type.name).to eq :Int + expect(type).to only_satisfy_predicates(:nullable?) + expect(type.unwrap_fully).to be schema.type_named(:Int) + expect(type.unwrap_non_null).to be type + end + + it "can model a non-nullable scalar" do + type = type_for(:non_null_int) + + expect(type.name).to eq :Int! + expect(type).to only_satisfy_predicates(:non_null?) + expect(type.unwrap_fully).to be schema.type_named(:Int) + expect(type.unwrap_non_null).to be schema.type_named(:Int) + end + + it "can model an embedded object" do + type = type_for(:color) + + expect(type.name).to eq :Color + expect(type).to only_satisfy_predicates(:nullable?, :object?, :embedded_object?) + expect(type.unwrap_fully).to be schema.type_named(:Color) + expect(type.unwrap_non_null).to be type + end + + it "can model a non-nullable embedded object" do + type = type_for(:non_null_color) + + expect(type.name).to eq :Color! + expect(type).to only_satisfy_predicates(:non_null?, :object?, :embedded_object?) + expect(type.unwrap_fully).to be schema.type_named(:Color) + expect(type.unwrap_non_null).to be schema.type_named(:Color) + end + + it "can model a list" do + type = type_for(:list_of_int) + + expect(type.name).to eq :"[Int]" + expect(type).to only_satisfy_predicates(:nullable?, :list?, :collection?) + expect(type.unwrap_fully).to be schema.type_named(:Int) + expect(type.unwrap_non_null).to be type + end + + it "can model a relay connection" do + type = type_for(:relay_connection) + + expect(type.name).to eq :PersonConnection + expect(type).to only_satisfy_predicates(:nullable?, :object?, :relay_connection?, :collection?) + expect(type.unwrap_fully).to be schema.type_named(:Person) + expect(type.unwrap_non_null).to be type + end + + it "can model a non-nullable relay connection" do + type = type_for(:non_null_relay_connection) + + expect(type.name).to eq :PersonConnection! + expect(type).to only_satisfy_predicates(:non_null?, :object?, :relay_connection?, :collection?) + expect(type.unwrap_fully).to be schema.type_named(:Person) + expect(type.unwrap_non_null).to be schema.type_named(:PersonConnection) + end + + it "can model a relay edge" do + type = type_for(:relay_edge) + + expect(type.name).to eq :PersonEdge + expect(type).to only_satisfy_predicates(:nullable?, :object?, :relay_edge?) + expect(type.unwrap_fully).to be schema.type_named(:PersonEdge) + expect(type.unwrap_non_null).to be type + end + + it "can model a non-nullable relay edge" do + type = type_for(:non_null_relay_edge) + + expect(type.name).to eq :PersonEdge! + expect(type).to only_satisfy_predicates(:non_null?, :object?, :relay_edge?) + expect(type.unwrap_fully).to be schema.type_named(:PersonEdge) + expect(type.unwrap_non_null).to be schema.type_named(:PersonEdge) + end + + it "can model a list of non-nullable scalars" do + type = type_for(:list_of_non_null_int) + + expect(type.name).to eq :"[Int!]" + expect(type).to only_satisfy_predicates(:nullable?, :list?, :collection?) + expect(type.unwrap_fully).to be schema.type_named(:Int) + expect(type.unwrap_non_null).to be type + end + + it "can model a non-nullable list of scalars" do + type = type_for(:non_null_list_of_int) + + expect(type.name).to eq :"[Int]!" + expect(type).to only_satisfy_predicates(:non_null?, :list?, :collection?) + expect(type.unwrap_fully).to be schema.type_named(:Int) + expect(type.unwrap_non_null.name).to eq :"[Int]" + end + + it "can model a non-nullable list of non-nullable scalars" do + type = type_for(:non_null_list_of_non_null_int) + + expect(type.name).to eq :"[Int!]!" + expect(type).to only_satisfy_predicates(:non_null?, :list?, :collection?) + expect(type.unwrap_fully).to be schema.type_named(:Int) + expect(type.unwrap_non_null.name).to eq :"[Int!]" + end + + it "can model an indexed type" do + type = type_for(:person) + + expect(type.name).to eq :Person + expect(type).to only_satisfy_predicates(:nullable?, :object?, :indexed_document?) + expect(type.unwrap_fully).to be schema.type_named(:Person) + expect(type.unwrap_non_null).to be type + end + + it "can model an indexed aggregation type" do + type = type_for(:indexed_aggregation) + + expect(type.name).to eq :PersonAggregation + expect(type).to only_satisfy_predicates(:nullable?, :object?, :indexed_aggregation?) + expect(type.unwrap_fully).to be schema.type_named(:PersonAggregation) + expect(type.unwrap_non_null).to be type + end + + it "can model a non-null indexed aggregation type" do + type = type_for(:non_null_indexed_aggregation) + + expect(type.name).to eq :PersonAggregation! + expect(type).to only_satisfy_predicates(:non_null?, :object?, :indexed_aggregation?) + expect(type.unwrap_fully).to be schema.type_named(:PersonAggregation) + expect(type.unwrap_non_null.name).to be :PersonAggregation + end + + it "can model a list of indexed aggregation type" do + type = type_for(:list_of_indexed_aggregation) + + expect(type.name).to eq :"[PersonAggregation]" + expect(type).to only_satisfy_predicates(:nullable?, :list?, :collection?) + expect(type.unwrap_fully).to be schema.type_named(:PersonAggregation) + expect(type.unwrap_non_null.name).to be :"[PersonAggregation]" + end + + it "can model a non-null list of indexed aggregation type" do + type = type_for(:non_null_list_of_indexed_aggregation) + + expect(type.name).to eq :"[PersonAggregation]!" + expect(type).to only_satisfy_predicates(:non_null?, :list?, :collection?) + expect(type.unwrap_fully).to be schema.type_named(:PersonAggregation) + expect(type.unwrap_non_null.name).to be :"[PersonAggregation]" + end + + it "can model a list of indexed type" do + type = type_for(:person_list) + + expect(type.name).to eq :"[Person]" + expect(type).to only_satisfy_predicates(:nullable?, :list?, :collection?) + expect(type.unwrap_fully).to be schema.type_named(:Person) + expect(type.unwrap_non_null).to be type + end + + it "can model a non-nullable indexed type" do + type = type_for(:non_null_person) + + expect(type.name).to eq :Person! + expect(type).to only_satisfy_predicates(:non_null?, :object?, :indexed_document?) + expect(type.unwrap_fully).to be schema.type_named(:Person) + expect(type.unwrap_non_null).to be schema.type_named(:Person) + end + + it "can model a union of embedded object types" do + type = type_for(:attribute) + + expect(type.name).to eq :Attribute + expect(type).to only_satisfy_predicates(:nullable?, :object?, :abstract?, :embedded_object?) + expect(type.unwrap_fully).to be schema.type_named(:Attribute) + expect(type.unwrap_non_null).to be type + end + + it "can model an indexed union of object types" do + type = type_for(:indexed_attribute) + + expect(type.name).to eq :IndexedAttribute + expect(type).to only_satisfy_predicates(:nullable?, :object?, :abstract?, :indexed_document?) + expect(type.unwrap_fully).to be schema.type_named(:IndexedAttribute) + expect(type.unwrap_non_null).to be type + end + + it "can model a non-nullable union of embedded object types" do + type = type_for(:non_null_attribute) + + expect(type.name).to eq :Attribute! + expect(type).to only_satisfy_predicates(:non_null?, :object?, :abstract?, :embedded_object?) + expect(type.unwrap_fully).to be schema.type_named(:Attribute) + expect(type.unwrap_non_null).to be schema.type_named(:Attribute) + end + + it "can model a union of indexed object types" do + type = type_for(:entity) + + expect(type.name).to eq :Entity + expect(type).to only_satisfy_predicates(:nullable?, :object?, :abstract?, :indexed_document?) + expect(type.unwrap_fully).to be schema.type_named(:Entity) + expect(type.unwrap_non_null).to be type + end + + it "can model a non-null union of indexed object types" do + type = type_for(:non_null_entity) + + expect(type.name).to eq :Entity! + expect(type).to only_satisfy_predicates(:non_null?, :object?, :abstract?, :indexed_document?) + expect(type.unwrap_fully).to be schema.type_named(:Entity) + expect(type.unwrap_non_null).to be schema.type_named(:Entity) + end + + it "can model an enum type" do + type = type_for(:size) + + expect(type.name).to eq :Size + expect(type).to only_satisfy_predicates(:nullable?, :enum?) + expect(type.unwrap_fully).to be type + expect(type.unwrap_non_null).to be type + end + + it "can model a non-null enum type" do + type = type_for(:non_null_size) + + expect(type.name).to eq :Size! + expect(type).to only_satisfy_predicates(:non_null?, :enum?) + expect(type.unwrap_fully).to be schema.type_named(:Size) + expect(type.unwrap_non_null).to be schema.type_named(:Size) + end + + it "can model a list of enums" do + type = type_for(:list_of_size) + + expect(type.name).to eq :"[Size]" + expect(type).to only_satisfy_predicates(:nullable?, :list?, :collection?) + expect(type.unwrap_fully).to be schema.type_named(:Size) + expect(type.unwrap_non_null).to be type + end + + it "can model an input type" do + type = schema.type_named(:SomeFilterInput) + + expect(type.name).to eq :SomeFilterInput + expect(type).to only_satisfy_predicates(:nullable?, :object?) + expect(type.unwrap_fully).to be type + expect(type.unwrap_non_null).to be type + end + + it "can model an embedded interface type" do + type = schema.type_named(:EmbeddedInterface) + + expect(type.name).to eq :EmbeddedInterface + expect(type).to only_satisfy_predicates(:nullable?, :object?, :embedded_object?, :abstract?) + expect(type.unwrap_fully).to be type + expect(type.unwrap_non_null).to be type + end + + it "can model a directly indexed interface type" do + type = schema.type_named(:DirectlyIndexedInterface) + + expect(type.name).to eq :DirectlyIndexedInterface + expect(type).to only_satisfy_predicates(:nullable?, :object?, :indexed_document?, :abstract?) + expect(type.unwrap_fully).to be type + expect(type.unwrap_non_null).to be type + end + + it "can model an indirectly indexed interface type" do + type = schema.type_named(:IndirectlyIndexedInterface) + + expect(type.name).to eq :IndirectlyIndexedInterface + expect(type).to only_satisfy_predicates(:nullable?, :object?, :indexed_document?, :abstract?) + expect(type.unwrap_fully).to be type + expect(type.unwrap_non_null).to be type + end + + predicates = %i[nullable? non_null? list? object? embedded_object? indexed_document? indexed_aggregation? relay_connection? relay_edge? abstract? collection? enum?] + matcher :only_satisfy_predicates do |*expected_predicates| + match do |type| + @satisfied_predicates = predicates.select { |p| type.public_send(p) } + @satisfied_predicates.sort == expected_predicates.sort + end + + # :nocov: -- only executed on a test failure + failure_message do |type| + parts = [message_part("expected #{type.inspect} to only satisfy", expected_predicates)] + + failed_to_satisfy_predicates = expected_predicates - @satisfied_predicates + if failed_to_satisfy_predicates.any? + parts << [message_part("It failed to satisfy", failed_to_satisfy_predicates)] + end + + extra_satisfied_predicates = @satisfied_predicates - expected_predicates + if extra_satisfied_predicates.any? + parts << [message_part("It also satisfied", extra_satisfied_predicates)] + end + + parts.join("\n\n") + end + + def message_part(intro, predicates) + "#{intro}:\n\n - #{predicates.join("\n - ")}" + end + # :nocov: + end + + def type_for(field_name) + schema.field_named(:WrappedTypes, field_name).type + end + end + + describe "#enum_value_named" do + let(:schema) do + define_schema do |schema| + schema.enum_type "ColorSpace" do |t| + t.values "rgb", "srgb" + end + end + end + + it "returns the same enum_value object returns by schema's `enum_value_named` method" do + from_type = schema.type_named(:ColorSpace).enum_value_named(:rgb) + from_schema = schema.enum_value_named(:ColorSpace, :rgb) + + expect(from_schema).to be_a(EnumValue).and be(from_type) + end + + it "supports the enum_value being named with a string or symbol" do + from_string = schema.type_named(:ColorSpace).enum_value_named("rgb") + from_symbol = schema.type_named(:ColorSpace).enum_value_named(:rgb) + + expect(from_symbol).to be_a(EnumValue).and be(from_string) + end + end + + describe "#field_named" do + let(:schema) do + define_schema do |s| + s.object_type "Color" do |t| + t.field "red", "Int!" + end + end + end + + it "returns the same field object returned from the schema's `field_named` method" do + from_type = schema.type_named(:Color).field_named(:red) + from_schema = schema.field_named(:Color, :red) + + expect(from_schema).to be_a(Field).and be(from_type) + end + + it "supports the field being named with a string or symbol" do + from_string = schema.type_named(:Color).field_named("red") + from_symbol = schema.type_named(:Color).field_named(:red) + + expect(from_symbol).to be_a(Field).and be(from_string) + end + end + + describe "#abstract?" do + let(:schema) do + define_schema do |s| + s.object_type "Color" do |t| + t.field "red", "Int!" + end + + s.object_type "Size" do |t| + t.implements "Named" + t.field "name", "String" + t.field "small", "Int!" + end + + s.object_type "Person" do |t| + t.implements "Named" + t.field "name", "String" + end + + s.interface_type "Named" do |t| + t.field "name", "String" + end + + s.union_type "Options" do |t| + t.subtypes "Color", "Size" + end + end + end + + it "returns true for unions" do + type = schema.type_named(:Options) + expect(type.abstract?).to be true + end + + it "returns true for interfaces" do + type = schema.type_named(:Named) + expect(type.abstract?).to be true + end + + it "returns false for objects" do + type = schema.type_named(:Size) + expect(type.abstract?).to be false + end + + it "returns false for scalars" do + type = schema.type_named(:Int) + expect(type.abstract?).to be false + end + end + + describe "#subtypes" do + let(:schema) do + define_schema do |s| + s.object_type "Color" do |t| + t.field "red", "Int!" + end + + s.object_type "Size" do |t| + t.implements "Named" + t.field "name", "String" + t.field "small", "Int!" + end + + s.object_type "Person" do |t| + t.implements "Named" + t.field "name", "String" + end + + s.interface_type "Named" do |t| + t.field "name", "String" + end + + s.union_type "Options" do |t| + t.subtypes "Color", "Size" + end + end + end + + it "returns [] for object types" do + type = schema.type_named(:Size) + expect(type.subtypes).to eq [] + end + + it "returns [] for scalar types" do + type = schema.type_named(:Int) + expect(type.subtypes).to eq [] + end + + it "returns the subtypes of a union" do + type = schema.type_named(:Options) + expect(type.subtypes).to contain_exactly(schema.type_named(:Color), schema.type_named(:Size)) + end + + it "returns the subtypes of an interface" do + type = schema.type_named(:Named) + expect(type.subtypes).to contain_exactly(schema.type_named(:Size), schema.type_named(:Person)) + end + end + + describe "#search_index_definitions" do + it "returns an empty array for a non-union type that is not indexed" do + search_index_definitions = search_index_definitions_from do |schema, type| + schema.object_type(type) {} + end + + expect(search_index_definitions).to eq [] + end + + it "returns an array of one IndexDefinition object for a non-union indexed document type with one datastore index" do + search_index_definitions = search_index_definitions_from do |schema, type| + schema.object_type type do |t| + t.field "id", "ID!" + t.index "things" + end + end + + expect(search_index_definitions.map(&:class)).to eq [DatastoreCore::IndexDefinition::Index] + expect(search_index_definitions.map(&:name)).to eq ["things"] + end + + it "includes the index definitions from the subtypes when it is a type union of indexed document types" do + search_index_definitions = search_index_definitions_from do |schema, type| + schema.object_type "T1" do |t| + t.field "id", "ID!" + t.index "t1" + end + + schema.object_type "T2" do |t| + t.field "id", "ID!" + t.index "t2" + end + + schema.object_type "T3" do |t| + t.field "id", "ID!" + t.index "t3" + end + + schema.object_type "T4" do |t| + t.field "id", "ID!" + t.index "t4" + end + + schema.union_type type do |t| + t.subtypes "T1", "T2", "T3", "T4" + t.index "union_index" + end + end + + expect(search_index_definitions.map(&:name)).to contain_exactly("t1", "t2", "t3", "t4", "union_index") + end + + it "deduplicates the index definitions before returning them" do + search_index_definitions = search_index_definitions_from do |schema, type| + schema.object_type "T1a" do |t| + t.field "id", "ID!" + t.index "t1" + end + + schema.object_type "T2" do |t| + t.field "id", "ID!" + t.index "t2" + end + + schema.object_type "T1b" do |t| + t.field "id", "ID!" + t.index "t1" + end + + schema.object_type "T4" do |t| + t.field "id", "ID!" + t.index "t4" + end + + schema.union_type type do |t| + t.subtypes "T1a", "T2", "T1b", "T4" + t.index "t4" + end + end + + expect(search_index_definitions.map(&:name)).to contain_exactly("t1", "t2", "t4") + end + + context "on an indexed aggregation type" do + it "returns the indices of the corresponding indexed document type" do + search_index_definitions = search_index_definitions_from type_name: "ThingAggregation" do |schema| + schema.object_type "Thing" do |t| + t.field "id", "ID!" + t.index "things" + end + end + + expect(search_index_definitions.map(&:class)).to eq [DatastoreCore::IndexDefinition::Index] + expect(search_index_definitions.map(&:name)).to eq ["things"] + end + + it "returns the set union of indices of the corresponding indexed union type when the source type is a union" do + search_index_definitions = search_index_definitions_from type_name: "ThingAggregation" do |schema| + schema.object_type "Entity" do |t| + t.field "id", "ID!" + t.index "entities" + end + + schema.object_type "Gadget" do |t| + t.field "id", "ID!" + t.index "gadgets" + end + + schema.union_type "Thing" do |t| + t.subtypes "Entity", "Gadget" + end + end + + expect(search_index_definitions.map(&:class)).to contain_exactly(DatastoreCore::IndexDefinition::Index, DatastoreCore::IndexDefinition::Index) + expect(search_index_definitions.map(&:name)).to contain_exactly("entities", "gadgets") + end + end + + def search_index_definitions_from(type_name: :TheType) + schema = define_schema do |s| + yield s, type_name + end + + schema.type_named(type_name).search_index_definitions + end + end + + describe "#hidden_from_queries?" do + it "returns `false` on a type that has no backing indexed types" do + schema = define_schema do |s| + s.object_type "Color" do |t| + t.field "name", "String" + end + end + + type = schema.type_named("Color") + + expect(type.hidden_from_queries?).to be false + end + + it "returns `false` on a type that has all index definitions accessible from queries on its backing indexed types" do + schema = define_schema(index_definitions: { + "colors" => config_index_def_of(query_cluster: "main") + }) do |s| + s.object_type "Color" do |t| + t.field "id", "ID!" + t.index "colors" + end + end + + type = schema.type_named("Color") + + expect(type.hidden_from_queries?).to be false + end + + it "returns `true` on a type that has all index definitions inaccessible from queries on its backing indexed types" do + schema = define_schema(index_definitions: { + "colors" => config_index_def_of(query_cluster: nil) + }) do |s| + s.object_type "Color" do |t| + t.field "id", "ID!" + t.index "colors" + end + end + + type = schema.type_named("Color") + + expect(type.hidden_from_queries?).to be true + end + + it "returns `true` on an indexed aggregation type based on a source union type that has all indices inaccessible" do + schema = define_schema(index_definitions: { + "entities" => config_index_def_of(query_cluster: nil), + "gadgets" => config_index_def_of(query_cluster: nil) + }) do |s| + s.object_type "Entity" do |t| + t.field "id", "ID!" + t.index "entities" + end + + s.object_type "Gadget" do |t| + t.field "id", "ID!" + t.index "gadgets" + end + + s.union_type "Thing" do |t| + t.subtypes "Entity", "Gadget" + end + end + + expect(schema.type_named("ThingAggregation").hidden_from_queries?).to eq true + end + + it "returns `false` on a type that has a mixture of accessible and inaccessible index definitions on its backing indexed types" do + schema = define_schema(index_definitions: { + "colors" => config_index_def_of(query_cluster: nil), + "sizes" => config_index_def_of(query_cluster: "main") + }) do |s| + s.object_type "Color" do |t| + t.field "id", "ID!" + t.index "colors" + end + + s.object_type "Size" do |t| + t.field "id", "ID!" + t.index "sizes" + end + + s.union_type "ColorOrSize" do |t| + t.subtypes "Color", "Size" + end + end + + color = schema.type_named("Color") + size = schema.type_named("Size") + color_or_size = schema.type_named("ColorOrSize") + + expect(color.hidden_from_queries?).to be true + expect(size.hidden_from_queries?).to be false + expect(color_or_size.hidden_from_queries?).to be false + end + end + + def define_schema(index_definitions: nil, **overrides, &schema_def) + build_graphql(schema_definition: schema_def, index_definitions: index_definitions, **overrides).schema + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql/schema_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/schema_spec.rb new file mode 100644 index 00000000..af71ac60 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql/schema_spec.rb @@ -0,0 +1,280 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/schema" +require "elastic_graph/support/monotonic_clock" + +module ElasticGraph + class GraphQL + RSpec.describe Schema do + it "can be instantiated with directives that have custom scalar arguments" do + define_schema do |schema| + schema.scalar_type "_FieldSet" do |t| + t.mapping type: "keyword" + t.json_schema type: "string" + end + + schema.object_type "Widget" do |t| + t.directive "key", fields: "id" + t.field "id", "ID!" + t.index "widgets" + end + + schema.raw_sdl <<~EOS + directive @key(fields: _FieldSet!) on OBJECT | INTERFACE + EOS + end + end + + describe "#type_named" do + let(:schema) do + define_schema do |s| + s.object_type "Color" + end + end + + it "finds a type by string name" do + expect(schema.type_named("Color")).to be_a GraphQL::Schema::Type + end + + it "finds a type by symbol name" do + expect(schema.type_named(:Color)).to be_a GraphQL::Schema::Type + end + + it "consistently returns the same type object" do + color1 = schema.type_named("Color") + color2 = schema.type_named("Color") + color3 = schema.type_named(:Color) + query = schema.type_named("Query") + + expect(color2).to be(color1) + expect(color3).to be(color1) + expect(query).not_to eq(color1) + end + + it "raises an error when a type is misspelled (suggesting the correct spelling)" do + expect { + schema.type_named("Collor") + }.to raise_error(Errors::NotFoundError, a_string_including("Collor", "Color")) + end + + it "raises an error when a type cannot be found (or any suggestion)" do + expect { + schema.type_named("Person") + }.to raise_error(Errors::NotFoundError, a_string_including("Person")) + end + end + + describe "#defined_types" do + it "returns a list containing all explicitly defined types (excluding built-ins)" do + schema = define_schema do |s| + s.enum_type "Options" do |t| + t.value "firstOption" + end + s.object_type "Color" + end + + expect(schema.defined_types).to include( + schema.type_named(:Options), + schema.type_named(:Color), + schema.type_named(:Query) + ).and exclude( + schema.type_named(:Int), + schema.type_named(:Float), + schema.type_named(:Boolean), + schema.type_named(:String), + schema.type_named(:ID) + ) + end + end + + describe "#indexed_document_types" do + it "returns a list containing all types defined as indexed types" do + schema = define_schema do |s| + define_indexed_type(s, "Person", "people") + define_indexed_type(s, "Widget", "widgets") + s.object_type "Options" + end + + expect(schema.indexed_document_types).to contain_exactly( + schema.type_named(:Person), + schema.type_named(:Widget) + ) + end + end + + describe "#document_type_stored_in" do + it "returns the type stored in the named index" do + schema = schema_with_indices("Person" => "people", "Widget" => "widgets") + + expect(schema.document_type_stored_in("people")).to eq(schema.type_named(:Person)) + expect(schema.document_type_stored_in("widgets")).to eq(schema.type_named(:Widget)) + end + + it "raises an exception if given an unrecognizd index name" do + schema = schema_with_indices("Person" => "people", "Widget" => "widgets") + + expect { + schema.document_type_stored_in("foobars") + }.to raise_error(Errors::NotFoundError, a_string_including("foobars")) + end + + it "raises a clear exception if given the name of a rollover index instead of the source index definition name" do + schema = schema_with_indices("Person" => "people", "Widget" => "widgets") + + expect { + schema.document_type_stored_in("widgets#{ROLLOVER_INDEX_INFIX_MARKER}2021-02") + }.to raise_error(ArgumentError, a_string_including("widgets#{ROLLOVER_INDEX_INFIX_MARKER}2021-02", "name of a rollover index")) + end + + it "raises an exception if two GraphQL types are configured to use the same index" do + schema = schema_with_indices("Person" => "widgets", "Widget" => "widgets") + + expect { + schema.document_type_stored_in("widgets") + }.to raise_error(Errors::SchemaError, a_string_including("widgets", "Person", "Widget")) + end + + def schema_with_indices(index_name_by_type) + define_schema do |s| + define_indexed_type(s, "Person", index_name_by_type.fetch("Person")) + define_indexed_type(s, "Widget", index_name_by_type.fetch("Widget")) + + s.object_type "Options" do + end + end + end + end + + describe "#enum_value_named" do + let(:schema) do + define_schema do |s| + s.enum_type "ColorSpace" do |t| + t.values "rgb", "srgb" + end + end + end + + it "finds an enum_value when given a type and enum_value name strings" do + expect(schema.enum_value_named("ColorSpace", "rgb")).to be_a GraphQL::Schema::EnumValue + end + + it "finds an enum_value when given a type and enum_value name symbols" do + expect(schema.enum_value_named(:ColorSpace, :rgb)).to be_a GraphQL::Schema::EnumValue + end + + it "consistently returns the same enum_value_object" do + enum_value1 = schema.enum_value_named(:ColorSpace, :rgb) + enum_value2 = schema.enum_value_named(:ColorSpace, :rgb) + enum_value3 = schema.enum_value_named("ColorSpace", "rgb") + other_field = schema.enum_value_named(:ColorSpace, :srgb) + + expect(enum_value2).to be(enum_value1) + expect(enum_value3).to be(enum_value1) + expect(other_field).not_to eq(enum_value1) + end + + it "raises an error when the type cannot be found" do + expect { + schema.enum_value_named(:ColorType, :name) + }.to raise_error(Errors::NotFoundError, /ColorSpace/) + end + + it "raises an error when the enum value cannot be found, suggesting a correction if it can find one" do + expect { + schema.enum_value_named(:ColorSpace, :srg) + }.to raise_error(Errors::NotFoundError, a_string_including("ColorSpace", "srg", "Possible alternatives", "srgb")) + end + + it "raises an error when the enum value cannot be found, with no suggestions if not close to any enum value name" do + expect { + schema.enum_value_named(:ColorSpace, :foo) + }.to raise_error(Errors::NotFoundError, a_string_including("ColorSpace", "foo").and(excluding("Possible alternatives"))) + end + end + + describe "#field_named" do + {"" => "return type", "FilterInput" => "input type"}.each do |suffix, type_description| + context "on a #{type_description}" do + let(:schema) do + define_schema do |s| + s.object_type "Color" do |t| + t.field "red", "Int!" + t.field "green", "Int!" + t.field "blue", "Int!" + end + end + end + + it "finds a field when given type and field name strings" do + expect(field_named("Color", "blue")).to be_a GraphQL::Schema::Field + end + + it "finds a field when given type and field name symbols" do + expect(field_named(:Color, :blue)).to be_a GraphQL::Schema::Field + end + + it "consistently returns the same field object" do + field1 = field_named(:Color, :red) + field2 = field_named(:Color, :red) + field3 = field_named("Color", "red") + other_field = field_named(:Color, :blue) + + expect(field2).to be(field1) + expect(field3).to be(field1) + expect(other_field).not_to eq(field1) + end + + it "raises an error when the type part of the given field name cannot be found" do + expect { + field_named(:Person, :name) + }.to raise_error(Errors::NotFoundError, /Person/) + end + + it "raises an error when the field part of the given field name cannot be found, suggesting a correction if possible" do + expect { + field_named(:Color, :gren) + }.to raise_error(Errors::NotFoundError, a_string_including("Color", "gren", "Possible alternatives", "green")) + end + + it "raises an error when the field part of the given field name cannot be found, with no suggestions if not close to any field names" do + expect { + field_named(:Color, :purple) + }.to raise_error(Errors::NotFoundError, a_string_including("Color", "purple").and(excluding("Possible alternatives"))) + end + + define_method :field_named do |type_name_root, field| + schema.field_named("#{type_name_root}#{suffix}", field) + end + end + end + end + + it "inspects nicely" do + schema = define_schema do |s| + define_indexed_type(s, "Component", "components") + define_indexed_type(s, "Address", "addresses") + end + + expect(schema.inspect).to eq schema.to_s + expect(schema.to_s).to eq "#" + end + + def define_schema(&schema_definition) + build_graphql(schema_definition: schema_definition).schema + end + + def define_indexed_type(schema, indexed_type, index_name, **index_options) + schema.object_type indexed_type do |t| + t.field "id", "ID!" + t.index index_name, **index_options + end + end + end + end +end diff --git a/elasticgraph-graphql/spec/unit/elastic_graph/graphql_spec.rb b/elasticgraph-graphql/spec/unit/elastic_graph/graphql_spec.rb new file mode 100644 index 00000000..a5605a43 --- /dev/null +++ b/elasticgraph-graphql/spec/unit/elastic_graph/graphql_spec.rb @@ -0,0 +1,107 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql" +require "elastic_graph/graphql/resolvers/graphql_adapter" + +module ElasticGraph + RSpec.describe GraphQL do + it "returns non-nil values from each attribute" do + expect_to_return_non_nil_values_from_all_attributes(build_graphql) + end + + describe ".from_parsed_yaml" do + it "builds a GraphQL instance from the contents of a YAML settings file" do + customization_block = lambda { |conn| } + graphql = GraphQL.from_parsed_yaml(parsed_test_settings_yaml, &customization_block) + + expect(graphql).to be_a(GraphQL) + expect(graphql.datastore_core.client_customization_block).to be(customization_block) + end + end + + describe "#load_dependencies_eagerly" do + it "loads dependencies eagerly" do + graphql = build_graphql + + expect(loaded_dependencies_of(graphql)).to exclude(:schema, :graphql_query_executor) + graphql.load_dependencies_eagerly + expect(loaded_dependencies_of(graphql)).to include(:schema, :graphql_query_executor) + end + + def loaded_dependencies_of(graphql) + graphql.instance_variables + .reject { |ivar| graphql.instance_variable_get(ivar).nil? } + .map { |ivar_name| ivar_name.to_s.delete_prefix("@").to_sym } + end + end + + describe "#schema" do + it "uses the injected `graphql_adapter` if provided" do + graphql_adapter = instance_double(GraphQL::Resolvers::GraphQLAdapter).as_null_object + + # Manually stub these method; otherwise we get odd warnings like this: + # /Users/myron/Development/sq-elasticgraph-ruby/bundle/ruby/2.7.0/gems/graphql-2.0.15/lib/graphql/schema/build_from_definition.rb:305: warning: ##coerce_result at /Users/myron/.rvm/rubies/ruby-2.7.5/lib/ruby/2.7.0/forwardable.rb:154 forwarding to private method RSpec::Mocks::InstanceVerifyingDouble#coerce_result + # + # I don't understand why are started getting these warnings but this fixes it. + allow(graphql_adapter).to receive(:coerce_input) { |_, value, _| value } + allow(graphql_adapter).to receive(:coerce_result) { |_, value, _| value } + end + + it "uses a real normal `graphql_adapter` if none is provided" do + expect(build_graphql.schema.send(:resolver)).to be_a ElasticGraph::GraphQL::Resolvers::GraphQLAdapter + end + end + + context "when `config.extension_modules` or runtime metadata graphql extension modules are configured" do + it "applies the extensions when the GraphQL instance is instantiated without impacting any other instances" do + extension_data = {"extension" => "data"} + + config_extension_module = Module.new do + define_method :graphql_schema_string do + super() + "\n# #{extension_data.inspect}" + end + end + + runtime_metadata_extension_module = Module.new do + define_method :runtime_metadata do + metadata = super() + metadata.with( + scalar_types_by_name: metadata.scalar_types_by_name.merge(extension_data) + ) + end + end + + extended_graphql = build_graphql( + extension_modules: [config_extension_module], + schema_definition: lambda do |schema| + schema.register_graphql_extension runtime_metadata_extension_module, defined_at: __FILE__ + define_schema_elements(schema) + end + ) + + normal_graphql = build_graphql( + schema_definition: lambda { |schema| define_schema_elements(schema) } + ) + + expect(extended_graphql.runtime_metadata.scalar_types_by_name).to include(extension_data) + expect(extended_graphql.graphql_schema_string).to include(extension_data.inspect) + + expect(normal_graphql.runtime_metadata.scalar_types_by_name).not_to include(extension_data) + expect(normal_graphql.graphql_schema_string).not_to include(extension_data.inspect) + end + + def define_schema_elements(schema) + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.index "widgets" + end + end + end + end +end diff --git a/elasticgraph-graphql_lambda/.rspec b/elasticgraph-graphql_lambda/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-graphql_lambda/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-graphql_lambda/.yardopts b/elasticgraph-graphql_lambda/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-graphql_lambda/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-graphql_lambda/Gemfile b/elasticgraph-graphql_lambda/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-graphql_lambda/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-graphql_lambda/LICENSE.txt b/elasticgraph-graphql_lambda/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-graphql_lambda/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-graphql_lambda/README.md b/elasticgraph-graphql_lambda/README.md new file mode 100644 index 00000000..359cea98 --- /dev/null +++ b/elasticgraph-graphql_lambda/README.md @@ -0,0 +1,3 @@ +# ElasticGraph::GraphQLLambda + +This gem wraps `elasticgraph-graphql` in order to run it from an AWS Lambda. diff --git a/elasticgraph-graphql_lambda/elasticgraph-graphql_lambda.gemspec b/elasticgraph-graphql_lambda/elasticgraph-graphql_lambda.gemspec new file mode 100644 index 00000000..ac79cf28 --- /dev/null +++ b/elasticgraph-graphql_lambda/elasticgraph-graphql_lambda.gemspec @@ -0,0 +1,20 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :lambda) do |spec, eg_version| + spec.summary = "ElasticGraph gem that wraps elasticgraph-graphql in an AWS Lambda." + + spec.add_dependency "elasticgraph-graphql", eg_version + spec.add_dependency "elasticgraph-lambda_support", eg_version + + spec.add_development_dependency "elasticgraph-elasticsearch", eg_version + spec.add_development_dependency "elasticgraph-query_registry", eg_version + spec.add_development_dependency "httpx", "~> 1.3" +end diff --git a/elasticgraph-graphql_lambda/lib/elastic_graph/graphql_lambda.rb b/elasticgraph-graphql_lambda/lib/elastic_graph/graphql_lambda.rb new file mode 100644 index 00000000..f7142971 --- /dev/null +++ b/elasticgraph-graphql_lambda/lib/elastic_graph/graphql_lambda.rb @@ -0,0 +1,20 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql" +require "elastic_graph/lambda_support" + +module ElasticGraph + # @private + module GraphQLLambda + # Builds an `ElasticGraph::GraphQL` instance from our lambda ENV vars. + def self.graphql_from_env + LambdaSupport.build_from_env(GraphQL) + end + end +end diff --git a/elasticgraph-graphql_lambda/lib/elastic_graph/graphql_lambda/graphql_endpoint.rb b/elasticgraph-graphql_lambda/lib/elastic_graph/graphql_lambda/graphql_endpoint.rb new file mode 100644 index 00000000..3bccaf7a --- /dev/null +++ b/elasticgraph-graphql_lambda/lib/elastic_graph/graphql_lambda/graphql_endpoint.rb @@ -0,0 +1,187 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/graphql" +require "uri" + +module ElasticGraph + module GraphQLLambda + # @private + class GraphQLEndpoint + # Used to add a timeout buffer so that the lambda timeout should generally not be reached, + # instead preferring our `timeout_in_ms` behavior to the harsher timeout imposed by lambda itself. + # We prefer this in order to have consistent timeout behavior, regardless of what timeout is + # reached. For example, we have designed our timeout logic to disconnect from the datastore + # (which causes it to kill the running query) but we do not know if the lambda-based timeout + # would also cause that. This buffer gives our lambda enough time to respond before the hard + # lambda timeout so that it should (hopefully) never get reached. + # + # Note we generally run with a 30 second overall lambda timeout so a single second of buffer + # still gives plenty of time to satisfy the query. + LAMBDA_TIMEOUT_BUFFER_MS = 1_000 + + # As per https://docs.aws.amazon.com/lambda/latest/dg/gettingstarted-limits.html, AWS Lambdas + # are limited to returning up to 6 MB responses. + # + # Note: 6 MB is technically 6291456 bytes, but the AWS log message when you exceed the limit mentions + # a limit of 6291556 (100 bytes larger!). Here we use the smaller threshold since it's the documented value. + LAMBDA_MAX_RESPONSE_PAYLOAD_BYTES = (_ = 2**20) * 6 + + def initialize(graphql) + @graphql_http_endpoint = graphql.graphql_http_endpoint + @logger = graphql.logger + @monotonic_clock = graphql.monotonic_clock + end + + def handle_request(event:, context:) + start_time_in_ms = @monotonic_clock.now_in_ms # should be the first line so our duration logging is accurate + request = request_from(event) + + response = @graphql_http_endpoint.process( + request, + max_timeout_in_ms: context.get_remaining_time_in_millis - LAMBDA_TIMEOUT_BUFFER_MS, + start_time_in_ms: start_time_in_ms + ) + + convert_response(response) + end + + private + + def request_from(event) + # The `GRAPHQL_LAMBDA_AWS_ARN_HEADER` header can be used to determine who the client is, which + # has security implications. Therefore, we need to make sure it can't be spoofed. Here we remove + # any header which, when normalized, is equivalent to that header. + headers = event.fetch("headers").reject do |key, _| + GraphQL::HTTPRequest.normalize_header_name(key) == GRAPHQL_LAMBDA_AWS_ARN_HEADER + end + + header_overrides = { + GRAPHQL_LAMBDA_AWS_ARN_HEADER => event.dig("requestContext", "identity", "userArn") + }.compact + + GraphQL::HTTPRequest.new( + url: url_from(event), + http_method: http_method_from(event), + headers: headers.merge(header_overrides), + body: event.fetch("body") + ) + end + + def convert_response(response) + if response.body.bytesize >= LAMBDA_MAX_RESPONSE_PAYLOAD_BYTES + response = content_too_large_response(response) + end + + {statusCode: response.status_code, body: response.body, headers: response.headers} + end + + def url_from(event) + uri = URI.join("/") + + # stage_name will be part of the path when a client tries to send a request to an API Gateway endpoint + # but will be omitted from the path in the actual event. For example a call to /stage-name/graphql + # will be passed in the event as `requestContext.stage` = "stage-name" and `path` = "/graphql". Here we are + # using stage_name to be placed back in as a prefix for the path. + # Note: stage is not expected to ever be nil or empty when invoked through API Gateway. Here we handle that case + # to be tolerant of it but we don't expect it to ever happen. + # + # The event format can be seen here: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format + # As you should be able to see, the stage name isn't included in the path. In this doc + # https://docs.aws.amazon.com/apigateway/latest/developerguide/how-to-call-api.html you should be able to see that stage_name is included in the + # base url for invoking a REST API. + stage_name = event.dig("requestContext", "stage") || "" + stage_name = "/" + stage_name unless stage_name == "" + + # It'll be `path` if it's an HTTP API with a v1.0 payload: + # https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html#1.0 + # + # And for a REST API, it'll also be `path`: + # https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format + uri.path = stage_name + event.fetch("path") do + # It'll be `rawPath` if it's an HTTP API with a v2.0 payload: + # https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html#2.0 + event.fetch("rawPath") + end + + # If it's an HTTP API with a v2.0 payload, it'll have `rawQueryString`, which we want to use if available: + # https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html#2.0 + uri.query = event.fetch("rawQueryString") do + # If it's an HTTP API with a v1.0 payload, it'll have `queryStringParameters` as a hash: + # https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html#1.0 + # + # And for a REST API, it'll also have `queryStringParameters`: + # https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format + event.fetch("queryStringParameters")&.then { |params| ::URI.encode_www_form(params) } + end + + uri.to_s + end + + def http_method_from(event) + # It'll be `httpMethod` if it's an HTTP API with a v1.0 payload: + # https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html#1.0 + # + # And for a REST API, it'll also be `httpMethod`: + # https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format + event.fetch("httpMethod") do + # Unfortunately, for an HTTP API with a v2.0 payload, the method is only available from `requestContext.http.method`: + # https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html#2.0 + event.fetch("requestContext").fetch("http").fetch("method") + end.downcase.to_sym + end + + # Responsible for building a response when the existing response is too large to + # return due to AWS Lambda response size limits. + # + # Note: an HTTP 413 status code[^1] would usually be appropriate, but we're not + # totally sure how API gateway will treat that (e.g. will it pass the response + # body through to the client?) and the GraphQL-over-HTTP spec recommends[^2] that + # we return a 200 in this case: + # + # > This section only applies when the response body is to use the + # > `application/json` media type. + # > + # > The server SHOULD use the `200` status code for every response to a well-formed + # > _GraphQL-over-HTTP request_, independent of any _GraphQL request error_ or + # > _GraphQL field error_ raised. + # > + # > Note: A status code in the `4xx` or `5xx` ranges or status code `203` (and maybe + # > others) could originate from intermediary servers; since the client cannot + # > determine if an `application/json` response with arbitrary status code is a + # > well-formed _GraphQL response_ (because it cannot trust the source) the server + # > must use `200` status code to guarantee to the client that the response has not + # > been generated or modified by an intermediary. + # > + # > ... + # > The server SHOULD NOT use a `4xx` or `5xx` status code for a response to a + # > well-formed _GraphQL-over-HTTP request_. + # > + # > Note: For compatibility with legacy servers, this specification allows the use + # > of `4xx` or `5xx` status codes for a failed well-formed _GraphQL-over-HTTP + # > request_ where the response uses the `application/json` media type, but it is + # > strongly discouraged. To use `4xx` and `5xx` status codes in these situations, + # > please use the `application/graphql-response+json` media type. + # + # At the time of this writing, ElasticGraph uses the `application/json` media type. + # We may want to migrate to `application/graphql-response+json` at some later point, + # at which time we can consider using 413 instead. + # + # [^1]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/413 + # [^2]: https://github.com/graphql/graphql-over-http/blob/4db4e501f0537a14fd324c455294056676e38e8c/spec/GraphQLOverHTTP.md#applicationjson + def content_too_large_response(response) + GraphQL::HTTPResponse.json(200, { + "errors" => [{ + "message" => "The query results were #{response.body.bytesize} bytes, which exceeds the max AWS Lambda response size (#{LAMBDA_MAX_RESPONSE_PAYLOAD_BYTES} bytes). Please update the query to request less data and try again." + }] + }) + end + end + end +end diff --git a/elasticgraph-graphql_lambda/lib/elastic_graph/graphql_lambda/lambda_function.rb b/elasticgraph-graphql_lambda/lib/elastic_graph/graphql_lambda/lambda_function.rb new file mode 100644 index 00000000..dfc4e415 --- /dev/null +++ b/elasticgraph-graphql_lambda/lib/elastic_graph/graphql_lambda/lambda_function.rb @@ -0,0 +1,39 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/lambda_support/lambda_function" + +module ElasticGraph + module GraphQLLambda + # @private + class LambdaFunction + prepend LambdaSupport::LambdaFunction + + def initialize + require "elastic_graph/graphql_lambda" + require "elastic_graph/graphql_lambda/graphql_endpoint" + + graphql = ElasticGraph::GraphQLLambda.graphql_from_env + + # ElasticGraph loads things lazily by default. We want to eagerly load + # the graphql gem, the GraphQL schema, etc. rather than waiting for the + # first request, since we want consistent response times. + graphql.load_dependencies_eagerly + + @graphql_endpoint = ElasticGraph::GraphQLLambda::GraphQLEndpoint.new(graphql) + end + + def handle_request(event:, context:) + @graphql_endpoint.handle_request(event: event, context: context) + end + end + end +end + +# Lambda handler for `elasticgraph-graphql_lambda`. +ExecuteGraphQLQuery = ElasticGraph::GraphQLLambda::LambdaFunction.new diff --git a/elasticgraph-graphql_lambda/sig/elastic_graph/graphql_lambda.rbs b/elasticgraph-graphql_lambda/sig/elastic_graph/graphql_lambda.rbs new file mode 100644 index 00000000..0328c735 --- /dev/null +++ b/elasticgraph-graphql_lambda/sig/elastic_graph/graphql_lambda.rbs @@ -0,0 +1,5 @@ +module ElasticGraph + module GraphQLLambda + def self.graphql_from_env: () -> GraphQL + end +end diff --git a/elasticgraph-graphql_lambda/sig/elastic_graph/graphql_lambda/graphql_endpoint.rbs b/elasticgraph-graphql_lambda/sig/elastic_graph/graphql_lambda/graphql_endpoint.rbs new file mode 100644 index 00000000..e0f9c194 --- /dev/null +++ b/elasticgraph-graphql_lambda/sig/elastic_graph/graphql_lambda/graphql_endpoint.rbs @@ -0,0 +1,28 @@ +module ElasticGraph + module GraphQLLambda + class GraphQLEndpoint + LAMBDA_TIMEOUT_BUFFER_MS: ::Integer + LAMBDA_MAX_RESPONSE_PAYLOAD_BYTES: ::Integer + type lambdaHTTPResponse = {statusCode: ::Integer, body: ::String, headers: ::Hash[::String, ::String]} + + def initialize: (GraphQL) -> void + + def handle_request: ( + event: ::Hash[::String, untyped], + context: LambdaContext + ) -> lambdaHTTPResponse + + private + + @graphql_http_endpoint: GraphQL::HTTPEndpoint + @logger: ::Logger + @monotonic_clock: Support::MonotonicClock + + def request_from: (::Hash[::String, untyped]) -> GraphQL::HTTPRequest + def convert_response: (GraphQL::HTTPResponse) -> lambdaHTTPResponse + def url_from: (::Hash[::String, untyped]) -> ::String + def http_method_from: (::Hash[::String, untyped]) -> ::Symbol + def content_too_large_response: (GraphQL::HTTPResponse) -> GraphQL::HTTPResponse + end + end +end diff --git a/elasticgraph-graphql_lambda/sig/elastic_graph/graphql_lambda/lambda_function.rbs b/elasticgraph-graphql_lambda/sig/elastic_graph/graphql_lambda/lambda_function.rbs new file mode 100644 index 00000000..9a0a09e1 --- /dev/null +++ b/elasticgraph-graphql_lambda/sig/elastic_graph/graphql_lambda/lambda_function.rbs @@ -0,0 +1,11 @@ +module ElasticGraph + module GraphQLLambda + class LambdaFunction + include LambdaSupport::LambdaFunction[GraphQLEndpoint::lambdaHTTPResponse] + include LambdaSupport::_LambdaFunctionClass[GraphQLEndpoint::lambdaHTTPResponse] + @graphql_endpoint: GraphQLEndpoint + end + end +end + +ExecuteGraphQLQuery: ElasticGraph::GraphQLLambda::LambdaFunction diff --git a/elasticgraph-graphql_lambda/sig/lambda_context.rbs b/elasticgraph-graphql_lambda/sig/lambda_context.rbs new file mode 100644 index 00000000..1dd5ee2b --- /dev/null +++ b/elasticgraph-graphql_lambda/sig/lambda_context.rbs @@ -0,0 +1,8 @@ +# Type signatures for `context` arg passed to AWS Lambda handlers. +# Note that we have only bothered to define signatures for the parts we use. +# +# Based on: +# https://github.com/aws/aws-lambda-ruby-runtime-interface-client/blob/v1.0.2/lib/aws_lambda_ric/lambda_context.rb +class LambdaContext + def get_remaining_time_in_millis: () -> ::Integer +end diff --git a/elasticgraph-graphql_lambda/spec/spec_helper.rb b/elasticgraph-graphql_lambda/spec/spec_helper.rb new file mode 100644 index 00000000..36d72d3c --- /dev/null +++ b/elasticgraph-graphql_lambda/spec/spec_helper.rb @@ -0,0 +1,102 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-graphql_lambda`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. + +module ElasticGraph + module GraphQLLambda + # https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html#1.0 + module APIGatewayV1HTTPAPI + def build_event(http_method:, path:, body:, headers:, user_arn:, request_context: {}) + uri = URI(path) + + # Note: this isn't the entire event; it's just the fields we care about. See link above for a full example. + { + "version" => "1.0", + "resource" => uri.path, + "path" => uri.path, + "httpMethod" => http_method.to_s.upcase, + "headers" => headers.transform_keys(&:downcase), + "multiValueHeaders" => headers.transform_keys(&:downcase).transform_values { |v| [v] }, + "queryStringParameters" => ::URI.decode_www_form(uri.query.to_s).to_h, + "multiValueQueryStringParameters" => ::URI.decode_www_form(uri.query.to_s).to_h.transform_values { |v| [v] }, + "requestContext" => { + "httpMethod" => http_method.to_s.upcase, + "path" => uri.path, + "protocol" => "HTTP/1.1", + "resourcePath" => uri.path, + "identity" => { + "userArn" => user_arn + } + }.merge(request_context), + "body" => body, + "isBase64Encoded" => false + } + end + end + + # https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html#2.0 + module APIGatewayV2HTTPAPI + def build_event(http_method:, path:, body:, headers:, user_arn:, request_context: {}) + uri = URI(path) + + # Note: this isn't the entire event; it's just the fields we care about. See link above for a full example. + # Note also that `userArn` isn't anywhere in this payload :(. + { + "version" => "2.0", + "rawPath" => uri.path, + "rawQueryString" => uri.query, + "headers" => headers.transform_keys(&:downcase), + "queryStringParameters" => ::URI.decode_www_form(uri.query.to_s).to_h, + "requestContext" => { + "http" => { + "method" => http_method.to_s.upcase, + "path" => uri.path, + "protocol" => "HTTP/1.1" + }, + "stage" => "test-stage" + }.merge(request_context), + "body" => body, + "isBase64Encoded" => false + } + end + end + + # https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format + module APIGatewayRestAPI + def build_event(http_method:, path:, body:, headers:, user_arn:, request_context: {}) + uri = URI(path) + + # Note: this isn't the entire event; it's just the fields we care about. See link above for a full example. + { + "resource" => uri.path, + "path" => uri.path, + "httpMethod" => http_method.to_s.upcase, + "headers" => headers.transform_keys(&:downcase), + "multiValueHeaders" => headers.transform_keys(&:downcase).transform_values { |v| [v] }, + "queryStringParameters" => ::URI.decode_www_form(uri.query.to_s).to_h, + "multiValueQueryStringParameters" => ::URI.decode_www_form(uri.query.to_s).to_h.transform_values { |v| [v] }, + "requestContext" => { + "httpMethod" => http_method.to_s.upcase, + "path" => uri.path, + "protocol" => "HTTP/1.1", + "resourcePath" => uri.path, + "identity" => { + "userArn" => user_arn + }, + "stage" => "test-stage" + }.merge(request_context), + "body" => body, + "isBase64Encoded" => false + } + end + end + end +end diff --git a/elasticgraph-graphql_lambda/spec/unit/elastic_graph/graphql_lambda/graphql_endpoint_spec.rb b/elasticgraph-graphql_lambda/spec/unit/elastic_graph/graphql_lambda/graphql_endpoint_spec.rb new file mode 100644 index 00000000..9bca5ad4 --- /dev/null +++ b/elasticgraph-graphql_lambda/spec/unit/elastic_graph/graphql_lambda/graphql_endpoint_spec.rb @@ -0,0 +1,277 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/graphql_lambda/graphql_endpoint" +require "elastic_graph/support/hash_util" +require "json" +require "uri" + +module ElasticGraph + module GraphQLLambda + RSpec.describe GraphQLEndpoint, :builds_graphql do + shared_examples_for "HTTP handling" do |supports_user_arn: true, supports_get: true| + let(:graphql) { build_graphql } + let(:endpoint) { GraphQLEndpoint.new(graphql) } + let(:processed_requests) { [] } + + let(:introspection_query) do + <<~IQUERY + query IntrospectionQuery { + __schema { + types { + kind + name + } + } + } + IQUERY + end + + before do + allow(graphql.graphql_http_endpoint).to receive(:process).and_wrap_original do |original, request, **options| + processed_requests << request + original.call(request, **options) + end + end + + it "processes the provided GraphQL query" do + response = handle_request(body: JSON.generate("query" => introspection_query)) + + expect(response).to include(statusCode: 200, body: a_hash_including("data")) + end + + context "when the response payload exceeds the AWS Lambda max response size" do + it "returns a clear error to avoid manifesting as a confusing API gateway failure" do + stub_const("#{GraphQLEndpoint.name}::LAMBDA_MAX_RESPONSE_PAYLOAD_BYTES", 10) + + response = handle_request(body: JSON.generate("query" => introspection_query)) + + expect(response).to include(statusCode: 200) + expect(::JSON.parse(response[:body]).dig("errors", 0, "message")).to match(/The query results were \d+ bytes, which exceeds the max AWS Lambda response size \(10 bytes\)/) + end + end + + if supports_get + it "supports executing queries submitted as a GET" do + response = handle_request(http_method: :get, path: "/?query=#{::URI.encode_www_form_component(introspection_query)}") + + expect(response).to include(statusCode: 200, body: a_hash_including("data")) + end + end + + it "respects the `#{TIMEOUT_MS_HEADER}` header, regardless of casing, returning a 504 Gateway Timeout when a datastore query times out" do + response = handle_widget_query_request_with_headers({TIMEOUT_MS_HEADER => "0"}) + expect_json_error_including(response, 504, "Search exceeded requested timeout.") + + response = handle_widget_query_request_with_headers({TIMEOUT_MS_HEADER.downcase => "0"}) + expect_json_error_including(response, 504, "Search exceeded requested timeout.") + + response = handle_widget_query_request_with_headers({TIMEOUT_MS_HEADER.upcase => "0"}) + expect_json_error_including(response, 504, "Search exceeded requested timeout.") + end + + it "passes a `max_timeout_in_ms` based on `context.get_remaining_time_in_millis`" do + timeout = max_timeout_used_for(lambda_time_remaining_ms: 15_400) + + expect(timeout).to eq(15_400 - GraphQLEndpoint::LAMBDA_TIMEOUT_BUFFER_MS) + end + + it "returns a 400 if the timeout header is not formatted correctly" do + response = handle_widget_query_request_with_headers({TIMEOUT_MS_HEADER => "zero"}) + + expect_json_error_including(response, 400, TIMEOUT_MS_HEADER, "zero", "invalid") + end + + it "returns a 400 if the request body is not valid JSON" do + response = handle_request(body: "not json") + + expect_json_error_including(response, 400, "is invalid JSON") + end + + it "responds reasonably if the request lacks a `query` field in the JSON" do + response = handle_request(body: "{}") + + expect_json_error_including(response, 200, "No query string") + end + + if supports_user_arn + it "passes the `#{GRAPHQL_LAMBDA_AWS_ARN_HEADER}` HTTP header so the configured client resolver can use it" do + user_arn = "arn:aws:sts::123456789:role/client_app" + headers = effective_http_headers_for_query(user_arn: user_arn) + + expect(headers).to include(GRAPHQL_LAMBDA_AWS_ARN_HEADER => user_arn) + end + + it "overwrites a `#{GRAPHQL_LAMBDA_AWS_ARN_HEADER}` HTTP header passed by the client so that clients can't spoof it" do + user_arn = "arn:aws:sts::123456789:role/client_app" + headers = effective_http_headers_for_query( + user_arn: user_arn, + headers: { + GRAPHQL_LAMBDA_AWS_ARN_HEADER => "arn:aws:sts::123456789:role/attacker", + GRAPHQL_LAMBDA_AWS_ARN_HEADER.downcase => "arn:aws:sts::123456789:role/attacker", + GRAPHQL_LAMBDA_AWS_ARN_HEADER.tr("-", "_") => "arn:aws:sts::123456789:role/attacker", + GRAPHQL_LAMBDA_AWS_ARN_HEADER.downcase.tr("-", "_") => "arn:aws:sts::123456789:role/attacker" + } + ) + + expect(headers.except("content-type")).to eq({GRAPHQL_LAMBDA_AWS_ARN_HEADER => user_arn}) + end + else + it "passes no value for the `#{GRAPHQL_LAMBDA_AWS_ARN_HEADER}` HTTP header since no value is available" do + headers = effective_http_headers_for_query(user_arn: nil) + + expect(headers).to exclude(GRAPHQL_LAMBDA_AWS_ARN_HEADER) + end + + it "drops a `#{GRAPHQL_LAMBDA_AWS_ARN_HEADER}` HTTP header passed by the client so that clients can't spoof it" do + headers = effective_http_headers_for_query( + user_arn: nil, + headers: { + GRAPHQL_LAMBDA_AWS_ARN_HEADER => "arn:aws:sts::123456789:role/attacker", + GRAPHQL_LAMBDA_AWS_ARN_HEADER.downcase => "arn:aws:sts::123456789:role/attacker", + GRAPHQL_LAMBDA_AWS_ARN_HEADER.tr("-", "_") => "arn:aws:sts::123456789:role/attacker", + GRAPHQL_LAMBDA_AWS_ARN_HEADER.downcase.tr("-", "_") => "arn:aws:sts::123456789:role/attacker" + } + ) + + expect(headers.except("content-type")).to eq({}) + end + end + + def max_timeout_used_for(**options) + used_timeout = nil + + allow(graphql.graphql_http_endpoint).to receive(:process).and_wrap_original do |original, request, **opts| + used_timeout = opts[:max_timeout_in_ms] + original.call(request, **opts) + end + + handle_request(body: JSON.generate("query" => introspection_query), **options) + + used_timeout + end + + def handle_widget_query_request_with_headers(headers) + body = JSON.generate("query" => "query { widgets { edges { node { id } } } }") + handle_request(body: body, headers: headers) + end + + def handle_request( + http_method: :post, + path: "/", + body: nil, + headers: {}, + user_arn: "arn:aws:sts::123456789:assumed-role/someone", + lambda_time_remaining_ms: 30_000, + endpoint: self.endpoint, + request_context: {} + ) + event = build_event( + http_method: http_method, + path: path, + body: body, + headers: headers.merge("Content-Type" => "application/json"), + user_arn: user_arn, + request_context: request_context + ) + + # AWS docs don't tell us what the class name is for the + # `context` object, and we don't have it available to verify against, anyway. So we gotta use + # a non-verifying double here. But https://docs.aws.amazon.com/lambda/latest/dg/ruby-context.html + # documents the available methods on the context object. + context = double("AWSLambdaContext", get_remaining_time_in_millis: lambda_time_remaining_ms) # standard:disable RSpec/VerifiedDoubles + + endpoint.handle_request(event: event, context: context) + end + + def expect_json_error_including(response, status_code, *parts) + expect(response).to include(statusCode: status_code) + expect(response[:headers]).to include("Content-Type" => "application/json") + expect(::JSON.parse(response[:body])).to include("errors" => [a_hash_including("message" => a_string_including(*parts))]) + end + + def effective_http_headers_for_query(user_arn:, headers: {}) + effective_headers = nil + + allow(graphql.graphql_http_endpoint).to receive(:process).and_wrap_original do |original, request, **options| + effective_headers = request.headers + original.call(request, **options) + end + + handle_request(body: JSON.generate("query" => introspection_query), user_arn: user_arn, headers: headers) + + effective_headers + end + end + + describe "an API Gateway HTTP API with the v1.0 payload format" do + include APIGatewayV1HTTPAPI + + context "with query params" do + include_examples "HTTP handling" + end + + context "with no query params" do + include_examples "HTTP handling", supports_get: false + + def build_event(...) + super(...).merge("queryStringParameters" => nil, "multiValueQueryStringParameters" => nil) + end + + after do + bad_requests = processed_requests.select { |r| r.url == "/?" } + expect(bad_requests).to be_empty, "Expected no requests to have a URL like `/?` but some did: #{bad_requests.inspect}" + end + end + end + + describe "an API Gateway HTTP API with the v2.0 payload format" do + include APIGatewayV2HTTPAPI + + context "with query params" do + include_examples "HTTP handling", supports_user_arn: false + end + + context "with no query params" do + include_examples "HTTP handling", supports_user_arn: false, supports_get: false + + def build_event(...) + super(...).merge("queryStringParameters" => nil) + end + + after do + bad_requests = processed_requests.select { |r| r.url == "/?" } + expect(bad_requests).to be_empty, "Expected no requests to have a URL like `/?` but some did: #{bad_requests.inspect}" + end + end + end + + describe "an API Gateway REST API with query params" do + include APIGatewayRestAPI + + context "with query params" do + include_examples "HTTP handling" + end + + context "with no query params" do + include_examples "HTTP handling", supports_get: false + + def build_event(...) + super(...).merge("queryStringParameters" => nil) + end + + after do + bad_requests = processed_requests.select { |r| r.url == "/?" } + expect(bad_requests).to be_empty, "Expected no requests to have a URL like `/?` but some did: #{bad_requests.inspect}" + end + end + end + end + end +end diff --git a/elasticgraph-graphql_lambda/spec/unit/elastic_graph/graphql_lambda/lambda_function_spec.rb b/elasticgraph-graphql_lambda/spec/unit/elastic_graph/graphql_lambda/lambda_function_spec.rb new file mode 100644 index 00000000..641c7c76 --- /dev/null +++ b/elasticgraph-graphql_lambda/spec/unit/elastic_graph/graphql_lambda/lambda_function_spec.rb @@ -0,0 +1,40 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/lambda_function" + +RSpec.describe "GraphQL lambda function" do + include_context "lambda function" + + # Doesn't matter which API Gateway version adapter we include here--we just need one of them. + include ElasticGraph::GraphQLLambda::APIGatewayV2HTTPAPI + + it "executes GraphQL queries" do + expect_loading_lambda_to_define_constant( + lambda: "elastic_graph/graphql_lambda/lambda_function.rb", + const: :ExecuteGraphQLQuery + ) do |lambda_function| + event = build_event( + http_method: :post, + path: "/graphql", + body: "query { __typename }", + headers: {"Content-Type" => "application/graphql"}, + user_arn: "arn:aws:iam::123456789012:user/username" + ) + # AWS docs don't tell us what the class name is for the + # `context` object, and we don't have it available to verify against, anyway. So we gotta use + # a non-verifying double here. But https://docs.aws.amazon.com/lambda/latest/dg/ruby-context.html + # documents the available methods on the context object. + context = double("AWSLambdaContext", get_remaining_time_in_millis: 10_000) # standard:disable RSpec/VerifiedDoubles + response = lambda_function.handle_request(event: event, context: context) + + expect(response.fetch(:statusCode)).to eq 200 + expect(::JSON.parse(response.fetch(:body))).to eq({"data" => {"__typename" => "Query"}}) + end + end +end diff --git a/elasticgraph-graphql_lambda/spec/unit/elastic_graph/graphql_lambda_spec.rb b/elasticgraph-graphql_lambda/spec/unit/elastic_graph/graphql_lambda_spec.rb new file mode 100644 index 00000000..06adb149 --- /dev/null +++ b/elasticgraph-graphql_lambda/spec/unit/elastic_graph/graphql_lambda_spec.rb @@ -0,0 +1,23 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql_lambda" +require "elastic_graph/spec_support/lambda_function" + +module ElasticGraph + RSpec.describe GraphQLLambda do + describe ".graphql_from_env" do + include_context "lambda function" + around { |ex| with_lambda_env_vars(&ex) } + + it "builds a graphql instance" do + expect(GraphQLLambda.graphql_from_env).to be_a(GraphQL) + end + end + end +end diff --git a/elasticgraph-health_check/.rspec b/elasticgraph-health_check/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-health_check/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-health_check/.yardopts b/elasticgraph-health_check/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-health_check/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-health_check/Gemfile b/elasticgraph-health_check/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-health_check/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-health_check/LICENSE.txt b/elasticgraph-health_check/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-health_check/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-health_check/README.md b/elasticgraph-health_check/README.md new file mode 100644 index 00000000..9d2785d8 --- /dev/null +++ b/elasticgraph-health_check/README.md @@ -0,0 +1,82 @@ +# ElasticGraph::HealthCheck + +Provides a component that can act as a health check for high availability deployments. The HealthCheck component +returns a summary status of either `healthy`, `degraded`, or `unhealthy` for the endpoint. + +The intended semantics of these statuses +map to the corresponding Envoy statuses, see +[the Envoy documentation for more details](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/health_checking), +but in short `degraded` maps to "endpoint is impaired, do not use unless you have no other choice" and `unhealthy` maps to "endpoint is hard +down/should not be used under any circumstances". + +The returned status is the worst of the status values from the individual sub-checks: +1. The datastore clusters' own health statuses. The datastore clusters reflect their status as green/yellow/red. See + [the Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html#cluster-health-api-response-body) + for details on the meaning of these statuses. + - `green` maps to `healthy`, `yellow` to `degraded`, and `red` to `unhealthy`. + +2. The recency of data present in ElasticGraph indices. The HealthCheck configuration specifies the expected "max recency" for items within an + index. + - If no records have been indexed within the specified period, the HealthCheck component will consider the index to be in a `degraded` status. + +As mentioned above, the returned status is the worst status of these two checks. E.g. if the datastore cluster(s) are all `green`, but a recency check fails, the +overall status will be `degraded`. If the recency checks pass, but at least one datastore cluster is `red`, an `unhealthy` status will be returned. + +## Integration + +To use, simply register the `EnvoyExtension` when defining your schema: + +```ruby +require(envoy_extension_path = "elastic_graph/health_check/envoy_extension") +schema.register_graphql_extension ElasticGraph::HealthCheck::EnvoyExtension, + defined_at: envoy_extension_path, + http_path_segment: "/_status" +``` + +## Configuration + +These checks are configurable. The following configuration will be used as an example: + +``` +health_check: + clusters_to_consider: ["widgets-cluster"] + data_recency_checks: + Widget: + timestamp_field: createdAt + expected_max_recency_seconds: 30 +``` + +- `clusters_to_consider` configures the first check (datastore cluster health), and specifies which clusters' health status is monitored. +- `data_recency_checks` configures the second check (data recency), and configures the recency check described above. In this example, if no new "Widgets" + are indexed for thirty seconds (perhaps because of an infrastructure issue), a `degraded` status will be returned. + - Note that this setting is most appropriate for types where you expect a steady stream of indexing (and where the absence of new records is indicative + of some kind of failure). + +## Behavior when datastore clusters are inaccessible + +A given ElasticGraph GraphQL endpoint does not necessarily have access to all datastore clusters - more specifically, the endpoint will only have access +to clusters present in the `datastore.clusters` configuration map. + +If a health check is configured for either a cluster or type that the GraphQL endpoint does not have access to, the respective check will be skipped. This is appropriate, +as since the GraphQL endpoint does not have access to the cluster/type, the cluster's/type's health is immaterial. + +For example, with the following configuration: + +``` +datastore: + clusters: + widgets-cluster: { ... } + # components-cluster: { ... } ### Not available, commented out. +health_check: + clusters_to_consider: ["widgets-cluster", "components-cluster"] + data_recency_checks: + Component: + timestamp_field: createdAt + expected_max_recency_seconds: 10 + Widget: + timestamp_field: createdAt + expected_max_recency_seconds: 30 +``` + +... the `components-cluster` datastore status health check will be skipped, as will the Component recency check. However the `widgets-cluster`/`Widget` health +checks will proceed as normal. diff --git a/elasticgraph-health_check/elasticgraph-health_check.gemspec b/elasticgraph-health_check/elasticgraph-health_check.gemspec new file mode 100644 index 00000000..5da1f0b9 --- /dev/null +++ b/elasticgraph-health_check/elasticgraph-health_check.gemspec @@ -0,0 +1,23 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :extension) do |spec, eg_version| + spec.summary = "An ElasticGraph extension that provides a health check for high availability deployments." + + spec.add_dependency "elasticgraph-datastore_core", eg_version + spec.add_dependency "elasticgraph-graphql", eg_version + spec.add_dependency "elasticgraph-support", eg_version + + spec.add_development_dependency "elasticgraph-admin", eg_version + spec.add_development_dependency "elasticgraph-elasticsearch", eg_version + spec.add_development_dependency "elasticgraph-indexer", eg_version + spec.add_development_dependency "elasticgraph-opensearch", eg_version + spec.add_development_dependency "elasticgraph-schema_definition", eg_version +end diff --git a/elasticgraph-health_check/lib/elastic_graph/health_check/config.rb b/elasticgraph-health_check/lib/elastic_graph/health_check/config.rb new file mode 100644 index 00000000..afec81be --- /dev/null +++ b/elasticgraph-health_check/lib/elastic_graph/health_check/config.rb @@ -0,0 +1,48 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module HealthCheck + class Config < ::Data.define( + # The list of clusters to perform datastore status health checks on. A `green` status maps to `healthy`, a + # `yellow` status maps to `degraded`, and a `red` status maps to `unhealthy`. The returned status is the minimum + # status from all clusters in the list (a `yellow` cluster and a `green` cluster will result in a `degraded` status). + # + # Example: ["cluster-one", "cluster-two"] + :clusters_to_consider, + # A map of types to perform recency checks on. If no new records for that type have been indexed within the specified + # period, a `degraded` status will be returned. + # + # Example: { Widget: { timestamp_field: createdAt, expected_max_recency_seconds: 30 }} + :data_recency_checks + ) + EMPTY = new([], {}) + + def self.from_parsed_yaml(config_hash) + config_hash = config_hash.fetch("health_check") { return EMPTY } + + new( + clusters_to_consider: config_hash.fetch("clusters_to_consider"), + data_recency_checks: config_hash.fetch("data_recency_checks").transform_values do |value_hash| + DataRecencyCheck.from(value_hash) + end + ) + end + + DataRecencyCheck = ::Data.define(:expected_max_recency_seconds, :timestamp_field) do + # @implements DataRecencyCheck + def self.from(config_hash) + new( + expected_max_recency_seconds: config_hash.fetch("expected_max_recency_seconds"), + timestamp_field: config_hash.fetch("timestamp_field") + ) + end + end + end + end +end diff --git a/elasticgraph-health_check/lib/elastic_graph/health_check/constants.rb b/elasticgraph-health_check/lib/elastic_graph/health_check/constants.rb new file mode 100644 index 00000000..40e5fa71 --- /dev/null +++ b/elasticgraph-health_check/lib/elastic_graph/health_check/constants.rb @@ -0,0 +1,46 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# Enumerates constants that are used from multiple places in ElasticGraph::HealthCheck. +module ElasticGraph + module HealthCheck + # List of datastore cluster health fields from: + # https://www.elastic.co/guide/en/elasticsearch/reference/7.10/cluster-health.html#cluster-health-api-response-body + # + # This is expressed as a constant so that we can use it in dynamic ways in a few places + # (such as in a test; we want an acceptance test to fetch all these fields to make + # sure they work, and having them defined this way makes that easier). + # + # To get this list, this javascript was used in the chrome console: + # + # Array.from(document.querySelectorAll('div.variablelist')[2].querySelectorAll(':scope > dl.variablelist > dt')).map(x => x.innerText) + # + # (Feel free to use/change that as needed if/when you update this list in the future based on a newer datastore version.) + # + # Note: `discovered_master` is a new boolean field that AWS OpenSearch seems to add to the cluster health response. + # It was observed on the response on 2022-04-18. + DATASTORE_CLUSTER_HEALTH_FIELDS = %i[ + cluster_name + status + timed_out + number_of_nodes + number_of_data_nodes + active_primary_shards + active_shards + relocating_shards + initializing_shards + unassigned_shards + delayed_unassigned_shards + number_of_pending_tasks + number_of_in_flight_fetch + task_max_waiting_in_queue_millis + active_shards_percent_as_number + discovered_master + ].to_set + end +end diff --git a/elasticgraph-health_check/lib/elastic_graph/health_check/envoy_extension.rb b/elasticgraph-health_check/lib/elastic_graph/health_check/envoy_extension.rb new file mode 100644 index 00000000..8b6e3303 --- /dev/null +++ b/elasticgraph-health_check/lib/elastic_graph/health_check/envoy_extension.rb @@ -0,0 +1,43 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/health_check/envoy_extension/graphql_http_endpoint_decorator" +require "elastic_graph/health_check/health_checker" + +module ElasticGraph + module HealthCheck + # An extension module that hooks into the HTTP endpoint to provide Envoy health checks. + module EnvoyExtension + def graphql_http_endpoint + @graphql_http_endpoint ||= + begin + http_path_segment = config.extension_settings.dig("health_check", "http_path_segment") + http_path_segment ||= runtime_metadata + .graphql_extension_modules + .find { |ext_mod| ext_mod.extension_class == EnvoyExtension } + &.extension_config + &.dig(:http_path_segment) + + if http_path_segment.nil? + raise ElasticGraph::Errors::ConfigSettingNotSetError, "Health check `http_path_segment` is not configured. " \ + "Either set under `health_check` in YAML config or pass it along if you register the `EnvoyExtension` " \ + "via `register_graphql_extension`." + end + + GraphQLHTTPEndpointDecorator.new( + super, + health_check_http_path_segment: http_path_segment, + health_checker: HealthChecker.build_from(self), + logger: logger + ) + end + end + end + end +end diff --git a/elasticgraph-health_check/lib/elastic_graph/health_check/envoy_extension/graphql_http_endpoint_decorator.rb b/elasticgraph-health_check/lib/elastic_graph/health_check/envoy_extension/graphql_http_endpoint_decorator.rb new file mode 100644 index 00000000..b590656e --- /dev/null +++ b/elasticgraph-health_check/lib/elastic_graph/health_check/envoy_extension/graphql_http_endpoint_decorator.rb @@ -0,0 +1,61 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" +require "elastic_graph/graphql/http_endpoint" +require "uri" + +module ElasticGraph + module HealthCheck + module EnvoyExtension + # Intercepts HTTP requests so that a health check can be performed if it's a GET request to the configured health check path. + # The HTTP response follows Envoy HTTP health check guidelines: + # + # https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/health_checking + class GraphQLHTTPEndpointDecorator < DelegateClass(GraphQL::HTTPEndpoint) + def initialize(http_endpoint, health_check_http_path_segment:, health_checker:, logger:) + super(http_endpoint) + @health_check_http_path_segment = health_check_http_path_segment.delete_prefix("/").delete_suffix("/") + @health_checker = health_checker + @logger = logger + end + + __skip__ = + def process(request, **) + if request.http_method == :get && URI(request.url).path.split("/").include?(@health_check_http_path_segment) + perform_health_check + else + super + end + end + + private + + RESPONSES_BY_HEALTH_STATUS_CATEGORY = { + healthy: [200, "Healthy!", {}], + unhealthy: [500, "Unhealthy!", {}], + # https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/health_checking#degraded-health + degraded: [200, "Degraded.", {"x-envoy-degraded" => "true"}] + } + + def perform_health_check + status = @health_checker.check_health + @logger.info status.to_loggable_description + + status, message, headers = RESPONSES_BY_HEALTH_STATUS_CATEGORY.fetch(status.category) + + GraphQL::HTTPResponse.new( + status_code: status, + headers: headers.merge("Content-Type" => "text/plain"), + body: message + ) + end + end + end + end +end diff --git a/elasticgraph-health_check/lib/elastic_graph/health_check/health_checker.rb b/elasticgraph-health_check/lib/elastic_graph/health_check/health_checker.rb new file mode 100644 index 00000000..e1559a74 --- /dev/null +++ b/elasticgraph-health_check/lib/elastic_graph/health_check/health_checker.rb @@ -0,0 +1,221 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/health_check/config" +require "elastic_graph/health_check/health_status" +require "elastic_graph/support/threading" +require "time" + +module ElasticGraph + module HealthCheck + class HealthChecker + # Static factory method that builds a HealthChecker from an ElasticGraph::GraphQL instance. + def self.build_from(graphql) + new( + schema: graphql.schema, + config: HealthCheck::Config.from_parsed_yaml(graphql.config.extension_settings), + datastore_search_router: graphql.datastore_search_router, + datastore_query_builder: graphql.datastore_query_builder, + datastore_clients_by_name: graphql.datastore_core.clients_by_name, + clock: graphql.clock, + logger: graphql.logger + ) + end + + def initialize( + schema:, + config:, + datastore_search_router:, + datastore_query_builder:, + datastore_clients_by_name:, + clock:, + logger: + ) + @schema = schema + @datastore_search_router = datastore_search_router + @datastore_query_builder = datastore_query_builder + @datastore_clients_by_name = datastore_clients_by_name + @clock = clock + @logger = logger + @indexed_document_types_by_name = @schema.indexed_document_types.to_h { |t| [t.name.to_s, t] } + + @config = validate_and_normalize_config(config) + end + + def check_health + recency_queries_by_type_name = @config.data_recency_checks.to_h do |type_name, recency_config| + [type_name, build_recency_query_for(type_name, recency_config)] + end + + recency_results_by_query, *cluster_healths = execute_in_parallel( + lambda { @datastore_search_router.msearch(recency_queries_by_type_name.values) }, + *@config.clusters_to_consider.map do |cluster| + lambda { [cluster, @datastore_clients_by_name.fetch(cluster).get_cluster_health] } + end + ) + + HealthStatus.new( + cluster_health_by_name: build_cluster_health_by_name(cluster_healths.to_h), + latest_record_by_type: build_latest_record_by_type(recency_results_by_query, recency_queries_by_type_name) + ) + end + + private + + def build_recency_query_for(type_name, recency_config) + type = @indexed_document_types_by_name.fetch(type_name) + + @datastore_query_builder.new_query( + search_index_definitions: type.search_index_definitions, + filter: build_index_optimization_filter_for(recency_config), + requested_fields: ["id", recency_config.timestamp_field], + document_pagination: {first: 1}, + sort: [{recency_config.timestamp_field => {"order" => "desc"}}] + ) + end + + # To make the recency query more optimal, we filter on the timestamp field. This can provide + # a couple optimizations: + # + # - If its a rollover index and the timestamp is the field we use for rollover, this allows + # the ElasticGraph query engine to hit only a subset of indices for better perf. + # - We've been told (by AWS support) that sorting a larger result set if more expensive than + # a small result set (presumably larger than filtering cost) so even if we can't limit what + # indices we hit with this, it should still be helpful. + # + # However, there's a bit of a risk of not actually finding the latest record if we include + # this filter. What we have here is a compromise: we "lookback" up to 100 times the + # `expected_max_recency_seconds`. For example, if that's set at 30, we'd search the last 3000 + # seconds of data, which should be plenty of lookback for most cases, while still allowing + # a filter optimization. Once the latest record is more than 100 times older than our threshold + # the exact age of it is less interesting, anyway. + def build_index_optimization_filter_for(recency_config) + lookback_timestamp = @clock.now - (recency_config.expected_max_recency_seconds * 100) + {recency_config.timestamp_field => {"gte" => lookback_timestamp.iso8601}} + end + + def execute_in_parallel(*lambdas) + Support::Threading.parallel_map(lambdas) { |l| l.call } + end + + def build_cluster_health_by_name(cluster_healths) + cluster_healths.transform_values do |health| + health_status_fields = DATASTORE_CLUSTER_HEALTH_FIELDS.to_h do |field_name| + [field_name, health[field_name.to_s]] + end + + HealthStatus::ClusterHealth.new(**health_status_fields) + end + end + + def build_latest_record_by_type(recency_results_by_query, recency_queries_by_type_name) + recency_queries_by_type_name.to_h do |type_name, query| + config = @config.data_recency_checks.fetch(type_name) + + latest_record = if (latest_doc = recency_results_by_query.fetch(query).first) + timestamp = ::Time.iso8601(latest_doc.fetch(config.timestamp_field)) + + HealthStatus::LatestRecord.new( + id: latest_doc.id, + timestamp: timestamp, + seconds_newer_than_required: timestamp - (@clock.now - config.expected_max_recency_seconds) + ) + end + + [type_name, latest_record] + end + end + + def validate_and_normalize_config(config) + unrecognized_cluster_names = config.clusters_to_consider - all_known_clusters + + # @type var errors: ::Array[::String] + errors = [] + + if unrecognized_cluster_names.any? + errors << "`health_check.clusters_to_consider` contains " \ + "unrecognized cluster names: #{unrecognized_cluster_names.join(", ")}" + end + + # Here, we determine which of the specified `clusters_to_consider` are actually available for datastore health checks (green/yellow/red). + # Before partitioning, we remove `unrecognized_cluster_names` as those will be reported through a separate error mechanism (above). + # + # Below, `available_clusters_to_consider` will replace `clusters_to_consider` in the returned `Config` instance. + available_clusters_to_consider, unavailable_clusters_to_consider = + (config.clusters_to_consider - unrecognized_cluster_names).partition { |it| @datastore_clients_by_name.key?(it) } + + if unavailable_clusters_to_consider.any? + @logger.warn("#{unavailable_clusters_to_consider.length} cluster(s) were unavailable for health-checking: #{unavailable_clusters_to_consider.join(", ")}") + end + + valid_type_names, invalid_type_names = config + .data_recency_checks.keys + .partition { |type| @indexed_document_types_by_name.key?(type) } + + if invalid_type_names.any? + errors << "Some `health_check.data_recency_checks` types are not recognized indexed types: " \ + "#{invalid_type_names.join(", ")}" + end + + # It is possible to configure a GraphQL endpoint that has a healthcheck set up on type A, but doesn't actually + # have access to the datastore cluster that backs type A. In that case, we want to skip the health check - if the endpoint + # can't access type A, its health (or unhealth) is immaterial. + # + # So below, filter to types that have all of their datastore clusters available for querying. + available_type_names, unavailable_type_names = valid_type_names.partition do |type_name| + @indexed_document_types_by_name[type_name].search_index_definitions.all? do |search_index_definition| + @datastore_clients_by_name.key?(search_index_definition.cluster_to_query.to_s) + end + end + + if unavailable_type_names.any? + @logger.warn("#{unavailable_type_names.length} type(s) were unavailable for health-checking: #{unavailable_type_names.join(", ")}") + end + + # @type var invalid_timestamp_fields_by_type: ::Hash[::String, ::String] + invalid_timestamp_fields_by_type = {} + # @type var normalized_data_recency_checks: ::Hash[::String, Config::DataRecencyCheck] + normalized_data_recency_checks = {} + + available_type_names.each do |type| + check = config.data_recency_checks.fetch(type) + field = @indexed_document_types_by_name + .fetch(type) + .fields_by_name[check.timestamp_field] + + if field&.type&.unwrap_fully&.name.to_s == "DateTime" + # Convert the config so that we have a reference to the index field name. + normalized_data_recency_checks[type] = check.with(timestamp_field: field.name_in_index.to_s) + else + invalid_timestamp_fields_by_type[type] = check.timestamp_field + end + end + + if invalid_timestamp_fields_by_type.any? + errors << "Some `health_check.data_recency_checks` entries have invalid timestamp fields: " \ + "#{invalid_timestamp_fields_by_type.map { |k, v| "#{k} (#{v})" }.join(", ")}" + end + + raise Errors::ConfigError, errors.join("\n\n") unless errors.empty? + config.with( + data_recency_checks: normalized_data_recency_checks, + clusters_to_consider: available_clusters_to_consider + ) + end + + def all_known_clusters + @all_known_clusters ||= @indexed_document_types_by_name.flat_map do |_, index_type| + index_type.search_index_definitions.flat_map do |it| + [it.cluster_to_query] + it.clusters_to_index_into + end + end + @datastore_clients_by_name.keys + end + end + end +end diff --git a/elasticgraph-health_check/lib/elastic_graph/health_check/health_status.rb b/elasticgraph-health_check/lib/elastic_graph/health_check/health_status.rb new file mode 100644 index 00000000..441e985a --- /dev/null +++ b/elasticgraph-health_check/lib/elastic_graph/health_check/health_status.rb @@ -0,0 +1,90 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/health_check/constants" + +module ElasticGraph + module HealthCheck + # Encapsulates all of the status information for an ElasticGraph GraphQL endpoint. + # Computes a `category` for the status of the ElasticGraph endpoint. + # + # - unhealthy: the endpoint should not be used + # - degraded: the endpoint can be used, but prefer a healthy endpoint over it + # - healthy: the endpoint should be used + class HealthStatus < ::Data.define(:cluster_health_by_name, :latest_record_by_type, :category) + def initialize(cluster_health_by_name:, latest_record_by_type:) + super( + cluster_health_by_name: cluster_health_by_name, + latest_record_by_type: latest_record_by_type, + category: compute_category(cluster_health_by_name, latest_record_by_type) + ) + end + + def to_loggable_description + latest_record_descriptions = latest_record_by_type + .sort_by(&:first) # sort by type name + .map { |type, record| record&.to_loggable_description(type) || "Latest #{type} (missing)" } + .map { |description| "- #{description}" } + + cluster_health_descriptions = cluster_health_by_name + .sort_by(&:first) # sort by cluster name + .map { |name, health| "\n- #{health.to_loggable_description(name)}" } + + <<~EOS.strip.gsub("\n\n\n", "\n") + HealthStatus: #{category} (checked #{cluster_health_by_name.size} clusters, #{latest_record_by_type.size} latest records) + #{latest_record_descriptions.join("\n")} + #{cluster_health_descriptions.join("\n")} + EOS + end + + private + + def compute_category(cluster_health_by_name, latest_record_by_type) + cluster_statuses = cluster_health_by_name.values.map(&:status) + return :unhealthy if cluster_statuses.include?("red") + + return :degraded if cluster_statuses.include?("yellow") + return :degraded if latest_record_by_type.values.any? { |v| v.nil? || v.too_old? } + + :healthy + end + + # Encapsulates the status information for a single datastore cluster. + ClusterHealth = ::Data.define(*DATASTORE_CLUSTER_HEALTH_FIELDS.to_a) do + # @implements ClusterHealth + + def to_loggable_description(name) + field_values = to_h.map { |field, value| " #{field}: #{value.inspect}" } + "#{name} cluster health (#{status}):\n#{field_values.join("\n")}" + end + end + + # Encapsulates information about the latest record of a type. + LatestRecord = ::Data.define( + :id, # the id of the record + :timestamp, # the record's timestamp + :seconds_newer_than_required # the recency of the record relative to expectation; positive == more recent + ) do + # @implements LatestRecord + def to_loggable_description(type) + rounded_age = seconds_newer_than_required.round(2).abs + + if too_old? + "Latest #{type} (too old): #{id} / #{timestamp.iso8601} (#{rounded_age}s too old)" + else + "Latest #{type} (recent enough): #{id} / #{timestamp.iso8601} (#{rounded_age}s newer than required)" + end + end + + def too_old? + seconds_newer_than_required < 0 + end + end + end + end +end diff --git a/elasticgraph-health_check/sig/elastic_graph/health_check/config.rbs b/elasticgraph-health_check/sig/elastic_graph/health_check/config.rbs new file mode 100644 index 00000000..ffd682fc --- /dev/null +++ b/elasticgraph-health_check/sig/elastic_graph/health_check/config.rbs @@ -0,0 +1,43 @@ +module ElasticGraph + module HealthCheck + class ConfigSupertype + attr_reader clusters_to_consider: ::Array[::String] + attr_reader data_recency_checks: ::Hash[::String, Config::DataRecencyCheck] + + def self.new: ( + ::Array[::String], + ::Hash[::String, Config::DataRecencyCheck] + ) -> Config | ( + clusters_to_consider: ::Array[::String], + data_recency_checks: ::Hash[::String, Config::DataRecencyCheck] + ) -> Config + + def with: ( + ?clusters_to_consider: ::Array[::String], + ?data_recency_checks: ::Hash[::String, Config::DataRecencyCheck]) -> Config + end + + class Config < ConfigSupertype + EMPTY: Config + extend _BuildableFromParsedYaml[Config] + + class DataRecencyCheck + attr_reader expected_max_recency_seconds: ::Integer + attr_reader timestamp_field: ::String + + def self.new: ( + expected_max_recency_seconds: ::Integer, + timestamp_field: ::String) -> DataRecencyCheck + def with: ( + ?expected_max_recency_seconds: ::Integer, + ?timestamp_field: ::String) -> DataRecencyCheck + + def self.from: (::Hash[::String, untyped]) -> DataRecencyCheck + end + end + end + + class Config + attr_reader health_check: HealthCheck::Config + end +end diff --git a/elasticgraph-health_check/sig/elastic_graph/health_check/constants.rbs b/elasticgraph-health_check/sig/elastic_graph/health_check/constants.rbs new file mode 100644 index 00000000..23a676a0 --- /dev/null +++ b/elasticgraph-health_check/sig/elastic_graph/health_check/constants.rbs @@ -0,0 +1,5 @@ +module ElasticGraph + module HealthCheck + DATASTORE_CLUSTER_HEALTH_FIELDS: ::Set[::Symbol] + end +end diff --git a/elasticgraph-health_check/sig/elastic_graph/health_check/envoy_extension.rbs b/elasticgraph-health_check/sig/elastic_graph/health_check/envoy_extension.rbs new file mode 100644 index 00000000..1d176ff9 --- /dev/null +++ b/elasticgraph-health_check/sig/elastic_graph/health_check/envoy_extension.rbs @@ -0,0 +1,6 @@ +module ElasticGraph + module HealthCheck + module EnvoyExtension: GraphQL + end + end +end diff --git a/elasticgraph-health_check/sig/elastic_graph/health_check/envoy_extension/graphql_http_endpoint_decorator.rbs b/elasticgraph-health_check/sig/elastic_graph/health_check/envoy_extension/graphql_http_endpoint_decorator.rbs new file mode 100644 index 00000000..d67ac193 --- /dev/null +++ b/elasticgraph-health_check/sig/elastic_graph/health_check/envoy_extension/graphql_http_endpoint_decorator.rbs @@ -0,0 +1,34 @@ +module ElasticGraph + module HealthCheck + module EnvoyExtension + class GraphQLHTTPEndpointDecoratorSupertype < GraphQL::HTTPEndpoint + def initialize: (GraphQL::HTTPEndpoint) -> void + end + + class GraphQLHTTPEndpointDecorator < GraphQLHTTPEndpointDecoratorSupertype + def initialize: ( + GraphQL::HTTPEndpoint, + health_check_http_path_segment: ::String, + health_checker: HealthChecker, + logger: ::Logger + ) -> void + + @health_check_http_path_segment: ::String + @health_checker: HealthChecker + @logger: ::Logger + + # def process: (GraphQL::HTTPRequest, **untyped) -> GraphQL::HTTPResponse + + private + + RESPONSES_BY_HEALTH_STATUS_CATEGORY: ::Hash[HealthCheck::HealthStatus::category, [ + ::Integer, + ::String, + ::Hash[::String, ::String] + ]] + + def perform_health_check: () -> GraphQL::HTTPResponse + end + end + end +end diff --git a/elasticgraph-health_check/sig/elastic_graph/health_check/health_checker.rbs b/elasticgraph-health_check/sig/elastic_graph/health_check/health_checker.rbs new file mode 100644 index 00000000..d9aa7922 --- /dev/null +++ b/elasticgraph-health_check/sig/elastic_graph/health_check/health_checker.rbs @@ -0,0 +1,46 @@ +module ElasticGraph + module HealthCheck + class HealthChecker + def self.build_from: (GraphQL) -> HealthChecker + + def initialize: ( + schema: GraphQL::Schema, + config: Config, + datastore_search_router: GraphQL::DatastoreSearchRouter, + datastore_query_builder: GraphQL::DatastoreQuery::Builder, + datastore_clients_by_name: ::Hash[::String, DatastoreCore::_Client], + clock: singleton(::Time), + logger: ::Logger + ) -> void + + def check_health: () -> HealthStatus + + private + + @schema: GraphQL::Schema + @datastore_search_router: GraphQL::DatastoreSearchRouter + @datastore_query_builder: GraphQL::DatastoreQuery::Builder + @datastore_clients_by_name: ::Hash[::String, DatastoreCore::_Client] + @clock: singleton(::Time) + @logger: ::Logger + @indexed_document_types_by_name: ::Hash[::String, GraphQL::Schema::Type] + @config: Config + + def build_recency_query_for: (::String, Config::DataRecencyCheck) -> GraphQL::DatastoreQuery + def build_index_optimization_filter_for: (Config::DataRecencyCheck) -> ::Hash[::String, untyped]? + def execute_in_parallel: (*::Proc) -> ::Array[untyped] + + def build_cluster_health_by_name: ( + ::Hash[::String, ::Hash[::String, untyped]]) -> ::Hash[::String, HealthStatus::ClusterHealth] + + def build_latest_record_by_type: ( + ::Hash[GraphQL::DatastoreQuery, GraphQL::DatastoreResponse::SearchResponse], + ::Hash[::String, GraphQL::DatastoreQuery]) -> ::Hash[::String, HealthStatus::LatestRecord?] + + @all_known_clusters: ::Array[::String]? + def all_known_clusters: () -> ::Array[::String] + + def validate_and_normalize_config: (Config) -> Config + end + end +end diff --git a/elasticgraph-health_check/sig/elastic_graph/health_check/health_status.rbs b/elasticgraph-health_check/sig/elastic_graph/health_check/health_status.rbs new file mode 100644 index 00000000..91bdb2a2 --- /dev/null +++ b/elasticgraph-health_check/sig/elastic_graph/health_check/health_status.rbs @@ -0,0 +1,71 @@ +module ElasticGraph + module HealthCheck + class HealthStatusSupertype + attr_reader cluster_health_by_name: ::Hash[::String, HealthStatus::ClusterHealth] + attr_reader latest_record_by_type: ::Hash[::String, HealthStatus::LatestRecord?] + attr_reader category: HealthStatus::category + + def initialize: ( + cluster_health_by_name: ::Hash[::String, HealthStatus::ClusterHealth], + latest_record_by_type: ::Hash[::String, HealthStatus::LatestRecord?], + category: HealthStatus::category + ) -> void + + def self.new: ( + cluster_health_by_name: ::Hash[::String, HealthStatus::ClusterHealth], + latest_record_by_type: ::Hash[::String, HealthStatus::LatestRecord?] + ) -> HealthStatus + + def with: ( + ?cluster_health_by_name: ::Hash[::String, HealthStatus::ClusterHealth], + ?latest_record_by_type: ::Hash[::String, HealthStatus::LatestRecord?] + ) -> HealthStatus + end + + class HealthStatus < HealthStatusSupertype + type category = :unhealthy | :healthy | :degraded + + def initialize: ( + cluster_health_by_name: ::Hash[::String, HealthStatus::ClusterHealth], + latest_record_by_type: ::Hash[::String, HealthStatus::LatestRecord?] + ) -> void + + def to_loggable_description: () -> ::String + + private + + def compute_category: ( + ::Hash[::String, HealthStatus::ClusterHealth], + ::Hash[::String, HealthStatus::LatestRecord?] + ) -> category + + class ClusterHealth + attr_reader status: ::String + def self.with: (**untyped) -> ClusterHealth + def to_loggable_description: (::String) -> ::String + def to_h: () -> ::Hash[::Symbol, untyped] + end + + class LatestRecord + attr_reader id: ::String + attr_reader timestamp: ::Time + attr_reader seconds_newer_than_required: ::Float + + def self.new: ( + id: ::String, + timestamp: ::Time, + seconds_newer_than_required: ::Float + ) -> LatestRecord + + def with: ( + ?id: ::String, + ?timestamp: ::Time, + ?seconds_newer_than_required: ::Float + ) -> LatestRecord + + def to_loggable_description: (::String) -> ::String + def too_old?: () -> bool + end + end + end +end diff --git a/elasticgraph-health_check/spec/acceptance/health_checker_spec.rb b/elasticgraph-health_check/spec/acceptance/health_checker_spec.rb new file mode 100644 index 00000000..c9cc7379 --- /dev/null +++ b/elasticgraph-health_check/spec/acceptance/health_checker_spec.rb @@ -0,0 +1,76 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/health_check/health_checker" +require "elastic_graph/support/hash_util" +require "yaml" + +module ElasticGraph + module HealthCheck + RSpec.describe "HealthChecker", :uses_datastore, :factories, :builds_graphql do + let(:now) { ::Time.iso8601("2022-02-14T12:30:00Z") } + let(:graphql) { build_graphql(extension_settings: Support::HashUtil.stringify_keys(extension_settings), clock: class_double(::Time, now: now)) } + let(:health_checker) { HealthChecker.build_from(graphql) } + let(:extension_settings) do + { + health_check: { + clusters_to_consider: ["main", "other2"], + data_recency_checks: { + Widget: { + expected_max_recency_seconds: 300, + timestamp_field: "created_at2" # use a field that has an alternate `name_in_index`. + }, + Component: { + expected_max_recency_seconds: 30, + timestamp_field: "created_at" + } + } + } + } + end + + it "returns health status", :expect_index_exclusions do + index_into( + graphql, + widget = build(:widget, id: "w1", created_at: (now - 20).iso8601), + component = build(:component, id: "c1", created_at: (now - 200).iso8601) + ) + + status = health_checker.check_health + + expect(status).to be_a HealthStatus + expect(status.category).to eq(:degraded) # Component latest record is too old + + expect(status.cluster_health_by_name).to include("main", "other2") + expect(status.cluster_health_by_name["main"]).to be_a(HealthStatus::ClusterHealth).and have_attributes(status: /(green|red|yellow)/) + expect(status.cluster_health_by_name["other2"]).to be_a(HealthStatus::ClusterHealth).and have_attributes(status: /(green|red|yellow)/) + + expect(status.latest_record_by_type).to eq({ + "Widget" => HealthStatus::LatestRecord.new( + id: widget.fetch(:id), + timestamp: ::Time.iso8601(widget.fetch(:created_at)), + seconds_newer_than_required: 280 # 300 - 20 + ), + "Component" => HealthStatus::LatestRecord.new( + id: component.fetch(:id), + timestamp: ::Time.iso8601(component.fetch(:created_at)), + seconds_newer_than_required: -170 # 30 - 200 + ) + }) + + # Verify that our query was optimized to exclude the pre-2022 indices. + expect(indices_excluded_from_searches("main").flatten).to contain_exactly( + "widgets_rollover__before_2019", + "widgets_rollover__2019", + "widgets_rollover__2020", + "widgets_rollover__2021" + ) + end + end + end +end diff --git a/elasticgraph-health_check/spec/spec_helper.rb b/elasticgraph-health_check/spec/spec_helper.rb new file mode 100644 index 00000000..107bf75f --- /dev/null +++ b/elasticgraph-health_check/spec/spec_helper.rb @@ -0,0 +1,11 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-health_check`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. diff --git a/elasticgraph-health_check/spec/unit/elastic_graph/health_check/config_spec.rb b/elasticgraph-health_check/spec/unit/elastic_graph/health_check/config_spec.rb new file mode 100644 index 00000000..6caa7a58 --- /dev/null +++ b/elasticgraph-health_check/spec/unit/elastic_graph/health_check/config_spec.rb @@ -0,0 +1,45 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/health_check/config" +require "yaml" + +module ElasticGraph + module HealthCheck + RSpec.describe Config do + it "builds from parsed YAML correctly" do + parsed_yaml = ::YAML.safe_load(<<~EOS) + clusters_to_consider: [widgets1, components2] + data_recency_checks: + Widget: + expected_max_recency_seconds: 30 + timestamp_field: created_at + EOS + + config = Config.from_parsed_yaml("health_check" => parsed_yaml) + + expect(config).to eq(Config.new( + clusters_to_consider: ["widgets1", "components2"], + data_recency_checks: { + "Widget" => Config::DataRecencyCheck.new( + expected_max_recency_seconds: 30, + timestamp_field: "created_at" + ) + } + )) + end + + it "returns an empty, benign config instance if the config settings have no `health_check` key" do + config = Config.from_parsed_yaml({}) + + expect(config.clusters_to_consider).to be_empty + expect(config.data_recency_checks).to be_empty + end + end + end +end diff --git a/elasticgraph-health_check/spec/unit/elastic_graph/health_check/health_checker_spec.rb b/elasticgraph-health_check/spec/unit/elastic_graph/health_check/health_checker_spec.rb new file mode 100644 index 00000000..375b6528 --- /dev/null +++ b/elasticgraph-health_check/spec/unit/elastic_graph/health_check/health_checker_spec.rb @@ -0,0 +1,507 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/datastore_response/search_response" +require "elastic_graph/health_check/health_checker" +require "elastic_graph/support/hash_util" + +module ElasticGraph + module HealthCheck + RSpec.describe HealthChecker, :capture_logs, :builds_graphql do + let(:cluster_names) { ["main", "other1", "other2"] } + let(:now) { ::Time.iso8601("2022-02-14T12:30:00Z") } + let(:datastore_query_body_by_type) { {} } + + attr_reader :example_datastore_health_response + + before(:context) do + @example_datastore_health_response = { + # Note: we intentionally have a different value for every field here, so that + # our assertions below can demonstrate that the returned values come from the + # datastore response fields. + "cluster_name" => "replace_me", + "status" => "yellow", + "timed_out" => false, + "number_of_nodes" => 1, + "number_of_data_nodes" => 2, + "active_primary_shards" => 3, + "active_shards" => 4, + "relocating_shards" => 5, + "initializing_shards" => 6, + "unassigned_shards" => 7, + "delayed_unassigned_shards" => 8, + "number_of_pending_tasks" => 9, + "number_of_in_flight_fetch" => 10, + "task_max_waiting_in_queue_millis" => 11, + "active_shards_percent_as_number" => 50.0, + "discovered_master" => true + } + + expect(@example_datastore_health_response.keys).to match_array(DATASTORE_CLUSTER_HEALTH_FIELDS.map(&:to_s)) + end + + describe "with valid configuration" do + let(:valid_config) do + Config.new( + clusters_to_consider: ["main", "other1"], + data_recency_checks: { + "Widget" => build_recency_check(timestamp_field: "created_at", expected_max_recency_seconds: 30), + "Component" => build_recency_check(timestamp_field: "created_at", expected_max_recency_seconds: 30) + } + ) + end + + it "queries and returns the cluster health information" do + status = build_health_checker(health_check: valid_config).check_health + + expect(status.cluster_health_by_name.keys).to contain_exactly("main", "other1") + + expect(status.cluster_health_by_name["main"]).to eq HealthStatus::ClusterHealth.new( + **Support::HashUtil.symbolize_keys(example_datastore_health_response).merge( + cluster_name: "main" + ) + ) + + expect(status.cluster_health_by_name["other1"]).to eq HealthStatus::ClusterHealth.new( + **Support::HashUtil.symbolize_keys(example_datastore_health_response).merge( + cluster_name: "other1" + ) + ) + end + + it "ignores extra health status fields returned by the datastore" do + status = build_health_checker(health_check: valid_config) do |health_response| + health_response.merge("extra_field" => "23") + end.check_health + + expect(status.cluster_health_by_name.keys).to contain_exactly("main", "other1") + + expect(status.cluster_health_by_name["main"]).to eq HealthStatus::ClusterHealth.new( + **Support::HashUtil.symbolize_keys(example_datastore_health_response).merge( + cluster_name: "main" + ) + ) + + expect(status.cluster_health_by_name["other1"]).to eq HealthStatus::ClusterHealth.new( + **Support::HashUtil.symbolize_keys(example_datastore_health_response).merge( + cluster_name: "other1" + ) + ) + end + + it "does not fail when the datastore health status response does not include one of our defined fields" do + status = build_health_checker(health_check: valid_config) do |health_response| + health_response.except("number_of_nodes") + end.check_health + + expect(status.cluster_health_by_name.keys).to contain_exactly("main", "other1") + + expect(status.cluster_health_by_name["main"]).to eq HealthStatus::ClusterHealth.new( + **Support::HashUtil.symbolize_keys(example_datastore_health_response).merge( + cluster_name: "main", + number_of_nodes: nil + ) + ) + + expect(status.cluster_health_by_name["other1"]).to eq HealthStatus::ClusterHealth.new( + **Support::HashUtil.symbolize_keys(example_datastore_health_response).merge( + cluster_name: "other1", + number_of_nodes: nil + ) + ) + end + + it "queries and returns the latest record info for the configured data recency checks" do + status = build_health_checker( + health_check: valid_config, + latest: { + "Widget" => {"id" => "w1", "created_at" => (now - 100).iso8601}, + "Component" => {"id" => "c1", "created_at" => (now - 50).iso8601} + } + ).check_health + + expect(status.latest_record_by_type.keys).to contain_exactly("Widget", "Component") + + expect(status.latest_record_by_type["Widget"]).to eq HealthStatus::LatestRecord.new( + id: "w1", + timestamp: now - 100, + seconds_newer_than_required: -70 # 30 - 100 + ) + + expect(status.latest_record_by_type["Component"]).to eq HealthStatus::LatestRecord.new( + id: "c1", + timestamp: now - 50, + seconds_newer_than_required: -20 # 30 - 50 + ) + end + + it "returns `nil` for a type's latest record if there is no data in that type's index" do + status = build_health_checker( + health_check: valid_config, + latest: {} + ).check_health + + expect(status.latest_record_by_type).to eq("Widget" => nil, "Component" => nil) + end + + it "only ever asks for one record" do + build_health_checker(health_check: valid_config).check_health + + # Note: while we build the query to only ask for 1, the current implementation of DatastoreQuery + # will ask for one more in order to implement `has_previous_page`/`has_next_page`. There's an + # unimplemented optimization to not ask for one more when the client hasn't asked for those fields. + # Here we want to tolerate that optimization being present or not so we allow the size to be 1 or 2. + expect(datastore_query_body_by_type.fetch("Widget")["size"]).to eq(1).or eq(2) + expect(datastore_query_body_by_type.fetch("Component")["size"]).to eq(1).or eq(2) + end + + it "sorts by the timestamp field descending in order to get the latest record" do + build_health_checker(health_check: valid_config).check_health + + expect(datastore_query_body_by_type.fetch("Widget")["sort"]).to start_with("created_at" => a_hash_including({"order" => "desc"})) + expect(datastore_query_body_by_type.fetch("Component")["sort"]).to start_with("created_at" => a_hash_including({"order" => "desc"})) + end + + it "only requests the timestamp field for optimal queries" do + build_health_checker(health_check: valid_config).check_health + + expect(datastore_query_body_by_type.fetch("Widget")["_source"]).to eq("includes" => ["created_at"]) + expect(datastore_query_body_by_type.fetch("Component")["_source"]).to eq("includes" => ["created_at"]) + end + + it "filters by the timestamp field on all types regardless of whether it is a rollover index or not, to optimize the query" do + build_health_checker(health_check: valid_config).check_health + + expect(datastore_query_body_by_type.fetch("Widget")["query"]).to match({ + "bool" => {"filter" => [ + {"range" => {"created_at" => {"gte" => a_value < (now - 100).iso8601}}} + ]} + }) + + expect(datastore_query_body_by_type.fetch("Component")["query"]).to match({ + "bool" => {"filter" => [ + {"range" => {"created_at" => {"gte" => a_value < (now - 100).iso8601}}} + ]} + }) + end + + context "when a configured `timestamp_field` has a different `name_in_index`" do + it "correctly uses the `name_in_index` for all index field references" do + config = valid_config.with(data_recency_checks: { + # created_at2 is defined in config/schema.rb with `name_in_index: "created_at" + "Widget" => build_recency_check(timestamp_field: "created_at2") + }) + + status = build_health_checker( + health_check: config, + latest: {"Widget" => {"id" => "w1", "created_at" => (now - 100).iso8601}} + ).check_health + + expect(status.latest_record_by_type).to eq("Widget" => HealthStatus::LatestRecord.new( + id: "w1", + timestamp: now - 100, + seconds_newer_than_required: -70 # 30 - 100 + )) + + query_body = datastore_query_body_by_type.fetch("Widget") + expect(query_body["sort"]).to start_with("created_at" => a_hash_including({"order" => "desc"})) + expect(query_body["_source"]).to eq("includes" => ["created_at"]) + expect(query_body["query"]).to match({ + "bool" => {"filter" => [ + {"range" => {"created_at" => {"gte" => a_value < (now - 100).iso8601}}} + ]} + }) + end + end + end + + describe "config validation" do + it "raises a clear error when instantiated with a `clusters_to_consider` option referencing a cluster that does not exist" do + expect { + build_health_checker(health_check: Config.new( + clusters_to_consider: ["main", "other2", "other3", "other4"], + data_recency_checks: {} + )) + }.to raise_error Errors::ConfigError, a_string_including( + "clusters_to_consider", + "unrecognized cluster names", + "other3", "other4" + ).and(excluding("main", "other2")) + end + + context "with subset of clusters available for querying" do + let(:cluster_names) { ["other2"] } + + # The situation here is that only "other2" is an accessible cluster. The healthcheck is configured to check "main". The + # Widget type references "main" as an "query" cluster - health_checker should not claim "main" is unrecognized. + it "uses query clusters on indexing definitions for recognizing clusters" do + expect { + health_checker = build_health_checker( + health_check: Config.new( + clusters_to_consider: ["main"], + data_recency_checks: { + "Widget" => build_recency_check(timestamp_field: "created_at") + } + ), + index_definitions: { + "widgets" => config_index_def_of(query_cluster: "main", index_into_clusters: ["other3"]) + }, + schema_definition: lambda do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime!" + t.index "widgets" do |i| + i.default_sort "created_at", :desc + end + end + end + ) + + status = health_checker.check_health + expect(status.cluster_health_by_name.keys).to be_empty # Can't check cluster health, as "main" not available + expect(status.latest_record_by_type.keys).to be_empty # .. and can't do recency check, as `widgets` relies on main. + }.to log_warning a_string_including("1 type(s) were unavailable for health-checking", "Widget", + "1 cluster(s) were unavailable for health-checking", "main") + end + + # The state here is only "other2" is an accessible cluster. The cluster health check is configured to check "main". The + # Widget type references "main" as an "index_into" cluster - health_checker should not claim "main" is unrecognized. + it "uses index_into clusters on indexing definitions for recognizing clusters" do + expect { + health_checker = build_health_checker( + health_check: Config.new( + clusters_to_consider: ["main"], + data_recency_checks: { + "Widget" => build_recency_check(timestamp_field: "created_at") + } + ), + index_definitions: { + "widgets" => config_index_def_of(query_cluster: "other3", index_into_clusters: ["main"]) + }, + schema_definition: lambda do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime!" + t.index "widgets" do |i| + i.default_sort "created_at", :desc + end + end + end + ) + + status = health_checker.check_health + expect(status.cluster_health_by_name.keys).to be_empty # Can't check cluster health, as "main" not available + expect(status.latest_record_by_type.keys).to be_empty # .. and can't do recency check, as `widgets` relies on `other3`, which is not available. + }.to log_warning a_string_including("1 type(s) were unavailable for health-checking", "Widget", + "1 cluster(s) were unavailable for health-checking", "main") + end + + it "ignores datastore health checks for clusters not available for querying" do + expect { + health_checker = build_health_checker( + health_check: Config.new( + clusters_to_consider: ["main", "other2"], # "main" is not available, only other2 is available. + data_recency_checks: {} + ) + ) + + status = health_checker.check_health + expect(status.latest_record_by_type.keys).to be_empty # No recency checks. + expect(status.cluster_health_by_name.keys).to contain_exactly("other2") # Just "other2", not "main" + }.to log_warning a_string_including("1 cluster(s) were unavailable for health-checking", "main") + end + + it "ignores recency health checks for types whose query clusters are not available for querying" do + # The state here is that the "Widget" type is backed by the "widgets" index, which depends on the "main" datastore cluster for querying. + # However, the graphql endpoint only has access to the "other2" cluster (see the `cluster_names` override above). So the recency health check + # for "Widget" should be skipped, but a warning should be logged. + # + # Separately, there is the "Component" type - which is backed by the "other2" cluster for queries - because that is available, it *should* be type-checked. + expect { + health_checker = build_health_checker( + health_check: Config.new( + clusters_to_consider: [], + data_recency_checks: { + "Widget" => build_recency_check(timestamp_field: "created_at"), + "Component" => build_recency_check(timestamp_field: "created_at") + } + ), + index_definitions: { + # The "query" cluster should be used for determining recency health check eligibility, not the indexing clusters. So to check that, only "other2" is made available, + # and so only the Component health check should be used. + "widgets" => config_index_def_of(query_cluster: "main", index_into_clusters: ["other2"]), + "components" => config_index_def_of(query_cluster: "other2", index_into_clusters: ["other2"]) + }, + schema_definition: lambda do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime!" + t.index "widgets" do |i| + i.default_sort "created_at", :desc + end + end + + schema.object_type "Component" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime!" + t.index "components" do |i| + i.default_sort "created_at", :desc + end + end + end + ) + + status = health_checker.check_health + expect(status.latest_record_by_type.keys).to contain_exactly("Component") # Widget should not be present, since its query datastore clusters can be accessed. + }.to log_warning a_string_including("1 type(s) were unavailable for health-checking", "Widget") + end + end + + it "raises a clear error when instantiated with `data_recency_checks` that name unknown types" do + expect { + build_health_checker(health_check: Config.new( + clusters_to_consider: ["main"], + data_recency_checks: { + "UnknownType1" => build_recency_check, + "UnknownType2" => build_recency_check, + "Widget" => build_recency_check + } + )) + }.to raise_error Errors::ConfigError, a_string_including( + "data_recency_checks", + "not recognized indexed types", + "UnknownType1", "UnknownType2" + ).and(excluding("Widget")) + end + + it "raises a clear error when instantiated with `data_recency_checks` that name types that are not indexed" do + expect { + build_health_checker(health_check: Config.new( + clusters_to_consider: ["main"], + data_recency_checks: { + "Component" => build_recency_check, + # Color is an enum type + "Color" => build_recency_check, + # DateTime is a scalar type + "DateTime" => build_recency_check, + # WidgetOptions is an embedded object type + "WidgetOptions" => build_recency_check + } + )) + }.to raise_error Errors::ConfigError, a_string_including( + "data_recency_checks", + "not recognized indexed types", + "Color", "DateTime", "WidgetOptions" + ).and(excluding("Component")) + end + + it "raises a clear error when instantiated with a data recency check timestamp field that does not exist" do + expect { + build_health_checker(health_check: Config.new( + clusters_to_consider: ["main"], + data_recency_checks: { + "Widget" => build_recency_check(timestamp_field: "created_at"), + "Component" => build_recency_check(timestamp_field: "generated_at") + } + )) + }.to raise_error Errors::ConfigError, a_string_including( + "data_recency_checks", + "invalid timestamp fields", + "Component", "generated_at" + ).and(excluding("Widget", "created_at")) + end + + it "raises a clear error when instantiated with a data recency check timestamp field that does not exist" do + expect { + build_health_checker(health_check: Config.new( + clusters_to_consider: ["main"], + data_recency_checks: { + "Widget" => build_recency_check(timestamp_field: "created_at"), + "Component" => build_recency_check(timestamp_field: "name") + } + )) + }.to raise_error Errors::ConfigError, a_string_including( + "data_recency_checks", + "invalid timestamp fields", + "Component", "name" + ).and(excluding("Widget", "created_at")) + end + end + + def build_health_checker(health_check:, latest: {}, index_definitions: nil, schema_definition: nil, &customize_health) + datastore_clients_by_name = cluster_names.to_h do |name| + [name, build_fake_datastore_client(name, latest, &customize_health)] + end + + health_check_settings = Support::HashUtil.stringify_keys(health_check.to_h.merge( + data_recency_checks: health_check.data_recency_checks.transform_values(&:to_h) + )) + + graphql = build_graphql( + clients_by_name: datastore_clients_by_name, + clock: class_double(::Time, now: now), + schema_definition: schema_definition, + index_definitions: index_definitions, + extension_settings: {"health_check" => health_check_settings} + ) + + HealthChecker.build_from(graphql) + end + + def build_fake_datastore_client(name, latest_records_by_type, &customize_health) + cluster_health = example_datastore_health_response.merge("cluster_name" => name) + cluster_health = customize_health.call(cluster_health) if customize_health + + stubbed_datastore_client(get_cluster_health: cluster_health).tap do |client| + allow(client).to receive(:msearch) do |request| + build_msearch_response(request, latest_records_by_type) + end + end + end + + def build_msearch_response(request, latest_record_by_type) + # Our query logic generates a payload with a mixture of string and symbol keys + # (it doesn't matter to the datastore since it serializes in JSON the same). + # Here we do not want to be mix and match (or be coupled to the current key form + # being used) so we normalize to string keys here. + normalized_request = Support::HashUtil.stringify_keys(request) + + responses = normalized_request.fetch("body").each_slice(2).map do |(headers, body)| + type = + case headers.fetch("index") + when /widget/ then "Widget" + when /component/ then "Component" + # :nocov: -- this `else` case is only hit when tests here have bugs + else raise "Unknown index: #{headers.fetch("index")}" + # :nocov: + end + + datastore_query_body_by_type[type] = body + + if (latest_values = latest_record_by_type[type]) + Support::HashUtil.deep_merge( + GraphQL::DatastoreResponse::SearchResponse::RAW_EMPTY, + {"hits" => {"hits" => [{"_id" => latest_values.fetch("id"), "_source" => latest_values}]}} + ) + else + GraphQL::DatastoreResponse::SearchResponse::RAW_EMPTY + end + end + + {"responses" => responses} + end + + def build_recency_check(expected_max_recency_seconds: 30, timestamp_field: "created_at") + Config::DataRecencyCheck.new( + expected_max_recency_seconds: expected_max_recency_seconds, + timestamp_field: timestamp_field + ) + end + end + end +end diff --git a/elasticgraph-health_check/spec/unit/elastic_graph/health_check/health_status_spec.rb b/elasticgraph-health_check/spec/unit/elastic_graph/health_check/health_status_spec.rb new file mode 100644 index 00000000..0d7f54bd --- /dev/null +++ b/elasticgraph-health_check/spec/unit/elastic_graph/health_check/health_status_spec.rb @@ -0,0 +1,274 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/health_check/health_status" +require "time" + +module ElasticGraph + module HealthCheck + RSpec.describe HealthStatus do + let(:new_enough_latest_record) { HealthStatus::LatestRecord.new("abc", ::Time.iso8601("2022-03-12T12:30:00Z"), 10) } + let(:too_old_latest_record) { HealthStatus::LatestRecord.new("abc", ::Time.iso8601("2022-03-12T12:30:00Z"), -12) } + let(:exact_moment_latest_record) { HealthStatus::LatestRecord.new("abc", ::Time.iso8601("2022-03-12T12:30:00Z"), 0) } + + let(:example_cluster_health) do + HealthStatus::ClusterHealth.new( + cluster_name: "my_cluster", + status: "green", + timed_out: false, + number_of_nodes: 1, + number_of_data_nodes: 2, + active_primary_shards: 3, + active_shards: 4, + relocating_shards: 5, + initializing_shards: 6, + unassigned_shards: 7, + delayed_unassigned_shards: 8, + number_of_pending_tasks: 9, + number_of_in_flight_fetch: 10, + task_max_waiting_in_queue_millis: 11, + active_shards_percent_as_number: 50.0, + discovered_master: true + ) + end + + describe "#category" do + context "when the status is empty (as happens when the health check is not configured)" do + it "returns :healthy as the safest default return value" do + status = HealthStatus.new(cluster_health_by_name: {}, latest_record_by_type: {}) + + expect(status.category).to eq :healthy + end + end + + context "when one of the clusters has a red status" do + it "returns :unhealthy, regardless of other clusters or latest records" do + status = HealthStatus.new( + cluster_health_by_name: { + "main" => example_cluster_health.with(status: "green"), + "other1" => example_cluster_health.with(status: "red"), + "other2" => example_cluster_health.with(status: "yellow") + }, + latest_record_by_type: { + "Widget" => new_enough_latest_record + } + ) + + expect(status.category).to eq :unhealthy + end + end + + context "when one of the clusters has a yellow status (and no clusters have a red status)" do + it "returns :degraded, regardless of other clusters or latest records" do + status = HealthStatus.new( + cluster_health_by_name: { + "main" => example_cluster_health.with(status: "green"), + "other1" => example_cluster_health.with(status: "yellow") + }, + latest_record_by_type: { + "Widget" => new_enough_latest_record + } + ) + + expect(status.category).to eq :degraded + end + end + + context "when one of the latest records is older than expected" do + it "returns :degraded, regardless of other latest records, so long as no clusters are red" do + status = HealthStatus.new( + cluster_health_by_name: { + "main" => example_cluster_health.with(status: "green"), + "other1" => example_cluster_health.with(status: "green") + }, + latest_record_by_type: { + "Widget" => too_old_latest_record, + "Component" => new_enough_latest_record + } + ) + + expect(status.category).to eq :degraded + end + end + + context "when one of the latest records is nil" do + it "returns :degraded, regardless of other latest records, so long as no clusters are red" do + status = HealthStatus.new( + cluster_health_by_name: { + "main" => example_cluster_health.with(status: "green"), + "other1" => example_cluster_health.with(status: "green") + }, + latest_record_by_type: { + "Widget" => nil, + "Component" => new_enough_latest_record + } + ) + + expect(status.category).to eq :degraded + end + end + + context "when all the clusters green and all the latest records are new enough" do + it "returns :healthy" do + status = HealthStatus.new( + cluster_health_by_name: { + "main" => example_cluster_health.with(status: "green"), + "other1" => example_cluster_health.with(status: "green") + }, + latest_record_by_type: { + "Widget" => new_enough_latest_record, + "Component" => new_enough_latest_record + } + ) + + expect(status.category).to eq :healthy + end + + it "considers a record that's exactly as old as the threshold to be new enough" do + status = HealthStatus.new( + cluster_health_by_name: { + "main" => example_cluster_health.with(status: "green"), + "other1" => example_cluster_health.with(status: "green") + }, + latest_record_by_type: { + "Widget" => new_enough_latest_record, + "Component" => exact_moment_latest_record + } + ) + + expect(status.category).to eq :healthy + end + end + + describe "#to_loggable_description" do + it "generates a readable description of the status details" do + status = HealthStatus.new( + cluster_health_by_name: { + "main" => example_cluster_health.with(status: "green"), + "other1" => example_cluster_health.with(status: "yellow") + }, + latest_record_by_type: { + "Widget" => new_enough_latest_record, + "Component" => exact_moment_latest_record + } + ) + + expect(status.to_loggable_description).to eq(<<~EOS.strip) + HealthStatus: degraded (checked 2 clusters, 2 latest records) + - Latest Component (recent enough): abc / 2022-03-12T12:30:00Z (0s newer than required) + - Latest Widget (recent enough): abc / 2022-03-12T12:30:00Z (10s newer than required) + + - main cluster health (green): + cluster_name: "my_cluster" + status: "green" + timed_out: false + number_of_nodes: 1 + number_of_data_nodes: 2 + active_primary_shards: 3 + active_shards: 4 + relocating_shards: 5 + initializing_shards: 6 + unassigned_shards: 7 + delayed_unassigned_shards: 8 + number_of_pending_tasks: 9 + number_of_in_flight_fetch: 10 + task_max_waiting_in_queue_millis: 11 + active_shards_percent_as_number: 50.0 + discovered_master: true + + - other1 cluster health (yellow): + cluster_name: "my_cluster" + status: "yellow" + timed_out: false + number_of_nodes: 1 + number_of_data_nodes: 2 + active_primary_shards: 3 + active_shards: 4 + relocating_shards: 5 + initializing_shards: 6 + unassigned_shards: 7 + delayed_unassigned_shards: 8 + number_of_pending_tasks: 9 + number_of_in_flight_fetch: 10 + task_max_waiting_in_queue_millis: 11 + active_shards_percent_as_number: 50.0 + discovered_master: true + EOS + end + + it "distinguishes between records that are too old, recent enough, or missing (and can skip the cluster health info if not available)" do + status = HealthStatus.new( + cluster_health_by_name: {}, + latest_record_by_type: { + "Widget" => new_enough_latest_record, + "Component" => too_old_latest_record, + "Part" => nil + } + ) + + expect(status.to_loggable_description).to eq(<<~EOS.strip) + HealthStatus: degraded (checked 0 clusters, 3 latest records) + - Latest Component (too old): abc / 2022-03-12T12:30:00Z (12s too old) + - Latest Part (missing) + - Latest Widget (recent enough): abc / 2022-03-12T12:30:00Z (10s newer than required) + EOS + end + + it "can skip the latest record info if not available" do + status = HealthStatus.new( + cluster_health_by_name: { + "main" => example_cluster_health.with(status: "green"), + "other1" => example_cluster_health.with(status: "yellow") + }, + latest_record_by_type: {} + ) + + expect(status.to_loggable_description).to eq(<<~EOS.strip) + HealthStatus: degraded (checked 2 clusters, 0 latest records) + - main cluster health (green): + cluster_name: "my_cluster" + status: "green" + timed_out: false + number_of_nodes: 1 + number_of_data_nodes: 2 + active_primary_shards: 3 + active_shards: 4 + relocating_shards: 5 + initializing_shards: 6 + unassigned_shards: 7 + delayed_unassigned_shards: 8 + number_of_pending_tasks: 9 + number_of_in_flight_fetch: 10 + task_max_waiting_in_queue_millis: 11 + active_shards_percent_as_number: 50.0 + discovered_master: true + + - other1 cluster health (yellow): + cluster_name: "my_cluster" + status: "yellow" + timed_out: false + number_of_nodes: 1 + number_of_data_nodes: 2 + active_primary_shards: 3 + active_shards: 4 + relocating_shards: 5 + initializing_shards: 6 + unassigned_shards: 7 + delayed_unassigned_shards: 8 + number_of_pending_tasks: 9 + number_of_in_flight_fetch: 10 + task_max_waiting_in_queue_millis: 11 + active_shards_percent_as_number: 50.0 + discovered_master: true + EOS + end + end + end + end + end +end diff --git a/elasticgraph-health_check/spec/unit/envoy_extension_spec.rb b/elasticgraph-health_check/spec/unit/envoy_extension_spec.rb new file mode 100644 index 00000000..cb3cc289 --- /dev/null +++ b/elasticgraph-health_check/spec/unit/envoy_extension_spec.rb @@ -0,0 +1,169 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/health_check/envoy_extension" + +module ElasticGraph + module HealthCheck + RSpec.describe EnvoyExtension, :builds_graphql, :capture_logs do + shared_examples_for "a health check endpoint" do + let(:processed_graphql_queries) { [] } + let(:health_status_log_line) { "HealthStatus: " } + + context "on a GET request to the health check HTTP path" do + let(:degraded_header) { "x-envoy-degraded" } + + it "checks health and returns a 200 if healthy" do + response = process(:get, "/health", with_configured_path_segment: "/health", cluster_status: "green") + + expect(response.status_code).to eq 200 + expect(response.body).to eq "Healthy!" + expect(response.headers.keys).to exclude(degraded_header) + + expect(processed_graphql_queries).to be_empty + expect(logged_output).to include(health_status_log_line) + end + + it "checks health and returns a 500 if unhealthy" do + response = process(:get, "/health", with_configured_path_segment: "/health", cluster_status: "red") + + expect(response.status_code).to eq 500 + expect(response.body).to eq "Unhealthy!" + expect(response.headers.keys).to exclude(degraded_header) + + expect(processed_graphql_queries).to be_empty + expect(logged_output).to include(health_status_log_line) + end + + it "checks health and returns a 200 with the degraded header if degraded" do + response = process(:get, "/health", with_configured_path_segment: "/health", cluster_status: "yellow") + + expect(response.status_code).to eq 200 + expect(response.body).to eq "Degraded." + expect(response.headers).to include(degraded_header => "true") + + expect(processed_graphql_queries).to be_empty + expect(logged_output).to include(health_status_log_line) + end + end + + it "processes the request as GraphQL for a non-GET request to the health check path" do + response = process(:post, "/health", body: "query { __typename }", with_configured_path_segment: "/health") + + expect(response.body).to eq %({"data":{}}) + expect(processed_graphql_queries).to contain_exactly("query { __typename }") + expect(logged_output).to exclude(health_status_log_line) + end + + it "processes the request as GraphQL for a GET request to a path that contains the specified segment within a larger segment" do + response = process(:get, "/foo/health_foo?#{URI.encode_www_form("query" => "query { __typename }")}", with_configured_path_segment: "/health") + + expect(response.body).to eq %({"data":{}}) + expect(processed_graphql_queries).to contain_exactly("query { __typename }") + expect(logged_output).to exclude(health_status_log_line) + end + + it "processes the request as a health check for a GET request to a path that has the configured segment as one of its segments" do + response = process(:get, "/foo/health/bar?#{URI.encode_www_form("query" => "query { __typename }")}", with_configured_path_segment: "/health") + + expect(response.status_code).to eq 200 + expect(response.body).to eq "Healthy!" + + expect(processed_graphql_queries).to be_empty + expect(logged_output).to include(health_status_log_line) + end + + it "ignores a trailing `/` when determining if a path segment matches" do + response = process(:get, "/foo/health/bar?#{URI.encode_www_form("query" => "query { __typename }")}", with_configured_path_segment: "health/") + + expect(response.status_code).to eq 200 + expect(response.body).to eq "Healthy!" + + expect(processed_graphql_queries).to be_empty + expect(logged_output).to include(health_status_log_line) + end + + it "raises an error if the `http_path_segment` is not configured" do + expect { + process(:get, "/health", with_configured_path_segment: nil) + }.to raise_error Errors::ConfigSettingNotSetError, a_string_including("Health check `http_path_segment` is not configured") + end + + def process(http_method, url, with_configured_path_segment:, body: nil, cluster_status: nil) + graphql = build_graphql_for_path(with_configured_path_segment) + + status = HealthCheck::HealthStatus.new( + cluster_health_by_name: { + "main" => HealthCheck::HealthStatus::ClusterHealth.new( + cluster_name: "my_cluster", + status: cluster_status, + timed_out: false, + number_of_nodes: 1, + number_of_data_nodes: 2, + active_primary_shards: 3, + active_shards: 4, + relocating_shards: 5, + initializing_shards: 6, + unassigned_shards: 7, + delayed_unassigned_shards: 8, + number_of_pending_tasks: 9, + number_of_in_flight_fetch: 10, + task_max_waiting_in_queue_millis: 11, + active_shards_percent_as_number: 50.0, + discovered_master: true + ) + }, + latest_record_by_type: {} + ) + + health_checker = instance_double(HealthCheck::HealthChecker, check_health: status) + allow(HealthCheck::HealthChecker).to receive(:build_from).with(graphql).and_return(health_checker) + + allow(graphql.graphql_query_executor).to receive(:execute) do |query_string, **options| + processed_graphql_queries << query_string + {"data" => {}} + end + + request = GraphQL::HTTPRequest.new( + http_method: http_method, + url: url, + headers: {"Content-Type" => "application/graphql"}, + body: body + ) + + graphql.graphql_http_endpoint.process(request) + end + end + + context "when enabled via `register_graphql_extension`" do + include_context "a health check endpoint" + + def build_graphql_for_path(http_path_segment) + config = {http_path_segment: http_path_segment}.compact + schema_artifacts = generate_schema_artifacts do |schema| + schema.register_graphql_extension(EnvoyExtension, defined_at: "elastic_graph/health_check/envoy_extension", **config) + end + + build_graphql(schema_artifacts: schema_artifacts) + end + end + + context "when enabled via YAML config" do + include_context "a health check endpoint" + + def build_graphql_for_path(http_path_segment) + build_graphql(extension_modules: [EnvoyExtension], extension_settings: {"health_check" => { + "clusters_to_consider" => [], + "data_recency_checks" => {}, + "http_path_segment" => http_path_segment + }.compact}) + end + end + end + end +end diff --git a/elasticgraph-indexer/.rspec b/elasticgraph-indexer/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-indexer/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-indexer/.yardopts b/elasticgraph-indexer/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-indexer/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-indexer/Gemfile b/elasticgraph-indexer/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-indexer/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-indexer/LICENSE.txt b/elasticgraph-indexer/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-indexer/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-indexer/README.md b/elasticgraph-indexer/README.md new file mode 100644 index 00000000..0ead491a --- /dev/null +++ b/elasticgraph-indexer/README.md @@ -0,0 +1 @@ +# ElasticGraph::Indexer diff --git a/elasticgraph-indexer/elasticgraph-indexer.gemspec b/elasticgraph-indexer/elasticgraph-indexer.gemspec new file mode 100644 index 00000000..2cb304b0 --- /dev/null +++ b/elasticgraph-indexer/elasticgraph-indexer.gemspec @@ -0,0 +1,24 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :core) do |spec, eg_version| + spec.summary = "ElasticGraph gem that provides APIs to robustly index data into a datastore." + + spec.add_dependency "elasticgraph-datastore_core", eg_version + spec.add_dependency "elasticgraph-json_schema", eg_version + spec.add_dependency "elasticgraph-schema_artifacts", eg_version + spec.add_dependency "elasticgraph-support", eg_version + spec.add_dependency "hashdiff", "~> 1.1" + + spec.add_development_dependency "elasticgraph-admin", eg_version + spec.add_development_dependency "elasticgraph-elasticsearch", eg_version + spec.add_development_dependency "elasticgraph-opensearch", eg_version + spec.add_development_dependency "elasticgraph-schema_definition", eg_version +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer.rb b/elasticgraph-indexer/lib/elastic_graph/indexer.rb new file mode 100644 index 00000000..e483ce8d --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer.rb @@ -0,0 +1,98 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core" +require "elastic_graph/indexer/config" +require "elastic_graph/support/from_yaml_file" + +module ElasticGraph + class Indexer + extend Support::FromYamlFile + + # @dynamic config, datastore_core, schema_artifacts, logger + attr_reader :config, :datastore_core, :schema_artifacts, :logger + + # A factory method that builds an Indexer instance from the given parsed YAML config. + # `from_yaml_file(file_name, &block)` is also available (via `Support::FromYamlFile`). + def self.from_parsed_yaml(parsed_yaml, &datastore_client_customization_block) + new( + config: Indexer::Config.from_parsed_yaml(parsed_yaml), + datastore_core: DatastoreCore.from_parsed_yaml(parsed_yaml, for_context: :indexer, &datastore_client_customization_block) + ) + end + + def initialize( + config:, + datastore_core:, + datastore_router: nil, + monotonic_clock: nil, + clock: nil + ) + @config = config + @datastore_core = datastore_core + @logger = datastore_core.logger + @datastore_router = datastore_router + @schema_artifacts = @datastore_core.schema_artifacts + @monotonic_clock = monotonic_clock + @clock = clock || ::Time + end + + def datastore_router + @datastore_router ||= begin + require "elastic_graph/indexer/datastore_indexing_router" + DatastoreIndexingRouter.new( + datastore_clients_by_name: datastore_core.clients_by_name, + mappings_by_index_def_name: schema_artifacts.index_mappings_by_index_def_name, + monotonic_clock: monotonic_clock, + logger: datastore_core.logger + ) + end + end + + def record_preparer_factory + @record_preparer_factory ||= begin + require "elastic_graph/indexer/record_preparer" + RecordPreparer::Factory.new(schema_artifacts) + end + end + + def processor + @processor ||= begin + require "elastic_graph/indexer/processor" + Processor.new( + datastore_router: datastore_router, + operation_factory: operation_factory, + indexing_latency_slo_thresholds_by_timestamp_in_ms: config.latency_slo_thresholds_by_timestamp_in_ms, + clock: @clock, + logger: datastore_core.logger + ) + end + end + + def operation_factory + @operation_factory ||= begin + require "elastic_graph/indexer/operation/factory" + Operation::Factory.new( + schema_artifacts: schema_artifacts, + index_definitions_by_graphql_type: datastore_core.index_definitions_by_graphql_type, + record_preparer_factory: record_preparer_factory, + logger: datastore_core.logger, + skip_derived_indexing_type_updates: config.skip_derived_indexing_type_updates, + configure_record_validator: nil + ) + end + end + + def monotonic_clock + @monotonic_clock ||= begin + require "elastic_graph/support/monotonic_clock" + Support::MonotonicClock.new + end + end + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/config.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/config.rb new file mode 100644 index 00000000..9d16078c --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/config.rb @@ -0,0 +1,48 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/indexer/event_id" + +module ElasticGraph + class Indexer + class Config < ::Data.define( + # Map of indexing latency thresholds (in milliseconds), keyed by the name of + # the indexing latency metric. When an event is indexed with an indexing latency + # exceeding the threshold, a warning with the event type, id, and version will + # be logged, so the issue can be investigated. + :latency_slo_thresholds_by_timestamp_in_ms, + # Setting that can be used to specify some derived indexing type updates that should be skipped. This + # setting should be a map keyed by the name of the derived indexing type, and the values should be sets + # of ids. This can be useful when you have a "hot spot" of a single derived document that is + # receiving a ton of updates. During a backfill (or whatever) you may want to skip the derived + # type updates. + :skip_derived_indexing_type_updates + ) + def self.from_parsed_yaml(hash) + hash = hash.fetch("indexer") + extra_keys = hash.keys - EXPECTED_KEYS + + unless extra_keys.empty? + raise Errors::ConfigError, "Unknown `indexer` config settings: #{extra_keys.join(", ")}" + end + + new( + latency_slo_thresholds_by_timestamp_in_ms: hash.fetch("latency_slo_thresholds_by_timestamp_in_ms"), + skip_derived_indexing_type_updates: (hash["skip_derived_indexing_type_updates"] || {}).transform_values(&:to_set) + ) + end + + EXPECTED_KEYS = members.map(&:to_s) + end + + # Steep weirdly expects them here... + # @dynamic initialize, config, datastore_core, schema_artifacts, datastore_router + # @dynamic record_preparer, processor, operation_factory + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/datastore_indexing_router.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/datastore_indexing_router.rb new file mode 100644 index 00000000..91247409 --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/datastore_indexing_router.rb @@ -0,0 +1,408 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/errors" +require "elastic_graph/datastore_core/index_config_normalizer" +require "elastic_graph/indexer/event_id" +require "elastic_graph/indexer/hash_differ" +require "elastic_graph/indexer/indexing_failures_error" +require "elastic_graph/support/threading" + +module ElasticGraph + class Indexer + # Responsible for routing datastore indexing requests to the appropriate cluster and index. + class DatastoreIndexingRouter + # In this class, we internally cache the datastore mapping for an index definition, so that we don't have to + # fetch the mapping from the datastore on each call to `bulk`. It rarely changes and ElasticGraph is designed so that + # mapping updates are applied before deploying the indexer with a new mapping. + # + # However, if an engineer forgets to apply a mapping update before deploying, they'll run into "mappings are incomplete" + # errors. They can updated the mapping to fix it, but the use of caching in this class could mean that the fix doesn't + # necessarily work right away. The app would have to be deployed or restarted so that the caches are cleared. That could + # be annoying. + # + # To address this issue, we're adding an expiration on the caching of the index mappings. Re-fetching the index + # mapping once every few minutes is no big deal and will allow the indexer to recover on its own after a mapping + # update has been applied without requiring a deploy or a restart. + # + # The expiration is a range so that, when we have many processes running, and they all started around the same time, + # (say, after a deploy!), they don't all expire their caches in sync, leading to spiky load on the datastore. Instead, + # the random distribution of expiration times will spread out the load. + MAPPING_CACHE_MAX_AGE_IN_MS_RANGE = (5 * 60 * 1000)..(10 * 60 * 1000) + + def initialize( + datastore_clients_by_name:, + mappings_by_index_def_name:, + monotonic_clock:, + logger: + ) + @datastore_clients_by_name = datastore_clients_by_name + @logger = logger + @monotonic_clock = monotonic_clock + @cached_mappings = {} + + @mappings_by_index_def_name = mappings_by_index_def_name.transform_values do |mappings| + DatastoreCore::IndexConfigNormalizer.normalize_mappings(mappings) + end + end + + # Proxies `client#bulk` by converting `operations` to their bulk + # form. Returns a hash between a cluster and a list of successfully applied operations on that cluster. + # + # For each operation, 1 of 4 things will happen, each of which will be treated differently: + # + # 1. The operation was successfully applied to the datastore and updated its state. + # The operation will be included in the successful operation of the returned result. + # 2. The operation could not even be attempted. For example, an `Update` operation + # cannot be attempted when the source event has `nil` for the field used as the source of + # the destination type's id. The returned result will not include this operation. + # 3. The operation was a no-op due to the external version not increasing. This happens when we + # process a duplicate or out-of-order event. The operation will be included in the returned + # result's list of noop results. + # 4. The operation failed outright for some other reason. The operation will be included in the + # returned result's failure results. + # + # It is the caller's responsibility to deal with any returned failures as this method does not + # raise an exception in that case. + # + # Note: before any operations are attempted, the datastore indices are validated for consistency + # with the mappings we expect, meaning that no bulk operations will be attempted if that is not up-to-date. + def bulk(operations, refresh: false) + # Before writing these operations, verify their destination index mapping are consistent. + validate_mapping_completeness_of!(:accessible_cluster_names_to_index_into, *operations.map(&:destination_index_def).uniq) + + # @type var ops_by_client: ::Hash[DatastoreCore::_Client, ::Array[_Operation]] + ops_by_client = ::Hash.new { |h, k| h[k] = [] } + # @type var unsupported_ops: ::Set[_Operation] + unsupported_ops = ::Set.new + + operations.reject { |op| op.to_datastore_bulk.empty? }.each do |op| + # Note: this intentionally does not use `accessible_cluster_names_to_index_into`. + # We want to fail with clear error if any clusters are inaccessible instead of silently ignoring + # the named cluster. The `IndexingFailuresError` provides a clear error. + cluster_names = op.destination_index_def.clusters_to_index_into + + cluster_names.each do |cluster_name| + if (client = @datastore_clients_by_name[cluster_name]) + ops_by_client[client] << op + else + unsupported_ops << op + end + end + + unsupported_ops << op if cluster_names.empty? + end + + unless unsupported_ops.empty? + raise IndexingFailuresError, + "The index definitions for #{unsupported_ops.size} operations " \ + "(#{unsupported_ops.map { |o| Indexer::EventID.from_event(o.event) }.join(", ")}) " \ + "were configured to be inaccessible. Check the configuration, or avoid sending " \ + "events of this type to this ElasticGraph indexer." + end + + ops_and_results_by_cluster = Support::Threading.parallel_map(ops_by_client) do |(client, ops)| + responses = client.bulk(body: ops.flat_map(&:to_datastore_bulk), refresh: refresh).fetch("items") + + # As per https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html#bulk-api-response-body, + # > `items` contains the result of each operation in the bulk request, in the order they were submitted. + # Thus, we can trust it has the same cardinality as `ops` and they can be zipped together. + ops_and_results = ops.zip(responses).map { |(op, response)| [op, op.categorize(response)] } + [client.cluster_name, ops_and_results] + end.to_h + + BulkResult.new(ops_and_results_by_cluster) + end + + # Return type encapsulating all of the results of the bulk call. + class BulkResult < ::Data.define(:ops_and_results_by_cluster, :noop_results, :failure_results) + def initialize(ops_and_results_by_cluster:) + results_by_category = ops_and_results_by_cluster.values + .flat_map { |ops_and_results| ops_and_results.map(&:last) } + .group_by(&:category) + + super( + ops_and_results_by_cluster: ops_and_results_by_cluster, + noop_results: results_by_category[:noop] || [], + failure_results: results_by_category[:failure] || [] + ) + end + + # Returns successful operations grouped by the cluster they were applied to. If there are any + # failures, raises an exception to alert the caller to them unless `check_failures: false` is passed. + # + # This is designed to prevent failures from silently being ignored. For example, in tests + # we often call `successful_operations` or `successful_operations_by_cluster_name` and don't + # bother checking `failure_results` (because we don't expect a failure). If there was a failure + # we want to be notified about it. + def successful_operations_by_cluster_name(check_failures: true) + if check_failures && failure_results.any? + raise IndexingFailuresError, "Got #{failure_results.size} indexing failure(s):\n\n" \ + "#{failure_results.map.with_index(1) { |result, idx| "#{idx}. #{result.summary}" }.join("\n\n")}" + end + + ops_and_results_by_cluster.transform_values do |ops_and_results| + ops_and_results.filter_map do |(op, result)| + op if result.category == :success + end + end + end + + # Returns a flat list of successful operations. If there are any failures, raises an exception + # to alert the caller to them unless `check_failures: false` is passed. + # + # This is designed to prevent failures from silently being ignored. For example, in tests + # we often call `successful_operations` or `successful_operations_by_cluster_name` and don't + # bother checking `failure_results` (because we don't expect a failure). If there was a failure + # we want to be notified about it. + def successful_operations(check_failures: true) + successful_operations_by_cluster_name(check_failures: check_failures).values.flatten(1).uniq + end + end + + # Given a list of operations (which can contain different types of operations!), queries the datastore + # to identify the source event versions stored on the corresponding documents. + # + # This was specifically designed to support dealing with malformed events. If an event is malformed we + # usually want to raise an exception, but if the document targeted by the malformed event is at a newer + # version in the index than the version number in the event, the malformed state of the event has + # already been superseded by a corrected event and we can just log a message instead. This method specifically + # supports that logic. + # + # If the datastore returns errors for any of the calls, this method will raise an exception. + # Otherwise, this method returns a nested hash: + # + # - The outer hash maps operations to an inner hash of results for that operation. + # - The inner hash maps datastore cluster/client names to the version number for that operation from the datastore cluster. + # + # Note that the returned `version` for an operation on a cluster can be `nil` (as when the document is not found, + # or for an operation type that doesn't store source versions). + # + # This nested structure is necessary because a single operation can target more than one datastore + # cluster, and a document may have different source event versions in different datastore clusters. + def source_event_versions_in_index(operations) + ops_by_client_name = operations.each_with_object(::Hash.new { |h, k| h[k] = [] }) do |op, ops_hash| + # Note: this intentionally does not use `accessible_cluster_names_to_index_into`. + # We want to fail with clear error if any clusters are inaccessible instead of silently ignoring + # the named cluster. The `IndexingFailuresError` provides a clear error. + cluster_names = op.destination_index_def.clusters_to_index_into + cluster_names.each { |cluster_name| ops_hash[cluster_name] << op } + end + + client_names_and_results = Support::Threading.parallel_map(ops_by_client_name) do |(client_name, all_ops)| + ops, unversioned_ops = all_ops.partition(&:versioned?) + + msearch_response = + if (client = @datastore_clients_by_name[client_name]) && ops.any? + body = ops.flat_map do |op| + # We only care about the source versions, but the way we get it varies. + include_version = + if op.destination_index_def.use_updates_for_indexing? + {_source: {includes: [ + "__versions.#{op.update_target.relationship}", + # The update_data script before ElasticGraph v0.8 used __sourceVersions[type] instead of __versions[relationship]. + # To be backwards-compatible we need to fetch the data at both paths. + # + # TODO: Drop this when we no longer need to maintain backwards-compatibility. + "__sourceVersions.#{op.event.fetch("type")}" + ]}} + else + {version: true, _source: false} + end + + [ + # Note: we intentionally search the entire index expression, not just an individual index based on a rollover timestamp. + # And we intentionally do NOT provide a routing value--we want to find the version, no matter what shard the document + # lives on. + # + # Since this `source_event_versions_in_index` is for handling malformed events, its possible that the + # rollover timestamp or routing value on the operation is wrong and that the correct document lives in + # a different shard and index than what the operation is targeted at. We want to search across all of them + # so that we will find it, regardless of where it lives. + {index: op.destination_index_def.index_expression_for_search}, + # Filter to the documents matching the id. + {query: {ids: {values: [op.doc_id]}}}.merge(include_version) + ] + end + + client.msearch(body: body) + else + # The named client doesn't exist, so we don't have any versions for the docs. + {"responses" => ops.map { |op| {"hits" => {"hits" => _ = []}} }} + end + + errors = msearch_response.fetch("responses").select { |res| res["error"] } + + if errors.empty? + versions_by_op = ops.zip(msearch_response.fetch("responses")).to_h do |(op, response)| + hits = response.fetch("hits").fetch("hits") + + if hits.size > 1 + # Got multiple results. The document is duplicated in multiple shards or indexes. Log a warning about this. + @logger.warn({ + "message_type" => "IdentifyDocumentVersionsGotMultipleResults", + "index" => hits.map { |h| h["_index"] }, + "routing" => hits.map { |h| h["_routing"] }, + "id" => hits.map { |h| h["_id"] }, + "version" => hits.map { |h| h["_version"] } + }) + end + + if op.destination_index_def.use_updates_for_indexing? + versions = hits.filter_map do |hit| + hit.dig("_source", "__versions", op.update_target.relationship, hit.fetch("_id")) || + # The update_data script before ElasticGraph v0.8 used __sourceVersions[type] instead of __versions[relationship]. + # To be backwards-compatible we need to fetch the data at both paths. + # + # TODO: Drop this when we no longer need to maintain backwards-compatibility. + hit.dig("_source", "__sourceVersions", op.event.fetch("type"), hit.fetch("_id")) + end + + [op, versions.uniq] + else + [op, hits.map { |h| h.fetch("_version") }.uniq] + end + end + + unversioned_ops_hash = unversioned_ops.to_h do |op| + [op, []] # : [_Operation, ::Array[::Integer]] + end + + [client_name, :success, versions_by_op.merge(unversioned_ops_hash)] + else + [client_name, :failure, errors] + end + end + + failures = client_names_and_results.flat_map do |(client_name, success_or_failure, results)| + if success_or_failure == :success + [] + else + results.map do |result| + "From cluster #{client_name}: #{::JSON.generate(result, space: " ")}" + end + end + end + + if failures.empty? + client_names_and_results.each_with_object(_ = {}) do |(client_name, _success_or_failure, results), accum| + results.each do |op, version| + accum[op] ||= _ = {} + accum[op][client_name] = version + end + end + else + raise Errors::IdentifyDocumentVersionsFailedError, "Got #{failures.size} failure(s) while querying the datastore " \ + "for document versions:\n\n#{failures.join("\n")}" + end + end + + # Queries the datastore mapping(s) for the given index definition(s) to verify that they are up-to-date + # with our schema artifacts, raising an error if the datastore mappings are missing fields that we + # expect. (Extra fields are allowed, though--we'll just ignore them). + # + # This is intended for use when you want a strong guarantee before proceeding that the indices are current, + # such as before indexing data, or after applying index updates (to "prove" that everything is how it should + # be). + # + # This correctly queries the datastore clusters specified via `index_into_clusters` in config, + # but ignores clusters specified via `query_cluster` (since this isn't intended to be used as part + # of the query flow). + # + # For a rollover template, this takes care of verifying the template itself and also any indices that originated + # from the template. + # + # Note also that this caches the datastore mappings, since this is intended to be used to verify an index + # before we index data into it, and we do not want to impose a huge performance penalty on that process (requiring + # multiple datastore requests before we index each document...). In general, the index mapping only changes + # when we make it change, and we deploy and restart ElasticGraph after any index mapping changes, so we do not + # need to worry about it mutating during the lifetime of a single process (particularly given the expense of doing + # so). + def validate_mapping_completeness_of!(index_cluster_name_method, *index_definitions) + diffs_by_cluster_and_index_name = index_definitions.reduce(_ = {}) do |accum, index_def| + accum.merge(mapping_diffs_for(index_def, index_cluster_name_method)) + end + + if diffs_by_cluster_and_index_name.any? + formatted_diffs = diffs_by_cluster_and_index_name.map do |(cluster_name, index_name), diff| + <<~EOS + On cluster `#{cluster_name}` and index/template `#{index_name}`: + #{diff} + EOS + end.join("\n\n") + + raise Errors::ConfigError, "Datastore index mappings are incomplete compared to the current schema. " \ + "The diff below uses the datastore index mapping as the base, and shows the expected mapping as a diff. " \ + "\n\n#{formatted_diffs}" + end + end + + private + + def mapping_diffs_for(index_definition, index_cluster_name_method) + expected_mapping = @mappings_by_index_def_name.fetch(index_definition.name) + + index_definition.public_send(index_cluster_name_method).flat_map do |cluster_name| + datastore_client = datastore_client_named(cluster_name) + + cached_mappings_for(index_definition, datastore_client).filter_map do |index, mapping_in_index| + if (diff = HashDiffer.diff(mapping_in_index, expected_mapping, ignore_ops: [:-])) + [[cluster_name, index.name], diff] + end + end + end.to_h + end + + def cached_mappings_for(index_definition, datastore_client) + key = [datastore_client, index_definition] # : [DatastoreCore::_Client, DatastoreCore::indexDefinition] + cached_mapping = @cached_mappings[key] ||= new_cached_mapping(fetch_mappings_from_datastore(index_definition, datastore_client)) + + return cached_mapping.mappings if @monotonic_clock.now_in_ms < cached_mapping.expires_at + + begin + fetch_mappings_from_datastore(index_definition, datastore_client).tap do |mappings| + @logger.info "Mapping cache expired for #{index_definition.name}; cleared it from the cache and re-fetched the mapping." + @cached_mappings[key] = new_cached_mapping(mappings) + end + rescue => e + @logger.warn <<~EOS + Mapping cache expired for #{index_definition.name}; attempted to re-fetch it but got an error[1]. Will continue using expired mapping information for now. + + [1] #{e.class}: #{e.message} + #{e.backtrace.join("\n")} + EOS + + # Update the cached mapping so that the expiration is reset. + @cached_mappings[key] = new_cached_mapping(cached_mapping.mappings) + + cached_mapping.mappings + end + end + + def fetch_mappings_from_datastore(index_definition, datastore_client) + # We need to also check any related indices... + indices_to_check = [index_definition] + index_definition.related_rollover_indices(datastore_client) + + indices_to_check.to_h do |index| + [index, index.mappings_in_datastore(datastore_client)] + end + end + + def new_cached_mapping(mappings) + CachedMapping.new(mappings, @monotonic_clock.now_in_ms + rand(MAPPING_CACHE_MAX_AGE_IN_MS_RANGE).to_i) + end + + def datastore_client_named(name) + @datastore_clients_by_name.fetch(name) + end + + CachedMapping = ::Data.define(:mappings, :expires_at) + end + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/event_id.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/event_id.rb new file mode 100644 index 00000000..b5c2dc08 --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/event_id.rb @@ -0,0 +1,32 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" + +module ElasticGraph + class Indexer + # A unique identifier for an event ingested by the indexer. As a string, takes the form of + # "[type]:[id]@v[version]", such as "Widget:123abc@v7". This format was designed to make it + # easy to put these ids in a comma-seperated list. + EventID = ::Data.define(:type, :id, :version) do + # @implements EventID + def self.from_event(event) + new(type: event["type"], id: event["id"], version: event["version"]) + end + + def to_s + "#{type}:#{id}@v#{version}" + end + end + + # Steep weirdly expects them here... + # @dynamic initialize, config, datastore_core, schema_artifacts, datastore_router, monotonic_clock + # @dynamic record_preparer_factory, processor, operation_factory, logger + # @dynamic self.from_parsed_yaml + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/failed_event_error.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/failed_event_error.rb new file mode 100644 index 00000000..ec8b1cb5 --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/failed_event_error.rb @@ -0,0 +1,83 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/indexer/event_id" + +module ElasticGraph + class Indexer + # Indicates an event that we attempted to process which failed for some reason. It may have + # failed due to a validation issue before we even attempted to write it to the datastore, or it + # could have failed in the datastore itself. + class FailedEventError < Errors::Error + # @dynamic main_message, event, operations + + # The "main" part of the error message (without the `full_id` portion). + attr_reader :main_message + + # The invalid event. + attr_reader :event + + # The operations that would have been returned by the `OperationFactory` if the event was valid. + # Note that sometimes an event is so malformed that we can't build any operations for it, but + # most of the time we can. + attr_reader :operations + + def self.from_failed_operation_result(result, all_operations_for_event) + new( + event: result.event, + operations: all_operations_for_event, + main_message: result.summary + ) + end + + def initialize(event:, operations:, main_message:) + @main_message = main_message + @event = event + @operations = operations + + super("#{full_id}: #{main_message}") + end + + # A filtered list of operations that have versions that can be compared against our event + # version. Not all operation types have a version (e.g. derived indexing `Update` operations don't). + def versioned_operations + @versioned_operations ||= operations.select(&:versioned?) + end + + def full_id + event_id = EventID.from_event(event).to_s + if (message_id = event["message_id"]) + "#{event_id} (message_id: #{message_id})" + else + event_id + end + end + + def id + event["id"] + end + + def op + event["op"] + end + + def type + event["type"] + end + + def version + event["version"] + end + + def record + event["record"] + end + end + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/hash_differ.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/hash_differ.rb new file mode 100644 index 00000000..85a701f2 --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/hash_differ.rb @@ -0,0 +1,37 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "hashdiff" + +module ElasticGraph + class Indexer + class HashDiffer + # Generates a string describing how `old` and `new` differ, similar to a git diff. + # `ignore_ops` can contain any of `:-`, `:+`, and `:~`; when provided those diff operations + # will be ignored. + def self.diff(old, new, ignore_ops: []) + ignore_op_strings = ignore_ops.map(&:to_s).to_set + + diffs = ::Hashdiff.diff(old, new) + .reject { |op, path, *vals| ignore_op_strings.include?(_ = op) } + + return if diffs.empty? + + diffs.map do |op, path, *vals| + suffix = if vals.one? + vals.first + else + vals.map { |v| "`#{v.inspect}`" }.join(" => ") + end + + "#{op} #{path}: #{suffix}" + end.join("\n") + end + end + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/indexing_failures_error.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/indexing_failures_error.rb new file mode 100644 index 00000000..eae31bb7 --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/indexing_failures_error.rb @@ -0,0 +1,28 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" + +module ElasticGraph + class Indexer + class IndexingFailuresError < Errors::Error + # Builds an `IndexingFailuresError` with a nicely formatted message for the given array of `FailedEventError`. + def self.for(failures:, events:) + summary = "Got #{failures.size} failure(s) from #{events.size} event(s):" + failure_details = failures.map.with_index { |failure, index| "#{index + 1}) #{failure.message}" } + + message_ids = failures.filter_map { |f| f.event["message_id"] }.uniq + if message_ids.any? + message_details = "These failures came from #{message_ids.size} message(s): #{message_ids.join(", ")}." + end + + new([summary, failure_details, message_details].compact.join("\n\n")) + end + end + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/indexing_preparers/integer.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/indexing_preparers/integer.rb new file mode 100644 index 00000000..43ae8494 --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/indexing_preparers/integer.rb @@ -0,0 +1,41 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class Indexer + module IndexingPreparers + class Integer + # Here we coerce an integer-valued float like `3.0` to a true integer (e.g. `3`). + # This is necessary because: + # + # 1. If a field is an integer in the datastore mapping, it does not tolerate it coming in + # as a float, even if it is integer-valued. + # 2. While we use JSON schema to validate event payloads before we get here, JSON schema + # cannot consistently enforce that we receive true integers for int fields. + # + # As https://json-schema.org/understanding-json-schema/reference/numeric.html#integer explains: + # + # > **Warning** + # > + # > The precise treatment of the “integer” type may depend on the implementation of your + # > JSON Schema validator. JavaScript (and thus also JSON) does not have distinct types + # > for integers and floating-point values. Therefore, JSON Schema can not use type alone + # > to distinguish between integers and non-integers. The JSON Schema specification + # > recommends, but does not require, that validators use the mathematical value to + # > determine whether a number is an integer, and not the type alone. Therefore, there + # > is some disagreement between validators on this point. For example, a JavaScript-based + # > validator may accept 1.0 as an integer, whereas the Python-based jsonschema does not. + def self.prepare_for_indexing(value) + integer = value.to_i + return integer if value == integer + raise Errors::IndexOperationError, "Cannot safely coerce `#{value.inspect}` to an integer" + end + end + end + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/indexing_preparers/no_op.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/indexing_preparers/no_op.rb new file mode 100644 index 00000000..7cab33de --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/indexing_preparers/no_op.rb @@ -0,0 +1,19 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class Indexer + module IndexingPreparers + class NoOp + def self.prepare_for_indexing(value) + value + end + end + end + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/indexing_preparers/untyped.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/indexing_preparers/untyped.rb new file mode 100644 index 00000000..904f7bcb --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/indexing_preparers/untyped.rb @@ -0,0 +1,22 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/untyped_encoder" + +module ElasticGraph + class Indexer + module IndexingPreparers + class Untyped + # Converts the given untyped value to a String so it can be indexed in a `keyword` field. + def self.prepare_for_indexing(value) + Support::UntypedEncoder.encode(value) + end + end + end + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/operation/count_accumulator.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/operation/count_accumulator.rb new file mode 100644 index 00000000..6ae951af --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/operation/count_accumulator.rb @@ -0,0 +1,166 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" + +module ElasticGraph + class Indexer + module Operation + # Responsible for maintaining state and accumulating list counts while we traverse the `data` we are preparing + # to update in the index. Much of the complexity here is due to the fact that we have 3 kinds of list fields: + # scalar lists, embedded object lists, and `nested` object lists. + # + # The Elasticsearch/OpenSearch `nested` type[^1] indexes objects of this type as separate hidden documents. As a result, + # each `nested` object type gets its own `__counts` field. In contrast, embedded object lists get flattened into separate + # entries (one per field path) in a flat map (with `dot_separated_path: values_at_path` entries) at the document root. + # + # We mirror this structure with our `__counts`: each document (either a root document, or a hidden `nested` document) + # gets its own `__counts` field, so we essentially have multiple "count parents". Each `__counts` field is a map, + # keyed by field paths, and containing the number of list elements at that field path after the flattening has + # occurred. + # + # The index mapping defines where the `__counts` fields go. This abstraction uses the mapping to determine when + # it needs to create a new "count parent". + # + # Note: instances of this class are "shallow immutable" (none of the attributes of an instance can be reassigned) + # but the `counts` attribute is itself a mutable hash--we use it to accumulate the list counts as we traverse the + # structure. + # + # [^1]: https://www.elastic.co/guide/en/elasticsearch/reference/8.9/nested.html + CountAccumulator = ::Data.define( + # Hash containing the counts we have accumulated so far. This hash gets mutated as we accumulate, + # and multiple accumulator instances share the same hash instance. However, a new `counts` hash will + # be created when we reach a new parent. + :counts, + # String describing our current location in the traversed structure relative to the current parent. + # This gets replaced on new accumulator instances as we traverse the data structure. + :path_from_parent, + # String describing our current location in the traversed structure relative to the overall document root. + # This gets replaced on new accumulator instances as we traverse the data structure. + :path_from_root, + # The index mapping at the current level of the structure when this accumulator instance was created. + # As we traverse new levels of the data structure, new `CountAccumulator` instances will be created with + # the `mapping` updated to reflect the new level of the structure we are at. + :mapping, + # Set of field paths to subfields of `LIST_COUNTS_FIELD` for the current source relationship. + # This will be used to determine which subfields of the `LIST_COUNTS_FIELD` are populated. + :list_counts_field_paths_for_source, + # Indicates if our current path is underneath a list; if so, `maybe_increment` will increment when called. + :has_list_ancestor + ) do + # @implements CountAccumulator + def self.merge_list_counts_into(params, mapping:, list_counts_field_paths_for_source:) + # Here we compute the counts of our list elements so that we can index it. + data = compute_list_counts_of(params.fetch("data"), CountAccumulator.new_parent( + # We merge in `type: nested` since the `nested` type indicates a new count accumulator parent and we want that applied at the root. + mapping.merge("type" => "nested"), + list_counts_field_paths_for_source + )) + + # The root `__counts` field needs special handling due to our `sourced_from` feature. Anything in `data` + # will overwrite what's in the specified fields when the script executes, but since there could be list + # fields from multiple sources, we need `__counts` to get merged properly. So here we "promote" it from + # `data.__counts` to being a root-level parameter. + params.merge( + "data" => data.except(LIST_COUNTS_FIELD), + LIST_COUNTS_FIELD => data[LIST_COUNTS_FIELD] + ) + end + + def self.compute_list_counts_of(value, parent_accumulator) + case value + when nil + value + when ::Hash + parent_accumulator.maybe_increment + parent_accumulator.process_hash(value) do |key, subvalue, accumulator| + [key, compute_list_counts_of(subvalue, accumulator[key])] + end + when ::Array + parent_accumulator.process_list(value) do |element, accumulator| + compute_list_counts_of(element, accumulator) + end + else + parent_accumulator.maybe_increment + value + end + end + + # Creates an initially empty accumulator instance for a new parent (either at the overall document + # root are at the root of a `nested` object). + def self.new_parent(mapping, list_counts_field_paths_for_source, path_from_root: nil) + count_field_prefix = path_from_root ? "#{path_from_root}.#{LIST_COUNTS_FIELD}." : "#{LIST_COUNTS_FIELD}." + + initial_counts = (mapping.dig("properties", LIST_COUNTS_FIELD, "properties") || {}).filter_map do |field, _| + [field, 0] if list_counts_field_paths_for_source.include?(count_field_prefix + field) + end.to_h + + new(initial_counts, nil, path_from_root, mapping, list_counts_field_paths_for_source, false) + end + + # Processes the given hash, beginning a new parent if need. A new parent is needed if the + # current mapping has a `__counts` field. + # + # Yields repeatedly (once per hash entry). We yield the entry key/value, and an accumulator + # instance (either the current `self` or a new parent). + # + # Afterwards, merges the resulting `__counts` into the hash before it's returned, as needed. + def process_hash(hash) + mapping_type = mapping["type"] + + # As we traverse through the JSON object structure, we also have to traverse through the + # condenseed mapping. Doing this requires that the `properties` of the index mapping + # match the fields of the JSON data structure. However, Elasticsearch/OpenSearch have a number of field + # types which can be represented as a JSON object in an indexing call, but which have no + # `properties` in the mapping. We can't successfully traverse through the JSON data and the + # mapping when we encounter these field types (since the mapping has no record of the + # subfields) so we must treat these types as a special case; we can't proceed, and we won't + # have any lists to count, anyway. + return hash if DATASTORE_PROPERTYLESS_OBJECT_TYPES.include?(mapping_type) + + # THe `nested` type indicates a new document level, so if it's not `nested`, we should process the hash without making a new parent. + return hash.to_h { |key, value| yield key, value, self } unless mapping_type == "nested" + + # ...but otherwise, we should make a new parent. + new_parent = CountAccumulator.new_parent(mapping, list_counts_field_paths_for_source, path_from_root: path_from_root) + updated_hash = hash.to_h { |key, value| yield key, value, new_parent } + + # If we have a LIST_COUNTS_FIELD at this level of our mapping, we should merge in the counts hash from the new parent. + if mapping.dig("properties", LIST_COUNTS_FIELD) + updated_hash.merge(LIST_COUNTS_FIELD => new_parent.counts) + else + updated_hash + end + end + + # Processes the given list, tracking the fact that subpaths have a list ancestor. + def process_list(list) + child_accumulator = with(has_list_ancestor: true) + list.map { |value| yield value, child_accumulator } + end + + # Increments the count at the current `path_from_parent` in the current parent's counts hash if we are under a list. + def maybe_increment + return unless has_list_ancestor + + key = path_from_parent.to_s + counts[key] = counts.fetch(key) + 1 + end + + # Creates a "child" accumulator at the given subpath. Should be used as we traverse the data structure. + def [](subpath) + with( + path_from_parent: path_from_parent ? "#{path_from_parent}#{LIST_COUNTS_FIELD_PATH_KEY_SEPARATOR}#{subpath}" : subpath, + path_from_root: path_from_root ? "#{path_from_root}.#{subpath}" : subpath, + mapping: mapping.fetch("properties").fetch(subpath) + ) + end + end + end + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/operation/factory.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/operation/factory.rb new file mode 100644 index 00000000..a7f93515 --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/operation/factory.rb @@ -0,0 +1,226 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/indexer/event_id" +require "elastic_graph/indexer/failed_event_error" +require "elastic_graph/indexer/operation/update" +require "elastic_graph/indexer/operation/upsert" +require "elastic_graph/indexer/record_preparer" +require "elastic_graph/json_schema/validator_factory" +require "elastic_graph/support/memoizable_data" + +module ElasticGraph + class Indexer + module Operation + class Factory < Support::MemoizableData.define( + :schema_artifacts, + :index_definitions_by_graphql_type, + :record_preparer_factory, + :logger, + :skip_derived_indexing_type_updates, + :configure_record_validator + ) + def build(event) + event = prepare_event(event) + + selected_json_schema_version = select_json_schema_version(event) { |failure| return failure } + + # Because the `select_json_schema_version` picks the closest-matching json schema version, the incoming + # event might not match the expected json_schema_version value in the json schema (which is a `const` field). + # This is by design, since we're picking a schema based on best-effort, so to avoid that by-design validation error, + # performing the envelope validation on a "patched" version of the event. + event_with_patched_envelope = event.merge({JSON_SCHEMA_VERSION_KEY => selected_json_schema_version}) + + if (error_message = validator(EVENT_ENVELOPE_JSON_SCHEMA_NAME, selected_json_schema_version).validate_with_error_message(event_with_patched_envelope)) + return build_failed_result(event, "event payload", error_message) + end + + failed_result = validate_record_returning_failure(event, selected_json_schema_version) + failed_result || BuildResult.success(build_all_operations_for( + event, + record_preparer_factory.for_json_schema_version(selected_json_schema_version) + )) + end + + private + + def select_json_schema_version(event) + available_json_schema_versions = schema_artifacts.available_json_schema_versions + + requested_json_schema_version = event[JSON_SCHEMA_VERSION_KEY] + + # First check that a valid value has been requested (a positive integer) + if !event.key?(JSON_SCHEMA_VERSION_KEY) + yield build_failed_result(event, JSON_SCHEMA_VERSION_KEY, "Event lacks a `#{JSON_SCHEMA_VERSION_KEY}`") + elsif !requested_json_schema_version.is_a?(Integer) || requested_json_schema_version < 1 + yield build_failed_result(event, JSON_SCHEMA_VERSION_KEY, "#{JSON_SCHEMA_VERSION_KEY} (#{requested_json_schema_version}) must be a positive integer.") + end + + # The requested version might not necessarily be available (if the publisher is deployed ahead of the indexer, or an old schema + # version is removed prematurely, or an indexer deployment is rolled back). So the behavior is to always pick the closest-available + # version. If there's an exact match, great. Even if not an exact match, if the incoming event payload conforms to the closest match, + # the event can still be indexed. + # + # This min_by block will take the closest version in the list. If a tie occurs, the first value in the list wins. The desired + # behavior is in the event of a tie (highly unlikely, there shouldn't be a gap in available json schema versions), the higher version + # should be selected. So to get that behavior, the list is sorted in descending order. + # + selected_json_schema_version = available_json_schema_versions.sort.reverse.min_by { |it| (requested_json_schema_version - it).abs } + + if selected_json_schema_version != requested_json_schema_version + logger.info({ + "message_type" => "ElasticGraphMissingJSONSchemaVersion", + "message_id" => event["message_id"], + "event_id" => EventID.from_event(event), + "event_type" => event["type"], + "requested_json_schema_version" => requested_json_schema_version, + "selected_json_schema_version" => selected_json_schema_version + }) + end + + if selected_json_schema_version.nil? + yield build_failed_result( + event, JSON_SCHEMA_VERSION_KEY, + "Failed to select json schema version. Requested version: #{event[JSON_SCHEMA_VERSION_KEY]}. \ + Available json schema versions: #{available_json_schema_versions.sort.join(", ")}" + ) + end + + selected_json_schema_version + end + + def validator(type, selected_json_schema_version) + factory = validator_factories_by_version[selected_json_schema_version] + factory.validator_for(type) + end + + def validator_factories_by_version + @validator_factories_by_version ||= ::Hash.new do |hash, json_schema_version| + factory = JSONSchema::ValidatorFactory.new( + schema: schema_artifacts.json_schemas_for(json_schema_version), + sanitize_pii: true + ) + factory = configure_record_validator.call(factory) if configure_record_validator + hash[json_schema_version] = factory + end + end + + # This copies the `id` from event into the actual record + # This is necessary because we want to index `id` as part of the record so that the datastore will include `id` in returned search payloads. + def prepare_event(event) + return event unless event["record"].is_a?(::Hash) && event["id"] + event.merge("record" => event["record"].merge("id" => event.fetch("id"))) + end + + def validate_record_returning_failure(event, selected_json_schema_version) + record = event.fetch("record") + graphql_type_name = event.fetch("type") + validator = validator(graphql_type_name, selected_json_schema_version) + + if (error_message = validator.validate_with_error_message(record)) + build_failed_result(event, "#{graphql_type_name} record", error_message) + end + end + + def build_failed_result(event, payload_description, validation_message) + message = "Malformed #{payload_description}. #{validation_message}" + + # Here we use the `RecordPreparer::Identity` record preparer because we may not have a valid JSON schema + # version number in this case (which is usually required to get a `RecordPreparer` from the factory), and + # we won't wind up using the record preparer for real on these operations, anyway. + operations = build_all_operations_for(event, RecordPreparer::Identity) + + BuildResult.failure(FailedEventError.new(event: event, operations: operations.to_set, main_message: message)) + end + + def build_all_operations_for(event, record_preparer) + upsert_operations(event, record_preparer) + update_operations(event, record_preparer) + end + + def upsert_operations(event, record_preparer) + type = event.fetch("type") do + # This key should only be missing on invalid events. We still want to build operations + # for the event (to put it in the `FailedEventError`) but in this case we can't build + # any because we don't know what indices to target. + return [] + end + + index_definitions_for(type).reject(&:use_updates_for_indexing?).map do |index_definition| + Upsert.new(event, index_definition, record_preparer) + end + end + + def update_operations(event, record_preparer) + # If `type` is missing or is not a known type (as indicated by `runtime_metadata` being nil) + # then we can't build a derived indexing type update operation. That case will only happen when we build + # operations for an `FailedEventError` rather than to execute. + return [] unless (type = event["type"]) + return [] unless (runtime_metadata = schema_artifacts.runtime_metadata.object_types_by_name[type]) + + runtime_metadata.update_targets.flat_map do |update_target| + ids_to_skip = skip_derived_indexing_type_updates.fetch(update_target.type, ::Set.new) + + index_definitions_for(update_target.type).flat_map do |destination_index_def| + operations = Update.operations_for( + event: event, + destination_index_def: destination_index_def, + record_preparer: record_preparer, + update_target: update_target, + destination_index_mapping: schema_artifacts.index_mappings_by_index_def_name.fetch(destination_index_def.name) + ) + + operations.reject do |op| + ids_to_skip.include?(op.doc_id).tap do |skipped| + if skipped + logger.info({ + "message_type" => "SkippingUpdate", + "message_id" => event["message_id"], + "update_target" => update_target.type, + "id" => op.doc_id, + "event_id" => EventID.from_event(event).to_s + }) + end + end + end + end + end + end + + def index_definitions_for(type) + # If `type` is missing or is not a known type (as indicated by not being in this hash) + # then we return an empty list. That case will only happen when we build + # operations for an `FailedEventError` rather than to execute. + index_definitions_by_graphql_type[type] || [] + end + + # :nocov: -- this should not be called. Instead, it exists to guard against wrongly raising an error from this class. + def raise(*args) + super("`raise` was called on `Operation::Factory`, but should not. Instead, use " \ + "`yield build_failed_result(...)` so that we can accumulate all invalid events and allow " \ + "the valid events to still be processed.") + end + # :nocov: + + # Return value from `build` that indicates what happened. + # - If it was successful, `operations` will be a non-empty array of operations and `failed_event_error` will be nil. + # - If there was a validation issue, `operations` will be an empty array and `failed_event_error` will be non-nil. + BuildResult = ::Data.define(:operations, :failed_event_error) do + # @implements BuildResult + def self.success(operations) + new(operations, nil) + end + + def self.failure(failed_event_error) + new([], failed_event_error) + end + end + end + end + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/operation/result.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/operation/result.rb new file mode 100644 index 00000000..66fde1ed --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/operation/result.rb @@ -0,0 +1,76 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer/event_id" + +module ElasticGraph + class Indexer + module Operation + # Describes the result of an operation. + # :category value will be one of: [:success, :noop, :failure] + Result = ::Data.define(:category, :operation, :description) do + # @implements Result + def self.success_of(operation) + Result.new( + category: :success, + operation: operation, + description: nil + ) + end + + def self.noop_of(operation, description) + Result.new( + category: :noop, + operation: operation, + description: description + ) + end + + def self.failure_of(operation, description) + Result.new( + category: :failure, + operation: operation, + description: description + ) + end + + def operation_type + operation.type + end + + def event + operation.event + end + + def event_id + EventID.from_event(event) + end + + def summary + # :nocov: -- `description == nil` case is not covered; not simple to test. + suffix = description ? "--#{description}" : nil + # :nocov: + "<#{operation.description} #{event_id} #{category}#{suffix}>" + end + + def inspect + parts = [ + self.class.name, + operation_type.inspect, + category.inspect, + event_id, + description + ].compact + + "#<#{parts.join(" ")}>" + end + alias_method :to_s, :inspect + end + end + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/operation/update.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/operation/update.rb new file mode 100644 index 00000000..ec3ba871 --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/operation/update.rb @@ -0,0 +1,160 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/errors" +require "elastic_graph/indexer/event_id" +require "elastic_graph/indexer/operation/count_accumulator" +require "elastic_graph/indexer/operation/result" +require "elastic_graph/support/hash_util" +require "elastic_graph/support/memoizable_data" + +module ElasticGraph + class Indexer + module Operation + class Update < Support::MemoizableData.define(:event, :prepared_record, :destination_index_def, :update_target, :doc_id, :destination_index_mapping) + # @dynamic event, destination_index_def, doc_id + + def self.operations_for( + event:, + destination_index_def:, + record_preparer:, + update_target:, + destination_index_mapping: + ) + return [] if update_target.for_normal_indexing? && !destination_index_def.use_updates_for_indexing? + + prepared_record = record_preparer.prepare_for_index(event["type"], event["record"] || {"id" => event["id"]}) + + Support::HashUtil + .fetch_leaf_values_at_path(prepared_record, update_target.id_source) + .reject { |id| id.to_s.strip.empty? } + .uniq + .map { |doc_id| new(event, prepared_record, destination_index_def, update_target, doc_id, destination_index_mapping) } + end + + def to_datastore_bulk + @to_datastore_bulk ||= [{update: metadata}, update_request] + end + + def categorize(response) + update = response.fetch("update") + status = update.fetch("status") + + if noop_result?(response) + noop_error_message = message_from_thrown_painless_exception(update) + &.delete_prefix(UPDATE_WAS_NOOP_MESSAGE_PREAMBLE) + + Result.noop_of(self, noop_error_message) + elsif (200..299).cover?(status) + Result.success_of(self) + else + error = update.fetch("error") + + further_detail = + if (more_detail = error["caused_by"]) + # Usually the type/reason details are nested an extra level (`caused_by.caused_by`) but sometimes + # it's not. I think it's nested when the script itself throws an exception where as it's unnested + # when the datastore is unable to run the script. + more_detail = more_detail["caused_by"] if more_detail.key?("caused_by") + " (#{more_detail["type"]}: #{more_detail["reason"]})" + else + "; full response: #{::JSON.pretty_generate(response)}" + end + + Result.failure_of(self, "#{update_target.script_id}(applied to `#{doc_id}`): #{error.fetch("reason")}#{further_detail}") + end + end + + def type + :update + end + + def description + if update_target.type == event.fetch("type") + "#{update_target.type} update" + else + "#{update_target.type} update (from #{event.fetch("type")})" + end + end + + def inspect + "#<#{self.class.name} event=#{EventID.from_event(event)} target=#{update_target.type}>" + end + alias_method :to_s, :inspect + + def versioned? + # We do not track source event versions when applying derived indexing updates, but we do for + # normal indexing updates, so if the update target is for normal indexing it's a versioned operation. + update_target.for_normal_indexing? + end + + private + + # The number of retries of the update script we'll have the datastore attempt on concurrent modification conflicts. + CONFLICT_RETRIES = 5 + + def metadata + { + _index: destination_index_def.index_name_for_writes(prepared_record, timestamp_field_path: update_target.rollover_timestamp_value_source), + _id: doc_id, + routing: destination_index_def.routing_value_for_prepared_record( + prepared_record, + route_with_path: update_target.routing_value_source, + id_path: update_target.id_source + ), + retry_on_conflict: CONFLICT_RETRIES + }.compact + end + + def update_request + { + script: {id: update_target.script_id, params: script_params}, + # We use a scripted upsert instead of formatting an upsert document because it creates + # for simpler code. To create the upsert document, we'd have to convert the param + # values to their "upsert form"--for example, for an `append_only_set` field, the param + # value is generally a single scalar value while in an upsert document it would need to + # be a list. By using `scripted_upsert`, we can always just pass the params in a consistent + # way, and rely on the script to handle the case where it is creating a brand new document. + scripted_upsert: true, + upsert: {} + } + end + + def noop_result?(response) + update = response.fetch("update") + error_message = message_from_thrown_painless_exception(update).to_s + error_message.start_with?(UPDATE_WAS_NOOP_MESSAGE_PREAMBLE) || update["result"] == "noop" + end + + def message_from_thrown_painless_exception(update) + update.dig("error", "caused_by", "caused_by", "reason") + end + + def script_params + initial_params = update_target.params_for( + doc_id: doc_id, + event: event, + prepared_record: prepared_record + ) + + # The normal indexing script uses `__counts`. Other indexing scripts (e.g. the ones generated + # for derived indexing) do not use `__counts` so there's no point in spending effort on computing + # it. Plus, the logic below raises an exception in that case, so it's important we avoid it. + return initial_params unless update_target.for_normal_indexing? + + CountAccumulator.merge_list_counts_into( + initial_params, + mapping: destination_index_mapping, + list_counts_field_paths_for_source: destination_index_def.list_counts_field_paths_for_source(update_target.relationship.to_s) + ) + end + end + end + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/operation/upsert.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/operation/upsert.rb new file mode 100644 index 00000000..450afb14 --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/operation/upsert.rb @@ -0,0 +1,71 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer/operation/result" +require "elastic_graph/support/hash_util" +require "elastic_graph/support/memoizable_data" + +module ElasticGraph + class Indexer + module Operation + Upsert = Support::MemoizableData.define(:event, :destination_index_def, :record_preparer) do + # @implements Upsert + + def to_datastore_bulk + @to_datastore_bulk ||= [{index: metadata}, prepared_record] + end + + def categorize(response) + index = response.fetch("index") + status = index.fetch("status") + + case status + when 200..299 + Result.success_of(self) + when 409 + Result.noop_of(self, index.fetch("error").fetch("reason")) + else + Result.failure_of(self, index.fetch("error").fetch("reason")) + end + end + + def doc_id + @doc_id ||= event.fetch("id") + end + + def type + :upsert + end + + def description + "#{event.fetch("type")} upsert" + end + + def versioned? + true + end + + private + + def metadata + @metadata ||= { + _index: destination_index_def.index_name_for_writes(prepared_record), + _id: doc_id, + version: event.fetch("version"), + version_type: "external", + routing: destination_index_def.routing_value_for_prepared_record(prepared_record) + }.compact + end + + def prepared_record + @prepared_record ||= record_preparer.prepare_for_index(event.fetch("type"), event.fetch("record")) + end + end + end + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/processor.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/processor.rb new file mode 100644 index 00000000..9d41f1c7 --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/processor.rb @@ -0,0 +1,137 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/indexer/event_id" +require "elastic_graph/indexer/indexing_failures_error" +require "time" + +module ElasticGraph + class Indexer + class Processor + def initialize( + datastore_router:, + operation_factory:, + logger:, + indexing_latency_slo_thresholds_by_timestamp_in_ms:, + clock: ::Time + ) + @datastore_router = datastore_router + @operation_factory = operation_factory + @clock = clock + @logger = logger + @indexing_latency_slo_thresholds_by_timestamp_in_ms = indexing_latency_slo_thresholds_by_timestamp_in_ms + end + + # Processes the given events, writing them to the datastore. If any events are invalid, an + # exception will be raised indicating why the events were invalid, but the valid events will + # still be written to the datastore. No attempt is made to provide atomic "all or nothing" + # behavior. + def process(events, refresh_indices: false) + failures = process_returning_failures(events, refresh_indices: refresh_indices) + return if failures.empty? + raise IndexingFailuresError.for(failures: failures, events: events) + end + + # Like `process`, but returns failures instead of raising an exception. + # The caller is responsible for handling the failures. + def process_returning_failures(events, refresh_indices: false) + factory_results_by_event = events.to_h { |event| [event, @operation_factory.build(event)] } + + factory_results = factory_results_by_event.values + + bulk_result = @datastore_router.bulk(factory_results.flat_map(&:operations), refresh: refresh_indices) + successful_operations = bulk_result.successful_operations(check_failures: false) + + calculate_latency_metrics(successful_operations, bulk_result.noop_results) + + all_failures = + factory_results.map(&:failed_event_error).compact + + bulk_result.failure_results.map do |result| + all_operations_for_event = factory_results_by_event.fetch(result.event).operations + FailedEventError.from_failed_operation_result(result, all_operations_for_event.to_set) + end + + categorize_failures(all_failures, events) + end + + private + + def categorize_failures(failures, events) + source_event_versions_by_cluster_by_op = @datastore_router.source_event_versions_in_index( + failures.flat_map { |f| f.versioned_operations.to_a } + ) + + superseded_failures, outstanding_failures = failures.partition do |failure| + failure.versioned_operations.size > 0 && failure.versioned_operations.all? do |op| + # Under normal conditions, we expect to get back only one version per operation per cluster. + # However, when a field used for routing or index rollover has mutated, we can wind up with + # multiple copies of the document in different indexes or shards. `source_event_versions_in_index` + # returns a list of found versions. + # + # We only need to consider the largest version when deciding if a failure has been supeseded or not. + # An event with a larger version is considered to be a full replacement for an earlier event for the + # same entity, so if we've processed an event for the same entity with a larger version, we can consider + # the failure superseded. + max_version_per_cluster = source_event_versions_by_cluster_by_op.fetch(op).values.map(&:max) + + # We only consider an event to be superseded if the document version in the datastore + # for all its versioned operations is greater than the version of the failing event. + max_version_per_cluster.all? { |v| v && v > failure.version } + end + end + + if superseded_failures.any? + superseded_ids = superseded_failures.map { |f| EventID.from_event(f.event).to_s } + @logger.warn( + "Ignoring #{superseded_ids.size} malformed event(s) because they have been superseded " \ + "by corrected events targeting the same id: #{superseded_ids.join(", ")}." + ) + end + + outstanding_failures + end + + def calculate_latency_metrics(successful_operations, noop_results) + current_time = @clock.now + successful_events = successful_operations.map(&:event).to_set + noop_events = noop_results.map(&:event).to_set + all_operations_events = successful_events + noop_events + + all_operations_events.each do |event| + latencies_in_ms_from = {} # : Hash[String, Integer] + slo_results = {} # : Hash[String, String] + + latency_timestamps = event.fetch("latency_timestamps", _ = {}) + latency_timestamps.each do |ts_name, ts_value| + metric_value = ((current_time - Time.iso8601(ts_value)) * 1000).round + + latencies_in_ms_from[ts_name] = metric_value + + if (threshold = @indexing_latency_slo_thresholds_by_timestamp_in_ms[ts_name]) + slo_results[ts_name] = (metric_value >= threshold) ? "bad" : "good" + end + end + + result = successful_events.include?(event) ? "success" : "noop" + + @logger.info({ + "message_type" => "ElasticGraphIndexingLatencies", + "message_id" => event["message_id"], + "event_type" => event.fetch("type"), + "event_id" => EventID.from_event(event).to_s, + JSON_SCHEMA_VERSION_KEY => event.fetch(JSON_SCHEMA_VERSION_KEY), + "latencies_in_ms_from" => latencies_in_ms_from, + "slo_results" => slo_results, + "result" => result + }) + end + end + end + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/record_preparer.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/record_preparer.rb new file mode 100644 index 00000000..adeef348 --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/record_preparer.rb @@ -0,0 +1,163 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" + +module ElasticGraph + class Indexer + class RecordPreparer + # Provides the ability to get a `RecordPreparer` for a specific JSON schema version. + class Factory + def initialize(schema_artifacts) + @schema_artifacts = schema_artifacts + + scalar_types_by_name = schema_artifacts.runtime_metadata.scalar_types_by_name + indexing_preparer_by_scalar_type_name = ::Hash.new do |hash, type_name| + hash[type_name] = scalar_types_by_name[type_name]&.load_indexing_preparer&.extension_class + end + + @preparers_by_json_schema_version = ::Hash.new do |hash, version| + hash[version] = RecordPreparer.new( + indexing_preparer_by_scalar_type_name, + build_type_metas_from(@schema_artifacts.json_schemas_for(version)) + ) + end + end + + # Gets the `RecordPreparer` for the given JSON schema version. + def for_json_schema_version(json_schema_version) + @preparers_by_json_schema_version[json_schema_version] + end + + # Gets the `RecordPreparer` for the latest JSON schema version. Intended primarily + # for use in tests for convenience. + def for_latest_json_schema_version + for_json_schema_version(@schema_artifacts.latest_json_schema_version) + end + + private + + def build_type_metas_from(json_schemas) + json_schemas.fetch("$defs").filter_map do |type, type_def| + next if type == EVENT_ENVELOPE_JSON_SCHEMA_NAME + + properties = type_def.fetch("properties") do + {} # : ::Hash[::String, untyped] + end # : ::Hash[::String, untyped] + + required_fields = type_def.fetch("required") do + [] # : ::Array[::String] + end # : ::Array[::String] + + eg_meta_by_field_name = properties.filter_map do |prop_name, prop| + eg_meta = prop["ElasticGraph"] + [prop_name, eg_meta] if eg_meta + end.to_h + + TypeMetadata.new( + name: type, + requires_typename: required_fields.include?("__typename"), + eg_meta_by_field_name: eg_meta_by_field_name + ) + end + end + end + + # An alternate `RecordPreparer` implementation that implements the identity function: + # it just echoes back the record it is given. + # + # This is intended only for use where a `RecordPreparer` is required but the data is not + # ultimately going to be sent to the datastore. For example, when an event is invalid, we + # still build operations for it, and the operations require a `RecordPreparer`, but we do + # not send them to the datastore. + module Identity + def self.prepare_for_index(type_name, record) + record + end + end + + def initialize(indexing_preparer_by_scalar_type_name, type_metas) + @indexing_preparer_by_scalar_type_name = indexing_preparer_by_scalar_type_name + @eg_meta_by_field_name_by_concrete_type = type_metas.to_h do |meta| + [meta.name, meta.eg_meta_by_field_name] + end + + @types_requiring_typename = type_metas.filter_map do |meta| + meta.name if meta.requires_typename + end.to_set + end + + # Prepares the given payload for being indexed into the named index. + # This allows any value or field name conversion to happen before we index + # the data, to support the few cases where we expect differences between + # the payload received by the ElasticGraph indexer, and the payload we + # send to the datastore. + # + # As part of preparing the data, we also drop any `record` fields that + # are not defined in our schema. This allows us to handle events that target + # multiple indices (e.g. v1 and v2) for the same type. The event can contain + # the set union of fields and this will take care of dropping any unsupported + # fields before we attempt to index the record. + # + # Note: this method does not mutate the given `record`. Instead it returns a + # copy with any updates applied to it. + def prepare_for_index(type_name, record) + prepare_value_for_indexing(record, type_name) + end + + private + + def prepare_value_for_indexing(value, type_name) + type_name = type_name.delete_suffix("!") + + return nil if value.nil? + + if (preparer = @indexing_preparer_by_scalar_type_name[type_name]) + return (_ = preparer).prepare_for_indexing(value) + end + + case value + when ::Array + element_type_name = type_name.delete_prefix("[").delete_suffix("]") + value.map { |v| prepare_value_for_indexing(v, element_type_name) } + when ::Hash + # `@eg_meta_by_field_name_by_concrete_type` does not have abstract types in it (e.g. type unions). + # Instead, it'll have each concrete subtype in it. + # + # If `type_name` is an abstract type, we need to look at the `__typename` field to see + # what the concrete subtype is. `__typename` is required on abstract types and indicates that. + eg_meta_by_field_name = @eg_meta_by_field_name_by_concrete_type.fetch(value["__typename"] || type_name) + + value.filter_map do |field_name, field_value| + if field_name == "__typename" + # We only want to include __typename if it we're dealing with a type that requires it. + # (This is the case for an abstract type, so it can differentiate between which subtype we have + [field_name, field_value] if @types_requiring_typename.include?(type_name) + elsif (eg_meta = eg_meta_by_field_name[field_name]) + [eg_meta.fetch("nameInIndex"), prepare_value_for_indexing(field_value, eg_meta.fetch("type"))] + end + end.to_h + else + # We won't have a registered preparer for enum types, since those aren't dumped in + # runtime metadata `scalar_types_by_name`, and we can just return the value as-is in + # this case. + value + end + end + + TypeMetadata = ::Data.define( + # The name of the type this metadata object is for. + :name, + # Indicates if this type requires a `__typename` field. + :requires_typename, + # The per-field ElasticGraph metadata, keyed by field name. + :eg_meta_by_field_name + ) + end + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/spec_support/event_matcher.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/spec_support/event_matcher.rb new file mode 100644 index 00000000..df3bad6c --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/spec_support/event_matcher.rb @@ -0,0 +1,44 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "json" + +# Defines an RSpec matcher that can be used to validate ElasticGraph events. +::RSpec::Matchers.define :be_a_valid_elastic_graph_event do |for_indexer:| + match do |event| + result = for_indexer + .operation_factory + .with(configure_record_validator: block_arg) + .build(event) + + @validation_failure = result.failed_event_error + !@validation_failure + end + + description do + "be a valid ElasticGraph event" + end + + failure_message do |event| + <<~EOS + expected the event[1] to #{description}, but it was invalid[2]. + + [1] #{::JSON.pretty_generate(event)} + + [2] #{@validation_failure.message} + EOS + end + + failure_message_when_negated do |event| + <<~EOS + expected the event[1] not to #{description}, but it was valid. + + [1] #{::JSON.pretty_generate(event)} + EOS + end +end diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/test_support/converters.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/test_support/converters.rb new file mode 100644 index 00000000..7b964722 --- /dev/null +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/test_support/converters.rb @@ -0,0 +1,36 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/support/hash_util" +require "json" + +module ElasticGraph + class Indexer + module TestSupport + module Converters + # Helper method for testing and generating fake data to convert a factory record into an event + def self.upsert_event_for(record) + { + "op" => "upsert", + "id" => record.fetch("id"), + "type" => record.fetch("__typename"), + "version" => record.fetch("__version"), + "record" => record.except("__typename", "__version", "__json_schema_version"), + JSON_SCHEMA_VERSION_KEY => record.fetch("__json_schema_version") + } + end + + # Helper method to create an array of events given an array of records + def self.upsert_events_for_records(records) + records.map { |record| upsert_event_for(Support::HashUtil.stringify_keys(record)) } + end + end + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer.rbs new file mode 100644 index 00000000..880b484e --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer.rbs @@ -0,0 +1,38 @@ +module ElasticGraph + class Indexer + attr_reader config: Config + attr_reader datastore_core: DatastoreCore + attr_reader schema_artifacts: schemaArtifacts + attr_reader logger: ::Logger + + extend _BuildableFromParsedYaml[Indexer] + extend Support::FromYamlFile[Indexer] + + def initialize: ( + config: Config, + datastore_core: DatastoreCore, + ?datastore_router: DatastoreIndexingRouter?, + ?monotonic_clock: Support::MonotonicClock?, + ?clock: singleton(::Time)? + ) -> void + + @datastore_router: DatastoreIndexingRouter? + def datastore_router: () -> DatastoreIndexingRouter + + @record_preparer_factory: RecordPreparer::Factory? + def record_preparer_factory: () -> RecordPreparer::Factory + + @processor: Processor? + def processor: () -> Processor + + @operation_factory: Operation::Factory? + def operation_factory: () -> Operation::Factory + + @monotonic_clock: Support::MonotonicClock? + def monotonic_clock: () -> Support::MonotonicClock + + private + + @clock: singleton(::Time) + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/config.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/config.rbs new file mode 100644 index 00000000..caa57496 --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/config.rbs @@ -0,0 +1,23 @@ +module ElasticGraph + class Indexer + class ConfigSupertype + attr_reader latency_slo_thresholds_by_timestamp_in_ms: ::Hash[::String, ::Integer] + attr_reader skip_derived_indexing_type_updates: ::Hash[::String, ::Set[::String]] + + def initialize: ( + latency_slo_thresholds_by_timestamp_in_ms: ::Hash[::String, ::Integer], + skip_derived_indexing_type_updates: ::Hash[::String, ::Set[::String]]) -> void + + def with: ( + ?latency_slo_thresholds_by_timestamp_in_ms: ::Hash[::String, ::Integer], + ?skip_derived_indexing_type_updates: ::Hash[::String, ::Set[::String]]) -> Config + + def self.members: () -> ::Array[::Symbol] + end + + class Config < ConfigSupertype + extend _BuildableFromParsedYaml[Config] + EXPECTED_KEYS: ::Array[::String] + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/datastore_indexing_router.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/datastore_indexing_router.rbs new file mode 100644 index 00000000..7af8557e --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/datastore_indexing_router.rbs @@ -0,0 +1,97 @@ +module ElasticGraph + class Indexer + class DatastoreIndexingRouter + MAPPING_CACHE_MAX_AGE_IN_MS_RANGE: ::Range[::Integer] + + def initialize: ( + datastore_clients_by_name: ::Hash[::String, DatastoreCore::_Client], + mappings_by_index_def_name: ::Hash[::String, untyped], + monotonic_clock: Support::MonotonicClock, + logger: ::Logger + ) -> void + + def bulk: (::Array[_Operation], ?refresh: bool) -> BulkResult + def source_event_versions_in_index: (::Array[_Operation]) -> ::Hash[_Operation, ::Hash[::String, ::Array[::Integer]]] + def validate_mapping_completeness_of!: ( + ::Symbol, + *DatastoreCore::indexDefinition + ) -> void + + private + + @datastore_clients_by_name: ::Hash[::String, DatastoreCore::_Client] + @mappings_by_index_def_name: ::Hash[::String, untyped] + @logger: ::Logger + @monotonic_clock: Support::MonotonicClock + @cached_mappings: ::Hash[ + [DatastoreCore::_Client, DatastoreCore::indexDefinition], + CachedMapping + ] + + def mapping_diffs_for: ( + DatastoreCore::indexDefinition, + ::Symbol + ) -> ::Hash[[::String, ::String], ::String] + + def cached_mappings_for: ( + DatastoreCore::indexDefinition, + DatastoreCore::_Client + ) -> ::Hash[DatastoreCore::indexDefinition, ::Hash[::String, untyped]] + + def fetch_mappings_from_datastore: ( + DatastoreCore::indexDefinition, + DatastoreCore::_Client + ) -> ::Hash[DatastoreCore::indexDefinition, ::Hash[::String, untyped]] + + def new_cached_mapping: ( + ::Hash[DatastoreCore::indexDefinition, ::Hash[::String, untyped]] + ) -> CachedMapping + + def datastore_client_named: (::String) -> DatastoreCore::_Client + + class BulkResultSupertype + attr_reader ops_and_results_by_cluster: ::Hash[::String, ::Array[[_Operation, Operation::Result]]] + attr_reader noop_results: ::Array[Operation::Result] + attr_reader failure_results: ::Array[Operation::Result] + + def initialize: ( + ops_and_results_by_cluster: ::Hash[::String, ::Array[[_Operation, Operation::Result]]], + noop_results: ::Array[Operation::Result], + failure_results: ::Array[Operation::Result] + ) -> void + + def with: ( + ?ops_and_results_by_cluster: ::Hash[::String, ::Array[[_Operation, Operation::Result]]] + ) -> BulkResult + end + + class BulkResult < BulkResultSupertype + def self.new: ( + ::Hash[::String, ::Array[[_Operation, Operation::Result]]] + ) -> BulkResult + + def initialize: ( + ops_and_results_by_cluster: ::Hash[::String, ::Array[[_Operation, Operation::Result]]] + ) -> void + + def successful_operations_by_cluster_name: ( + ?check_failures: bool + ) -> ::Hash[::String, ::Array[_Operation]] + + def successful_operations: ( + ?check_failures: bool + ) -> ::Array[_Operation] + end + + class CachedMapping + attr_reader mappings: ::Hash[DatastoreCore::indexDefinition, ::Hash[::String, untyped]] + attr_reader expires_at: ::Integer + + def initialize: ( + ::Hash[DatastoreCore::indexDefinition, ::Hash[::String, untyped]], + ::Integer + ) -> void + end + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/event_id.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/event_id.rbs new file mode 100644 index 00000000..5b0aa8e3 --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/event_id.rbs @@ -0,0 +1,17 @@ +module ElasticGraph + class Indexer + type eventIDString = ::String + + class EventID + attr_reader type: ::String + attr_reader id: ::String + attr_reader version: ::Integer + + def initialize: (type: ::String, id: ::String, version: ::Integer) -> void + def self.new: + (type: ::String, id: ::String, version: ::Integer) -> EventID + | (::String, ::String, ::Integer) -> EventID + def self.from_event: (event) -> EventID + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/failed_event_error.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/failed_event_error.rbs new file mode 100644 index 00000000..4bdd74e6 --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/failed_event_error.rbs @@ -0,0 +1,26 @@ +module ElasticGraph + class Indexer + class FailedEventError < Errors::Error + attr_reader main_message: ::String + attr_reader event: ::Hash[::String, untyped] + attr_reader operations: ::Set[_Operation] + attr_reader versioned_operations: ::Array[_Operation] + attr_reader id: ::String + attr_reader full_id: ::String + attr_reader op: ::String + attr_reader version: ::Integer + attr_reader record: ::Hash[::String, untyped]? + + def self.from_failed_operation_result: ( + Operation::Result, + ::Set[_Operation] + ) -> FailedEventError + + def initialize: ( + event: ::Hash[::String, untyped], + operations: ::Set[_Operation], + main_message: ::String + ) -> void + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/hash_differ.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/hash_differ.rbs new file mode 100644 index 00000000..398243cc --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/hash_differ.rbs @@ -0,0 +1,7 @@ +module ElasticGraph + class Indexer + class HashDiffer + def self.diff: [K, V] (::Hash[K, V], ::Hash[K, V], ?ignore_ops: ::Array[:+ | :- | :~]) -> ::String? + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/indexing_failures_error.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/indexing_failures_error.rbs new file mode 100644 index 00000000..f0a530a3 --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/indexing_failures_error.rbs @@ -0,0 +1,10 @@ +module ElasticGraph + class Indexer + class IndexingFailuresError < Errors::Error + def self.for: ( + failures: ::Array[Indexer::FailedEventError], + events: ::Array[::Hash[::String, untyped]] + ) -> IndexingFailuresError + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/indexing_preparers/integer.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/indexing_preparers/integer.rbs new file mode 100644 index 00000000..d615b9b1 --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/indexing_preparers/integer.rbs @@ -0,0 +1,9 @@ +module ElasticGraph + class Indexer + module IndexingPreparers + class Integer + extend SchemaArtifacts::_IndexingPreparer[untyped, ::Integer] + end + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/indexing_preparers/no_op.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/indexing_preparers/no_op.rbs new file mode 100644 index 00000000..800721b2 --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/indexing_preparers/no_op.rbs @@ -0,0 +1,9 @@ +module ElasticGraph + class Indexer + module IndexingPreparers + class NoOp + extend SchemaArtifacts::_IndexingPreparer[untyped, untyped] + end + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/indexing_preparers/untyped.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/indexing_preparers/untyped.rbs new file mode 100644 index 00000000..183e9c75 --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/indexing_preparers/untyped.rbs @@ -0,0 +1,9 @@ +module ElasticGraph + class Indexer + module IndexingPreparers + class Untyped + extend SchemaArtifacts::_IndexingPreparer[untyped, ::String?] + end + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/operation.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/operation.rbs new file mode 100644 index 00000000..5e57adb7 --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/operation.rbs @@ -0,0 +1,17 @@ +module ElasticGraph + class Indexer + type operationType = :upsert | :delete | :update + type datastoreBulkPayload = ::Array[::Hash[::Symbol | ::String, untyped]] + + interface _Operation + def to_datastore_bulk: () -> datastoreBulkPayload + def event: () -> event + def destination_index_def: () -> DatastoreCore::indexDefinition + def categorize: (::Hash[::String, untyped]) -> Operation::Result + def doc_id: () -> ::String + def type: () -> operationType + def description: () -> ::String + def versioned?: () -> bool + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/operation/count_accumulator.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/operation/count_accumulator.rbs new file mode 100644 index 00000000..cec4dccf --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/operation/count_accumulator.rbs @@ -0,0 +1,60 @@ +module ElasticGraph + class Indexer + module Operation + class CountAccumulator + def self.merge_list_counts_into: ( + ::Hash[::String, untyped], + mapping: ::Hash[::String, untyped], + list_counts_field_paths_for_source: ::Set[::String] + ) -> ::Hash[::String, untyped] + + def self.compute_list_counts_of: + [T] (T, CountAccumulator) -> T | + [T] (::Hash[::String, T], CountAccumulator) -> ::Hash[::String, T] | + [T] (::Array[T], CountAccumulator) -> ::Array[T] + + attr_reader counts: ::Hash[::String, ::Integer] + attr_reader path_from_parent: ::String? + attr_reader path_from_root: ::String? + attr_reader mapping: ::Hash[::String, untyped] + attr_reader list_counts_field_paths_for_source: ::Set[::String] + attr_reader has_list_ancestor: bool + + def initialize: ( + ::Hash[::String, ::Integer], + ::String?, + ::String?, + ::Hash[::String, untyped], + ::Set[::String], + bool + ) -> void + + def with: ( + ?counts: ::Hash[::String, ::Integer], + ?path_from_parent: ::String?, + ?path_from_root: ::String?, + ?mapping: ::Hash[::String, untyped], + ?list_counts_field_paths_for_source: ::Set[::String], + ?has_list_ancestor: bool + ) -> CountAccumulator + + def self.new_parent: ( + ::Hash[::String, untyped], + ::Set[::String], + ?path_from_root: ::String? + ) -> CountAccumulator + + def process_hash: (::Hash[::String, untyped]) { + (::String, untyped, CountAccumulator) -> [::String, untyped] + } -> ::Hash[::String, untyped] + + def process_list: (::Array[untyped]) { + (untyped, CountAccumulator) -> untyped + } -> ::Array[untyped] + + def maybe_increment: () -> void + def []: (::String) -> CountAccumulator + end + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/operation/factory.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/operation/factory.rbs new file mode 100644 index 00000000..029c4414 --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/operation/factory.rbs @@ -0,0 +1,72 @@ +module ElasticGraph + class Indexer + type event = ::Hash[::String, untyped] + type validator = JSONSchema::Validator + type validatorFactory = JSONSchema::ValidatorFactory + + module Operation + class FactorySupertype + attr_reader schema_artifacts: schemaArtifacts + attr_reader index_definitions_by_graphql_type: ::Hash[::String, ::Array[DatastoreCore::_IndexDefinition]] + attr_reader record_preparer_factory: RecordPreparer::Factory + attr_reader logger: ::Logger + attr_reader skip_derived_indexing_type_updates: ::Hash[::String, ::Set[::String]] + attr_reader configure_record_validator: (^(validatorFactory) -> validatorFactory)? + + def initialize: ( + schema_artifacts: schemaArtifacts, + index_definitions_by_graphql_type: ::Hash[::String, ::Array[DatastoreCore::_IndexDefinition]], + record_preparer_factory: RecordPreparer::Factory, + logger: ::Logger, + skip_derived_indexing_type_updates: ::Hash[::String, ::Set[::String]], + configure_record_validator: (^(validatorFactory) -> validatorFactory)? + ) -> void + + def with: ( + ?schema_artifacts: schemaArtifacts, + ?index_definitions_by_graphql_type: ::Hash[::String, ::Array[DatastoreCore::_IndexDefinition]], + ?record_preparer_factory: RecordPreparer::Factory, + ?logger: ::Logger, + ?skip_derived_indexing_type_updates: ::Hash[::String, ::Set[::String]], + ?configure_record_validator: (^(validatorFactory) -> validatorFactory)? + ) -> instance + end + + class Factory < FactorySupertype + def build: (event) -> BuildResult + + private + + def validator: (::String, ::Integer) -> JSONSchema::Validator + + @validator_factories_by_version: ::Hash[::Integer, JSONSchema::ValidatorFactory]? + def validator_factories_by_version: () -> ::Hash[::Integer, JSONSchema::ValidatorFactory] + + def select_json_schema_version: (event) { (BuildResult) -> bot } -> (::Integer | bot) + def prepare_event: (event) -> event + def validate_record_returning_failure: (event, ::Integer) -> BuildResult? + def build_failed_result: (event, ::String, ::String) -> BuildResult + def build_all_operations_for: (event, _RecordPreparer) -> ::Array[_Operation] + def upsert_operations: (event, _RecordPreparer) -> ::Array[_Operation] + def update_operations: (event, _RecordPreparer) -> ::Array[_Operation] + def index_definitions_for: (::String) -> ::Array[DatastoreCore::_IndexDefinition] + def raise: (*untyped) -> void + + class BuildResult + attr_reader operations: ::Array[_Operation] + attr_reader failed_event_error: FailedEventError? + + def initialize: (::Array[_Operation], FailedEventError?) -> void + + def with: ( + ?operations: ::Array[_Operation], + ?failed_event_error: FailedEventError? + ) -> BuildResult + + def self.success: (::Array[_Operation]) -> BuildResult + def self.failure: (FailedEventError) -> BuildResult + end + end + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/operation/result.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/operation/result.rbs new file mode 100644 index 00000000..7973c11d --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/operation/result.rbs @@ -0,0 +1,27 @@ +module ElasticGraph + class Indexer + module Operation + class Result + type category = :failure | :noop | :success + + attr_reader category: category + attr_reader operation: _Operation + attr_reader description: ::String? + + def initialize: ( + category: category, + operation: _Operation, + description: ::String?) -> void + + attr_reader operation_type: operationType + attr_reader event: ::Hash[::String, untyped] + attr_reader event_id: EventID + attr_reader summary: ::String + + def self.success_of: (_Operation) -> Result + def self.noop_of: (_Operation, ::String?) -> Result + def self.failure_of: (_Operation, ::String?) -> Result + end + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/operation/update.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/operation/update.rbs new file mode 100644 index 00000000..2dd94540 --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/operation/update.rbs @@ -0,0 +1,65 @@ +module ElasticGraph + class Indexer + module Operation + class UpdateSupertype + attr_reader event: event + attr_reader prepared_record: ::Hash[::String, untyped] + attr_reader destination_index_def: DatastoreCore::_IndexDefinition + attr_reader update_target: SchemaArtifacts::RuntimeMetadata::UpdateTarget + attr_reader doc_id: ::String + attr_reader destination_index_mapping: ::Hash[::String, untyped] + + def initialize: ( + event, + ::Hash[::String, untyped], + DatastoreCore::_IndexDefinition, + SchemaArtifacts::RuntimeMetadata::UpdateTarget, + ::String, + ::Hash[::String, untyped] + ) -> void + + def self.with: ( + event: event, + prepared_record: ::Hash[::String, untyped], + destination_index_def: DatastoreCore::_IndexDefinition, + update_target: SchemaArtifacts::RuntimeMetadata::UpdateTarget, + doc_id: ::String, + destination_index_mapping: ::Hash[::String, untyped] + ) -> Update + + def with: ( + ?event: event, + ?prepared_record: ::Hash[::String, untyped], + ?destination_index_def: DatastoreCore::_IndexDefinition, + ?update_target: SchemaArtifacts::RuntimeMetadata::UpdateTarget, + ?doc_id: ::String, + ?destination_index_mapping: ::Hash[::String, untyped] + ) -> Update + end + + class Update < UpdateSupertype + include _Operation + + def self.operations_for: ( + event: event, + destination_index_def: DatastoreCore::_IndexDefinition, + record_preparer: _RecordPreparer, + update_target: SchemaArtifacts::RuntimeMetadata::UpdateTarget, + destination_index_mapping: ::Hash[::String, untyped] + ) -> ::Array[Update] + + private + + CONFLICT_RETRIES: ::Integer + def metadata: () -> ::Hash[::Symbol, untyped] + + def update_request: () -> ::Hash[::Symbol, untyped] + def noop_result?: (::Hash[::String, untyped]) -> bool + def message_from_thrown_painless_exception: (::Hash[::String, untyped]) -> ::String? + def script_params: () -> ::Hash[::String, untyped] + + @to_datastore_bulk: datastoreBulkPayload? + end + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/operation/upsert.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/operation/upsert.rbs new file mode 100644 index 00000000..e7850273 --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/operation/upsert.rbs @@ -0,0 +1,26 @@ +module ElasticGraph + class Indexer + module Operation + class Upsert + include _Operation + attr_reader record_preparer: _RecordPreparer + + def initialize: (event, DatastoreCore::_IndexDefinition, _RecordPreparer) -> void + + private + + @metadata: ::Hash[::Symbol, untyped]? + def metadata: () -> ::Hash[::Symbol, untyped] + + attr_reader prepared_record: ::Hash[::String, untyped] + + @to_datastore_bulk: datastoreBulkPayload? + @metadata: ::Hash[::Symbol, untyped]? + @doc_id: ::String? + @routing_value_for_event: ::String? + + def ignore_custom_routing_for_event?: (::String) -> bool + end + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/processor.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/processor.rbs new file mode 100644 index 00000000..feedbe13 --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/processor.rbs @@ -0,0 +1,27 @@ +module ElasticGraph + class Indexer + class Processor + def initialize: ( + datastore_router: DatastoreIndexingRouter, + operation_factory: Operation::Factory, + logger: ::Logger, + indexing_latency_slo_thresholds_by_timestamp_in_ms: ::Hash[::String, ::Integer], + ?clock: singleton(::Time) + ) -> void + + def process: (::Array[event], ?refresh_indices: bool) -> void + def process_returning_failures: (::Array[event], ?refresh_indices: bool) -> ::Array[FailedEventError] + + private + + @datastore_router: DatastoreIndexingRouter + @operation_factory: Operation::Factory + @logger: ::Logger + @indexing_latency_slo_thresholds_by_timestamp_in_ms: ::Hash[::String, ::Integer] + @clock: singleton(::Time) + + def categorize_failures: (::Array[FailedEventError], ::Array[event]) -> ::Array[FailedEventError] + def calculate_latency_metrics: (::Array[_Operation], ::Array[Operation::Result]) -> void + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/record_preparer.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/record_preparer.rbs new file mode 100644 index 00000000..5a52f691 --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/record_preparer.rbs @@ -0,0 +1,64 @@ +module ElasticGraph + class Indexer + interface _RecordPreparer + def prepare_for_index: (::String, ::Hash[::String, untyped]) -> ::Hash[::String, untyped] + end + + class RecordPreparer + type egMetaByFieldHash = ::Hash[::String, {"type" => ::String, "nameInIndex" => ::String}] + type egMetaByFieldByTypeHash = ::Hash[::String, egMetaByFieldHash] + + class Factory + def initialize: (schemaArtifacts) -> void + + def for_json_schema_version: (::Integer) -> RecordPreparer + def for_latest_json_schema_version: () -> RecordPreparer + + private + + @schema_artifacts: schemaArtifacts + @preparers_by_json_schema_version: ::Hash[::Integer, RecordPreparer] + + def build_type_metas_from: (::Hash[::String, untyped]) -> ::Array[TypeMetadata] + end + + include _RecordPreparer + + module Identity + extend _RecordPreparer + end + + def initialize: ( + ::Hash[::String, SchemaArtifacts::RuntimeMetadata::extensionClass?], + ::Array[TypeMetadata] + ) -> void + + private + + INTEGRAL_TYPE_NAMES: ::Set[::String] + @indexing_preparer_by_scalar_type_name: ::Hash[::String, SchemaArtifacts::RuntimeMetadata::extensionClass?] + @eg_meta_by_field_name_by_concrete_type: egMetaByFieldByTypeHash + @types_requiring_typename: ::Set[::String] + + def prepare_value_for_indexing: (untyped, ::String) -> untyped + + class TypeMetadata + attr_reader name: ::String + attr_reader requires_typename: bool + attr_reader eg_meta_by_field_name: egMetaByFieldHash + + def initialize: ( + name: ::String, + requires_typename: bool, + eg_meta_by_field_name: egMetaByFieldHash + ) -> void + + def with: ( + ?name: ::String, + ?requires_typename: bool, + ?eg_meta_by_field_name: egMetaByFieldHash + ) -> TypeMetadata + end + end + end +end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/test_support/converters.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/test_support/converters.rbs new file mode 100644 index 00000000..61ff910e --- /dev/null +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/test_support/converters.rbs @@ -0,0 +1,13 @@ +module ElasticGraph + class Indexer + module TestSupport + module Converters + def self.upsert_event_for: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + + def self.upsert_events_for_records: ( + ::Array[::Hash[::String | ::Symbol, untyped]] + ) -> ::Array[::Hash[::String, untyped]] + end + end + end +end diff --git a/elasticgraph-indexer/spec/acceptance/derived_indexing_types_spec.rb b/elasticgraph-indexer/spec/acceptance/derived_indexing_types_spec.rb new file mode 100644 index 00000000..7ce12d05 --- /dev/null +++ b/elasticgraph-indexer/spec/acceptance/derived_indexing_types_spec.rb @@ -0,0 +1,296 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + RSpec.describe "A derived indexing type", :uses_datastore, :factories, :capture_logs do + shared_examples_for "derived indexing" do + let(:indexer) { build_indexer } + + it "maintains derived fields, handling nested source and destination fields as needed" do + # Index only 1 record initially, so we can verify the state of the document when it is + # first inserted. This is important because the metadata available to `ctx` in our script + # is a bit different for an update to an existing document vs the insertion of a new one. + # Previously, we had a bug that was hidden because we didn't verify JUST the result of + # processing one source document. + w1 = index_records(widget("LARGE", "RED", "USD", name: "foo1", tags: ["b1", "c2", "a3"], fee_currencies: ["CAD", "GBP"])).first + + expect_payload_from_lookup_and_search({ + "id" => "USD", + "name" => "United States Dollar", + "details" => {"symbol" => "$", "unit" => "dollars"}, + "widget_names2" => ["foo1"], + "widget_tags" => ["a3", "b1", "c2"], + "widget_fee_currencies" => ["CAD", "GBP"], + "widget_options" => { + "colors" => ["RED"], + "sizes" => ["LARGE"] + }, + "nested_fields" => { + "max_widget_cost" => w1.fetch("record").fetch("cost").fetch("amount_cents") + }, + "oldest_widget_created_at" => w1.fetch("record").fetch("created_at") + }) + + # Now index a lot more documents so we can verify that we maintain a sorted list of unique values. + widgets = index_records( + widget("SMALL", "RED", "USD", name: "bar1", tags: ["d4", "d4", "e5"]), + widget("LARGE", "RED", "USD", name: "foo1", tags: [], fee_currencies: ["CAD", "USD"]), + widget("SMALL", "BLUE", "CAD", name: "bazz", tags: ["a6"]), + widget("LARGE", "BLUE", "USD", name: "bar2", tags: ["a6", "a5"]), + widget("SMALL", "BLUE", "USD", name: "foo1", tags: []), + widget("LARGE", "BLUE", "USD", name: "foo2", tags: []), + widget(nil, nil, "USD", name: nil, tags: []), # nils scalars should be ignored. + widget(nil, nil, "USD", name: nil, options: nil, tags: []), # ...as should nil parent objects + widget("MEDIUM", "GREEN", nil, name: "foo3", tags: ["z12"]), # ...as should events with `nil` for the derived indexing type id + widget("MEDIUM", "GREEN", "", name: "foo3", tags: ["z12"]), # ...as should events with empty string for the derived indexing type id + widget("SMALL", "RED", "USD", name: "", tags: ["g8"]) # but empty string values can be put in the set. It's odd but seems more correct then not allowing it. + ) + + usd_widgets = ([w1] + widgets).select { |w| w.dig("record", "cost", "currency") == "USD" } + + expect_payload_from_lookup_and_search({ + "id" => "USD", + "name" => "United States Dollar", + "details" => {"symbol" => "$", "unit" => "dollars"}, + "widget_names2" => ["", "bar1", "bar2", "foo1", "foo2"], + "widget_tags" => ["a3", "a5", "a6", "b1", "c2", "d4", "e5", "g8"], + "widget_fee_currencies" => ["CAD", "GBP", "USD"], + "widget_options" => { + "colors" => ["BLUE", "RED"], + "sizes" => ["LARGE", "SMALL"] + }, + "nested_fields" => { + "max_widget_cost" => usd_widgets.map { |w| w.fetch("record").fetch("cost").fetch("amount_cents") }.max + }, + "oldest_widget_created_at" => usd_widgets.map { |w| w.fetch("record").fetch("created_at") }.min + }) + end + + it "creates the derived document with empty field values when indexing a source document that lacks field values" do + w1 = widget(nil, nil, "GBP", name: nil, tags: [], cost_currency_name: nil, cost_currency_symbol: nil, cost_currency_unit: "dollars") + w1[:cost][:amount_cents] = nil + + index_records(w1) + + expect_payload_from_lookup_and_search({ + "id" => "GBP", + "name" => nil, + "details" => {"symbol" => nil, "unit" => "dollars"}, + "widget_names2" => [], + "widget_tags" => [], + "widget_fee_currencies" => [], + "widget_options" => { + "colors" => [], + "sizes" => [] + }, + "nested_fields" => { + "max_widget_cost" => nil + }, + "oldest_widget_created_at" => w1.fetch(:created_at) + }) + end + + it "logs the noop result when a DerivedIndexUpdate operation results in no state change in the datastore" do + w1 = widget(nil, nil, "GBP", name: "widget1", tags: []) + index_records(w1) + + expect { index_records(w1) }.to change { logged_output }.from(a_string_excluding("noop")).to(a_string_including("noop")) + + expect_payload_from_lookup_and_search({ + "id" => "GBP", + "name" => "British Pound Sterling", + "details" => {"symbol" => "£", "unit" => "pounds"}, + "widget_names2" => ["widget1"], + "widget_tags" => [], + "widget_fee_currencies" => [], + "widget_options" => { + "colors" => [], + "sizes" => [] + }, + "nested_fields" => { + "max_widget_cost" => w1.fetch(:cost).fetch(:amount_cents) + }, + "oldest_widget_created_at" => w1.fetch(:created_at) + }) + end + + describe "`immutable_value` fields" do + it "does not allow it change value" do + index_records( + widget("LARGE", "RED", "USD", cost_currency_name: "United States Dollar", cost_currency_unit: "dollars", cost_currency_symbol: "$") + ) + + expect_payload_from_lookup_and_search({ + "id" => "USD", + "name" => "United States Dollar", + "details" => {"unit" => "dollars", "symbol" => "$"} + }) + + expect { + index_records( + widget("LARGE", "RED", "USD", cost_currency_name: "US Dollar", cost_currency_unit: "dollar", cost_currency_symbol: "US$") + ) + }.to raise_error(Indexer::IndexingFailuresError, a_string_including( + "Field `name` cannot be changed (United States Dollar => US Dollar).", + "Field `details.unit` cannot be changed (dollars => dollar).", + "Field `details.symbol` cannot be changed ($ => US$)." + )) + + expect_payload_from_lookup_and_search({ + "id" => "USD", + "name" => "United States Dollar", + "details" => {"unit" => "dollars", "symbol" => "$"} + }) + end + + # Here we disable VCR because we are dealing with `version` numbers. + # To guarantee that our `router.bulk` calls index the operations, we + # use monotonically increasing `version` values based on the current + # system time clock, and have configured VCR to match requests that only + # differ on the `version` values. However, when VCR is playing back the + # response will contain the `version` from when the cassette was recorded, + # which will differ from the version we are dealing with on this run of the + # test. + # + # To avoid odd, confusing failures, we just disable VCR here. + it "ignores an event that tries to change the value if that event has already been superseded by a corrected event with a greater version", :no_vcr do + # Original widget. + widget_v1 = widget("LARGE", "RED", "USD", cost_currency_symbol: "$", id: "w1", workspace_id: "wid23") + + # Updated widget, which wrongly tries to change the currency symbol of USD. + widget_v2 = widget("LARGE", "RED", "USD", cost_currency_symbol: "US$", id: "w1", workspace_id: "wid23", __version: widget_v1.fetch(:__version) + 1) + widget_v2_event_id = Indexer::EventID.from_event(Indexer::TestSupport::Converters.upsert_events_for_records([widget_v2]).first).to_s + + # Later updated widget, which does not try to change the currency symbol. + widget_v3 = widget("LARGE", "RED", "USD", cost_currency_symbol: "$", id: "w1", workspace_id: "wid23", __version: widget_v1.fetch(:__version) + 2, name: "3rd version") + + index_records(widget_v1) + + # When our widget with a changed currency symbol is processed, we should get an error since changing it is not allowed. + expect { + index_records(widget_v2) + }.to raise_error( + Indexer::IndexingFailuresError, + a_string_including("Field `details.symbol` cannot be changed ($ => US$).", widget_v2_event_id) + ) + + index_records(widget_v3) + + # ...but if we retry after the event has been superseded by a corrected event, it just logs a warning instead. + expect { + index_records(widget_v2) + }.to log_warning(a_string_including("superseded by corrected events", widget_v2_event_id)) + end + + it "allows it to be set to `null` unless `nullable: false` was passed on the definition" do + expect { + index_records( + # The `unit` derivation was defined with `nullable: false`. + widget("LARGE", "RED", "USD", cost_currency_name: "United States Dollar", cost_currency_unit: nil) + ) + }.to raise_error(Indexer::IndexingFailuresError, a_string_including("#{DERIVED_INDEX_FAILURE_MESSAGE_PREAMBLE}: Field `details.unit` cannot be set to `null`, but the source event contains no value for it. Remove `nullable: false` from the `immutable_value` definition to allow this.")) + + expect(fetch_from_index("WidgetCurrency", "USD")).to eq nil + + index_records( + # ...but the `name` derivation does not have `nullable: false`. + widget("LARGE", "RED", "USD", cost_currency_name: nil, cost_currency_unit: "dollars") + ) + + expect_payload_from_lookup_and_search({ + "id" => "USD", + "name" => nil, + "details" => {"unit" => "dollars", "symbol" => "$"} + }) + end + + it "allows a one-time change from `null` to a non-null value if `can_change_from_null: true` was passed on the definition" do + # When an immutable value defined with `can_change_from_null: true` is initially indexed as `null`.... + index_records(widget("LARGE", "RED", "USD", cost_currency_symbol: nil)) + expect_payload_from_lookup_and_search({"id" => "USD", "details" => {"unit" => "dollars", "symbol" => nil}}) + + # ...we allow it to change to a non-null value once... + index_records(widget("LARGE", "RED", "USD", cost_currency_symbol: "$")) + expect_payload_from_lookup_and_search({"id" => "USD", "details" => {"unit" => "dollars", "symbol" => "$"}}) + + # ...and ignore any attempts to change it back to null. + index_records(widget("LARGE", "RED", "USD", cost_currency_symbol: nil)) + expect_payload_from_lookup_and_search({"id" => "USD", "details" => {"unit" => "dollars", "symbol" => "$"}}) + + # ...and don't allow it to be changed to another value. + expect { + index_records(widget("LARGE", "RED", "USD", cost_currency_symbol: "US$")) + }.to raise_error(Indexer::IndexingFailuresError, a_string_including( + "Field `details.symbol` cannot be changed ($ => US$)." + )) + expect_payload_from_lookup_and_search({"id" => "USD", "details" => {"unit" => "dollars", "symbol" => "$"}}) + end + + it "does not allow nullable fields to change from null if `can_change_from_null: true` wasn't passed on the definition" do + index_records(widget("LARGE", "RED", "USD", cost_currency_name: nil)) + expect_payload_from_lookup_and_search({"id" => "USD", "name" => nil}) + + expect { + index_records(widget("LARGE", "RED", "USD", cost_currency_name: "US Dollar")) + }.to raise_error(Indexer::IndexingFailuresError, a_string_including( + "Field `name` cannot be changed (null => US Dollar).", + "Set `can_change_from_null: true` on the `immutable_value` definition to allow this." + )) + expect_payload_from_lookup_and_search({"id" => "USD", "name" => nil}) + end + end + + def expect_payload_from_lookup_and_search(payload) + doc = fetch_from_index("WidgetCurrency", payload.fetch("id")) + expect(doc).to include(payload) + + search_response = search_by_type_and_id("WidgetCurrency", [payload.fetch("id")]) + expect(search_response.size).to eq(1) + expect(search_response.first.fetch("_source")).to include(payload) + end + + def widget(size, color, currency, fee_currencies: [], **widget_attributes) + widget_attributes = { + options: build(:widget_options, color: color, size: size), + cost: (build(:money, currency: currency) if currency), + fees: fee_currencies.map { |c| build(:money, currency: c) } + }.merge(widget_attributes) + + build(:widget, **widget_attributes) + end + + def fetch_from_index(type, id) + currency = ElasticGraphSpecSupport::CURRENCIES_BY_CODE.fetch(id) + index_name = indexer.datastore_core.index_definitions_by_graphql_type.fetch(type).first + .index_name_for_writes({"introduced_on" => currency.fetch(:introduced_on)}) + + result = search_index_by_id(index_name, [id]) + result.first&.fetch("_source") + end + + def search_by_type_and_id(type, ids) + index_name = indexer.datastore_core.index_definitions_by_graphql_type.fetch(type).first.index_expression_for_search + search_index_by_id(index_name, ids) + end + + def search_index_by_id(index_name, ids, **metadata) + main_datastore_client.msearch(body: [{index: index_name, **metadata}, { + query: {bool: {filter: [{terms: {id: ids}}]}} + }]).dig("responses", 0, "hits", "hits") + end + end + + context "when `use_updates_for_indexing?` is set to true", use_updates_for_indexing: true do + include_examples "derived indexing" + end + + context "when `use_updates_for_indexing?` is set to false", use_updates_for_indexing: false do + include_examples "derived indexing" + end + end +end diff --git a/elasticgraph-indexer/spec/acceptance/list_fields_spec.rb b/elasticgraph-indexer/spec/acceptance/list_fields_spec.rb new file mode 100644 index 00000000..f8d72ed0 --- /dev/null +++ b/elasticgraph-indexer/spec/acceptance/list_fields_spec.rb @@ -0,0 +1,500 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + RSpec.describe "Indexing into list fields", :uses_datastore, :factories, :capture_logs do + let(:indexer) { build_indexer } + + it "indexes counts of any list fields so we can later use it for filtering" do + sponsors = Array.new(10) { build(:sponsor) } + team = build_upsert_event( + :team, + id: "t1", + past_names: ["Pilots", "Sonics", "Rainiers"], + won_championships_at: [], + forbes_valuations: [90_000_000], + details: build(:team_details, uniform_colors: %w[teal blue silver]), + current_players: [ + build(:player, nicknames: ["The Kid", "JRod"], seasons: [ + build(:player_season, awards: ["MVP", "Rookie of the Year", "Cy Young"]), + build(:player_season, awards: ["Gold Glove"]) + ], sponsors: sponsors[..4]), # 5 sponsors + build(:player, nicknames: ["Jerry"], seasons: [ + build(:player_season, awards: ["Silver Slugger"]) + ], sponsors: sponsors[5..6]) # 2 sponsors + ], + seasons: [ + build(:team_season, notes: %w[A B C], won_games_at: [], players: [ + build(:player, nicknames: ["Neo"], seasons: [ + build(:player_season, awards: ["MVP"]), + build(:player_season, awards: ["RoY"]) + ], sponsors: sponsors[7..9]), # 3 sponsors + build(:player, nicknames: ["Dwight"], seasons: [ + build(:player_season, awards: ["Gold Glove"]) + ], sponsors: sponsors[9..]) # 1 sponsor + ]), + # This next team_season has `started_at: nil` so we can verify that the count of those values has one less as a result. + build(:team_season, notes: %w[D], players: [], won_games_at: [], started_at: nil) + ], + sponsors: [build(:sponsor)] + ) + + indexer.processor.process([team], refresh_indices: true) + + team_counts = indexed_team_counts.fetch("t1") + + expect(team_counts.keys).to contain_exactly( + LIST_COUNTS_FIELD, + "current_players_nested", "current_players_object", + "seasons_nested", "seasons_object", + "the_nested_fields", "nested_fields2" + ) + + expect(team_counts.fetch(LIST_COUNTS_FIELD)).to eq({ + "current_players_nested" => 2, + "current_players_object" => 2, + "current_players_object|affiliations" => 2, + "current_players_object|affiliations|sponsorships_nested" => 7, + "current_players_object|affiliations|sponsorships_object" => 7, + "current_players_object|affiliations|sponsorships_object|annual_total" => 7, + "current_players_object|affiliations|sponsorships_object|annual_total|amount_cents" => 7, + "current_players_object|affiliations|sponsorships_object|annual_total|currency" => 7, + "current_players_object|affiliations|sponsorships_object|sponsor_id" => 7, + "current_players_object|name" => 2, + "current_players_object|nicknames" => 3, + "current_players_object|seasons_nested" => 3, + "current_players_object|seasons_object" => 3, + "current_players_object|seasons_object|awards" => 5, # 3 + 1 + 1 + "current_players_object|seasons_object|games_played" => 3, + "current_players_object|seasons_object|year" => 3, + "details|uniform_colors" => 3, + "forbes_valuations" => 1, + "forbes_valuation_moneys_nested" => 1, + "forbes_valuation_moneys_object" => 1, + "forbes_valuation_moneys_object|amount_cents" => 1, + "forbes_valuation_moneys_object|currency" => 1, + "nested_fields2|current_players" => 2, + "nested_fields2|forbes_valuation_moneys" => 1, + "nested_fields2|the_seasons" => 2, + "past_names" => 3, + "seasons_nested" => 2, + "seasons_object" => 2, + "seasons_object|count" => 2, + "seasons_object|notes" => 4, # 3 + 1 + "seasons_object|players_nested" => 2, # 2 + 0 + "seasons_object|players_object" => 2, # 2 + 0 + "seasons_object|players_object|affiliations" => 2, + "seasons_object|players_object|affiliations|sponsorships_nested" => 4, + "seasons_object|players_object|affiliations|sponsorships_object" => 4, + "seasons_object|players_object|affiliations|sponsorships_object|annual_total" => 4, + "seasons_object|players_object|affiliations|sponsorships_object|annual_total|amount_cents" => 4, + "seasons_object|players_object|affiliations|sponsorships_object|annual_total|currency" => 4, + "seasons_object|players_object|affiliations|sponsorships_object|sponsor_id" => 4, + "seasons_object|players_object|name" => 2, + "seasons_object|players_object|nicknames" => 2, + "seasons_object|players_object|seasons_nested" => 3, + "seasons_object|players_object|seasons_object" => 3, + "seasons_object|players_object|seasons_object|awards" => 3, + "seasons_object|players_object|seasons_object|games_played" => 3, + "seasons_object|players_object|seasons_object|year" => 3, + "seasons_object|started_at" => 1, + "seasons_object|the_record" => 2, + "seasons_object|the_record|first_win_on" => 2, + "seasons_object|the_record|last_win_date" => 2, + "seasons_object|the_record|loss_count" => 2, + "seasons_object|the_record|win_count" => 2, + "seasons_object|won_games_at" => 0, + "seasons_object|year" => 2, + "the_nested_fields|current_players" => 2, + "the_nested_fields|forbes_valuation_moneys" => 1, + "the_nested_fields|the_seasons" => 2, + "won_championships_at" => 0 + }) + + expect(team_counts.fetch("current_players_nested")).to eq [ + { + LIST_COUNTS_FIELD => { + "affiliations|sponsorships_nested" => 5, + "affiliations|sponsorships_object" => 5, + "affiliations|sponsorships_object|annual_total" => 5, + "affiliations|sponsorships_object|annual_total|amount_cents" => 5, + "affiliations|sponsorships_object|annual_total|currency" => 5, + "affiliations|sponsorships_object|sponsor_id" => 5, + "nicknames" => 2, + "seasons_nested" => 2, + "seasons_object" => 2, + "seasons_object|awards" => 4, + "seasons_object|games_played" => 2, + "seasons_object|year" => 2 + }, + "seasons_nested" => [ + {LIST_COUNTS_FIELD => {"awards" => 3}}, + {LIST_COUNTS_FIELD => {"awards" => 1}} + ] + }, + { + LIST_COUNTS_FIELD => { + "nicknames" => 1, + "affiliations|sponsorships_nested" => 2, + "affiliations|sponsorships_object" => 2, + "affiliations|sponsorships_object|annual_total" => 2, + "affiliations|sponsorships_object|annual_total|amount_cents" => 2, + "affiliations|sponsorships_object|annual_total|currency" => 2, + "affiliations|sponsorships_object|sponsor_id" => 2, + "seasons_nested" => 1, + "seasons_object" => 1, + "seasons_object|awards" => 1, + "seasons_object|games_played" => 1, + "seasons_object|year" => 1 + }, + "seasons_nested" => [ + {LIST_COUNTS_FIELD => {"awards" => 1}} + ] + } + ] + + expect(team_counts.fetch("current_players_object")).to eq [ + { + "seasons_nested" => [ + {LIST_COUNTS_FIELD => {"awards" => 3}}, + {LIST_COUNTS_FIELD => {"awards" => 1}} + ] + }, + { + "seasons_nested" => [ + {LIST_COUNTS_FIELD => {"awards" => 1}} + ] + } + ] + + expect(team_counts.fetch("seasons_nested")).to eq [ + { + LIST_COUNTS_FIELD => { + "notes" => 3, + "players_nested" => 2, + "players_object" => 2, + "players_object|affiliations" => 2, + "players_object|affiliations|sponsorships_nested" => 4, + "players_object|affiliations|sponsorships_object" => 4, + "players_object|affiliations|sponsorships_object|annual_total" => 4, + "players_object|affiliations|sponsorships_object|annual_total|amount_cents" => 4, + "players_object|affiliations|sponsorships_object|annual_total|currency" => 4, + "players_object|affiliations|sponsorships_object|sponsor_id" => 4, + "players_object|name" => 2, + "players_object|nicknames" => 2, + "players_object|seasons_nested" => 3, + "players_object|seasons_object" => 3, + "players_object|seasons_object|awards" => 3, + "players_object|seasons_object|games_played" => 3, + "players_object|seasons_object|year" => 3, + "won_games_at" => 0 + }, + "players_nested" => [ + { + LIST_COUNTS_FIELD => { + "affiliations|sponsorships_nested" => 3, + "affiliations|sponsorships_object" => 3, + "affiliations|sponsorships_object|annual_total" => 3, + "affiliations|sponsorships_object|annual_total|amount_cents" => 3, + "affiliations|sponsorships_object|annual_total|currency" => 3, + "affiliations|sponsorships_object|sponsor_id" => 3, + "nicknames" => 1, + "seasons_nested" => 2, + "seasons_object" => 2, + "seasons_object|awards" => 2, + "seasons_object|games_played" => 2, + "seasons_object|year" => 2 + }, + "seasons_nested" => [ + {LIST_COUNTS_FIELD => {"awards" => 1}}, + {LIST_COUNTS_FIELD => {"awards" => 1}} + ] + }, + { + LIST_COUNTS_FIELD => { + "affiliations|sponsorships_nested" => 1, + "affiliations|sponsorships_object" => 1, + "affiliations|sponsorships_object|annual_total" => 1, + "affiliations|sponsorships_object|annual_total|amount_cents" => 1, + "affiliations|sponsorships_object|annual_total|currency" => 1, + "affiliations|sponsorships_object|sponsor_id" => 1, + "nicknames" => 1, + "seasons_nested" => 1, + "seasons_object" => 1, + "seasons_object|awards" => 1, + "seasons_object|games_played" => 1, + "seasons_object|year" => 1 + }, + "seasons_nested" => [ + {LIST_COUNTS_FIELD => {"awards" => 1}} + ] + } + ], + "players_object" => [ + { + "seasons_nested" => [ + {LIST_COUNTS_FIELD => {"awards" => 1}}, + {LIST_COUNTS_FIELD => {"awards" => 1}} + ] + }, + { + "seasons_nested" => [ + {LIST_COUNTS_FIELD => {"awards" => 1}} + ] + } + ] + }, + { + LIST_COUNTS_FIELD => { + "notes" => 1, + "players_nested" => 0, + "players_object" => 0, + "players_object|affiliations" => 0, + "players_object|affiliations|sponsorships_nested" => 0, + "players_object|affiliations|sponsorships_object" => 0, + "players_object|affiliations|sponsorships_object|annual_total" => 0, + "players_object|affiliations|sponsorships_object|annual_total|amount_cents" => 0, + "players_object|affiliations|sponsorships_object|annual_total|currency" => 0, + "players_object|affiliations|sponsorships_object|sponsor_id" => 0, + "players_object|name" => 0, + "players_object|nicknames" => 0, + "players_object|seasons_nested" => 0, + "players_object|seasons_object" => 0, + "players_object|seasons_object|awards" => 0, + "players_object|seasons_object|games_played" => 0, + "players_object|seasons_object|year" => 0, + "won_games_at" => 0 + } + } + ] + + expect(team_counts.fetch("seasons_object")).to eq [ + { + "players_nested" => [ + { + LIST_COUNTS_FIELD => { + "affiliations|sponsorships_nested" => 3, + "affiliations|sponsorships_object" => 3, + "affiliations|sponsorships_object|annual_total" => 3, + "affiliations|sponsorships_object|annual_total|amount_cents" => 3, + "affiliations|sponsorships_object|annual_total|currency" => 3, + "affiliations|sponsorships_object|sponsor_id" => 3, + "nicknames" => 1, + "seasons_nested" => 2, + "seasons_object" => 2, + "seasons_object|awards" => 2, + "seasons_object|games_played" => 2, + "seasons_object|year" => 2 + }, + "seasons_nested" => [ + {LIST_COUNTS_FIELD => {"awards" => 1}}, + {LIST_COUNTS_FIELD => {"awards" => 1}} + ] + }, + { + LIST_COUNTS_FIELD => { + "affiliations|sponsorships_nested" => 1, + "affiliations|sponsorships_object" => 1, + "affiliations|sponsorships_object|annual_total" => 1, + "affiliations|sponsorships_object|annual_total|amount_cents" => 1, + "affiliations|sponsorships_object|annual_total|currency" => 1, + "affiliations|sponsorships_object|sponsor_id" => 1, + "nicknames" => 1, + "seasons_nested" => 1, + "seasons_object" => 1, + "seasons_object|awards" => 1, + "seasons_object|games_played" => 1, + "seasons_object|year" => 1 + }, + "seasons_nested" => [ + {LIST_COUNTS_FIELD => {"awards" => 1}} + ] + } + ], + "players_object" => [ + { + "seasons_nested" => [ + {LIST_COUNTS_FIELD => {"awards" => 1}}, + {LIST_COUNTS_FIELD => {"awards" => 1}} + ] + }, + { + "seasons_nested" => [ + {LIST_COUNTS_FIELD => {"awards" => 1}} + ] + } + ] + }, + {} + ] + + expect(team_counts.fetch("the_nested_fields")).to eq({ + "current_players" => [ + { + "__counts" => { + "affiliations|sponsorships_nested" => 5, + "affiliations|sponsorships_object" => 5, + "affiliations|sponsorships_object|annual_total" => 5, + "affiliations|sponsorships_object|annual_total|amount_cents" => 5, + "affiliations|sponsorships_object|annual_total|currency" => 5, + "affiliations|sponsorships_object|sponsor_id" => 5, + "nicknames" => 2, + "seasons_nested" => 2, + "seasons_object" => 2, + "seasons_object|awards" => 4, + "seasons_object|games_played" => 2, + "seasons_object|year" => 2 + }, + "seasons_nested" => [ + {"__counts" => {"awards" => 3}}, + {"__counts" => {"awards" => 1}} + ] + }, + { + "__counts" => { + "nicknames" => 1, + "affiliations|sponsorships_nested" => 2, + "affiliations|sponsorships_object" => 2, + "affiliations|sponsorships_object|annual_total" => 2, + "affiliations|sponsorships_object|annual_total|amount_cents" => 2, + "affiliations|sponsorships_object|annual_total|currency" => 2, + "affiliations|sponsorships_object|sponsor_id" => 2, + "seasons_nested" => 1, + "seasons_object" => 1, + "seasons_object|awards" => 1, + "seasons_object|games_played" => 1, + "seasons_object|year" => 1 + }, + "seasons_nested" => [ + {"__counts" => {"awards" => 1}} + ] + } + ], + "the_seasons" => [ + { + "__counts" => { + "notes" => 3, + "players_nested" => 2, + "players_object" => 2, + "players_object|affiliations" => 2, + "players_object|affiliations|sponsorships_nested" => 4, + "players_object|affiliations|sponsorships_object" => 4, + "players_object|affiliations|sponsorships_object|annual_total" => 4, + "players_object|affiliations|sponsorships_object|annual_total|amount_cents" => 4, + "players_object|affiliations|sponsorships_object|annual_total|currency" => 4, + "players_object|affiliations|sponsorships_object|sponsor_id" => 4, + "players_object|name" => 2, + "players_object|nicknames" => 2, + "players_object|seasons_nested" => 3, + "players_object|seasons_object" => 3, + "players_object|seasons_object|awards" => 3, + "players_object|seasons_object|games_played" => 3, + "players_object|seasons_object|year" => 3, + "won_games_at" => 0 + }, + "players_nested" => [ + { + "__counts" => { + "affiliations|sponsorships_nested" => 3, + "affiliations|sponsorships_object" => 3, + "affiliations|sponsorships_object|annual_total" => 3, + "affiliations|sponsorships_object|annual_total|amount_cents" => 3, + "affiliations|sponsorships_object|annual_total|currency" => 3, + "affiliations|sponsorships_object|sponsor_id" => 3, + "nicknames" => 1, + "seasons_nested" => 2, + "seasons_object" => 2, + "seasons_object|awards" => 2, + "seasons_object|games_played" => 2, + "seasons_object|year" => 2 + }, + "seasons_nested" => [ + {"__counts" => {"awards" => 1}}, + {"__counts" => {"awards" => 1}} + ] + }, + { + "__counts" => { + "affiliations|sponsorships_nested" => 1, + "affiliations|sponsorships_object" => 1, + "affiliations|sponsorships_object|annual_total" => 1, + "affiliations|sponsorships_object|annual_total|amount_cents" => 1, + "affiliations|sponsorships_object|annual_total|currency" => 1, + "affiliations|sponsorships_object|sponsor_id" => 1, + "nicknames" => 1, + "seasons_nested" => 1, + "seasons_object" => 1, + "seasons_object|awards" => 1, + "seasons_object|games_played" => 1, + "seasons_object|year" => 1 + }, + "seasons_nested" => [ + {"__counts" => {"awards" => 1}} + ] + } + ], + "players_object" => [ + { + "seasons_nested" => [ + {"__counts" => {"awards" => 1}}, + {"__counts" => {"awards" => 1}} + ] + }, + { + "seasons_nested" => [ + {"__counts" => {"awards" => 1}} + ] + } + ] + }, + { + "__counts" => { + "notes" => 1, + "players_nested" => 0, + "players_object" => 0, + "players_object|affiliations" => 0, + "players_object|affiliations|sponsorships_nested" => 0, + "players_object|affiliations|sponsorships_object" => 0, + "players_object|affiliations|sponsorships_object|annual_total" => 0, + "players_object|affiliations|sponsorships_object|annual_total|amount_cents" => 0, + "players_object|affiliations|sponsorships_object|annual_total|currency" => 0, + "players_object|affiliations|sponsorships_object|sponsor_id" => 0, + "players_object|name" => 0, + "players_object|nicknames" => 0, + "players_object|seasons_nested" => 0, + "players_object|seasons_object" => 0, + "players_object|seasons_object|awards" => 0, + "players_object|seasons_object|games_played" => 0, + "players_object|seasons_object|year" => 0, + "won_games_at" => 0 + } + } + ] + }) + end + + def indexed_team_counts + main_datastore_client.msearch(body: [{index: "teams_rollover__*"}, {}]).dig("responses", 0, "hits", "hits").to_h do |hit| + [hit.fetch("_id"), get_counts_from(hit.fetch("_source"))] + end + end + + def get_counts_from(hash) + hash.filter_map do |key, value| + if key == LIST_COUNTS_FIELD + [key, value] + elsif value.is_a?(::Array) && value.first.is_a?(::Hash) + mapped_values = value.map { |v| get_counts_from(v) } + [key, mapped_values] unless mapped_values.all?(&:empty?) + elsif value.is_a?(::Hash) && (sub_counts = get_counts_from(value)).any? + [key, sub_counts] + end + end.to_h + end + end +end diff --git a/elasticgraph-indexer/spec/acceptance/multi_source_indexing_spec.rb b/elasticgraph-indexer/spec/acceptance/multi_source_indexing_spec.rb new file mode 100644 index 00000000..724176ba --- /dev/null +++ b/elasticgraph-indexer/spec/acceptance/multi_source_indexing_spec.rb @@ -0,0 +1,199 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + RSpec.describe "Multi-source indexing", :uses_datastore, :factories, :capture_logs do + let(:indexer) { build_indexer } + + it "ingests data from multiple source types into a single document, regardless of the ingestion order" do + options = build(:widget_options, size: "LARGE") + usd_10 = build(:money, currency: "USD", amount_cents: 10) + usd_20 = build(:money, currency: "USD", amount_cents: 20) + widget_v7 = build_upsert_event(:widget, id: "w1", name: "Pre-Thingy", component_ids: ["c23", "c47", "c56"], tags: ["a"], workspace_id: "ws3", options: options, cost: usd_10, __version: 7) + widget_v8 = build_upsert_event(:widget, id: "w1", name: "Thingy", component_ids: ["c23", "c47", "c56"], tags: ["d", "e", "f"], workspace_id: "ws3", options: options, cost: usd_20, __version: 8) + old_widget = build_upsert_event(:widget, id: "w1", name: "Old Name", component_ids: ["c23", "c47", "c56"], tags: ["b", "c"], workspace_id: "ws3", options: options, cost: usd_10, __version: 6) + + component1 = build_upsert_event(:component, id: "c23", __version: 1, name: "C", tags: ["a", "b"], part_ids: ["p1"]) + component2 = build_upsert_event(:component, id: "c56", __version: 2, name: "D", tags: ["a", "b"], part_ids: ["p1"]) + component3 = build_upsert_event(:component, id: "c78", __version: 3, name: "E", tags: ["a", "b"], part_ids: ["p1"]) + + # ingest component1 before the related widget + indexer.processor.process([component1], refresh_indices: true) + + # ingest widget after related component1 but before related component2 + indexer.processor.process([widget_v7], refresh_indices: true) + + # ingest an updated widget (with a changed name) + indexer.processor.process([widget_v8], refresh_indices: true) + + # ingest component2 after related widge, and ingest standalone component3 + indexer.processor.process([component2, component3], refresh_indices: true) + + # ingest an old version of the widget (with a different name); it should be ignored + indexer.processor.process([old_widget], refresh_indices: true) + + components = search_components + + # The components index should have the 3 we indexed plus the one materialized from the widget reference. + expect(components.keys).to contain_exactly("c23", "c47", "c56", "c78") + + # This component was indexed before the related widget, and should be fully filled in. + expect(components["c23"]).to eq({ + "__sources" => ["__self", "widget"], + "__versions" => { + "__self" => {"c23" => 1}, + "widget" => {"w1" => 8} + }, + LIST_COUNTS_FIELD => { + "tags" => 2, + "part_ids" => 1, + "widget_tags" => 3 + }, + "id" => "c23", + "name" => "C", + "part_ids" => ["p1"], + "tags" => ["a", "b"], + "widget_name" => "Thingy", + "widget_workspace_id" => "ws3", + "widget_size" => "LARGE", + "widget_tags" => ["d", "e", "f"], + "widget_cost" => {"currency" => "USD", "amount_cents" => 20} + }) + + # This component was never indexed, but the widget event should have materialized it with `nil` values for all attributes. + expect(components["c47"]).to eq({ + "__sources" => ["widget"], + "__versions" => { + "widget" => {"w1" => 8} + }, + LIST_COUNTS_FIELD => { + "widget_tags" => 3 + }, + "id" => "c47", + "name" => nil, + "part_ids" => nil, + "tags" => nil, + "widget_name" => "Thingy", + "widget_workspace_id" => "ws3", + "widget_size" => "LARGE", + "widget_tags" => ["d", "e", "f"], + "widget_cost" => {"currency" => "USD", "amount_cents" => 20} + }) + + # This component was indexed after the related widget, and should be fully filled in. + expect(components["c56"]).to eq({ + "__sources" => ["__self", "widget"], + "__versions" => { + "__self" => {"c56" => 2}, + "widget" => {"w1" => 8} + }, + LIST_COUNTS_FIELD => { + "part_ids" => 1, + "tags" => 2, + "widget_tags" => 3 + }, + "id" => "c56", + "name" => "D", + "part_ids" => ["p1"], + "tags" => ["a", "b"], + "widget_name" => "Thingy", + "widget_workspace_id" => "ws3", + "widget_size" => "LARGE", + "widget_tags" => ["d", "e", "f"], + "widget_cost" => {"currency" => "USD", "amount_cents" => 20} + }) + + # The related widget for this component was never indexed, so it's missing the widget data. + expect(components["c78"]).to eq({ + "__sources" => ["__self"], + "__versions" => { + "__self" => {"c78" => 3} + }, + LIST_COUNTS_FIELD => { + "part_ids" => 1, + "tags" => 2 + }, + "id" => "c78", + "name" => "E", + "part_ids" => ["p1"], + "tags" => ["a", "b"], + "widget_name" => nil, + "widget_workspace_id" => nil, + "widget_size" => nil, + "widget_tags" => nil, + "widget_cost" => nil + }) + end + + it "does not allow mutations of relationships used by `sourced_from`, since allowing such a mutation would break ElasticGraph's 'ingest in any order' guaranteees" do + widget1 = build_upsert_event(:widget, id: "w1", component_ids: ["c23"]) + widget2 = build_upsert_event(:widget, id: "w2", component_ids: ["c23"]) + + indexer.processor.process([widget1], refresh_indices: true) + + expect { + indexer.processor.process([widget2], refresh_indices: true) + }.to raise_error Indexer::IndexingFailuresError, a_string_including( + "Cannot update document c23 with data from related widget w2 because the related widget has apparently changed (was: [w1]), " \ + "but mutations of relationships used with `sourced_from` are not supported because allowing it could break ElasticGraph's " \ + "out-of-order processing guarantees." + ) + end + + it "is compatible with custom shard routing and rollover indices, so long as `equivalent_field` is used on the schema definition" do + timestamp_in_2023 = "2023-08-09T10:12:14Z" + timestamp_in_2021 = "2021-08-09T10:12:14Z" + + widget = build_upsert_event(:widget, id: "w1", workspace_id: "wid_23", created_at: timestamp_in_2023) + workspace = build_upsert_event(:widget_workspace, id: "wid_23", name: "Garage", created_at: timestamp_in_2021, widget: { + id: "w1", + created_at: timestamp_in_2023 + }) + + indexer.processor.process([widget, workspace], refresh_indices: true) + + indexed_widget_source = search("widgets").dig(0, "_source") + expect(indexed_widget_source).to include({ + "id" => "w1", + "created_at" => timestamp_in_2023, + "workspace_id2" => "wid_23", + "workspace_name" => "Garage" # the sourced_from field copied from the `WidgetWorkspace` + }) + end + + def search_components + search("components").to_h do |hit| + source = hit.fetch("_source") + + [ + hit.fetch("_id"), + { + "id" => source.fetch("id"), + "name" => source.dig("name"), + "part_ids" => source.dig("part_ids"), + "tags" => source.dig("tags"), + "widget_cost" => source.dig("widget_cost"), + "widget_name" => source.dig("widget_name"), + "widget_size" => source.dig("widget_size"), + "widget_tags" => source.dig("widget_tags"), + "widget_workspace_id" => source.dig("widget_workspace_id3"), + "__versions" => source.dig("__versions"), + "__sources" => source.dig("__sources"), + LIST_COUNTS_FIELD => source.dig(LIST_COUNTS_FIELD) + } + ] + end + end + + def search(index_prefix) + main_datastore_client + .msearch(body: [{index: "#{index_prefix}*"}, {}]) + .dig("responses", 0, "hits", "hits") + end + end +end diff --git a/elasticgraph-indexer/spec/acceptance/schema_evolution_spec.rb b/elasticgraph-indexer/spec/acceptance/schema_evolution_spec.rb new file mode 100644 index 00000000..57ebb725 --- /dev/null +++ b/elasticgraph-indexer/spec/acceptance/schema_evolution_spec.rb @@ -0,0 +1,462 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer" +require "elastic_graph/schema_definition/rake_tasks" + +module ElasticGraph + RSpec.describe "Indexing schema evolution", :uses_datastore, :factories, :capture_logs, :in_temp_dir, :rake_task do + let(:path_to_schema) { "config/schema.rb" } + + before do + ::FileUtils.mkdir_p "config" + end + + shared_examples "schema evolution" do + context "when the schema has evolved to gain a new field" do + it "can ingest an event published before that field existed" do + write_address_schema_def(json_schema_version: 1) + dump_artifacts + + address_1_event = build_address_event_without_geolocation + address_2_event = build_address_event_without_geolocation + + boot_indexer.processor.process([address_1_event], refresh_indices: true) + + write_address_schema_def(json_schema_version: 2, address_extras: "t.field 'geo_location', 'GeoLocation'") + dump_artifacts + + indexer_with_geo_location = boot_indexer + indexer_with_geo_location.processor.process([address_2_event], refresh_indices: true) + + expect(search_for_ids("addresses")).to contain_exactly( + address_1_event.fetch("id"), + address_2_event.fetch("id") + ) + end + + def build_address_event_without_geolocation + build_upsert_event(:address, __exclude_fields: [:geo_location]).tap do |event| + # Verify that our `__exclude_fields` option worked correctly since the correctness of this test depends on it. + expect(event.fetch("record").keys).to exclude("geo_location") + end + end + end + + context "when the fields used for routing and index rollover have been renamed" do + it "can ingest an event published before the renames" do + write_widget_schema_def( + json_schema_version: 1, + widget_extras: <<~EOS, + t.field "created_at", "DateTime!" + t.field "name", "String" + EOS + widgets_index_config: <<~EOS + i.rollover :yearly, "created_at" + i.route_with "name" + EOS + ) + dump_artifacts + + write_widget_schema_def( + json_schema_version: 2, + widget_extras: <<~EOS, + t.field "created_at2", "DateTime!", name_in_index: "created_at" do |f| + f.renamed_from "created_at" + end + t.field "name2", "String", name_in_index: "name" do |f| + f.renamed_from "name" + end + EOS + widgets_index_config: <<~EOS + i.rollover :yearly, "created_at2" + i.route_with "name2" + EOS + ) + dump_artifacts + + v1_event = build_widget(json_schema_version: 1) do |widget| + widget.slice("id", "name", "created_at") + end + + v2_event = build_widget(json_schema_version: 2) do |widget| + widget.slice("id").merge( + "name2" => widget.fetch("name"), + "created_at2" => widget.fetch("created_at") + ) + end + + expect { + boot_indexer.processor.process([v1_event, v2_event], refresh_indices: true) + }.not_to raise_error + end + + def build_widget(json_schema_version:) + event = build_upsert_event(:widget, __json_schema_version: json_schema_version) + event.merge("record" => (yield event.fetch("record"))) + end + end + + context "when a schema has evolved to lose a new field and we are running on a new datastore cluster that doesn't have the removed field in its mapping" do + it "can still ingest old schema version events that have the removed field" do + write_address_schema_def(json_schema_version: 1, address_extras: "t.field 'deprecated', 'String'") + dump_artifacts + + # Attempt to drop the field; ElasticGraph should give us an error due to it still existing in the old JSON schema version. + write_address_schema_def(json_schema_version: 2) + expect { dump_artifacts }.to abort_with a_string_including( + "The `Address.deprecated` field (which existed in JSON schema version 1) no longer exists in the current schema definition." + ) + + # Try again after indicating the field has been deleted. + write_address_schema_def(json_schema_version: 2, address_extras: "t.deleted_field 'deprecated'") + dump_artifacts + + event = build_upsert_event(:address, id: "abc", deprecated: "foo", __json_schema_version: 1) + expect(event.dig("record", "deprecated")).to eq("foo") + + boot_indexer.processor.process([event], refresh_indices: true) + + expect(get_address_payload("abc")).to include("id" => "abc").and exclude("deprecated") + end + + def get_address_payload(id) + search("addresses").find { |h| h.fetch("_id") == id }.fetch("_source") + end + end + + context "when a type referenced from a `nested` field is renamed as part of evolving the schema" do + let(:existing_team_schema_def) do + ::File.read(::File.join(CommonSpecHelpers::REPO_ROOT, "config", "schema", "teams.rb")) + end + + it "can index events for either JSON schema version" do + # Write v1 schema with a public `wins` JsonSafeLong field called `wins` in the index + write_teams_schema_def(json_schema_version: 1) do |team_def| + expect(team_def).to include('t.field "seasons_nested", "[TeamSeason!]!"') + team_def + end + dump_artifacts + + # Write v2 schema with a public `wins` Int field with an alternate name in the index (to allow the type to change) + write_teams_schema_def(json_schema_version: 2) do |team_def| + safe_replace(team_def, "TeamSeason", "SeasonOfATeam").then do |updated| + safe_replace( + updated, + 'schema.object_type "SeasonOfATeam" do |t|', + <<~EOS + schema.object_type "SeasonOfATeam" do |t| + t.renamed_from "TeamSeason" + EOS + ) + end + end + dump_artifacts + + # The bug this test was written to cover relates to `__typename`, and was only triggered + # when a nested field's type included `__typename` even though it's not required to be + # included at that part of the JSON schema. So here we verify that the factory includes that. + expect(build(:team_season)).to include(__typename: "TeamSeason") + + v1_event = build_upsert_event(:team, __json_schema_version: 1) + v2_event = build_upsert_event(:team, __json_schema_version: 2) + .then { |event| ::JSON.generate(event) } + # Fix the event to align with the v2 schema, since `build_upsert_event` doesn't automatically + # know that the `__typename` should be `SeasonOfATeam` instead of `TeamSeason`. + .then { |json| json.gsub("TeamSeason", "SeasonOfATeam") } + .then { |json| ::JSON.parse(json) } + + expect { + boot_indexer.processor.process([v1_event, v2_event], refresh_indices: true) + }.not_to raise_error + end + end + + context "when a field under a `nested` field uses `name_in_index` as part of evolving the schema" do + let(:existing_team_schema_def) do + ::File.read(::File.join(CommonSpecHelpers::REPO_ROOT, "config", "schema", "teams.rb")) + end + + context "when `name_in_index` is added to an existing field to change the internal index field while keeping the same public field" do + it "can index events for either JSON schema version" do + # Write v1 schema with a public `wins` JsonSafeLong field called `wins` in the index + write_teams_schema_def(json_schema_version: 1) do |team_def| + safe_replace( + team_def, + 't.field "wins", "Int", name_in_index: "win_count"', + 't.field "wins", "JsonSafeLong"' + ) + end + dump_artifacts + + # Write v2 schema with a public `wins` Int field with an alternate name in the index (to allow the type to change) + write_teams_schema_def(json_schema_version: 2) do |team_def| + expect(team_def).to include('t.field "wins", "Int", name_in_index: "win_count"') + team_def + end + dump_artifacts + + v1_event = build_upsert_event(:team, __json_schema_version: 1) + v2_event = build_upsert_event(:team, __json_schema_version: 2) + + expect { + boot_indexer.processor.process([v1_event, v2_event], refresh_indices: true) + }.not_to raise_error + end + end + + context "when a public field is renamed and `name_in_index` is used to make the new field keep reading and writing the existing index field" do + it "can index events for either JSON schema version" do + # Write a v1 schema with a `full_name` field that is called `name` in the index + write_teams_schema_def(json_schema_version: 1) do |team_def| + safe_replace( + team_def, + 't.field "name", "String"', + 't.field "full_name", "String", name_in_index: "name"' + ) + end + dump_artifacts + + # Attempt to write a v2 schema with the `full_name` field renamed to `name` (while keeping the same field name in the index) + # ElasticGraph should raise an error since it's not sure what to do with that field on old events. + write_teams_schema_def(json_schema_version: 2) do |team_def| + expect(team_def).to include 't.field "name", "String"' + team_def + end + expect { dump_artifacts }.to abort_with a_string_including( + "The `Player.full_name` field (which existed in JSON schema version 1) no longer exists in the current schema definition." + ) + + write_teams_schema_def(json_schema_version: 2) do |team_def| + safe_replace( + team_def, + 't.field "name", "String"', + <<~EOS + t.field "name", "String" do |f| + f.renamed_from "full_name" + end + EOS + ) + end + dump_artifacts + + v1_event = build_upsert_event(:team, __json_schema_version: 1) + v1_event = ::JSON.parse(::JSON.generate(v1_event).gsub('"name":', '"full_name":')) + v2_event = build_upsert_event(:team, __json_schema_version: 2) + + expect { + boot_indexer.processor.process([v1_event, v2_event], refresh_indices: true) + }.not_to raise_error + end + end + + context "when an embedded type that was in an old JSON schema has been dropped" do + it "correctly ignores that type's data when ingesting old events" do + write_teams_schema_def(json_schema_version: 1) do |team_def| + # V1 has a `team_details: TeamDetails` field. + expect(team_def).to include('schema.object_type "TeamDetails"', 't.field "details", "TeamDetails"') + team_def + end + dump_artifacts + + # Attempt to write a v2 schema with the `TeamDetails` type and referencing field removed. + # ElasticGraph should raise an error since it's not sure what to do with that field/type on old events. + write_teams_schema_def(json_schema_version: 2) do |team_def| + safe_replace( + safe_replace(team_def, 't.field "details", "TeamDetails"', ""), + /schema.object_type "TeamDetails".*?\bend\n/m, + "" + ) + end + expect { + dump_artifacts + }.to abort_with a_string_including( + "The `Team.details` field (which existed in JSON schema version 1) no longer exists in the current schema definition.", + "The `TeamDetails` type (which existed in JSON schema version 1) no longer exists in the current schema definition." + ) + + write_teams_schema_def(json_schema_version: 2) do |team_def| + safe_replace( + safe_replace(team_def, 't.field "details", "TeamDetails"', 't.deleted_field "details"'), + /schema.object_type "TeamDetails".*?\bend\n/m, + 'schema.deleted_type "TeamDetails"' + ) + end + dump_artifacts + + v1_event = build_upsert_event(:team, __json_schema_version: 1) + + expect { + boot_indexer.processor.process([v1_event], refresh_indices: true) + }.not_to raise_error + end + end + + context "when an indexed type that was in an old JSON schema has been dropped" do + it "ignores an old event attempting to upsert that type" do + write_address_schema_def(json_schema_version: 1, schema_extras: <<~EOS) + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "current_name", "String" + t.field "league", "String" + t.field "formed_on", "Date" + t.index "teams" do |i| + i.route_with "league" + i.rollover :yearly, "formed_on" + end + end + EOS + dump_artifacts + + # Attempt to drop the `Team` indexed type--ElasticGraph should fail indicating it is still on the v1 schema. + write_address_schema_def(json_schema_version: 2) + expect { + dump_artifacts + }.to abort_with a_string_including( + "The `Team` type (which existed in JSON schema version 1) no longer exists in the current schema definition." + ) + + write_address_schema_def(json_schema_version: 2, schema_extras: 'schema.deleted_type "Team"') + dump_artifacts + + v1_event = build_upsert_event(:team, __json_schema_version: 1) + boot_indexer.processor.process([v1_event], refresh_indices: true) + + expect(search_for_ids("teams")).to be_empty + end + end + end + + def write_teams_schema_def(json_schema_version:) + # Comment out some lines that lead to schema dump warnings (we don't want the warnings in the output). + schema_def_contents = existing_team_schema_def.gsub(/^\s+t\.field schema\.state.schema_elements.count/, "#") + schema_def_contents = yield schema_def_contents + + ::File.write(path_to_schema, <<~EOS) + ElasticGraph.define_schema do |schema| + schema.json_schema_version #{json_schema_version} + + # Money is referenced by the team schema but is defined in the widgets schema so we have duplicate it here. + schema.object_type "Money" do |t| + t.field "currency", "String!" + t.field "amount_cents", "Int" + end + end + + #{schema_def_contents} + EOS + end + + def write_address_schema_def(json_schema_version:, address_extras: "", schema_extras: "") + # This is a pared down schema definition of our normal test schema `Address` type. + ::File.write(path_to_schema, <<~EOS) + ElasticGraph.define_schema do |schema| + schema.json_schema_version #{json_schema_version} + + schema.object_type "Address" do |t| + t.field "id", "ID!" + t.field "full_address", "String!" + #{address_extras} + t.index "addresses" + end + + #{schema_extras} + end + EOS + end + + def write_widget_schema_def(json_schema_version:, widget_extras: "", widgets_index_config: "") + # This is a pared down schema definition of our normal test schema `Address` type. + ::File.write(path_to_schema, <<~EOS) + ElasticGraph.define_schema do |schema| + schema.json_schema_version #{json_schema_version} + + schema.object_type "Widget" do |t| + t.field "id", "ID!" + #{widget_extras} + t.index "widgets", number_of_shards: 10 do |i| + #{widgets_index_config} + end + end + end + EOS + end + + # Like `gsub` but also asserts that the expected pattern is in the string, so that we know that it actually replaced something. + def safe_replace(string, pattern, replace_with) + if pattern.is_a?(::String) + expect(string).to include(pattern) + else + expect(string).to match(pattern) + end + + string.gsub(pattern, replace_with) + end + + def dump_artifacts + run_rake "schema_artifacts:dump" do |output| + SchemaDefinition::RakeTasks.new( + schema_element_name_form: :snake_case, + index_document_sizes: true, + path_to_schema: path_to_schema, + schema_artifacts_directory: "config/schema/artifacts", + enforce_json_schema_version: true, + output: output + ) + end + end + end + + context "when `use_updates_for_indexing` is `true`" do + include_examples "schema evolution" + + def boot_indexer + super(use_updates_for_indexing: true) + end + end + + context "when `use_updates_for_indexing` is `false`" do + include_examples "schema evolution" + + def boot_indexer + super(use_updates_for_indexing: false) + end + end + + def boot_indexer(use_updates_for_indexing:) + overrides = { + "datastore" => { + "index_definitions" => { + "addresses" => { + "use_updates_for_indexing" => use_updates_for_indexing + }, + "teams" => { + "use_updates_for_indexing" => use_updates_for_indexing + } + } + } + } + + settings = CommonSpecHelpers.parsed_test_settings_yaml + settings = Support::HashUtil.deep_merge(settings, {"schema_artifacts" => {"directory" => "config/schema/artifacts"}}) + settings = Support::HashUtil.deep_merge(settings, overrides) + + Indexer.from_parsed_yaml(settings) + end + + def search_for_ids(index_prefix) + search(index_prefix).map { |h| h.fetch("_id") } + end + + def search(index_prefix) + main_datastore_client + .msearch(body: [{index: "#{index_prefix}*"}, {}]) + .dig("responses", 0, "hits", "hits") + end + end +end diff --git a/elasticgraph-indexer/spec/integration/elastic_graph/indexer/datastore_indexing_router_spec.rb b/elasticgraph-indexer/spec/integration/elastic_graph/indexer/datastore_indexing_router_spec.rb new file mode 100644 index 00000000..79ae9768 --- /dev/null +++ b/elasticgraph-indexer/spec/integration/elastic_graph/indexer/datastore_indexing_router_spec.rb @@ -0,0 +1,467 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer/datastore_indexing_router" +require "elastic_graph/support/monotonic_clock" + +module ElasticGraph + class Indexer + RSpec.describe DatastoreIndexingRouter, :uses_datastore, :capture_logs do + # Here we disable VCR because we are dealing with `version` numbers. + # To guarantee that our `router.bulk` calls index the operations, we + # use monotonically increasing `version` values based on the current + # system time clock, and have configured VCR to match requests that only + # differ on the `version` values. However, when VCR is playing back the + # response will contain the `version` from when the cassette was recorded, + # which will differ from the version we are dealing with on this run of the + # test. + # + # To avoid odd, confusing failures, we just disable VCR here. + describe "#source_event_versions_in_index", :factories, :no_vcr do + shared_examples_for "source_event_versions_in_index" do + let(:indexer) { build_indexer } + let(:router) { indexer.datastore_router } + let(:operation_factory) { indexer.operation_factory } + + it "looks up the document version for each of the specified operations, returning a map of versions by operation" do + test_documents_of_type(:address) do |op| + expect(uses_custom_routing?(op)).to eq false + end + end + + it "queries the version from the correct shard when the index uses custom shard routing" do + test_documents_of_type(:widget) do |op| + expect(uses_custom_routing?(op)).to eq true + end + end + + it "returns an empty list of versions when only given an unversioned operation" do + unversioned_op = build_expecting_success(build_upsert_event(:widget)).find { |op| !op.versioned? } + expect(unversioned_op).to be_a(Operation::Update) + + expect { + versions_by_cluster_by_op = router.source_event_versions_in_index([unversioned_op]) + + expect(versions_by_cluster_by_op).to eq({unversioned_op => {"main" => []}}) + }.not_to change { datastore_requests("main") } + end + + it "finds the document on any shard, even if it differs from what the operation's routing key would route to" do + op1 = build_primary_indexing_op(:widget, id: "mutated_routing_key", workspace_id: "wid1") + + results = router.bulk([op1], refresh: true) + expect(results.successful_operations_by_cluster_name).to match("main" => a_collection_containing_exactly(op1)) + + op2 = build_primary_indexing_op(:widget, id: "mutated_routing_key", workspace_id: "wid2") + versions_by_cluster_by_op = router.source_event_versions_in_index([op2]) + + expect(versions_by_cluster_by_op.keys).to contain_exactly(op2) + expect(versions_by_cluster_by_op[op2]).to eq("main" => [op1.event.fetch("version")]) + end + + it "finds the document on any index, even if it differs from the operation's target index" do + op1 = build_primary_indexing_op(:widget, id: "mutated_rollover_timestamp", created_at: "2019-12-03T00:00:00Z") + + results = router.bulk([op1], refresh: true) + expect(results.successful_operations_by_cluster_name).to match("main" => a_collection_containing_exactly(op1)) + + op2 = build_primary_indexing_op(:widget, id: "mutated_rollover_timestamp", created_at: "2023-12-03T00:00:00Z") + versions_by_cluster_by_op = router.source_event_versions_in_index([op2]) + + expect(versions_by_cluster_by_op.keys).to contain_exactly(op2) + expect(versions_by_cluster_by_op[op2]).to eq("main" => [op1.event.fetch("version")]) + end + + it "logs a warning and returns all versions if multiple copies of the document are found" do + op1 = build_primary_indexing_op(:widget, id: "mutated_routing_and_timestamp", workspace_id: "wid1", created_at: "2019-12-03T00:00:00Z") + op2 = build_primary_indexing_op(:widget, id: "mutated_routing_and_timestamp", workspace_id: "wid2", created_at: "2023-12-03T00:00:00Z", __version: op1.event.fetch("version") + 1) + + results = router.bulk([op1, op2], refresh: true) + expect(results.successful_operations_by_cluster_name).to match("main" => a_collection_containing_exactly(op1, op2)) + + expect { + versions_by_cluster_by_op = router.source_event_versions_in_index([op1]) + expect(versions_by_cluster_by_op.keys).to contain_exactly(op1) + expect(versions_by_cluster_by_op[op1]).to match("main" => a_collection_containing_exactly( + op1.event.fetch("version"), + op2.event.fetch("version") + )) + + versions_by_cluster_by_op = router.source_event_versions_in_index([op2]) + expect(versions_by_cluster_by_op.keys).to contain_exactly(op2) + expect(versions_by_cluster_by_op[op2]).to match("main" => a_collection_containing_exactly( + op1.event.fetch("version"), + op2.event.fetch("version") + )) + }.to log_warning a_string_including("IdentifyDocumentVersionsGotMultipleResults") + + expect(logged_jsons_of_type("IdentifyDocumentVersionsGotMultipleResults")).to contain_exactly( + a_hash_including( + "id" => ["mutated_routing_and_timestamp", "mutated_routing_and_timestamp"], + "routing" => a_collection_containing_exactly("wid1", "wid2"), + "index" => a_collection_containing_exactly("widgets_rollover__after_2021", "widgets_rollover__2019") + ), + a_hash_including( + "id" => ["mutated_routing_and_timestamp", "mutated_routing_and_timestamp"], + "routing" => a_collection_containing_exactly("wid1", "wid2"), + "index" => a_collection_containing_exactly("widgets_rollover__after_2021", "widgets_rollover__2019") + ) + ) + end + end + + context "when `use_updates_for_indexing?` is set to false", use_updates_for_indexing: false do + include_examples "source_event_versions_in_index" do + it "supports all types of operations" do + op1 = build_primary_indexing_op(:widget) + expect(op1).to be_a Operation::Upsert + + op2 = build_expecting_success(build_upsert_event(:widget)).last + expect(op2).to be_a Operation::Update + + results = router.bulk([op1, op2], refresh: true) + expect(results.successful_operations_by_cluster_name).to match("main" => a_collection_containing_exactly(op1, op2)) + + versions_by_cluster_by_op = router.source_event_versions_in_index([op1, op2]) + expect(versions_by_cluster_by_op.keys).to contain_exactly(op1, op2) + expect(versions_by_cluster_by_op[op1]).to eq("main" => [op1.event.fetch("version")]) + expect(versions_by_cluster_by_op[op2]).to match("main" => []) # the derived document doesn't keep track of source event versions + end + + def build_primary_indexing_op(type, **overrides) + event = build_upsert_event(type, **overrides) + ops = build_expecting_success(event).grep(Operation::Upsert) + expect(ops.size).to eq(1) + ops.first + end + + def uses_custom_routing?(op) + op.to_datastore_bulk.first.fetch(:index).key?(:routing) + end + end + end + + shared_examples_for "source_event_versions_in_index when `use_updates_for_indexing?` is set to true" do + include_examples "source_event_versions_in_index" do + it "supports both primary indexing operations and derived indexing operations" do + derived_update, self_update = build_expecting_success(build_upsert_event(:widget)) + expect(derived_update.update_target.type).to eq("WidgetCurrency") + expect(self_update.update_target.type).to eq("Widget") + + results = router.bulk([derived_update, self_update], refresh: true) + expect(results.successful_operations_by_cluster_name).to match("main" => a_collection_containing_exactly(derived_update, self_update)) + + versions_by_cluster_by_op = router.source_event_versions_in_index([derived_update, self_update]) + expect(versions_by_cluster_by_op.keys).to contain_exactly(derived_update, self_update) + expect(versions_by_cluster_by_op[self_update]).to eq("main" => [derived_update.event.fetch("version")]) + + # The derived document doesn't keep track of `__versions` so it doesn't have a version it can return. + expect(versions_by_cluster_by_op[derived_update]).to eq("main" => []) + end + + def uses_custom_routing?(op) + op.to_datastore_bulk.first.fetch(:update).key?(:routing) + end + + def build_primary_indexing_op(type, **overrides) + event = build_upsert_event(type, **overrides) + ops = build_expecting_success(event).select { |op| op.update_target.for_normal_indexing? } + expect(ops.size).to eq(1) + ops.first + end + end + end + + context "when `use_updates_for_indexing?` is set to true (using the version of the `update_data` script from EG v0.8+)", use_updates_for_indexing: true do + include_examples "source_event_versions_in_index when `use_updates_for_indexing?` is set to true" + end + + context "when `use_updates_for_indexing?` is set to true (using the version of the `update_data` script from EG < v0.8)", use_updates_for_indexing: true do + include_examples "source_event_versions_in_index when `use_updates_for_indexing?` is set to true" + + def build_indexer(**options) + super(use_old_update_script: true, **options) + end + end + + def test_documents_of_type(type, &block) + op1 = build_primary_indexing_op(type).tap(&block) + op2 = build_primary_indexing_op(type).tap(&block) + op3 = build_primary_indexing_op(type).tap(&block) + + results = router.bulk([op1, op2], refresh: true) + + expect(results.successful_operations_by_cluster_name).to match("main" => a_collection_containing_exactly(op1, op2)) + + versions_by_cluster_by_op = router.source_event_versions_in_index([]) + expect(versions_by_cluster_by_op).to eq({}) + + versions_by_cluster_by_op = router.source_event_versions_in_index([op1, op2, op3]) + expect(versions_by_cluster_by_op.keys).to contain_exactly(op1, op2, op3) + expect(versions_by_cluster_by_op[op1]).to eq("main" => [op1.event.fetch("version")]) + expect(versions_by_cluster_by_op[op2]).to eq("main" => [op2.event.fetch("version")]) + expect(versions_by_cluster_by_op[op3]).to eq("main" => []) + end + + def build_expecting_success(event, **options) + result = operation_factory.build(event, **options) + # :nocov: -- our norm is to have no failure + raise result.failed_event_error if result.failed_event_error + # :nocov: + result.operations + end + end + + describe "#validate_mapping_completeness_of!" do + shared_examples_for "validate_mapping_completeness_of!" do |up_to_date_index_name, rollover:| + let(:monontonic_now_time) { 100_000 } + let(:monotonic_clock) { instance_double(Support::MonotonicClock, now_in_ms: monontonic_now_time) } + let(:person_schema_definition) do + lambda do |schema| + schema.object_type "Degree" do |t| + t.field "title", "String" + end + + schema.object_type "Person" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime" + t.field "graduate_degrees", "[Degree!]!" do |f| + f.mapping type: "object" + end + t.field "postgraduate_degrees", "[Degree!]!" do |f| + f.mapping type: "nested" + end + t.index unique_index_name do |i| + i.rollover :monthly, "created_at" if rollover + end + end + end + end + + it "does not raise an error when the schema is up-to-date in the datastore, and caches that fact so we don't have to re-query the datastore over and over for this" do + index_def, router = index_def_and_router_for(up_to_date_index_name) + + expect { + router.validate_mapping_completeness_of!(:all_accessible_cluster_names, index_def) + }.to make_datastore_calls("main") + + expect { + router.validate_mapping_completeness_of!(:all_accessible_cluster_names, index_def) + }.to make_no_datastore_calls("main") + end + + it "raises an error when the schema is not up-to-date in the datastore, and caches that fact for a period so we don't have to re-query the datastore over and over for this", :expect_warning_logging do + index_def, router = index_def_and_router_for(unique_index_name, schema_definition: person_schema_definition) do |config| + config.with(index_definitions: { + unique_index_name => config_index_def_of(index_into_clusters: ["main"]) + }) + end + + simulate_mapping_fetch_network_failure = false + allow(index_def).to receive(:mappings_in_datastore).and_wrap_original do |original_method, *args, **options| + raise Errors::RequestExceededDeadlineError, "Timed out" if simulate_mapping_fetch_network_failure + original_method.call(*args, **options) + end + + now_in_ms = monontonic_now_time + allow(monotonic_clock).to receive(:now_in_ms) { now_in_ms } + + cache_expiration_message_snippet = "Mapping cache expired for #{unique_index_name}" + + expect { + router.validate_mapping_completeness_of!(:all_accessible_cluster_names, index_def) + }.to raise_error(a_string_including("mappings are incomplete", unique_index_name, "+ properties")) + .and make_datastore_calls("main") + + expect { + router.validate_mapping_completeness_of!(:all_accessible_cluster_names, index_def) + }.to raise_error(a_string_including("mappings are incomplete", unique_index_name, "+ properties")) + .and make_no_datastore_calls("main") + + expect(logged_output).not_to include(cache_expiration_message_snippet, "Errors::RequestExceededDeadlineError") + + now_in_ms += DatastoreIndexingRouter::MAPPING_CACHE_MAX_AGE_IN_MS_RANGE.max + 1 + simulate_mapping_fetch_network_failure = true + + # After the cache expiration time has elapsed, it should attempt to refetch the mapping. + expect { + router.validate_mapping_completeness_of!(:all_accessible_cluster_names, index_def) + }.to raise_error(a_string_including("mappings are incomplete", unique_index_name, "+ properties")) + + # While that fetching failed, it should merely be reflected in a log message. + expect(logged_warnings.join).to include(cache_expiration_message_snippet, "got an error", "Errors::RequestExceededDeadlineError") + flush_logs + + now_in_ms += DatastoreIndexingRouter::MAPPING_CACHE_MAX_AGE_IN_MS_RANGE.max / 2 + + # ...and it should not attempt another refetch until the cache age elapses again. + expect { + router.validate_mapping_completeness_of!(:all_accessible_cluster_names, index_def) + }.to raise_error(a_string_including("mappings are incomplete", unique_index_name, "+ properties")) + .and make_no_datastore_calls("main") + + expect(logged_output).not_to include(cache_expiration_message_snippet, "Errors::RequestExceededDeadlineError") + + now_in_ms += DatastoreIndexingRouter::MAPPING_CACHE_MAX_AGE_IN_MS_RANGE.max + simulate_mapping_fetch_network_failure = false + + # Now that the cache age has passed, it should attempt to refetch it again. + expect { + router.validate_mapping_completeness_of!(:all_accessible_cluster_names, index_def) + }.to raise_error(a_string_including("mappings are incomplete", unique_index_name, "+ properties")) + .and make_datastore_calls("main") + + expect(logged_output).to include(cache_expiration_message_snippet).and exclude("got an error", "Errors::RequestExceededDeadlineError") + end + + it "validates the appropriate datastore clusters based on the passed argument" do + index_def, router = index_def_and_router_for(unique_index_name, schema_definition: person_schema_definition) do |config| + config.with(index_definitions: { + unique_index_name => config_index_def_of( + index_into_clusters: ["other1", "other2"], + query_cluster: "main" + ) + }) + end + + expect { + router.validate_mapping_completeness_of!(:accessible_cluster_names_to_index_into, index_def) + }.to raise_error(a_string_including("mappings are incomplete", unique_index_name, "+ properties")) + .and make_no_datastore_calls("main") + .and make_datastore_calls("other1") + .and make_datastore_calls("other2") + + expect { + router.validate_mapping_completeness_of!(:accessible_cluster_names_to_index_into, index_def) + }.to raise_error(a_string_including("mappings are incomplete", unique_index_name, "+ properties")) + .and make_no_datastore_calls("main") + .and make_no_datastore_calls("other1") + .and make_no_datastore_calls("other2") + + expect { + router.validate_mapping_completeness_of!(:all_accessible_cluster_names, index_def) + }.to raise_error(a_string_including("mappings are incomplete", unique_index_name, "+ properties")) + .and make_datastore_calls("main") + .and make_no_datastore_calls("other1") + .and make_no_datastore_calls("other2") + end + + define_method :index_def_config_and_router_for do |index_name, **options, &block| + indexer = build_indexer(monotonic_clock: monotonic_clock, **options, &block) + index_def = indexer.datastore_core.index_definitions_by_name.fetch(index_name) + + expect(index_def.rollover_index_template?).to eq(rollover) + index_config = + if rollover + indexer.schema_artifacts.index_templates.fetch(index_name) + else + indexer.schema_artifacts.indices.fetch(index_name) + end + + [index_def, index_config, indexer.datastore_router] + end + + def index_def_and_router_for(index_name, **options, &block) + index_def, _config, router = index_def_config_and_router_for(index_name, **options, &block) + [index_def, router] + end + end + + context "with a non-rollover index" do + include_examples "validate_mapping_completeness_of!", "addresses", rollover: false do + it "tolerates the index having an extra field that is not in the schema artifacts since we just ignore it and Elasticsearch/OpenSearch do not allow mapping removals" do + index_def, index_config, router = index_def_config_and_router_for(unique_index_name, schema_definition: person_schema_definition) do |config| + config.with(index_definitions: { + unique_index_name => config_index_def_of(index_into_clusters: ["main"]) + }) + end + + props_with_extra_field = index_config.dig("mappings", "properties").merge( + "name" => {"type" => "keyword"} + ) + + expect { + index_config = index_config.merge( + "mappings" => index_config["mappings"].merge("properties" => props_with_extra_field) + ) + }.to change { index_config.dig("mappings", "properties").keys } + .from(a_collection_excluding("name")) + .to(a_collection_including("name")) + + main_datastore_client.create_index(index: index_def.name, body: index_config) + + router.validate_mapping_completeness_of!(:all_accessible_cluster_names, index_def) + end + end + end + + context "with a rollover index" do + include_examples "validate_mapping_completeness_of!", "widgets", rollover: true do + it "also validates any related indices (e.g. concrete indices created from the rollover template)" do + index_def, router = index_def_and_router_for(unique_index_name, schema_definition: person_schema_definition) do |config| + config.with(index_definitions: { + unique_index_name => config_index_def_of(setting_overrides_by_timestamp: { + "2020-01-01T00:00:00Z" => {} + }) + }) + end + + expect(index_def.related_rollover_indices(main_datastore_client).map(&:name)).to eq [ + "#{unique_index_name}_rollover__2020-01" + ] + + expect { + router.validate_mapping_completeness_of!(:all_accessible_cluster_names, index_def) + }.to raise_error(a_string_including( + "On cluster `main` and index/template `#{unique_index_name}`", + "On cluster `main` and index/template `#{unique_index_name}_rollover__2020-01`" + )) + end + + it "tolerates the index having an extra field that is not in the schema artifacts since we just ignore it and Elasticsearch/OpenSearch do not allow mapping removals" do + index_def, index_config, router = index_def_config_and_router_for(unique_index_name, schema_definition: person_schema_definition) do |config| + config.with(index_definitions: { + unique_index_name => config_index_def_of(index_into_clusters: ["main"]) + }) + end + + props_with_extra_field = index_config.dig("template", "mappings", "properties").merge( + "name" => {"type" => "keyword"} + ) + + expect { + index_config = index_config.merge("template" => { + "mappings" => index_config.dig("template", "mappings").merge("properties" => props_with_extra_field), + "settings" => index_config.dig("template", "settings") + }) + }.to change { index_config.dig("template", "mappings", "properties").keys } + .from(a_collection_excluding("name")) + .to(a_collection_including("name")) + + main_datastore_client.put_index_template(name: index_def.name, body: index_config) + + router.validate_mapping_completeness_of!(:all_accessible_cluster_names, index_def) + end + end + end + + it "can be passed multiple index definitions to verify them all" do + indexer = build_indexer + widgets_index = indexer.datastore_core.index_definitions_by_name.fetch("widgets") + addresses_index = indexer.datastore_core.index_definitions_by_name.fetch("addresses") + + datastore_requests("main").clear + + indexer.datastore_router.validate_mapping_completeness_of!(:all_accessible_cluster_names, widgets_index, addresses_index) + + expect(datastore_requests("main").map(&:description).join("\n")).to include("GET /_index_template/widgets", "GET /addresses") + end + end + end + end +end diff --git a/elasticgraph-indexer/spec/integration/elastic_graph/indexer/processor_spec.rb b/elasticgraph-indexer/spec/integration/elastic_graph/indexer/processor_spec.rb new file mode 100644 index 00000000..830fd721 --- /dev/null +++ b/elasticgraph-indexer/spec/integration/elastic_graph/indexer/processor_spec.rb @@ -0,0 +1,386 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer/processor" + +module ElasticGraph + class Indexer + RSpec.describe Processor, :uses_datastore, :factories, :capture_logs do + shared_examples_for "processor examples" do |operation_verb| + let(:indexer) { build_indexer } + + context "process non-rollover upsert events" do + describe "upserts" do + let(:component_1_old) { build_upsert_event(:component, id: "123", name: "old_name") } + let(:component_1_new) { build_upsert_event(:component, id: "123", name: "new_name", __version: component_1_old.fetch("version") + 1) } + let(:component_2) { build_upsert_event(:component, id: "456", name: "old_name") } + + it "overwrites earlier versions of a document with a later version of the same document" do + process_batches([component_1_old], [component_1_new]) + response = search + + expect(get_component_names_from_response(response)).to contain_exactly("new_name") + end + + it "ignores earlier versions of a document that are processed after later versions" do + process_batches([component_1_new], [component_1_old]) + + response = search + + expect(get_component_names_from_response(response)).to contain_exactly("new_name") + end + + it "still overwrites earlier versions with later versions when both are in the same batch" do + process_batches([component_1_old, component_1_new]) + + response = search + + expect(get_component_names_from_response(response)).to contain_exactly("new_name") + end + + it "still ignores earlier versions of a document when both are in the same batch" do + process_batches([component_1_new, component_1_old]) + + response = search + + expect(get_component_names_from_response(response)).to contain_exactly("new_name") + end + + it "tolerates integer-valued-but-float-typed version values" do + # Here we use the monotonically increasing version number from `build_upsert_event` but convert it to a float. + # This is necessary to avoid confusing errors where version numbers on deleted documents "stick around" on + # the index for some indeterminate period of time after we delete all documents. + event = build_upsert_event(:component, name: "version_as_float") + event = event.merge("version" => event.fetch("version").to_f) + + process_batches([event]) + + response = search + + expect(get_component_names_from_response(response)).to contain_exactly("version_as_float") + end + + # Here we disable VCR because we are dealing with `version` numbers. + # To guarantee that our `router.bulk` calls index the operations, we + # use monotonically increasing `version` values based on the current + # system time clock, and have configured VCR to match requests that only + # differ on the `version` values. However, when VCR is playing back the + # response will contain the `version` from when the cassette was recorded, + # which will differ from the version we are dealing with on this run of the + # test. + # + # To avoid odd, confusing failures, we just disable VCR here. + context "when an event is malformed", :no_vcr do + let(:valid_event_1) { build_upsert_event(:component, id: "c678", name: "valid1") } + let(:malformed_event) { build_upsert_event(:component, id: "c789", name: 17) } # name is an integer instead of a string as expected + let(:valid_event_2) { build_upsert_event(:component, id: "c890", name: "valid2") } + + it "raises an error so the message goes into the DLQ and the issue is surfaced to the oncall engineer, while allowing the valid events to be indexed" do + expect { + process_batches([valid_event_1, malformed_event, valid_event_2]) + }.to raise_error IndexingFailuresError, a_string_including("c789").and(excluding("c678", "c890")) + + response = search + + expect(get_component_names_from_response(response)).to contain_exactly("valid1", "valid2") + end + + it "ignores the malformed event if it has been superseded by an indexed event with the same id and a greater version" do + process_batches([make_valid(malformed_event, version_offset: -1)]) + expect { process_batches([malformed_event]) }.to raise_error IndexingFailuresError, a_string_including("c789") + + process_batches([make_valid(malformed_event, version_offset: 0)]) + expect { process_batches([malformed_event]) }.to raise_error IndexingFailuresError, a_string_including("c789") + + process_batches([make_valid(malformed_event, version_offset: 1)]) + expect { process_batches([malformed_event]) }.to log_warning a_string_including( + "Ignoring 1 malformed event", + EventID.from_event(malformed_event).to_s + ) + + response = search + expect(get_component_names_from_response(response)).to contain_exactly("same_id_valid_event") + end + + it "ignores the version of a derived indexing type update when determining if an event has been superseded since the derived document's version is not related to the event version" do + valid_widget = build_upsert_event(:widget, id: "widget_34512") + superseded_invalid_widget = update_event(valid_widget, version_offset: -1) do |record| + record.merge("name" => 18) + end + + process_batches([valid_widget]) + + expect { process_batches([superseded_invalid_widget]) }.to log_warning a_string_including( + "Ignoring 1 malformed event", + EventID.from_event(superseded_invalid_widget).to_s + ) + end + + def make_valid(event, version_offset:) + update_event(event, version_offset: version_offset) do |record| + record.merge("name" => "same_id_valid_event") + end + end + + def update_event(event, version_offset:, &update) + event.merge( + "version" => event.fetch("version") + version_offset, + "record" => update.call(event.fetch("record")) + ) + end + end + end + end + + context "process rollover upsert events" do + describe "upserts" do + let(:widget_2019_06_02_old) { build_upsert_event(:widget, id: "123", workspace_id: "ws123", name: "2019_06_02_old_name", created_at: "2019-06-02T12:00:00Z") } + let(:widget_2019_06_02_new) { build_upsert_event(:widget, id: "123", workspace_id: "ws123", name: "2019_06_02_new_name", created_at: "2019-06-02T12:00:00Z", __version: widget_2019_06_02_old.fetch("version") + 1) } + let(:widget_2020_10_02) { build_upsert_event(:widget, id: "456", name: "2020_10_02_old_name", created_at: "2020-10-02T12:00:00Z") } + + it "writes to different indices/years based on `created_at`" do + process_batches([widget_2019_06_02_old, widget_2020_10_02]) + response = search(index: "widgets_rollover__*") + + expect(indexes_from_results(response)).to contain_exactly("widgets_rollover__2019", "widgets_rollover__2020") + expect(source_field_values(response, "id")).to contain_exactly("123", "456") + end + + it "overwrites earlier versions of a document with a later version of the same document" do + process_batches([widget_2019_06_02_old], [widget_2019_06_02_new]) + response = search(index: "widgets_rollover__*") + + expect(indexes_from_results(response)).to contain_exactly("widgets_rollover__2019") + expect(source_field_values(response, "name")).to contain_exactly("2019_06_02_new_name") + end + + it "ignores earlier versions of a document that are processed after later versions" do + process_batches([widget_2019_06_02_new], [widget_2019_06_02_old]) + + response = search(index: "widgets_rollover__*") + + expect(indexes_from_results(response)).to contain_exactly("widgets_rollover__2019") + expect(source_field_values(response, "name")).to contain_exactly("2019_06_02_new_name") + end + + def indexes_from_results(response) + response.dig("hits", "hits").map { |h| h["_index"] } + end + + def source_field_values(response, field) + response.dig("hits", "hits").map { |h| h.dig("_source", field) } + end + end + end + end + + context "when `use_updates_for_indexing?` is set to false", use_updates_for_indexing: false do + include_examples "processor examples", "upsert" + end + + context "when `use_updates_for_indexing?` is set to true", use_updates_for_indexing: true do + include_examples "processor examples", "update" do + it "can safely mix `version` values that use `int` vs `long` primitive types inside the datastore JVM" do + component_1 = build_upsert_event(:component, id: "version-mix-12", name: "name1", __version: 1) + component_2 = build_upsert_event(:component, id: "version-mix-12", name: "name2", __version: 2**61) + component_3 = build_upsert_event(:component, id: "version-mix-12", name: "name3", __version: 3) + + process_batches([component_1], [component_2]) + process_batches([component_3]) + + response = search + expect(get_component_names_from_response(response)).to contain_exactly("name2") + end + end + end + + # TODO: drop these backwards compatibility when we no longer need to maintain compatibility with the old version of the script. + context "on a system that's been using `use_updates_for_indexing: true` with the initial v0.8 update data script", use_updates_for_indexing: true do + let(:component_id) { "c12" } + let(:new_script_indexer) { build_indexer(use_old_update_script: false) } + let(:old_script_indexer) { build_indexer(use_old_update_script: true) } + + context "when a new event has a greater version than a previously processed event" do + let(:old_event) { build_component_event(name: "old", version: 1) } + let(:new_event) { build_component_event(name: "new", version: 2) } + + it "overwrites the data when old event used old script and new event uses new script" do + process_batches([old_event], via: old_script_indexer) + + expect { + process_batches([new_event], via: new_script_indexer) + }.to change_name_and_version_metadata( + from: { + "name" => "old", + "__sourceVersions" => {"Component" => {component_id => 1}}, + "__versions" => {SELF_RELATIONSHIP_NAME => {component_id => 1}} + }, + to: { + "name" => "new", + "__sourceVersions" => {"Component" => {component_id => 1}}, + "__versions" => {SELF_RELATIONSHIP_NAME => {component_id => 2}} + } + ) + end + + it "overwrites the data when old event used new script and new event uses old script" do + process_batches([old_event], via: new_script_indexer) + + expect { + process_batches([new_event], via: old_script_indexer) + }.to change_name_and_version_metadata( + from: { + "name" => "old", + "__versions" => {SELF_RELATIONSHIP_NAME => {component_id => 1}} + }, + to: { + "name" => "new", + "__sourceVersions" => {"Component" => {component_id => 2}}, + "__versions" => {SELF_RELATIONSHIP_NAME => {component_id => 2}} + } + ) + end + end + + context "when a new event has an equal version to a previously processed event" do + let(:old_event) { build_component_event(name: "old", version: 1) } + let(:new_event) { build_component_event(name: "new", version: 1) } + + it "leaves the data unchanged when old event used old script and new event uses new script" do + process_batches([old_event], via: old_script_indexer) + + expect { + process_batches([new_event], via: new_script_indexer) + }.to leave_name_and_version_metadata_unchanged_from( + "name" => "old", + "__sourceVersions" => {"Component" => {component_id => 1}}, + "__versions" => {SELF_RELATIONSHIP_NAME => {component_id => 1}} + ) + end + + it "leaves the data unchanged when old event used new script and new event uses old script" do + process_batches([old_event], via: new_script_indexer) + + expect { + process_batches([new_event], via: old_script_indexer) + }.to leave_name_and_version_metadata_unchanged_from( + "name" => "old", + "__versions" => {SELF_RELATIONSHIP_NAME => {component_id => 1}} + ) + end + end + + context "when a new event has a lesser version than a previously processed event" do + let(:old_event) { build_component_event(name: "old", version: 2) } + let(:new_event) { build_component_event(name: "new", version: 1) } + + it "leaves the data unchanged when old event used old script and new event uses new script" do + process_batches([old_event], via: old_script_indexer) + + expect { + process_batches([new_event], via: new_script_indexer) + }.to leave_name_and_version_metadata_unchanged_from( + "name" => "old", + "__sourceVersions" => {"Component" => {component_id => 2}}, + "__versions" => {SELF_RELATIONSHIP_NAME => {component_id => 2}} + ) + end + + it "leaves the data unchanged when old event used new script and new event uses old script" do + process_batches([old_event], via: new_script_indexer) + + expect { + process_batches([new_event], via: old_script_indexer) + }.to leave_name_and_version_metadata_unchanged_from( + "name" => "old", + "__versions" => {SELF_RELATIONSHIP_NAME => {component_id => 2}} + ) + end + + it "considers the max version from `__sourceVersions` and `__versions` to be the document's version when deciding whether to process an update" do + expected_state1 = { + "name" => "name1", + "__versions" => {SELF_RELATIONSHIP_NAME => {component_id => 1}} + } + + expected_state2 = { + "name" => "name2", + "__versions" => {SELF_RELATIONSHIP_NAME => {component_id => 2}}, + "__sourceVersions" => {"Component" => {component_id => 2}} + } + + expected_state3 = { + "name" => "name3", + "__versions" => {SELF_RELATIONSHIP_NAME => {component_id => 3}}, + "__sourceVersions" => {"Component" => {component_id => 2}} + } + + # 1) Successfully index using the new script... + process_batches([build_component_event(name: "name1", version: 1)], via: new_script_indexer) + + # 2) Then successfully index using the old script. Notably, the old script sets the version on `__sourceVersions` + # but not on `__versions`, leaving `__versions` with an out-of-date value. + expect { + process_batches([build_component_event(name: "name2", version: 2)], via: old_script_indexer) + }.to change_name_and_version_metadata(from: expected_state1, to: expected_state2) + + # 3) Then try to index using the new script with an event that doesn't have a greater version. It should + # leave things unchanged. (Which requires it to use the version found in `__sourceVersions`, not the + # version found in` __versions`). + expect { + process_batches([build_component_event(name: "name3", version: 2)], via: new_script_indexer) + }.to leave_name_and_version_metadata_unchanged_from(expected_state2) + + # 4) Finally try to index using the new script with an event that has a greater version. It should succeed. + expect { + process_batches([build_component_event(name: "name3", version: 3)], via: new_script_indexer) + }.to change_name_and_version_metadata(from: expected_state2, to: expected_state3) + end + end + + def build_component_event(name:, version:) + build_upsert_event(:component, id: component_id, name: name, __version: version) + end + + def change_name_and_version_metadata(from:, to:) + change { indexed_name_and_version_metadata }.from(from).to(to) + end + + def leave_name_and_version_metadata_unchanged_from(expected_name_and_version_metadata) + maintain { indexed_name_and_version_metadata } + .from(expected_name_and_version_metadata) + end + + def indexed_name_and_version_metadata + results = search.dig("hits", "hits") + .select { |h| h["_index"] == "components" } + .map { |h| h.dig("_source").slice("name", "__versions", "__sourceVersions") } + + expect(results.size).to eq(1) + results.first + end + end + + def get_component_names_from_response(response) + response.dig("hits", "hits") + .select { |h| h["_index"] == "components" } + .map { |h| h.dig("_source", "name") } + end + + def process_batches(*batches, via: indexer) + batches.each do |batch| + via.processor.process(batch, refresh_indices: true) + end + end + + def search(index: "*") + main_datastore_client.msearch(body: [{index: index}, {}]).dig("responses", 0) + end + end + end +end diff --git a/elasticgraph-indexer/spec/spec_helper.rb b/elasticgraph-indexer/spec/spec_helper.rb new file mode 100644 index 00000000..a61d8ce0 --- /dev/null +++ b/elasticgraph-indexer/spec/spec_helper.rb @@ -0,0 +1,105 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-indexer`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. + +require "delegate" + +module ElasticGraph + module IndexerSpecHelpers + class UseOldUpdateScripts < ::SimpleDelegator + def script_id + OLD_INDEX_DATA_UPDATE_SCRIPT_ID + end + + def for_normal_indexing? + true + end + end + + def with_use_updates_for_indexing(config, use_updates_for_indexing) + config.with(index_definitions: config.index_definitions.transform_values do |index_def| + index_def.with(use_updates_for_indexing: use_updates_for_indexing) + end) + end + + def build_indexer(use_old_update_script: false, **options, &block) + return super(**options, &block) unless use_old_update_script + + schema_artifacts = SchemaArtifacts::FromDisk.new( + ::File.join(CommonSpecHelpers::REPO_ROOT, "config", "schema", "artifacts"), + :indexer + ) + + schema_artifacts.runtime_metadata.object_types_by_name.each do |name, object_type| + object_type.update_targets.map! do |update_target| + if update_target.for_normal_indexing? + UseOldUpdateScripts.new(update_target) + else + update_target + end + end + end + + super(schema_artifacts: schema_artifacts, **options, &block) + end + end + + module UseUpdatesForIndexingTrue + def build_indexer(**options) + super do |config| + with_use_updates_for_indexing(config, true) + end + end + end + + module UseUpdatesForIndexingFalse + def build_indexer(**options) + super do |config| + with_use_updates_for_indexing(config, false) + end + end + + module WithFactories + # standard:disable Lint/UnderscorePrefixedVariableName + def build(type, __version: nil, **attributes) + # The default strategy our factories use for `__version` works for `use_updates_for_indexing: true`, + # but not for `use_updates_for_indexing: false`. When `use_updates_for_indexing` is `false`, our indexing + # calls compare the document's version against the event version. Our "delete all documents" cleanup + # that gets performed before every integration or acceptance test impacts this as well: the datastore + # remembers the `version` of a deleted document, and if you try indexing a new payload for that document + # with a lower version, it'll reject it. During local development, we do not restart the datastore between + # every `rspec` run (that would slow us down a ton...), which means that the datastore's memory of the + # versions of documents deleted in a prior test run can impact a later test run. + # + # Here we use a monotonic clock to guarantee that every factory-generated record has a higher version than + # every previously generated record--including ones generated on prior test runs. Note that on OS X it appears + # that the system clock does not return nanosecond precision times (the last 3 digits are consistently 000...) = + # but we trust that will never generate multiple factory records for the same record type and id on the exact + # same microsecond so it should be sufficient. + __version ||= Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond) + + super(type, __version: __version, **attributes) + end + # standard:enable Lint/UnderscorePrefixedVariableName + end + end + + RSpec.configure do |config| + config.define_derived_metadata(absolute_file_path: %r{/elasticgraph-indexer/}) do |meta| + meta[:builds_indexer] = true + end + + config.prepend IndexerSpecHelpers, absolute_file_path: %r{/elasticgraph-indexer/} + config.include UseUpdatesForIndexingTrue, use_updates_for_indexing: true + config.include UseUpdatesForIndexingFalse, use_updates_for_indexing: false + config.prepend UseUpdatesForIndexingFalse::WithFactories, use_updates_for_indexing: false, factories: true + end +end diff --git a/elasticgraph-indexer/spec/support/indexing_preparer.rb b/elasticgraph-indexer/spec/support/indexing_preparer.rb new file mode 100644 index 00000000..9c75c5b7 --- /dev/null +++ b/elasticgraph-indexer/spec/support/indexing_preparer.rb @@ -0,0 +1,69 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# Provides test harness support for testing indexing preparers. Instead of calling +# an index preparer directly, this tests it via the overall `RecordPreparer`, which +# gives us confidence that the index preparer is used as expected from the `RecordPreparer`. +# For example, to see your indexing preparer being used, this requires that it is registered +# on your scalar type correctly, whereas if this directly called your indexing preparer +# it wouldn't require it to be correctly registered. +RSpec.shared_context "indexing preparer support" do |scalar_type| + before(:context) do + @record_preparer = build_indexer(clients_by_name: {}, schema_definition: lambda do |schema| + schema.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "scalar", scalar_type + t.field "array_of_scalar", "[#{scalar_type}]" + t.field "array_of_array_of_scalar", "[[#{scalar_type}]]" + t.field "array_of_object", "[Object]" do |f| + f.mapping type: "object" + end + t.index "my_type" + end + + schema.object_type "Object" do |t| + t.field "scalar", scalar_type + end + end).record_preparer_factory.for_latest_json_schema_version + end + + def prepare_scalar_value(value) + prepare_field_value("scalar", value) + end + + def prepare_array_values(values) + prepare_field_value("array_of_scalar", values) + end + + def prepare_array_of_array_of_values(values) + prepare_field_value("array_of_array_of_scalar", values) + end + + def prepare_array_of_objects_of_values(values) + input_objects = values.map { |v| {"scalar" => v} } + output_objects = prepare_field_value("array_of_object", input_objects) + expect(output_objects.map(&:keys)).to all eq(["scalar"]) + output_objects.map { |o| o.fetch("scalar") } + end + + private + + def prepare_field_value(field, value) + record = @record_preparer.prepare_for_index("MyType", build_record(field => value)) + record.fetch(field) + end + + def build_record(overrides) + { + "id" => "some-id", + "scalar" => nil, + "array_of_scalar" => nil, + "array_of_array_of_scalar" => nil + }.merge(overrides) + end +end diff --git a/elasticgraph-indexer/spec/support/multiple_version_support.rb b/elasticgraph-indexer/spec/support/multiple_version_support.rb new file mode 100644 index 00000000..04d7689f --- /dev/null +++ b/elasticgraph-indexer/spec/support/multiple_version_support.rb @@ -0,0 +1,38 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/schema_definition_helpers" + +module ElasticGraph + class Indexer + ::RSpec.shared_context "MultipleVersionSupport" do + include_context "SchemaDefinitionHelpers" + + def build_indexer_with_multiple_schema_versions(schema_versions:) + results_by_version = schema_versions.to_h do |json_schema_version, prior_def| + results = define_schema(schema_element_name_form: :snake_case, json_schema_version: json_schema_version, &prior_def) + [json_schema_version, results] + end + + json_schemas_by_version = results_by_version.to_h do |version, results| + [version, results.json_schemas_for(version)] + end + + artifacts = results_by_version.fetch(results_by_version.keys.max) + + allow(artifacts).to receive(:available_json_schema_versions).and_return(json_schemas_by_version.keys.to_set) + allow(artifacts).to receive(:latest_json_schema_version).and_return(json_schemas_by_version.keys.max) + allow(artifacts).to receive(:json_schemas_for) do |version| + json_schemas_by_version.fetch(version) + end + + build_indexer(schema_artifacts: artifacts) + end + end + end +end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/config_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/config_spec.rb new file mode 100644 index 00000000..f04e10d2 --- /dev/null +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/config_spec.rb @@ -0,0 +1,36 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer/config" +require "yaml" + +module ElasticGraph + class Indexer + RSpec.describe Config do + it "raises an error when given an unrecognized config setting" do + expect { + Config.from_parsed_yaml("indexer" => { + "latency_slo_thresholds_by_timestamp_in_ms" => {}, + "fake_setting" => 23 + }) + }.to raise_error Errors::ConfigError, a_string_including("fake_setting") + end + + it "converts the values of `skip_derived_indexing_type_updates` to a set" do + config = Config.from_parsed_yaml("indexer" => { + "latency_slo_thresholds_by_timestamp_in_ms" => {}, + "skip_derived_indexing_type_updates" => { + "WidgetCurrency" => ["USD"] + } + }) + + expect(config.skip_derived_indexing_type_updates).to eq("WidgetCurrency" => ["USD"].to_set) + end + end + end +end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/datastore_indexing_router_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/datastore_indexing_router_spec.rb new file mode 100644 index 00000000..b10aa0c1 --- /dev/null +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/datastore_indexing_router_spec.rb @@ -0,0 +1,777 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer" +require "elastic_graph/constants" +require "elastic_graph/elasticsearch/client" +require "elastic_graph/indexer/datastore_indexing_router" +require "elastic_graph/indexer/operation/factory" + +module ElasticGraph + class Indexer + RSpec.describe DatastoreIndexingRouter, :capture_logs do + let(:main_datastore_client) { instance_spy(Elasticsearch::Client, cluster_name: "main") } + let(:other_datastore_client) { instance_spy(Elasticsearch::Client, cluster_name: "other") } + let(:indexer) { build_indexer } + let(:router) { indexer.datastore_router } + let(:noop_version_conflict_reason) { "[123]: version conflict, current version [534319179481001] is higher or equal to the one provided [534319179481000]" } + + describe "#source_event_versions_in_index" do + shared_examples_for "source_event_versions_in_index" do + before do + stub_msearch_on(main_datastore_client, "main") + stub_msearch_on(other_datastore_client, "other") + end + + let(:requested_docs_by_client) { ::Hash.new { |h, k| h[k] = [] } } + let(:stubbed_versions_by_index_and_id) { {} } + let(:widget_primary_indexing_op) { new_primary_indexing_op({"type" => "Widget", "id" => "1", "version" => 1, "record" => {"id" => "1", "some_field" => "value", "created_at" => "2021-08-24T23:30:00Z", "workspace_id" => "ws123"}}) } + let(:component_primary_indexing_op) { new_primary_indexing_op({"type" => "Component", "id" => "7", "version" => 1, "record" => {"id" => "1", "some_field" => "value"}}) } + let(:widget_derived_update_op) do + new_operation( + {"type" => "Widget", "id" => "4", "version" => 1, "record" => {"id" => "1", "currency" => "USD", "name" => "thing1"}}, + destination_index_def: indexer.datastore_core.index_definitions_by_name.fetch("widget_currencies"), + update_target: indexer.schema_artifacts.runtime_metadata.object_types_by_name.fetch("Widget").update_targets.first, + doc_id: "USD" + ) + end + + it "does not make a request to the datastore if the operations list is empty" do + expect(router.source_event_versions_in_index([])).to eq({}) + + expect(main_datastore_client).not_to have_received(:msearch) + expect(other_datastore_client).not_to have_received(:msearch) + end + + it "raises an error when the datastore returns unexpected errors" do + allow(main_datastore_client).to receive(:msearch) do |request| + # 4 elements = searches for 2 documents since its a search header + search body for each. + expect(request.fetch(:body).size).to eq(4) + + { + "responses" => [ + # These are example failures that we got while implementing the `source_event_versions_in_index` logic before we had it entirely correct. + # These specific errors should no longer happen but we are using them here as examples of what failures look like. + {"status" => 400, "error" => {"root_cause" => [{"type" => "null_pointer_exception", "reason" => "type must not be null"}], "type" => "null_pointer_exception", "reason" => "type must not be null"}}, + {"status" => 400, "error" => {"root_cause" => [{"type" => "index_not_found_exception", "reason" => "no such index [widgets]", "resource.type" => "index_expression", "resource.id" => "widgets", "index_uuid" => "_na_", "index" => "widgets"}], "type" => "index_not_found_exception", "reason" => "no such index [widgets]", "resource.type" => "index_expression", "resource.id" => "widgets", "index_uuid" => "_na_", "index" => "widgets"}} + ] + } + end + + expect { + router.source_event_versions_in_index([widget_primary_indexing_op, component_primary_indexing_op]) + }.to raise_error Errors::IdentifyDocumentVersionsFailedError, a_string_including( + "null_pointer_exception", "index_not_found_exception" + ) + end + + context "when configured to index types into separate clusters" do + let(:indexer) { build_indexer(index_to_clusters: {"components" => {"index_into_clusters" => ["main", "other"]}}) } + + it "queries the version on the appropriate clusters for each operation" do + stubbed_versions_by_index_and_id[["widgets", widget_primary_indexing_op.doc_id]] = 17 + stubbed_versions_by_index_and_id[["components", component_primary_indexing_op.doc_id]] = 27 + + results = router.source_event_versions_in_index([widget_primary_indexing_op, component_primary_indexing_op]) + + expect(results).to eq( + widget_primary_indexing_op => {"main" => [17]}, + component_primary_indexing_op => {"main" => [27], "other" => [27]} + ) + + expect(requested_docs_by_client.keys).to contain_exactly("main", "other") + expect(requested_docs_by_client["main"]).to contain_exactly( + doc_version_request_for(widget_primary_indexing_op), + doc_version_request_for(component_primary_indexing_op) + ) + expect(requested_docs_by_client["other"]).to contain_exactly( + doc_version_request_for(component_primary_indexing_op) + ) + end + end + + context "when a type is configured with a cluster name that is not itself configured" do + let(:indexer) do + build_indexer(index_to_clusters: { + "components" => {"index_into_clusters" => ["undefined"]}, + "widgets" => {"index_into_clusters" => ["main"]} + }) + end + + it "avoids querying the unconfigured cluster, and returns `nil` for the version" do + stubbed_versions_by_index_and_id[["widgets", widget_primary_indexing_op.doc_id]] = 17 + stubbed_versions_by_index_and_id[["components", component_primary_indexing_op.doc_id]] = 27 + + results = router.source_event_versions_in_index([widget_primary_indexing_op, component_primary_indexing_op]) + + expect(results).to eq( + widget_primary_indexing_op => {"main" => [17]}, + component_primary_indexing_op => {"undefined" => []} + ) + end + end + end + + context "when `use_updates_for_indexing?` is set to false" do + def build_indexer(**options, &block) + super(use_updates_for_indexing: false, **options, &block) + end + + include_context "source_event_versions_in_index" do + it "supports all types of operations" do + stubbed_versions_by_index_and_id[["widgets", widget_primary_indexing_op.doc_id]] = 17 + stubbed_versions_by_index_and_id[["widget_currencies", widget_derived_update_op.doc_id]] = 33 + stubbed_versions_by_index_and_id[["components", component_primary_indexing_op.doc_id]] = 27 + + # Note: we intentionally mix the operations which are ignored and the non-ignored operations to force + # the implementation to handle them correctly. + results = router.source_event_versions_in_index([widget_primary_indexing_op, widget_derived_update_op, component_primary_indexing_op]) + + expect(requested_docs_by_client.keys).to contain_exactly("main") + expect(requested_docs_by_client["main"]).to contain_exactly( + doc_version_request_for(widget_primary_indexing_op), + doc_version_request_for(component_primary_indexing_op) + ) + + expect(results.keys).to contain_exactly(widget_primary_indexing_op, component_primary_indexing_op, widget_derived_update_op) + expect(results[widget_primary_indexing_op]).to eq("main" => [17]) + expect(results[component_primary_indexing_op]).to eq("main" => [27]) + expect(results[widget_derived_update_op]).to eq("main" => []) # as an unversioned op we return an empty list + end + + def new_primary_indexing_op(event) + new_operation(event) + end + + def stub_msearch_on(client, client_name) + allow(client).to receive(:msearch) do |request| + requested_docs = request.dig(:body).each_slice(2).map do |(search_header, search_body)| + # verify we avoid requesting fields we don't need to identify the version + expect(search_body).to include(_source: false, version: true) + + [search_header.fetch(:index), search_body.fetch(:query).fetch(:ids).fetch(:values).first] + end + + requested_docs_by_client[client_name].concat(requested_docs) + + responses = requested_docs.map do |(index, id)| + version = stubbed_versions_by_index_and_id[[index, id]] + hit = {"_index" => index, "_type" => "_doc", "_id" => id, "_version" => version} + {"hits" => {"hits" => [hit].compact}} + end + + {"responses" => responses} + end + end + end + end + + context "when `use_updates_for_indexing?` is set to true" do + def build_indexer(**options, &block) + super(use_updates_for_indexing: true, **options, &block) + end + + include_context "source_event_versions_in_index" do + it "supports normal updates and derived indexing type update operations" do + stubbed_versions_by_index_and_id[["widgets", widget_primary_indexing_op.doc_id]] = 17 + stubbed_versions_by_index_and_id[["widget_currencies", widget_derived_update_op.doc_id]] = 33 + stubbed_versions_by_index_and_id[["components", component_primary_indexing_op.doc_id]] = 27 + + # Note: we intentionally mix the operations which are ignored and the non-ignored operations to force + # the implementation to handle them correctly. + results = router.source_event_versions_in_index([widget_primary_indexing_op, widget_derived_update_op, component_primary_indexing_op]) + + expect(requested_docs_by_client.keys).to contain_exactly("main") + expect(requested_docs_by_client["main"]).to contain_exactly( + doc_version_request_for(widget_primary_indexing_op), + doc_version_request_for(component_primary_indexing_op) + ) + + expect(results.keys).to contain_exactly(widget_primary_indexing_op, component_primary_indexing_op, widget_derived_update_op) + expect(results[widget_primary_indexing_op]).to eq("main" => [17]) + expect(results[component_primary_indexing_op]).to eq("main" => [27]) + + # The derived document doesn't keep track of `__versions` so it doesn't have a version it can return. + expect(results[widget_derived_update_op]).to eq("main" => []) + end + + def new_primary_indexing_op(event) + update_targets = indexer + .schema_artifacts + .runtime_metadata + .object_types_by_name + .fetch(event.fetch("type")) + .update_targets + .select { |ut| ut.type == event.fetch("type") } + + expect(update_targets.size).to eq(1) + index_def = indexer.datastore_core.index_definitions_by_graphql_type.fetch(event.fetch("type")).first + + Operation::Update.new( + event: event, + prepared_record: indexer.record_preparer_factory.for_latest_json_schema_version.prepare_for_index( + event.fetch("type"), + event.fetch("record") + ), + destination_index_def: index_def, + update_target: update_targets.first, + doc_id: event.fetch("id"), + destination_index_mapping: indexer.schema_artifacts.index_mappings_by_index_def_name.fetch(index_def.name) + ) + end + + def stub_msearch_on(client, client_name) + allow(client).to receive(:msearch) do |request| + requested_docs = request.dig(:body).each_slice(2).map do |(search_header, search_body)| + # verify we avoid requesting fields we don't need to identify the version + expect(search_body.dig(:_source, :includes)).to include(a_string_starting_with("__versions.")) + + [search_header.fetch(:index), search_body.fetch(:query).fetch(:ids).fetch(:values).first] + end + + requested_docs_by_client[client_name].concat(requested_docs) + + responses = requested_docs.map do |(index, id)| + version = stubbed_versions_by_index_and_id[[index, id]] + + relationship = { + "widgets" => SELF_RELATIONSHIP_NAME, + "components" => SELF_RELATIONSHIP_NAME, + "widget_currencies" => "currency" + }.fetch(index) + + hit = { + "_index" => index, "_type" => "_doc", "_id" => id, "_source" => { + "__versions" => {relationship => {id => version}} + } + } + + {"hits" => {"hits" => [hit]}} + end + + {"responses" => responses} + end + end + end + end + + def doc_version_request_for(op) + [ + op.destination_index_def.index_expression_for_search, + op.doc_id + ] + end + end + + describe "#bulk" do + # `Router#bulk` delegates to `validate_mapping_completeness_of!` and essentially treats it as a collaborator. + # Here we express it as a collaborator by essentially providing an alias for it. + let(:index_mapping_checker) { router } + + before do + allow(index_mapping_checker).to receive(:validate_mapping_completeness_of!) + allow(main_datastore_client).to receive(:bulk) { |request| respond_to_datastore_client_bulk_request(request) } + allow(other_datastore_client).to receive(:bulk) { |request| respond_to_datastore_client_bulk_request(request) } + end + + let(:widget_derived_update_op) do + new_operation( + {"type" => "Widget", "id" => "1", "version" => 1, "record" => {"id" => "1", "currency" => "USD", "name" => "thing1"}}, + destination_index_def: indexer.datastore_core.index_definitions_by_name.fetch("widget_currencies"), + update_target: indexer.schema_artifacts.runtime_metadata.object_types_by_name.fetch("Widget").update_targets.reject(&:for_normal_indexing?).first, + doc_id: "USD" + ) + end + + let(:operations) do + [ + new_operation({"type" => "Widget", "id" => "1", "version" => 1, "record" => {"id" => "1", "some_field" => "value"}}), + new_operation({"type" => "Widget", "id" => "2", "version" => 1, "record" => {"id" => "2", "some_field" => "value"}}), + new_operation({"type" => "Component", "id" => "1", "version" => 1, "record" => {"id" => "1", "some_field" => "value"}}), + widget_derived_update_op + ] + end + + it "transforms the provided operations into an appropriate bulk body" do + bulk_request = nil + allow(main_datastore_client).to receive(:bulk) do |request| + bulk_request = request + respond_to_datastore_client_bulk_request(request) + end + + router.bulk(operations, refresh: true) + + expect(bulk_request).to include(:refresh, :body) + expect(bulk_request[:refresh]).to eq(true) + + submitted_body = bulk_request[:body] + expect(submitted_body.size).to eq(8) + + expect(submitted_body[0]).to eq({update: {_id: "1", _index: "widgets", retry_on_conflict: 5}}) + expect(submitted_body[1]).to include( + script: a_hash_including(id: INDEX_DATA_UPDATE_SCRIPT_ID, params: a_hash_including("data")), + scripted_upsert: true, + upsert: {} + ) + + expect(submitted_body[2]).to eq({update: {_id: "2", _index: "widgets", retry_on_conflict: 5}}) + expect(submitted_body[3]).to include( + script: a_hash_including(id: INDEX_DATA_UPDATE_SCRIPT_ID, params: a_hash_including("data")), + scripted_upsert: true, + upsert: {} + ) + + expect(submitted_body[4]).to eq({update: {_id: "1", _index: "components", retry_on_conflict: 5}}) + expect(submitted_body[5]).to include( + script: a_hash_including(id: INDEX_DATA_UPDATE_SCRIPT_ID, params: a_hash_including("data")), + scripted_upsert: true, + upsert: {} + ) + + expect(submitted_body[6]).to eq({update: {_id: "USD", _index: "widget_currencies", retry_on_conflict: 5}}) + expect(submitted_body[7]).to include( + scripted_upsert: true, + upsert: {}, + script: a_hash_including( + id: /WidgetCurrency_from_Widget_/, + params: {"data" => {"name" => ["thing1"]}, "id" => "USD"} + ) + ) + end + + it "ignores operations that convert to an empty `to_datastore_bulk`" do + index_op = new_operation({"type" => "Widget", "id" => "1", "version" => 1, "record" => {"id" => "1", "some_field" => "value"}}) + # Note: technically, no operation implementations return an empty `to_datastore_bulk` at this point, + # but this is still useful behavior for the router to have, so we're using a test double here. + destination_index_def = indexer.datastore_core.index_definitions_by_name.fetch("widget_currencies") + empty_update_op = instance_double( + Indexer::Operation::Update, + to_datastore_bulk: [], + destination_index_def: destination_index_def + ) + + result = router.bulk([index_op, empty_update_op], refresh: true).successful_operations_by_cluster_name + + expected_successful_ops = { + "main" => [index_op] + } + expect(result).to eq expected_successful_ops + + # The empty update should not be attempted at all + expect(main_datastore_client).to have_received(:bulk).with( + body: index_op.to_datastore_bulk, + refresh: true + ) + end + + it "returns failures if `client#bulk` returns any error other than `version_conflict_engine_exception`" do + # Make sure the stubbed response ONLY contains keys that match the `filter_path` in DATASTORE_BULK_FILTER_PATH + fake_resp = { + "items" => [ + {"update" => {"status" => 500, "error" => {"reason" => "ERROR"}}}, + success_item, + success_item, + success_item + ] + } + + allow(main_datastore_client).to receive(:bulk).and_return(fake_resp) + + result = router.bulk(operations, refresh: true) + + expect(result.failure_results.size).to eq 1 + expect(result.failure_results.first).to be_a Operation::Result + expect(result.failure_results.first.operation).to eq(operations.first) + expect(result.failure_results.first.inspect).to include("ERROR; full response") + end + + it "prevents failures from being silently ignored via a `check_failures` argument on the success methods" do + fake_resp = { + "items" => [ + {"update" => {"status" => 500, "error" => {"reason" => "ERROR"}}}, + success_item, + success_item, + success_item + ] + } + + allow(main_datastore_client).to receive(:bulk).and_return(fake_resp) + + result = router.bulk(operations) + + expect { + result.successful_operations + }.to raise_error IndexingFailuresError, a_string_including("1 indexing failure", "ERROR") + + expect(result.successful_operations(check_failures: false)).not_to be_empty + + expect { + result.successful_operations_by_cluster_name + }.to raise_error IndexingFailuresError, a_string_including("1 indexing failure", "ERROR") + + expect(result.successful_operations_by_cluster_name(check_failures: false)).not_to be_empty + end + + it "converts a version conflict for an update operation into a noop result" do + # Make sure the stubbed response ONLY contains keys that match the `filter_path` in DATASTORE_BULK_FILTER_PATH + fake_resp = { + "items" => [ + success_item, + success_item, + success_item, + noop_item + ] + } + + allow(main_datastore_client).to receive(:bulk).and_return(fake_resp) + + result = router.bulk(operations, refresh: true) + + expected_successful_ops = { + "main" => [operations[0], operations[1], operations[2]] + } + + expect(result.successful_operations_by_cluster_name).to eq expected_successful_ops + expect(result.noop_results).to eq [Operation::Result.noop_of( + operations.last, + nil + )] + end + + it "avoids the I/O cost of writing to the datastore when given an empty list of bulk operations" do + results = router.bulk([]).successful_operations_by_cluster_name + + expect(results).to eq({}) + expect(main_datastore_client).not_to have_received(:bulk) + end + + it "validates the index mapping consistency of the destination index of each operation before performing the bulk request" do + call_sequence = [] + allow(index_mapping_checker).to receive(:validate_mapping_completeness_of!) do |index_cluster_name_method, *index_defs| + call_sequence << [:validate_indices, index_cluster_name_method, index_defs] + end + + allow(main_datastore_client).to receive(:bulk) do |request| + call_sequence << :bulk + respond_to_datastore_client_bulk_request(request) + end + + router.bulk(operations) + + expect(call_sequence).to eq [ + [:validate_indices, :accessible_cluster_names_to_index_into, operations.map(&:destination_index_def).uniq], + :bulk + ] + end + + it "includes the exception class and message in the return failure result when scripted updates fail" do + # Make sure the stubbed response ONLY contains keys that match the `filter_path` in DATASTORE_BULK_FILTER_PATH + fake_resp = {"items" => [ + { + "update" => { + "status" => 400, + "error" => { + "reason" => "failed to execute script", + "caused_by" => { + "caused_by" => { + "type" => "illegal_argument_exception", + "reason" => "value was null, which is not allowed" + } + } + } + } + } + ]} + + allow(main_datastore_client).to receive(:bulk).and_return(fake_resp) + + failure = only_failure_from(router.bulk([widget_derived_update_op])) + expect(failure.operation).to eq(widget_derived_update_op) + expect(failure.description).to include( + "update_WidgetCurrency_from_Widget_", + "(applied to `USD`): failed to execute script (illegal_argument_exception: value was null, which is not allowed)" + ) + end + + it "gracefully handles script update errors having `caused_by` details instead of `caused_by.caused_by` details" do + fake_resp = {"items" => [ + { + "update" => { + "status" => 400, + "error" => { + "type" => "illegal_argument_exception", + "reason" => "failed to execute script", + "caused_by" => { + "type" => "resource_not_found_exception", + "reason" => "unable to find script [some_script_id] in cluster state" + } + } + } + } + ]} + + allow(main_datastore_client).to receive(:bulk).and_return(fake_resp) + + failure = only_failure_from(router.bulk([widget_derived_update_op])) + expect(failure.operation).to eq(widget_derived_update_op) + expect(failure.description).to include( + "update_WidgetCurrency_from_Widget_", + "(applied to `USD`): failed to execute script (resource_not_found_exception: unable to find script [some_script_id] in cluster state)" + ) + end + + it "gracefully handles script update errors having no `caused_by` details" do + # Make sure the stubbed response ONLY contains keys that match the `filter_path` in DATASTORE_BULK_FILTER_PATH + fake_resp = {"items" => [ + { + "update" => { + "status" => 400, + "error" => { + "reason" => "failed for an unknown reason" + } + } + } + ]} + + allow(main_datastore_client).to receive(:bulk).and_return(fake_resp) + + failure = only_failure_from(router.bulk([widget_derived_update_op])) + expect(failure.operation).to eq(widget_derived_update_op) + expect(failure.description).to include( + "update_WidgetCurrency_from_Widget_", + "(applied to `USD`): failed for an unknown reason; full response: {", + "status\": 400" + ) + end + + def only_failure_from(result) + expect(result.failure_results.size).to eq(1) + failure = result.failure_results.first + + expect(failure).to be_a(Operation::Result) + expect(failure.category).to eq(:failure) + + failure + end + + context "when configured to index types into separate clusters" do + let(:indexer) { build_indexer(index_to_clusters: {"components" => {"index_into_clusters" => ["other"]}}) } + let(:widget_operations) { [operations[0], operations[1], operations[3]] } + let(:component_operations) { [operations[2]] } + + it "runs the operation for each type using the appropriate client" do + successful_ops = router.bulk(operations, refresh: true).successful_operations_by_cluster_name + + expect(successful_ops.keys).to contain_exactly("main", "other") + expect(successful_ops.fetch("main")).to eq(widget_operations) + expect(successful_ops.fetch("other")).to eq(component_operations) + end + + it "only returns successful operations across each cluster" do + # Make sure the stubbed response ONLY contains keys that match the `filter_path` in DATASTORE_BULK_FILTER_PATH + main_fake_resp = { + "items" => [ + success_item, + noop_item, + success_item + ] + } + + # Make sure the stubbed response ONLY contains keys that match the `filter_path` in DATASTORE_BULK_FILTER_PATH + other_fake_resp = { + "items" => [ + noop_item + ] + } + + allow(main_datastore_client).to receive(:bulk).and_return(main_fake_resp) + allow(other_datastore_client).to receive(:bulk).and_return(other_fake_resp) + + result = router.bulk(operations, refresh: true) + successful_ops = result.successful_operations_by_cluster_name + + expect(successful_ops.fetch("main")).to contain_exactly(operations[0], operations[3]) + expect(successful_ops.fetch("other")).to be_empty + expect(result.noop_results.size).to be > 0 + end + end + + context "when a type is configured to index into multiple clusters" do + let(:indexer) do + build_indexer(index_to_clusters: { + "components" => {"index_into_clusters" => ["main", "other"]}, + "widgets" => {"index_into_clusters" => ["main", "other"]} + }) + end + + it "successfully runs operations using multiple clients" do + component_op = new_operation({"type" => "Component", "id" => "1", "version" => 1, "record" => {"id" => "1", "some_field" => "value"}}) + widget_op = new_operation({"type" => "Widget", "id" => "1", "version" => 1, "record" => {"id" => "1", "some_field" => "value"}}) + successful_ops = router.bulk([component_op, widget_op], refresh: true).successful_operations_by_cluster_name + + expect(successful_ops.keys).to contain_exactly("main", "other") + expect(successful_ops["main"]).to contain_exactly(component_op, widget_op) + expect(successful_ops["other"]).to contain_exactly(component_op, widget_op) + end + + it "only returns operation on clusters it successfully ran on" do + # Make sure the stubbed response ONLY contains keys that match the `filter_path` in DATASTORE_BULK_FILTER_PATH + other_fake_resp = { + "items" => [ + noop_item, + success_item + ] + } + + allow(other_datastore_client).to receive(:bulk).and_return(other_fake_resp) + + component_op = new_operation({"type" => "Component", "id" => "1", "version" => 1, "record" => {"id" => "1", "some_field" => "value"}}) + widget_op = new_operation({"type" => "Widget", "id" => "1", "version" => 1, "record" => {"id" => "1", "some_field" => "value"}}) + + result = router.bulk([component_op, widget_op], refresh: true) + successful_ops = result.successful_operations_by_cluster_name + + expect(successful_ops.keys).to contain_exactly("main", "other") + expect(successful_ops["main"]).to contain_exactly(component_op, widget_op) + expect(successful_ops["other"]).to contain_exactly(widget_op) + expect(result.noop_results.size).to be > 0 + end + end + + context "when a type is configured to index into no clusters" do + let(:indexer) do + build_indexer(index_to_clusters: { + "components" => {"index_into_clusters" => []}, + "widgets" => {"index_into_clusters" => ["main"]} + }) + end + + it "fails with a clear error before any calls to the datastore are made, because we don't want to drop the event on the floor, and want to treat the batch consistently" do + expect_inaccessible_error + end + end + + context "when a type is configured with a cluster name that is not itself configured" do + let(:indexer) do + build_indexer(index_to_clusters: { + "components" => {"index_into_clusters" => ["main", "undefined"]}, + "widgets" => {"index_into_clusters" => ["main"]} + }) + end + + it "fails with a clear error before any calls to the datastore are made, because we don't want to drop the event on the floor, and want to treat the batch consistently" do + expect_inaccessible_error + end + end + + def expect_inaccessible_error + component_op1 = new_operation({"type" => "Component", "id" => "1", "version" => 1, "record" => {"id" => "1", "some_field" => "value"}}) + component_op2 = new_operation({"type" => "Component", "id" => "2", "version" => 1, "record" => {"id" => "2", "some_field" => "value"}}) + widget_op1 = new_operation({"type" => "Widget", "id" => "1", "version" => 1, "record" => {"id" => "1", "some_field" => "value"}}) + widget_op2 = new_operation({"type" => "Widget", "id" => "2", "version" => 1, "record" => {"id" => "2", "some_field" => "value"}}) + + expect { + router.bulk([widget_op1, component_op1, component_op2, widget_op2], refresh: true) + }.to raise_error IndexingFailuresError, a_string_including("configured to be inaccessible", "Component:1@v1", "Component:2@v1") + + expect(main_datastore_client).not_to have_received(:bulk) + end + + def noop_item + # Make sure the stubbed response ONLY contains keys that match the `filter_path` in DATASTORE_BULK_FILTER_PATH + { + "update" => { + "status" => 200, + "result" => "noop" + } + } + end + + def success_item + {"update" => {"status" => 200}} + end + end + + def new_operation(event, update_target: nil, **overrides) + update_target ||= begin + update_targets = indexer + .schema_artifacts + .runtime_metadata + .object_types_by_name + .fetch(event.fetch("type")) + .update_targets + .select { |ut| ut.type == event.fetch("type") } + + expect(update_targets.size).to eq(1) + update_targets.first + end + + index_defs = indexer.datastore_core.index_definitions_by_graphql_type.fetch(event.fetch("type")) + expect(index_defs.size).to eq 1 + index_def = index_defs.first + + arguments = { + event: event, + prepared_record: indexer.record_preparer_factory.for_latest_json_schema_version.prepare_for_index( + event.fetch("type"), + event.fetch("record") + ), + destination_index_def: index_def, + update_target: update_target, + doc_id: event.fetch("id"), + destination_index_mapping: indexer.schema_artifacts.index_mappings_by_index_def_name.fetch(index_def.name) + }.merge(overrides) + + Operation::Update.new(**arguments) + end + + def respond_to_datastore_client_bulk_request(request) + operation_actions = request + .fetch(:body) + .filter_map { |hash| hash.keys.first.to_s if [[:index], [:update]].include?(hash.keys) } + + items = operation_actions.map { |action| success_item } + # Make sure the stubbed response ONLY contains keys that match the `filter_path` in DATASTORE_BULK_FILTER_PATH + {"items" => items} + end + + def build_indexer(index_to_clusters: {}, use_updates_for_indexing: true) + super(clients_by_name: {"main" => main_datastore_client, "other" => other_datastore_client}, schema_definition: lambda do |schema| + schema.object_type "Component" do |t| + t.field "id", "ID!" + t.field "some_field", "String!" + t.index "components" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "currency", "String" + t.field "name", "String" + t.field "some_field", "String!" + t.index "widgets" + t.derive_indexed_type_fields "WidgetCurrency", from_id: "currency" do |derive| + derive.append_only_set "widget_names", from: "name" + end + end + + schema.object_type "WidgetCurrency" do |t| + t.field "id", "ID!" + t.field "widget_names", "[String!]!" + t.index "widget_currencies" + end + end) do |config| + config.with(index_definitions: config.index_definitions.merge( + index_to_clusters.to_h do |index, clusters| + [index, config_index_def_of(index_into_clusters: clusters["index_into_clusters"])] + end + )).then { |c| with_use_updates_for_indexing(c, use_updates_for_indexing) } + end + end + end + end +end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/event_id_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/event_id_spec.rb new file mode 100644 index 00000000..62864daf --- /dev/null +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/event_id_spec.rb @@ -0,0 +1,34 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer/event_id" + +module ElasticGraph + class Indexer + RSpec.describe EventID do + describe ".from_event", :factories do + it "builds it from an event payload" do + event = build_upsert_event(:widget, id: "abc", __version: 12) + event_id = EventID.from_event(event) + + expect(event_id.type).to eq "Widget" + expect(event_id.id).to eq "abc" + expect(event_id.version).to eq 12 + end + end + + describe "#to_s" do + it "converts to the string form" do + event_id = EventID.new(type: "Widget", id: "1234", version: 7) + + expect(event_id.to_s).to eq "Widget:1234@v7" + end + end + end + end +end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/hash_differ_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/hash_differ_spec.rb new file mode 100644 index 00000000..fac69245 --- /dev/null +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/hash_differ_spec.rb @@ -0,0 +1,50 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer/hash_differ" + +module ElasticGraph + class Indexer + RSpec.describe HashDiffer do + describe ".diff" do + it "returns nil when given two identical hashes" do + expect(HashDiffer.diff({a: 1}, {a: 1})).to eq nil + end + + it "returns a multi line string describing the difference" do + diff = HashDiffer.diff( + {a: 1, b: 2, d: 5}, + {b: 3, c: 4, d: 5} + ) + + expect(diff).to eq(<<~EOS.chomp) + - a: 1 + ~ b: `2` => `3` + + c: 4 + EOS + end + + it "ignores the kinds of differences specified in `ignore:`" do + diff = HashDiffer.diff( + {a: 1, b: 2, d: 5}, + {b: 3, c: 4, d: 5}, + ignore_ops: [:-, :~] + ) + + expect(diff).to eq(<<~EOS.chomp) + + c: 4 + EOS + end + + it "returns nil when all present diff ops are ignored" do + expect(HashDiffer.diff({}, {a: 1}, ignore_ops: [:+])).to eq nil + end + end + end + end +end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/indexing_preparers/integer_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/indexing_preparers/integer_spec.rb new file mode 100644 index 00000000..b47da991 --- /dev/null +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/indexing_preparers/integer_spec.rb @@ -0,0 +1,69 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "support/indexing_preparer" + +module ElasticGraph + class Indexer + module IndexingPreparers + RSpec.describe "Integral types" do + %w[Int JsonSafeLong LongString].each do |type| + describe "for the `#{type}` type" do + include_context "indexing preparer support", type + + it "coerces an integer-valued float to a true integer to satisfy the datastore (necessary since JSON schema doesn't validate this)" do + expect(prepare_scalar_value(3.0)).to eq(3).and be_an ::Integer + end + + it "leaves a true integer value unchanged" do + expect(prepare_scalar_value(4)).to eq(4).and be_an ::Integer + end + + it "leaves a `nil` value unchanged" do + expect(prepare_scalar_value(nil)).to eq(nil) + end + + it "applies the value coercion logic to each element of an array" do + expect(prepare_array_values([1.0, 3.0, 5.0])).to eq([1, 3, 5]).and all be_an ::Integer + expect(prepare_array_values([nil, nil])).to eq([nil, nil]) + end + + it "respects the index-preparation logic recursively at each level of a nested array" do + results = prepare_array_of_array_of_values([ + [1.0, 2], + [2.0, 3], + [3.0, 4], + [nil, 2] + ]) + + expect(results).to eq([[1, 2], [2, 3], [3, 4], [nil, 2]]) + expect(results.flatten.compact).to all be_an ::Integer + end + + it "respects the index-preparation rule recursively at each level of an object within an array" do + expect(prepare_array_of_objects_of_values([1.0, 3.0, 5.0])).to eq([1, 3, 5]).and all be_an ::Integer + expect(prepare_array_of_objects_of_values([nil, nil])).to eq([nil, nil]) + end + + it "raises an exception when given a true floating point number" do + expect { + prepare_scalar_value(3.1) + }.to raise_error Errors::IndexOperationError, a_string_including("3.1") + end + + it "raises an exception when given a string integer" do + expect { + prepare_scalar_value("17") + }.to raise_error Errors::IndexOperationError, a_string_including("17") + end + end + end + end + end + end +end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/indexing_preparers/no_op_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/indexing_preparers/no_op_spec.rb new file mode 100644 index 00000000..a82b131f --- /dev/null +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/indexing_preparers/no_op_spec.rb @@ -0,0 +1,21 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer/indexing_preparers/no_op" + +module ElasticGraph + class Indexer + module IndexingPreparers + RSpec.describe NoOp do + it "echoes the given value back unchanged" do + expect(NoOp.prepare_for_indexing(:anything)).to eq :anything + end + end + end + end +end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/indexing_preparers/untyped_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/indexing_preparers/untyped_spec.rb new file mode 100644 index 00000000..d2d9c7a7 --- /dev/null +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/indexing_preparers/untyped_spec.rb @@ -0,0 +1,69 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "support/indexing_preparer" + +module ElasticGraph + class Indexer + module IndexingPreparers + RSpec.describe "Untyped" do + include_context "indexing preparer support", "Untyped" + + it "dumps an integer as a string so it can be indexed as a keyword" do + expect(prepare_scalar_value(3)).to eq "3" + end + + it "dumps a float as a string so it can be indexed as a keyword" do + expect(prepare_scalar_value(3.14)).to eq "3.14" + end + + it "drops excess zeroes on a float to convert it to a canonical form" do + expect(prepare_scalar_value(3.2100000)).to eq("3.21") + end + + it "dumps a boolean as a string so it can be indexed as a keyword" do + expect(prepare_scalar_value(true)).to eq "true" + expect(prepare_scalar_value(false)).to eq "false" + end + + it "quotes strings so that it is parseable as JSON" do + expect(prepare_scalar_value("true")).to eq '"true"' + expect(prepare_scalar_value("3")).to eq '"3"' + end + + it "dumps `nil` as `nil` so that the index field remains unset" do + expect(prepare_scalar_value(nil, validate_roundtrip: false)).to eq nil + end + + it "dumps an array as a compact JSON string so it can be indexed as a keyword" do + expect(prepare_scalar_value([1, true, "hello"])).to eq('[1,true,"hello"]') + end + + it "dumps an array as a compact JSON string so it can be indexed as a keyword" do + expect(prepare_scalar_value([1, true, "hello"])).to eq('[1,true,"hello"]') + end + + it "orders object keys alphabetically when dumping it to normalize to a canonical form" do + expect(prepare_scalar_value({"b" => 1, "a" => 2, "c" => 3})).to eq('{"a":2,"b":1,"c":3}') + end + + # Override `prepare_scalar_value` to enforce an invariant: parsing the resulting JSON string + # should always produce the original value + prepend Module.new { + def prepare_scalar_value(original_value, validate_roundtrip: true) + super(original_value).tap do |prepared_value| + if validate_roundtrip + expect(::JSON.parse(prepared_value)).to eq(original_value) + end + end + end + } + end + end + end +end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/count_accumulator_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/count_accumulator_spec.rb new file mode 100644 index 00000000..9cfad178 --- /dev/null +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/count_accumulator_spec.rb @@ -0,0 +1,312 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/indexer/operation/update" + +module ElasticGraph + class Indexer + module Operation + RSpec.describe CountAccumulator do + describe "counts indexed on the root document" do + it "includes counts for each list field" do + counts = root_counts_for_team_event({"past_names" => ["a", "b", "c"], "seasons_object" => []}) + + expect(counts).to include({"past_names" => 3, "seasons_object" => 0}) + end + + it "includes a count of zero for nested subfields which we do not have data for" do + counts = root_counts_for_team_event({"past_names" => ["a", "b", "c"], "seasons_object" => []}) + + expect(counts).to include({ + "current_players_nested" => 0, + "current_players_object" => 0, + "current_players_object|name" => 0, + "current_players_object|nicknames" => 0, + "current_players_object|seasons_nested" => 0, + "current_players_object|seasons_object" => 0, + "current_players_object|seasons_object|awards" => 0, + "current_players_object|seasons_object|games_played" => 0, + "current_players_object|seasons_object|year" => 0, + "details|uniform_colors" => 0, + "forbes_valuations" => 0, + "forbes_valuation_moneys_nested" => 0, + "forbes_valuation_moneys_object" => 0, + "forbes_valuation_moneys_object|amount_cents" => 0, + "forbes_valuation_moneys_object|currency" => 0, + "seasons_nested" => 0, + "seasons_object|count" => 0, + "seasons_object|notes" => 0, + "seasons_object|players_nested" => 0, + "seasons_object|players_object" => 0, + "seasons_object|players_object|name" => 0, + "seasons_object|players_object|nicknames" => 0, + "seasons_object|players_object|seasons_nested" => 0, + "seasons_object|players_object|seasons_object" => 0, + "seasons_object|players_object|seasons_object|awards" => 0, + "seasons_object|players_object|seasons_object|games_played" => 0, + "seasons_object|players_object|seasons_object|year" => 0, + "seasons_object|started_at" => 0, + "seasons_object|won_games_at" => 0, + "seasons_object|year" => 0, + "won_championships_at" => 0 + }) + end + + it "omits the count for list fields which get omitted from the bulk payload" do + counts = root_counts_for_team_event({"past_names" => ["a", "b", "c"], "unknown_list_field" => []}) + + expect(counts).to include({"past_names" => 3}) + end + + it "counts embedded fields, using `#{LIST_COUNTS_FIELD_PATH_KEY_SEPARATOR}` as a path separator" do + counts = root_counts_for_team_event({"details" => {"uniform_colors" => ["a", "b"]}}) + + expect(counts).to include({"details|uniform_colors" => 2}) + end + + it "does not attempt to compute any counts from fields which use specialized index field types that are objects in JSON but have no properties in the mapping" do + counts = root_counts_for_team_event({"past_names" => ["a", "b", "c"], "stadium_location" => {"latitude" => 47.6, "longitude" => -122.3}}) + + expect(counts).to include({"past_names" => 3}) + end + + it "counts lists-of-objects-of-lists when the object is not `nested`" do + # `seasons` is defined with `type: "object" + counts = root_counts_for_team_event({"seasons_object" => [ + {"notes" => ["a", "b"]}, + {"notes" => []}, + {"notes" => ["c"]}, + {"notes" => ["d", "e", "f", "g"]} + ]}) + + expect(counts).to include({ + "seasons_object" => 4, + "seasons_object|notes" => 7 # 2 + 0 + 1 + 4 = 7 + }) + end + + it "only counts the size of a `nested` object list, without considering the counts of its inner lists" do + counts = root_counts_for_team_event({"current_players_nested" => [ + {"nicknames" => ["a", "b"], "seasons_object" => []}, + {"nicknames" => ["a"], "seasons_object" => []} + ]}) + + expect(counts).to include({"current_players_nested" => 2}) + end + + it "accumulates the count of single valued fields within objects in a list" do + counts = root_counts_for_team_event({"seasons_object" => [ + {"year" => 2020}, + {"year" => 2021} + ]}) + + expect(counts).to include({ + "seasons_object" => 2, + "seasons_object|year" => 2 + }) + end + + it "does not count `nil` scalar values" do + counts = root_counts_for_team_event({"seasons_object" => [ + {"year" => 2020}, + {"year" => 2021}, + {"year" => nil} + ]}) + + expect(counts).to include({ + "seasons_object" => 3, + "seasons_object|year" => 2 + }) + end + + it "does not count `nil` object values" do + counts = root_counts_for_team_event({"seasons_object" => [ + {"year" => 2020}, + {"year" => 2021}, + nil + ]}) + + expect(counts).to include({ + "seasons_object" => 2, + "seasons_object|year" => 2 + }) + end + + it "omits the `__counts` field when processing an indexed type that has no list fields" do + data = { + "id" => "abc", + "name" => "Acme", + "created_at" => "2023-10-12T00:00:00Z" + } + + params = script_params_for(data: data, source_type: "Manufacturer", destination_type: "Manufacturer") + + expect(params[LIST_COUNTS_FIELD]).to eq nil + expect(params["data"]).to exclude(LIST_COUNTS_FIELD) + end + end + + describe "counts indexed under a `nested` list field" do # `current_players` is a `nested` list field + it "counts the list elements on the `nested` documents" do + counts = current_players_counts_for_team_event({"current_players_nested" => [ + {"nicknames" => ["a", "b"]}, + {"nicknames" => ["c"]} + ]}) + + expect(counts).to match [ + a_hash_including({"nicknames" => 2}), + a_hash_including({"nicknames" => 1}) + ] + end + + it "counts embedded lists summing the counts" do + counts = current_players_counts_for_team_event({"current_players_nested" => [ + {"seasons_object" => [ + {"awards" => ["a", "b"]}, + {"awards" => ["c"]} + ]}, + {"seasons_object" => [{ + "awards" => [] + }]} + ]}) + + expect(counts).to match [ + a_hash_including({"seasons_object" => 2, "seasons_object|awards" => 3}), + a_hash_including({"seasons_object" => 1, "seasons_object|awards" => 0}) + ] + end + + it "does not attempt to count any subfields of a nested object that has no list fields" do + params = script_params_for_team_event({"forbes_valuation_moneys_nested" => [ + {"currency" => "USD", "amount_cents" => 525}, + {"currency" => "USD", "amount_cents" => 725} + ]}) + + expect(params.dig("data", "forbes_valuation_moneys_nested")).to eq [ + {"currency" => "USD", "amount_cents" => 525}, + {"currency" => "USD", "amount_cents" => 725} + ] + end + + def current_players_counts_for_team_event(data) + script_params_for_team_event(data).dig("data", "current_players_nested").map do |player| + player.fetch(LIST_COUNTS_FIELD) + end + end + end + + describe "counts indexed under an `object` list field" do # `seasons` is an `object` list field + it "does not have any indexed counts because the list gets flattened at the root" do + seasons = seasons_for_team_event({"seasons_object" => [ + {"notes" => ["a", "b"]}, + {"notes" => []} + ]}) + + expect(seasons).to match [ + a_hash_including({"notes" => ["a", "b"]}), + a_hash_including({"notes" => []}) + ] + end + + it "still generates a `#{LIST_COUNTS_FIELD}` field on any `nested` list fields under the `object` list field" do + seasons = seasons_for_team_event({"seasons_object" => [ + {"players_nested" => [ + {"seasons_object" => [ + {"awards" => ["a", "b", "c"]}, + {"awards" => ["d", "e"]} + ]}, + {"seasons_object" => [ + {"awards" => ["d"]} + ]} + ]}, + {"players_nested" => []}, + {"players_nested" => [ + {"seasons_object" => []} + ]}, + {"players_nested" => [ + {"seasons_object" => [ + {"awards" => []} + ]} + ]} + ]}) + + expect(seasons).to match [ + {"players_nested" => [ + {LIST_COUNTS_FIELD => a_hash_including({"seasons_object" => 2, "seasons_object|awards" => 5}), "seasons_object" => [ + {"awards" => ["a", "b", "c"]}, + {"awards" => ["d", "e"]} + ]}, + {LIST_COUNTS_FIELD => a_hash_including({"seasons_object" => 1, "seasons_object|awards" => 1}), "seasons_object" => [ + {"awards" => ["d"]} + ]} + ]}, + {"players_nested" => []}, + {"players_nested" => [ + {LIST_COUNTS_FIELD => a_hash_including({"seasons_object" => 0}), "seasons_object" => []} + ]}, + {"players_nested" => [ + {LIST_COUNTS_FIELD => a_hash_including({"seasons_object" => 1, "seasons_object|awards" => 0}), "seasons_object" => [ + {"awards" => []} + ]} + ]} + ] + end + + def seasons_for_team_event(data) + script_params_for_team_event(data).dig("data", "seasons_object") + end + end + + context "for a derived indexing operation", :factories do + it "does not attempt to compute any `#{LIST_COUNTS_FIELD}` because the derived indexing script ignores the `#{LIST_COUNTS_FIELD}` parameter" do + params = script_params_for(source_type: "Widget", destination_type: "WidgetCurrency", data: { + "cost_currency_introduced_on" => "1980-01-01", + "cost_currency_primary_continent" => "North America", + "tags" => ["t1", "t2"] + }) + + expect(params).to exclude(LIST_COUNTS_FIELD) + end + end + + def root_counts_for_team_event(data) + script_params_for_team_event(data).fetch(LIST_COUNTS_FIELD) + end + + def script_params_for_team_event(data) + script_params_for(data: data, source_type: "Team", destination_type: "Team") + end + + def script_params_for(data:, source_type:, destination_type:, indexer: build_indexer) + # `league` and `formed_on` are required in `data` because they are used for routing and rollover. + data = data.merge({"league" => "MLB", "formed_on" => "1950-01-01"}) + + destination_index_def = indexer.datastore_core.index_definitions_by_graphql_type.fetch(destination_type).first + update_target = indexer + .schema_artifacts + .runtime_metadata + .object_types_by_name + .fetch(source_type) + .update_targets.find { |ut| ut.type == destination_type } + + update = Update.new( + event: {"type" => source_type, "record" => data}, + destination_index_def: destination_index_def, + prepared_record: indexer.record_preparer_factory.for_latest_json_schema_version.prepare_for_index(source_type, data), + update_target: update_target, + doc_id: "the-id", + destination_index_mapping: indexer.schema_artifacts.index_mappings_by_index_def_name.fetch(destination_index_def.name) + ) + + update.to_datastore_bulk.dig(1, :script, :params) + end + end + end + end +end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/factory_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/factory_spec.rb new file mode 100644 index 00000000..8e9fedb9 --- /dev/null +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/factory_spec.rb @@ -0,0 +1,502 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer" +require "elastic_graph/constants" +require "elastic_graph/indexer/operation/factory" +require "json" + +module ElasticGraph + class Indexer + module Operation + RSpec.describe Factory, :capture_logs do + describe "#build", :factories do + shared_examples_for "building operations" do + let(:indexer) { build_indexer } + let(:component_index_definition) { index_def_named("components") } + + it "generates a primary indexing operation" do + event = build_upsert_event(:component, id: "1", __version: 1) + + expect(build_expecting_success(event)).to eq([new_primary_indexing_operation(event)]) + end + + it "also generates derived index update operations for an upsert event for the source type of a derived indexing type" do + event = build_upsert_event(:widget, id: "1", __version: 1) + formatted_event = { + "op" => "upsert", + "id" => "1", + "type" => "Widget", + "version" => 1, + "record" => event["record"], + JSON_SCHEMA_VERSION_KEY => 1 + } + + expect(build_expecting_success(event)).to contain_exactly( + new_primary_indexing_operation(formatted_event, index_def_named("widgets")), + widget_currency_derived_update_operation_for(formatted_event) + ) + end + + context "when the indexer is configured to skip updates for certain derived indexing types and ids" do + let(:indexer) do + build_indexer(skip_derived_indexing_type_updates: { + "WidgetCurrency" => ["USD"], + "SomeOtherType" => ["CAD"] + }) + end + + it "skips generating a derived indexing update when the id is configured to be skipped" do + usd_event = build_upsert_event(:widget, cost: build(:money, currency: "USD")) + + expect(build_expecting_success(usd_event)).to contain_exactly( + new_primary_indexing_operation(usd_event, index_def_named("widgets")) + ) + + expect(logged_jsons_of_type("SkippingUpdate").size).to eq 1 + end + + it "still generates a derived indexing update for ids that are not configured for this derived, even if those ids are configured for another derived indexing type" do + cad_event = build_upsert_event(:widget, cost: build(:money, currency: "CAD")) + + expect(build_expecting_success(cad_event)).to contain_exactly( + new_primary_indexing_operation(cad_event, index_def_named("widgets")), + widget_currency_derived_update_operation_for(cad_event) + ) + + expect(logged_jsons_of_type("SkippingUpdate").size).to eq 0 + end + end + + it "generates a primary indexing operation for a single index with latency metrics" do + event = build_upsert_event(:component, id: "1", __version: 1) + latency_timestamps = {"latency_timestamps" => {"created_in_esperanto_at" => "2012-04-23T18:25:43.511Z"}} + + expect(build_expecting_success(event.merge(latency_timestamps))).to eq([new_primary_indexing_operation({ + "op" => "upsert", + "id" => "1", + "type" => "Component", + "version" => 1, + "record" => event["record"], + JSON_SCHEMA_VERSION_KEY => 1 + }.merge(latency_timestamps))]) + end + + it 'notifies an error when latency metrics contain keys that violate regex "^\\w+_at$"' do + valid_event = build_upsert_event(:component, id: "1", __version: 1) + invalid_event = valid_event.merge({ + "latency_timestamps" => { + "created_in_esperanto_at" => "2012-04-23T18:25:43.511Z", + "bad metric with spaces _at" => "2012-04-20T18:25:43.511Z", + "bad_metric" => "2012-04-20T18:25:43.511Z" + } + }) + + expect_failed_event_error(invalid_event, "/latency_timestamps/bad_metric", "bad metric with spaces _at") + end + + it "notifies an error when latency metrics contain values that are not ISO8601 date-time" do + valid_event = build_upsert_event(:component, id: "1", __version: 1) + invalid_event = valid_event.merge({ + "latency_timestamps" => { + "created_in_esperanto_at" => "2012-04-23T18:25:43.511Z", + "bad_metric_at" => "malformed datetime" + } + }) + + expect_failed_event_error(invalid_event, "/latency_timestamps/bad_metric") + end + + it "notifies an error on version number less than 1" do + event = build_upsert_event(:widget, __version: -1) + + expect_failed_event_error(event, "/properties/version") + end + + it "notifies an error on version number greater than 2^63 - 1" do + event = build_upsert_event(:widget, __version: 2**64) + + expect_failed_event_error(event, "/properties/version") + end + + it "notifies an error on unknown graphql type" do + event = { + "op" => "upsert", + "id" => "1", + "type" => "MyOwnInvalidGraphQlType", + "version" => 1, + JSON_SCHEMA_VERSION_KEY => 1, + "record" => {"field1" => "value1", "field2" => "value2", "id" => "1"} + } + + # We can't build any operations when the `type` is unknown. We don't know what index to target! + expect_failed_event_error(event, "/properties/type", expect_no_ops: true) + end + + it "notifies an error on non-indexed graphql type" do + event = { + "op" => "upsert", + "id" => "1", + "type" => "WidgetOptions", + "version" => 1, + JSON_SCHEMA_VERSION_KEY => 1, + "record" => {"field1" => "value1", "field2" => "value2", "id" => "1"} + } + + expect(indexer.datastore_core.index_definitions_by_graphql_type.fetch(event.fetch("type"), [])).to be_empty + + # We can't build any operations when the `type` isn't an indexed type. We don't know what index to target! + expect_failed_event_error(event, "/properties/type", expect_no_ops: true) + end + + it "notifies an error on invalid operation" do + event = build_upsert_event(:widget).merge("op" => "invalid_op") + + expect_failed_event_error(event, "/properties/op") + end + + it "notifies an error on missing operation" do + event = build_upsert_event(:widget).except("op") + + expect_failed_event_error(event, "missing_keys", "op") + end + + it "notifies an error on missing record for upsert" do + event = build_upsert_event(:component).except("record") + + expect_failed_event_error(event, "/then") + end + + it "notifies an error on missing id" do + event = build_upsert_event(:component).except("id") + + expect_failed_event_error(event, "missing_keys", "id") + end + + it "notifies an error on missing type" do + event = build_upsert_event(:component).except("type") + + # We can't build any operations when the `type` isn't in the event. We don't know what index to target! + expect_failed_event_error(event, "missing_keys", "type", expect_no_ops: true) + end + + it "notifies an error on missing version" do + event = build_upsert_event(:component).except("version") + + expect_failed_event_error(event, "missing_keys", "version") + end + + it "notifies an error on missing `#{JSON_SCHEMA_VERSION_KEY}`" do + event = build_upsert_event(:component).except(JSON_SCHEMA_VERSION_KEY) + + expect_failed_event_error(event, JSON_SCHEMA_VERSION_KEY) + end + + it "notifies an error on wrong field types" do + event = { + "op" => "upsert", + "id" => 1, + JSON_SCHEMA_VERSION_KEY => 1, + "type" => [], + "version" => "1", + "record" => "" + } + + # This event is too malformed to build any operations for. + expect_failed_event_error(event, "/properties/type", "/properties/id", "/properties/version", "/properties/record", expect_no_ops: true) + end + + it "notifies an error when given a record that does not satisfy the type's JSON schema, while avoiding revealing PII" do + event = build_upsert_event(:component, id: "1", __version: 1) + event["record"]["name"] = 123 + + message = expect_failed_event_error(event, "Malformed", "Component", "name") + expect(message).to include("Malformed").and exclude("123") + end + + it "requires that a custom shard routing field have a non-empty value" do + good_widget = build_upsert_event(:widget, workspace_id: "good_value") + bad_widget1 = build_upsert_event(:widget, workspace_id: nil) # routing value can't be nil + bad_widget2 = build_upsert_event(:widget, workspace_id: "") # routing value can't be an empty string + bad_widget3 = build_upsert_event(:widget, workspace_id: " ") # routing value can't be entirely whitespace + + expect(build_expecting_success(good_widget).size).to eq(2) + + expect_failed_event_error(bad_widget1, "/workspace_id") + expect_failed_event_error(bad_widget2, "/workspace_id") + expect_failed_event_error(bad_widget3, "/workspace_id") + end + + it "allows the validator to be configured with a block" do + event_with_extra_field = build_upsert_event(:widget, extra_field: 17) + + expect { + build_expecting_success(event_with_extra_field) + }.not_to raise_error + + expect { + build_expecting_success(event_with_extra_field) { |v| v.with_unknown_properties_disallowed } + }.to raise_error FailedEventError, a_string_including("extra_field") + end + + context "when the indexer has json schemas v2 and v4 (v4 adds yellow color)" do + before do + # With the "real" version one as a baseline, create a separate version with a small schema change. + # Tests will then specify the desired json_schema_version in the event payload to test the schema-choosing + # behavior of the `factory` class. + schemas = { + 2 => indexer.schema_artifacts.json_schemas_for(1), + 4 => ::Marshal.load(::Marshal.dump(indexer.schema_artifacts.json_schemas_for(1))).tap do |it| + it["$defs"]["Color"]["enum"] << "YELLOW" + end + } + + allow(indexer.schema_artifacts).to receive(:available_json_schema_versions).and_return(schemas.keys.to_set) + allow(indexer.schema_artifacts).to receive(:latest_json_schema_version).and_return(schemas.keys.max) + allow(indexer.schema_artifacts).to receive(:json_schemas_for) do |version| + ::Marshal.load(::Marshal.dump(schemas.fetch(version))).tap do |schema| + schema[JSON_SCHEMA_VERSION_KEY] = version + schema["$defs"]["ElasticGraphEventEnvelope"]["properties"][JSON_SCHEMA_VERSION_KEY]["const"] = version + end + end + end + + it "validates against an older version of a json schema if specified" do + # YELLOW doesn't exist in schema version 2. So expect an error when json_schema_version is set to 2. + event = build_upsert_event(:widget, id: "1", __version: 1, __json_schema_version: 2) + event["record"]["options"]["color"] = "YELLOW" + + expect_failed_event_error(event, "/options/color") + end + + it "validates against the latest version of a json schema if specified" do + event = build_upsert_event(:widget, id: "1", __version: 1, __json_schema_version: 4) + event["record"]["options"]["color"] = "YELLOW" + + expect(build_expecting_success(event)).to include(new_primary_indexing_operation({ + "op" => "upsert", + "id" => "1", + "type" => "Widget", + "version" => 1, + "record" => event["record"], + JSON_SCHEMA_VERSION_KEY => 4 + }, index_def_named("widgets"))) + end + + it "validates against the closest version if the requested version is newer than what's available" do + # 5 is closest to "4", validation should match behavior from version "4" - YELLOW should pass validation. + event = build_upsert_event(:widget, id: "1", __version: 1, __json_schema_version: 5) + event["record"]["options"]["color"] = "YELLOW" + + expect(build_expecting_success(event)).to include(new_primary_indexing_operation({ + "op" => "upsert", + "id" => "1", + "type" => "Widget", + "version" => 1, + "record" => event["record"], + JSON_SCHEMA_VERSION_KEY => 5 # Originally-specified version. + }, index_def_named("widgets"))) + + expect(logged_jsons_of_type("ElasticGraphMissingJSONSchemaVersion").last).to include( + "event_id" => "Widget:1@v1", + "event_type" => "Widget", + "requested_json_schema_version" => 5, + "selected_json_schema_version" => 4 + ) + end + + it "validates against the closest version if the requested version older than what's available" do + # 1 is closest to "2", validation should match behavior from version "2" - YELLOW should fail validation. + event = build_upsert_event(:widget, id: "1", __version: 1, __json_schema_version: 1).merge("message_id" => "m123") + event["record"]["options"]["color"] = "YELLOW" + + # Should fail, but should still log the version mismatch as well. + expect_failed_event_error( + event, + "Malformed Widget record", + "1", + "/options/color" + ) + + expect(logged_jsons_of_type("ElasticGraphMissingJSONSchemaVersion").last).to include( + "event_id" => "Widget:1@v1", + "message_id" => "m123", + "event_type" => "Widget", + "requested_json_schema_version" => 1, + "selected_json_schema_version" => 2 + ) + end + + it "validates against a version newer than what's requested, if the requested version is equidistant from two available versions" do + event = build_upsert_event(:widget, id: "1", __version: 1, __json_schema_version: 3) + event["record"]["options"]["color"] = "YELLOW" + + expect(build_expecting_success(event)).to include(new_primary_indexing_operation({ + "op" => "upsert", + "id" => "1", + "type" => "Widget", + "version" => 1, + "record" => event["record"], + JSON_SCHEMA_VERSION_KEY => 3 # Originally-specified version. + }, index_def_named("widgets"))) + + expect(logged_jsons_of_type("ElasticGraphMissingJSONSchemaVersion").last).to include( + "event_id" => "Widget:1@v1", + "event_type" => "Widget", + "requested_json_schema_version" => 3, + "selected_json_schema_version" => 4 + ) + end + + it "notifies an error if an invalid (e.g. negative) json_schema_version is specified" do + event = build_upsert_event(:widget, id: "1", __version: 1, __json_schema_version: -1) + + expect_failed_event_error(event, "must be a positive integer", "(-1)") + end + + it "notifies an error if it's unable to select a json_schema_version" do + event = build_upsert_event(:component, id: "1", __version: 1) + event["record"]["name"] = 123 + + fake_empty_schema_artifacts = instance_double( + "ElasticGraph::SchemaArtifacts::FromDisk", + available_json_schema_versions: Set[], + runtime_metadata: indexer.schema_artifacts.runtime_metadata, + indices: indexer.schema_artifacts.indices, + index_templates: indexer.schema_artifacts.index_templates, + index_mappings_by_index_def_name: indexer.schema_artifacts.index_mappings_by_index_def_name + ) + + operation_factory = build_indexer(schema_artifacts: fake_empty_schema_artifacts).operation_factory + + expect_failed_event_error(event, "Failed to select json schema version", factory: operation_factory) + end + end + end + + context "when `use_updates_for_indexing?` is set to false", use_updates_for_indexing: false do + include_examples "building operations" do + def new_primary_indexing_operation(event, index_def = component_index_definition, idxr = indexer) + Upsert.new( + event, + index_def, + idxr.record_preparer_factory.for_latest_json_schema_version + ) + end + end + end + + context "when `use_updates_for_indexing?` is set to true", use_updates_for_indexing: true do + include_examples "building operations" do + it "also generates an update operation for related types that have fields `sourced_from` this event type" do + event = build_upsert_event(:widget, id: "1", __version: 1, component_ids: ["c1", "c2", "c3"]) + + operations = build_expecting_success(event).select { |op| op.is_a?(Operation::Update) && op.update_target.type == "Component" } + + expect(operations.size).to eq(3) + expect(operations.map(&:event)).to all eq event + expect(operations.map(&:destination_index_def)).to all eq index_def_named("components") + expect(operations.map(&:doc_id)).to contain_exactly("c1", "c2", "c3") + end + + def new_primary_indexing_operation(event, index_def = component_index_definition, idxr = indexer) + update_targets = idxr + .schema_artifacts + .runtime_metadata + .object_types_by_name + .fetch(event.fetch("type")) + .update_targets + .select { |ut| ut.type == event.fetch("type") } + + expect(update_targets.size).to eq(1) + + Update.new( + event: event, + prepared_record: indexer.record_preparer_factory.for_latest_json_schema_version.prepare_for_index( + event.fetch("type"), + event.fetch("record") + ), + destination_index_def: index_def, + update_target: update_targets.first, + doc_id: event.fetch("id"), + destination_index_mapping: idxr.schema_artifacts.index_mappings_by_index_def_name.fetch(index_def.name) + ) + end + end + end + + def expect_failed_event_error(event, *error_message_snippets, factory: indexer.operation_factory, expect_no_ops: false) + result = factory.build(event) + + error_operations = factory.send(:build_all_operations_for, event, RecordPreparer::Identity) + + # We expect/want `build_all_operations_for` to return operations in nearly all cases. + # There are a few cases where it can't return any operations, so we make the test pass + # `expect_no_ops` to opt-in to allowing that here. + if expect_no_ops + expect(error_operations).to be_empty + else + expect(error_operations).not_to be_empty + end + + # When the event is invalid it should return an empty list of operations. + expect(result.operations).to eq([]) + + failure = result.failed_event_error + + expect(failure).to be_an(FailedEventError) + expect(failure.event).to eq(event) + expect(failure.operations).to match_array(error_operations) + expect(failure.message).to include(event_id_from(event), *error_message_snippets) + expect(failure.main_message).to include(*error_message_snippets).and exclude(event_id_from(event)) + expect(failure).to have_attributes( + id: event["id"], + type: event["type"], + op: event["op"], + version: event["version"], + record: event["record"] + ) + + failure.message # to allow the caller to assert on the message further + end + + def event_id_from(event) + Indexer::EventID.from_event(event).to_s + end + end + + def build_expecting_success(event, **options, &configure_record_validator) + result = indexer + .operation_factory + .with(configure_record_validator: configure_record_validator) + .build(event, **options) + + raise result.failed_event_error if result.failed_event_error + result.operations + end + + def widget_currency_derived_update_operation_for(event) + operations = Update.operations_for( + event: event, + destination_index_def: index_def_named("widget_currencies"), + record_preparer: indexer.record_preparer_factory.for_latest_json_schema_version, + update_target: indexer.schema_artifacts.runtime_metadata.object_types_by_name.fetch("Widget").update_targets.first, + destination_index_mapping: indexer.schema_artifacts.index_mappings_by_index_def_name.fetch("widget_currencies") + ) + + expect(operations.size).to be < 2 + operations.first + end + + def index_def_named(index_def_name) + indexer.datastore_core.index_definitions_by_name.fetch(index_def_name) + end + end + end + end +end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/update_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/update_spec.rb new file mode 100644 index 00000000..d21814e9 --- /dev/null +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/update_spec.rb @@ -0,0 +1,541 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/indexer" +require "elastic_graph/indexer/operation/update" +require "elastic_graph/spec_support/runtime_metadata_support" +require "json" + +module ElasticGraph + class Indexer + module Operation + RSpec.describe Update do + include SchemaArtifacts::RuntimeMetadata::RuntimeMetadataSupport + + let(:indexer) { build_indexer } + let(:event) do + { + "op" => "upsert", + "id" => "3", + "type" => "Widget", + "version" => 1, + "record" => { + "workspace_id" => "17", + "workspace_ids" => ["17", "18", "19", "17", "17"], + "name" => "thing1", + "created_at" => "2021-06-10T12:30:00Z", + "size" => 3, + "embedded_values" => { + "workspace_id" => "embedded_workspace_id", + "name" => "embedded_name" + } + } + } + end + + describe "#versioned?" do + it "returns `false` for a derived indexing update since we don't keep track of source versions on the derived document" do + update = update_with_update_target(derived_indexing_update_target_with(type: "WidgetCurrency")) + + expect(update.versioned?).to be false + end + + it "returns `true` for a normal indexing update since we keep track of versions in `__versions`" do + update = update_with_update_target(normal_indexing_update_target_with(type: "Widget")) + + expect(update.versioned?).to be true + end + end + + it "has a readable `#inspect` and `#to_s`" do + update = update_with_update_target(derived_indexing_update_target_with(type: "WidgetWorkspace")) + + expect(update.inspect).to eq("#") + expect(update.to_s).to eq(update.inspect) + end + + describe "#to_datastore_bulk" do + it "returns the bulk form of an update request based on the derived index configuration" do + indexer = indexer_with_widget_workspace_index_definition do |index| + # no customization + end + + operations = operations_for_indexer(indexer) + + expect(operations.size).to eq(1) + expect(operations.flat_map(&:to_datastore_bulk)).to eq [ + {update: {_id: "17", _index: "widget_workspaces", retry_on_conflict: Update::CONFLICT_RETRIES}}, + { + script: {id: operations.first.update_target.script_id, params: { + "data" => {"name" => ["thing1"]}, + "id" => "17" + }}, + scripted_upsert: true, + upsert: {} + } + ] + end + + it "includes any metadata params defined on the update target" do + indexer = indexer_with_widget_workspace_index_definition do |index| + # no customization + end + + operations = operations_for_indexer(indexer) + expect(operations.size).to eq(1) + operation = operations.first.with(update_target: normal_indexing_update_target_with( + type: "Widget", + data_params: {"name" => dynamic_param_with(source_path: "name", cardinality: :one)}, + metadata_params: { + "staticValue" => static_param_with(47), + "sourceType" => dynamic_param_with(source_path: "type", cardinality: :one) + } + )) + + expect(operation.to_datastore_bulk).to eq [ + {update: {_id: "17", _index: "widget_workspaces", retry_on_conflict: Update::CONFLICT_RETRIES}}, + { + script: {id: INDEX_DATA_UPDATE_SCRIPT_ID, params: { + "data" => {"name" => "thing1"}, + "id" => "17", + "staticValue" => 47, + "sourceType" => "Widget", + LIST_COUNTS_FIELD => {"sizes" => 0, "widget_names" => 0} + }}, + scripted_upsert: true, + upsert: {} + } + ] + end + + it "returns no datastore bulk actions if the source document lacks the id field of the update target" do + indexer = indexer_with_widget_workspace_index_definition do |index| + # no customization + end + + operations = operations_for_indexer(indexer, event: event.merge("record" => event.fetch("record").merge("workspace_id" => nil))) + + expect(operations).to eq [] + end + + it "returns no datastore bulk actions if the source document has an empty string for the id field of the update target" do + indexer = indexer_with_widget_workspace_index_definition do |index| + # no customization + end + + operations = operations_for_indexer(indexer, event: event.merge("record" => event.fetch("record").merge("workspace_id" => ""))) + + expect(operations).to eq [] + end + + it "returns no datastore bulk actions if the source document has only whitespace for the id field of the update target" do + indexer = indexer_with_widget_workspace_index_definition do |index| + # no customization + end + + operations = operations_for_indexer(indexer, event: event.merge("record" => event.fetch("record").merge("workspace_id" => " "))) + + expect(operations).to eq [] + end + + it "still returns an update even if all params are empty so the script can still create a record of a derived indexing type (in case we are seeing it for the first time)" do + indexer = indexer_with_widget_workspace_index_definition do |index| + # no customization + end + + operations = operations_for_indexer(indexer, event: event.merge("record" => event.fetch("record").merge("name" => nil))) + + expect(operations.size).to eq(1) + expect(operations.flat_map(&:to_datastore_bulk)).to eq [ + {update: {_id: "17", _index: "widget_workspaces", retry_on_conflict: Update::CONFLICT_RETRIES}}, + { + script: {id: operations.first.update_target.script_id, params: { + "data" => {"name" => []}, + "id" => "17" + }}, + scripted_upsert: true, + upsert: {} + } + ] + end + + it "supports a nested id_source field" do + indexer = indexer_with_widget_workspace_index_definition(id_source: "embedded_values.workspace_id") do |index| + # no customization + end + + operations = operations_for_indexer(indexer) + + expect(operations.size).to eq(1) + expect(operations.flat_map(&:to_datastore_bulk)).to eq [ + {update: {_id: "embedded_workspace_id", _index: "widget_workspaces", retry_on_conflict: Update::CONFLICT_RETRIES}}, + { + script: {id: operations.first.update_target.script_id, params: { + "data" => {"name" => ["thing1"]}, + "id" => "embedded_workspace_id" + }}, + scripted_upsert: true, + upsert: {} + } + ] + end + + it "tolerates the record lacking anything at the `source_path` (e.g. for an event published before the field was added to the schema)" do + indexer = indexer_with_widget_workspace_index_definition(set_field_source: "embedded_values.missing_field") do |index| + # no customization + end + + operations = operations_for_indexer(indexer) + + expect(operations.size).to eq(1) + operation = operations.first.with(update_target: operations.first.update_target.with(data_params: { + "embedded_values.missing_field" => dynamic_param_with(source_path: "embedded_values.missing_field", cardinality: :many), + "name" => dynamic_param_with(source_path: "some_field_that_is_not_in_record", cardinality: :one) + })) + expect(operation.to_datastore_bulk).to eq [ + {update: {_id: "17", _index: "widget_workspaces", retry_on_conflict: Update::CONFLICT_RETRIES}}, + { + script: {id: operations.first.update_target.script_id, params: { + "data" => {"embedded_values.missing_field" => [], "name" => nil}, + "id" => "17" + }}, + scripted_upsert: true, + upsert: {} + } + ] + end + + it "gets the param value from the `source_path` rather than the param name if they differ" do + indexer = indexer_with_widget_workspace_index_definition do |index| + # no customization + end + + operations = operations_for_indexer(indexer) + expect(operations.size).to eq(1) + + operation = operations.first.with(update_target: operations.first.update_target.with(data_params: { + # Here we've swapped the source_paths with the param names. + "embedded_values" => dynamic_param_with(source_path: "name", cardinality: :many), + "name" => dynamic_param_with(source_path: "embedded_values", cardinality: :one) + })) + + expect(operation.to_datastore_bulk).to eq [ + {update: {_id: "17", _index: "widget_workspaces", retry_on_conflict: Update::CONFLICT_RETRIES}}, + { + script: {id: operations.first.update_target.script_id, params: { + "data" => { + "embedded_values" => ["thing1"], + "name" => { + "name" => "embedded_name", + "workspace_id" => "embedded_workspace_id" + } + }, + "id" => "17" + }}, + scripted_upsert: true, + upsert: {} + } + ] + end + + it "prepares the record to be indexed before extracting params so that we pass values to the script in the same form we index them" do + indexer = indexer_with_widget_workspace_index_definition(set_field: "sizes", set_field_source: "size") do |index| + # no customization + end + + operations = operations_for_indexer(indexer, event: event.merge( + "record" => event.fetch("record").merge( + "size" => 4.0 + ) + )) + + expect(operations.size).to eq(1) + expect(operations.flat_map(&:to_datastore_bulk)).to match [ + {update: {_id: "17", _index: "widget_workspaces", retry_on_conflict: Update::CONFLICT_RETRIES}}, + { + script: {id: operations.first.update_target.script_id, params: { + # Float-typed integer values are coerced to true ints before indexing + "data" => {"size" => [an_instance_of(::Integer).and(eq_to(4))]}, + "id" => "17" + }}, + scripted_upsert: true, + upsert: {} + } + ] + end + + it "returns an update operation for each of the unique id values when there is a list of ids available" do + indexer = indexer_with_widget_workspace_index_definition(id_source: "workspace_ids") do |index| + # no customization + end + + operations = operations_for_indexer(indexer) + + expect(operations.size).to eq(3) + expect(operations.flat_map(&:to_datastore_bulk)).to eq [ + {update: {_id: "17", _index: "widget_workspaces", retry_on_conflict: Update::CONFLICT_RETRIES}}, + { + script: {id: operations.first.update_target.script_id, params: { + "data" => {"name" => ["thing1"]}, + "id" => "17" + }}, + scripted_upsert: true, + upsert: {} + }, + {update: {_id: "18", _index: "widget_workspaces", retry_on_conflict: Update::CONFLICT_RETRIES}}, + { + script: {id: operations.first.update_target.script_id, params: { + "data" => {"name" => ["thing1"]}, + "id" => "18" + }}, + scripted_upsert: true, + upsert: {} + }, + {update: {_id: "19", _index: "widget_workspaces", retry_on_conflict: Update::CONFLICT_RETRIES}}, + { + script: {id: operations.first.update_target.script_id, params: { + "data" => {"name" => ["thing1"]}, + "id" => "19" + }}, + scripted_upsert: true, + upsert: {} + } + ] + end + + context "when the derived index is a rollover index" do + let(:event) do + base = super() + base.merge("record" => base.fetch("record").merge( + "workspace_created_at" => "1995-04-23T00:23:45Z" + )) + end + + it "targets the index identified by the rollover timestamp field" do + indexer = indexer_with_widget_workspace_index_definition derived_index_rollover_with: "workspace_created_at" do |index| + index.rollover :yearly, "was_created_at" + end + + operations = operations_for_indexer(indexer) + + expect(operations.flat_map(&:to_datastore_bulk).first).to eq({update: { + _id: "17", + _index: "widget_workspaces_rollover__1995", + retry_on_conflict: Update::CONFLICT_RETRIES + }}) + end + end + + context "when the derived index uses custom routing" do + let(:event) do + base = super() + base.merge("record" => base.fetch("record").merge( + "num" => 3.0 # an integer-valued-float that the record preparer normalizes + )) + end + + it "includes the `routing` value in the update calls" do + indexer = indexer_with_widget_workspace_index_definition derived_index_route_with: "embedded_values.name" do |index| + index.route_with "other_id" + end + + operations = operations_for_indexer(indexer) + + expect(operations.flat_map(&:to_datastore_bulk).first).to eq({update: { + _id: "17", + _index: "widget_workspaces", + routing: "embedded_name", + retry_on_conflict: Update::CONFLICT_RETRIES + }}) + end + + it "uses the prepared record to get the routing value so that the data is formatted for the datastore" do + indexer = indexer_with_widget_workspace_index_definition derived_index_route_with: "num" do |index| + index.route_with "num" + end + + operations = operations_for_indexer(indexer) + + expect(operations.flat_map(&:to_datastore_bulk).first).to eq({update: { + _id: "17", + _index: "widget_workspaces", + routing: "3", + retry_on_conflict: Update::CONFLICT_RETRIES + }}) + end + + it "correctly falls back to the id from `id_source` when the routing value is an ignored routing value" do + indexer = indexer_with_widget_workspace_index_definition( + derived_index_route_with: "embedded_values.name", + config_overrides: { + index_definitions: { + "widgets" => config_index_def_of, + "widget_workspaces" => config_index_def_of(ignore_routing_values: ["embedded_name"]) + } + } + ) do |index| + index.route_with "other_id" + end + + operations = operations_for_indexer(indexer) + + expect(operations.flat_map(&:to_datastore_bulk).first).to eq({update: { + _id: "17", + _index: "widget_workspaces", + routing: "17", + retry_on_conflict: Update::CONFLICT_RETRIES + }}) + end + end + + def operations_for_indexer(indexer, event: self.event) + update_target = indexer.schema_artifacts.runtime_metadata.object_types_by_name.fetch("Widget").update_targets.first + index_defs_by_name = indexer.datastore_core.index_definitions_by_name + + Update.operations_for( + event: event, + destination_index_def: index_defs_by_name.fetch("widget_workspaces"), + record_preparer: indexer.record_preparer_factory.for_latest_json_schema_version, + update_target: update_target, + destination_index_mapping: indexer.schema_artifacts.index_mappings_by_index_def_name.fetch("widget_workspaces") + ) + end + + def indexer_with_widget_workspace_index_definition( + id_source: "workspace_id", + set_field: "widget_names", + set_field_source: "name", + derived_index_route_with: nil, + derived_index_rollover_with: nil, + config_overrides: {}, + &block + ) + indexer_with_schema(**config_overrides) do |schema| + schema.object_type "WidgetWorkspace" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "other_id", "ID" + t.field "num", "Int" + t.field "widget_names", "[String!]!" + t.field "was_created_at", "DateTime" # intentionally different from `created_at` since that's used on `Widget` + t.field "sizes", "[Int!]!" + t.index "widget_workspaces", &block + end + + schema.object_type "WidgetEmbeddedValues" do |t| + t.field "workspace_id", "ID" + t.field "name", "String" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "workspace_id", "ID" + t.field "workspace_ids", "[ID!]!" + t.field "name", "String" + t.field "num", "Int" + t.field "size", "Int" + t.field "created_at", "DateTime" + t.field "workspace_created_at", "DateTime" + t.field "embedded_values", "WidgetEmbeddedValues" + t.index "widgets" do |i| + i.rollover :monthly, "created_at" + i.route_with "workspace_id" + end + + t.derive_indexed_type_fields( + "WidgetWorkspace", + from_id: id_source, + route_with: derived_index_route_with, + rollover_with: derived_index_rollover_with + ) do |derive| + derive.append_only_set set_field, from: set_field_source + end + end + end + end + + def indexer_with_schema(**overrides, &block) + build_indexer(schema_definition: block, **overrides) + end + end + + describe "#categorize" do + let(:operation) do + update_with_update_target(derived_indexing_update_target_with(script_id: "some_update_script"), doc_id: "some_doc_id") + end + + it "categorizes a response as :success if the status code is 2xx and result is not noop" do + response = {"update" => {"status" => 200}} + + result = operation.categorize(response) + + expect(result).to be_an_update_result_with(category: :success, event: event, description: nil) + end + + it "categorizes a response as :noop if result is noop" do + response = {"update" => {"status" => 200, "result" => "noop"}} + + result = operation.categorize(response) + + expect(result).to be_an_update_result_with(category: :noop, event: event, description: nil) + end + + it "categorizes a response as :noop if the script threw an exception with our noop preamble in the message" do + response = {"update" => {"status" => 500, "error" => { + "reason" => "an exception was thrown", + "caused_by" => {"caused_by" => { + "reason" => "#{UPDATE_WAS_NOOP_MESSAGE_PREAMBLE}the version was too low" + }} + }}} + + result = operation.categorize(response) + + expect(result).to be_an_update_result_with(category: :noop, event: event, description: "the version was too low") + end + + it "categorizes a response as :failure if not :noop or :success" do + response = {"update" => {"status" => 500, "error" => {"reason" => "an exception was thrown"}}} + + result = operation.categorize(response) + + expect(result).to be_an_update_result_with( + category: :failure, + event: event, + description: <<~EOS.strip + some_update_script(applied to `some_doc_id`): an exception was thrown; full response: { + "update": { + "status": 500, + "error": { + "reason": "an exception was thrown" + } + } + } + EOS + ) + end + end + + def be_an_update_result_with(**attributes) + be_a(Result).and have_attributes(operation_type: :update, **attributes) + end + + def update_with_update_target(update_target, doc_id: event.fetch("id")) + Update.new( + event: event, + prepared_record: nil, + destination_index_def: nil, + update_target: update_target, + doc_id: doc_id, + destination_index_mapping: {} + ) + end + end + end + end +end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/upsert_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/upsert_spec.rb new file mode 100644 index 00000000..0ddd7bbb --- /dev/null +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/upsert_spec.rb @@ -0,0 +1,289 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer" +require "elastic_graph/indexer/operation/upsert" +require "json" + +module ElasticGraph + class Indexer + module Operation + RSpec.describe Upsert, :factories do + let(:indexer) { build_indexer } + let(:component_index_definition) { indexer.datastore_core.index_definitions_by_name.fetch("components") } + + describe "#versioned?" do + it "always returns `true` since all `Upsert`s use datastore external versioning" do + event = build_upsert_event(:component, id: "1", __version: 1) + upsert = new_upsert(event, component_index_definition) + + expect(upsert.versioned?).to be true + end + end + + describe "#to_datastore_bulk" do + it "generates an upsert for a single index" do + event = build_upsert_event(:component, id: "1", __version: 1) + + expect(new_upsert(event, component_index_definition).to_datastore_bulk).to eq([ + {index: {_index: "components", _id: "1", version: 1, version_type: "external"}}, + event["record"] + ]) + end + + it "includes a `routing` value based on the configured routing field if the index is using custom routing" do + indexer = define_indexer do |s| + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index "my_type" do |i| + i.route_with "name" + end + end + end + + event = { + "op" => "upsert", + "id" => "1", + "type" => "MyType", + "version" => 1, + "record" => { + "id" => "1", + "name" => "Bob" + } + } + + index_def = indexer.datastore_core.index_definitions_by_name.fetch("my_type") + + expect(new_upsert(event, index_def, indexer).to_datastore_bulk).to eq([ + {index: {_index: "my_type", _id: "1", version: 1, version_type: "external", routing: "Bob"}}, + event["record"] + ]) + end + + it "sets the routing value correctly when the routing field has an alternate `name_in_index`" do + indexer = define_indexer do |s| + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "name", "String", name_in_index: "name_alt" + t.index "my_type" do |i| + i.route_with "name" + end + end + end + + event = { + "op" => "upsert", + "id" => "1", + "type" => "MyType", + "version" => 1, + "record" => { + "id" => "1", + "name" => "Bob" + } + } + + index_def = indexer.datastore_core.index_definitions_by_name.fetch("my_type") + + expect(new_upsert(event, index_def, indexer).to_datastore_bulk).to eq([ + {index: {_index: "my_type", _id: "1", version: 1, version_type: "external", routing: "Bob"}}, + {"id" => "1", "name_alt" => "Bob"} + ]) + end + + it "uses id for `routing` value if intended value is configured to be ignored by the index" do + ignored_value = "ignored_value" + indexer = build_indexer( + index_definitions: {"my_type" => config_index_def_of(ignore_routing_values: [ignored_value])}, + schema_definition: lambda do |s| + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index "my_type" do |i| + i.route_with "name" + end + end + end + ) + + event = { + "op" => "upsert", + "id" => "1", + "type" => "MyType", + "version" => 1, + "record" => { + "id" => "1", + "name" => ignored_value + } + } + + index_def = indexer.datastore_core.index_definitions_by_name.fetch("my_type") + + expect(new_upsert(event, index_def, indexer).to_datastore_bulk).to eq([ + {index: {_index: "my_type", _id: "1", version: 1, version_type: "external", routing: "1"}}, + event["record"] + ]) + end + + it "supports nested routing fields" do + indexer = define_indexer do |s| + s.object_type "NestedFields" do |t| + t.field "name", "String" + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "nested_fields", "NestedFields" + t.index "my_type" do |i| + i.route_with "nested_fields.name" + end + end + end + + event = { + "op" => "upsert", + "id" => "1", + "type" => "MyType", + "version" => 1, + "record" => { + "id" => "1", + "nested_fields" => {"name" => "Bob"} + } + } + + index_def = indexer.datastore_core.index_definitions_by_name.fetch("my_type") + + expect(new_upsert(event, index_def, indexer).to_datastore_bulk).to eq([ + {index: {_index: "my_type", _id: "1", version: 1, version_type: "external", routing: "Bob"}}, + event["record"] + ]) + end + + it "prepares the record to be indexed" do + indexer = define_indexer do |s| + s.object_type "WidgetOptions" do |t| + t.field "size", "Int" + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "options", "WidgetOptions" + t.index "my_type" + end + end + + event = { + "op" => "upsert", + "id" => "1", + "type" => "MyType", + "version" => 1, + "record" => { + "id" => "1", + "options" => { + "size" => 3.0 + } + } + } + index_def = indexer.datastore_core.index_definitions_by_name.fetch("my_type") + + expect(new_upsert(event, index_def, indexer).to_datastore_bulk.last).to match( + { + "id" => "1", + "options" => { + # Float-typed integer values are coerced to true ints before indexing + "size" => an_instance_of(::Integer).and(eq_to(3)) + } + } + ) + end + + it "raises an exception upon missing id" do + event = { + "type" => "Component", + "version" => 1, + "record" => {"field1" => "value1", "field2" => "value2"} + } + + expect { + new_upsert(event, component_index_definition).to_datastore_bulk + }.to raise_error(KeyError) + end + + it "raises an exception upon missing record" do + event = { + "id" => "1", + "type" => "Component", + "version" => 1 + } + + expect { + new_upsert(event, component_index_definition).to_datastore_bulk + }.to raise_error(KeyError) + end + end + + describe "#categorize" do + it "categorizes a response as a :success if the status code is 2xx" do + event = {"id" => "1", "type" => "Component", "version" => 1} + response = {"index" => {"status" => 200}} + upsert = new_upsert(event, component_index_definition) + + result = upsert.categorize(response) + + expect(result).to be_an_upsert_result_with( + category: :success, + event: event, + description: nil, + inspect: "#" + ) + end + + it "categorizes a response as :noop if the status code is 409" do + event = {"id" => "1", "type" => "Component", "version" => 1} + response = {"index" => {"status" => 409, "error" => {"reason" => "[Z0wCia1lbmd0u80n2ewAzQd8uaB]: version conflict, current version [30001] is higher or equal to the one provided [20001]"}}} + upsert = new_upsert(event, component_index_definition) + + result = upsert.categorize(response) + + expect(result).to be_an_upsert_result_with( + category: :noop, + event: event, + description: "[Z0wCia1lbmd0u80n2ewAzQd8uaB]: version conflict, current version [30001] is higher or equal to the one provided [20001]" + ) + end + + it "categorizes a response as :failure if not :noop or :success" do + event = {"id" => "1", "type" => "Component", "version" => 1} + response = {"index" => {"status" => 500, "error" => {"reason" => "[Z0wCia1lbmd0u80n2ewAzQd8uaB]: version conflict, current version [30001] is higher or equal to the one provided [20001]"}}} + upsert = new_upsert(event, component_index_definition) + + result = upsert.categorize(response) + + expect(result).to be_an_upsert_result_with( + category: :failure, + event: event, + description: "[Z0wCia1lbmd0u80n2ewAzQd8uaB]: version conflict, current version [30001] is higher or equal to the one provided [20001]" + ) + end + end + + def define_indexer(&block) + build_indexer(schema_definition: block) + end + + def new_upsert(event, index_def = component_index_definition, idxr = indexer) + Upsert.new(event, index_def, idxr.record_preparer_factory.for_latest_json_schema_version) + end + + def be_an_upsert_result_with(**attributes) + be_a(Result).and have_attributes(operation_type: :upsert, **attributes) + end + end + end + end +end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/processor_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/processor_spec.rb new file mode 100644 index 00000000..d9dd35b6 --- /dev/null +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/processor_spec.rb @@ -0,0 +1,451 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer" +require "elastic_graph/indexer/test_support/converters" +require "elastic_graph/indexer/processor" +require "elastic_graph/indexer/datastore_indexing_router" +require "elastic_graph/support/hash_util" +require "json" + +module ElasticGraph + class Indexer + RSpec.describe Processor do + describe ".process", :factories, :capture_logs do + shared_examples_for ".process method" do |operation_verb| + let(:clock) { class_double(Time, now: Time.iso8601("2020-09-15T12:30:00Z")) } + let(:datastore_router) { instance_spy(ElasticGraph::Indexer::DatastoreIndexingRouter) } + let(:component_to_ignore) { build_upsert_event(:component, id: ignored_event_id.id, __version: ignored_event_id.version) } + let(:indexer) do + build_indexer_with( + latency_thresholds: { + "originated_at" => 150_000, + "touched_by_foo_at" => 200_000 + } + ) + end + + before do + allow(datastore_router).to receive(:bulk) do |ops, **options| + ops_and_results = ops.map { |op| [op, Operation::Result.success_of(op)] } + DatastoreIndexingRouter::BulkResult.new({"main" => ops_and_results}) + end + + allow(datastore_router).to receive(:source_event_versions_in_index) do |ops| + ops.to_h do |op| + [op, {"main" => []}] + end + end + end + + it "calls router.bulk" do + component = build_upsert_event(:component, id: "123", __version: 1) + address = build_upsert_event(:address, id: "123", __version: 1) + + process([component, address]) + + expect(datastore_router).to have_received(:bulk).with( + [ + new_primary_indexing_operation(component.merge("record" => component["record"].merge("id" => component.fetch("id")))), + new_primary_indexing_operation(address.merge("record" => address["record"].merge("id" => address.fetch("id")))) + ], + refresh: true + ) + end + + context "when `router.bulk` returns some failures" do + let(:component1) { build_upsert_event(:component, id: "c123", __version: 1) } + let(:component2) { build_upsert_event(:component, id: "c234", __version: 1) } + let(:component3) { build_upsert_event(:component, id: "c345", __version: 1) } + + before do + allow(datastore_router).to receive(:bulk) do |ops, **options| + expect(ops.map(&:event)).to eq([component1, component2, component3]) + + DatastoreIndexingRouter::BulkResult.new({"main" => [ + [ops[0], Operation::Result.success_of(ops[0])], + [ops[1], Operation::Result.failure_of(ops[1], "overloaded!")], + [ops[2], Operation::Result.success_of(ops[2])] + ]}) + end + end + + it "raises `IndexingFailuresError` so the events can be retried later, while still logging what got processed successfully" do + expect { + process([component1, component2, component3]) + }.to raise_error( + IndexingFailuresError, + a_string_including( + "Got 1 failure(s) from 3 event(s)", "#{operation_verb} Component:c234@v1 failure--overloaded" + ).and(excluding("c123", "c345")) + ) + end + + it "allows the caller to handle the failures if they call `process_returning_failures` instead of `process`" do + failures = process_returning_failures([component1, component2, component3]) + + expect(failures).to all be_a FailedEventError + expect(failures.map(&:id)).to contain_exactly("c234") + end + end + + describe "latency metrics" do + it "extracts latency metrics from events" do + component = upsert_event_with_latency_timestamps(:component, 36, 72) + address = upsert_event_with_latency_timestamps(:address, 108, 144) + + process([component, address]) + + expect(logged_jsons_of_type("ElasticGraphIndexingLatencies")).to match([ + a_hash_including( + "event_type" => "Component", + "latencies_in_ms_from" => { + "originated_at" => 36000, + "touched_by_foo_at" => 72000 + }, + "slo_results" => { + "originated_at" => "good", + "touched_by_foo_at" => "good" + } + ), + a_hash_including( + "event_type" => "Address", + "latencies_in_ms_from" => { + "originated_at" => 108000, + "touched_by_foo_at" => 144000 + }, + "slo_results" => { + "originated_at" => "good", + "touched_by_foo_at" => "good" + } + ) + ]) + end + + it "fully identifies each event and message in the logged `ElasticGraphIndexingLatencies` message" do + component = upsert_event_with_latency_timestamps(:component, 36, 72).merge("message_id" => "m1") + process([component]) + + expect(logged_jsons_of_type("ElasticGraphIndexingLatencies").first).to include( + "event_id" => "Component:#{component.fetch("id")}@v#{component.fetch("version")}", + "message_id" => "m1" + ) + end + + it "avoids double-emitting metrics for multiple operations from the same event" do + widget = upsert_event_with_latency_timestamps(:widget, 36, 72) + + process([widget]) + + expect(logged_jsons_of_type("ElasticGraphIndexingLatencies")).to match([ + a_hash_including( + "event_type" => "Widget", + "latencies_in_ms_from" => { + "originated_at" => 36000, + "touched_by_foo_at" => 72000 + }, + "slo_results" => { + "originated_at" => "good", + "touched_by_foo_at" => "good" + } + ) + ]) + end + + it "emits latency metrics for all events, including those that did not update the datastore" do + component1 = upsert_event_with_latency_timestamps(:component, 36, 72) + address = upsert_event_with_latency_timestamps(:address, 108, 144) + component2 = upsert_event_with_latency_timestamps(:component, 12, 24, id: "no_op_update") + + allow(datastore_router).to receive(:bulk) do |ops, **options| + # simulate the update with id == `no_op_update` being an ignored event due to the version not increasing + ops_and_results = ops.map do |op| + result = + if op.event.fetch("id") == "no_op_update" + Operation::Result.noop_of(op, "was a noop") + else + Operation::Result.success_of(op) + end + + [op, result] + end + + DatastoreIndexingRouter::BulkResult.new({"main" => ops_and_results}) + end + + process([component1, address, component2]) + + expect(logged_jsons_of_type("ElasticGraphIndexingLatencies")).to match([ + a_hash_including( + "event_type" => "Component", + "latencies_in_ms_from" => { + "originated_at" => 36000, + "touched_by_foo_at" => 72000 + }, + "slo_results" => { + "originated_at" => "good", + "touched_by_foo_at" => "good" + }, + "result" => "success" + ), + a_hash_including( + "event_type" => "Address", + "latencies_in_ms_from" => { + "originated_at" => 108000, + "touched_by_foo_at" => 144000 + }, + "slo_results" => { + "originated_at" => "good", + "touched_by_foo_at" => "good" + }, + "result" => "success" + ), + a_hash_including( + "event_type" => "Component", + "latencies_in_ms_from" => { + "originated_at" => 12000, + "touched_by_foo_at" => 24000 + }, + "slo_results" => { + "originated_at" => "good", + "touched_by_foo_at" => "good" + }, + "result" => "noop" + ) + ]) + end + + it "logs true slo_results for events with latencies exceeding the configured thresholds" do + # thresholds set in `let(:indexer)` are 150 and 200 seconds. + no_outliers = upsert_event_with_latency_timestamps(:component, 36, 72, id: "good") + originated_at_outlier = upsert_event_with_latency_timestamps(:component, 151, 72, id: "bad1", __version: 7) + touched_by_foo_outlier = upsert_event_with_latency_timestamps(:component, 36, 201, id: "bad2", __version: 2) + both_exact_outliers = upsert_event_with_latency_timestamps(:component, 150, 200, id: "bad_both", __version: 3) + + process([no_outliers, originated_at_outlier, touched_by_foo_outlier, both_exact_outliers]) + + logged_jsons = logged_jsons_of_type("ElasticGraphIndexingLatencies") + + conditions = [ + { + substring: "bad1@v7", + expected: { + "originated_at" => "bad", + "touched_by_foo_at" => "good" + } + }, + { + substring: "bad2@v2", + expected: { + "touched_by_foo_at" => "bad", + "originated_at" => "good" + } + }, + { + substring: "bad_both@v3", + expected: { + "originated_at" => "bad", + "touched_by_foo_at" => "bad" + } + } + ] + + conditions.each do |condition| + expect(logged_jsons).to include( + a_hash_including( + "event_id" => a_string_including(condition[:substring]), + "slo_results" => condition[:expected] + ) + ) + end + end + + it "omits timestamps from `slo_results` when they have no configured threshold" do + no_outliers = upsert_event_with_latency_timestamps(:component, 36, 72, id: "good") + originated_at_outlier = upsert_event_with_latency_timestamps(:component, 151, 72, id: "bad1", __version: 7) + touched_by_foo_outlier = upsert_event_with_latency_timestamps(:component, 36, 201, id: "bad2", __version: 2) + both_exact_outliers = upsert_event_with_latency_timestamps(:component, 150, 200, id: "bad_both", __version: 3) + + indexer = build_indexer_with(latency_thresholds: { + "originated_at" => 150_000 + }) + + indexer.processor.process([no_outliers, originated_at_outlier, touched_by_foo_outlier, both_exact_outliers], refresh_indices: true) + + logged_jsons = logged_jsons_of_type("ElasticGraphIndexingLatencies") + + conditions = [ + { + substring: "bad1@v7", + expected: { + "originated_at" => "bad" + } + }, + { + substring: "bad2@v2", + expected: { + "originated_at" => "good" + } + }, + { + substring: "bad_both@v3", + expected: { + "originated_at" => "bad" + } + } + ] + + conditions.each do |condition| + expect(logged_jsons).to include( + a_hash_including( + "event_id" => a_string_including(condition[:substring]), + "slo_results" => condition[:expected] + ) + ) + end + end + + def upsert_event_with_latency_timestamps(entity_type, originated_at_offset, touched_by_foo_at_offset, **options) + build_upsert_event(entity_type, **options).merge({ + "latency_timestamps" => { + "originated_at" => (clock.now - originated_at_offset).iso8601, + "touched_by_foo_at" => (clock.now - touched_by_foo_at_offset).iso8601 + } + }) + end + end + + context "when the list of events includes some invalid ones" do + let(:good_component) { build_upsert_event(:component, id: "123", __version: 1) } + let(:good_address) { build_upsert_event(:address, id: "456", __version: 1) } + + let(:events) do + [ + good_component, + make_component_bad(good_component).merge("id" => "234"), + good_address, + good_address.merge("type" => "Color", "id" => "345") # Color is not a valid `type` + ] + end + + it "allows the valid events to be written, raising an error describing the invalid events" do + expect { + process(events) + }.to raise_error IndexingFailuresError, a_string_including( + "2 failure(s) from 4 event(s)", + "1) Component:234@v1: Malformed Component record", + "2) Color:345@v1: Malformed event payload" + ) + + expect(datastore_router).to have_received(:bulk).with( + [ + new_primary_indexing_operation(good_component), + new_primary_indexing_operation(good_address) + ], + refresh: true + ) + end + + it "mentions the `message_id` in the exception message if its available" do + expect { + events_with_msg_ids = events.map.with_index(1) { |e, index| e.merge("message_id" => "m#{index}") } + process(events_with_msg_ids) + }.to raise_error IndexingFailuresError, a_string_including( + "2 failure(s) from 4 event(s)", + "1) Component:234@v1 (message_id: m2): Malformed Component record", + "2) Color:345@v1 (message_id: m4): Malformed event payload" + ) + end + + it "allows the caller to handle the failures if they call `process_returning_failures` instead of `process`" do + failures = process_returning_failures(events) + + expect(failures).to all be_a FailedEventError + expect(failures.map(&:id)).to contain_exactly("234", "345") + + expect(datastore_router).to have_received(:bulk).with( + [ + new_primary_indexing_operation(good_component), + new_primary_indexing_operation(good_address) + ], + refresh: true + ) + end + + def make_component_bad(component) + component.merge("record" => component["record"].merge( + "name" => 17 # must be an string + )) + end + end + end + + context "when `use_updates_for_indexing?` is set to false", use_updates_for_indexing: false do + include_examples ".process method", "upsert" do + def new_primary_indexing_operation(event) + index_defs = indexer.datastore_core.index_definitions_by_graphql_type.fetch(event.fetch("type")) + expect(index_defs.size).to eq 1 + Operation::Upsert.new(event, index_defs.first, indexer.record_preparer_factory.for_latest_json_schema_version) + end + end + end + + context "when `use_updates_for_indexing?` is set to true", use_updates_for_indexing: true do + include_examples ".process method", "update" do + def new_primary_indexing_operation(event) + update_targets = indexer + .schema_artifacts + .runtime_metadata + .object_types_by_name + .fetch(event.fetch("type")) + .update_targets + .select { |ut| ut.type == event.fetch("type") } + + expect(update_targets.size).to eq(1) + + index_name = { + "Component" => "components", + "Address" => "addresses" + }.fetch(event.fetch("type")) + + index_def = indexer.datastore_core.index_definitions_by_name.fetch(index_name) + + Operation::Update.new( + event: event, + prepared_record: indexer.record_preparer_factory.for_latest_json_schema_version.prepare_for_index( + event.fetch("type"), + event.fetch("record") + ), + destination_index_def: index_def, + update_target: update_targets.first, + doc_id: event.fetch("id"), + destination_index_mapping: indexer.schema_artifacts.index_mappings_by_index_def_name.fetch(index_def.name) + ) + end + end + end + + def build_indexer_with(latency_thresholds:) + build_indexer( + clock: clock, + datastore_router: datastore_router, + latency_slo_thresholds_by_timestamp_in_ms: latency_thresholds + ) + end + + def process(*events) + indexer.processor.process(*events, refresh_indices: true) + end + + def process_returning_failures(*events) + indexer.processor.process_returning_failures(*events, refresh_indices: true) + end + end + end + end +end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb new file mode 100644 index 00000000..feb58f0f --- /dev/null +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb @@ -0,0 +1,396 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer/record_preparer" +require "elastic_graph/spec_support/schema_definition_helpers" +require "support/multiple_version_support" + +module ElasticGraph + class Indexer + RSpec.describe RecordPreparer::Factory do + include_context "MultipleVersionSupport" + + let(:factory_with_multiple_versions) do + build_indexer_with_multiple_schema_versions(schema_versions: { + 1 => lambda do |schema| + schema.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index "my_type" + end + end, + + 2 => lambda do |schema| + schema.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.index "my_type" + end + end + }).record_preparer_factory + end + + describe "#for_json_schema_version" do + it "memoizes `RecordPreparer` since they are immutable and that saves on memory" do + for_v1 = factory_with_multiple_versions.for_json_schema_version(1) + for_v2 = factory_with_multiple_versions.for_json_schema_version(2) + + expect(for_v1).not_to eq(for_v2) + expect(factory_with_multiple_versions.for_json_schema_version(1)).to be for_v1 + end + end + + describe "#for_latest_json_schema_version" do + it "returns the record preparer for the latest JSON schema version" do + for_v2 = factory_with_multiple_versions.for_json_schema_version(2) + + expect(factory_with_multiple_versions.for_latest_json_schema_version).to be for_v2 + end + end + end + + RSpec.describe RecordPreparer do + describe "#prepare_for_index" do + it "tolerates a `nil` value where an object would usually be" do + preparer = build_preparer do |s| + s.enum_type "Color" do |t| + t.value "BLUE" + t.value "GREEN" + t.value "RED" + end + + s.object_type "WidgetOptions" do |t| + t.field "color", "Color" + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "options", "WidgetOptions" + t.index "my_type" + end + end + + record = preparer.prepare_for_index("MyType", {"id" => "1", "options" => nil}) + + expect(record).to eq({"id" => "1", "options" => nil}) + end + + it "leaves enum values unchanged (notable since enum types aren't recorded in runtime metadata `scalar_types_by_name`)" do + preparer = build_preparer do |s| + s.enum_type "Color" do |t| + t.value "BLUE" + t.value "GREEN" + t.value "RED" + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "color", "Color" + t.index "my_type" + end + end + + record = preparer.prepare_for_index("MyType", {"id" => "1", "color" => "GREEN"}) + + expect(record).to eq({"id" => "1", "color" => "GREEN"}) + end + + it "drops excess fields not defined in the schema" do + preparer = build_preparer do |s| + s.object_type "Position" do |t| + t.field "x", "Float!" + t.field "y", "Float!" + end + + s.object_type "Component" do |t| + t.field "created_at", "String!" + t.field "id", "ID!" + t.field "name", "String!" + t.field "position", "Position!" + t.index "components" + end + end + + record = { + "id" => "1", + "created_at" => "2019-06-01T12:00:00Z", + "name" => "my_component", + "extra_field1" => { + "field1" => { + "field2" => 3, + "field3" => { + "field4" => 4 + }, + "field6" => "value" + }, + "field7" => "5" + }, + "extra_field2" => 2, + "position" => { + "x" => 1.1, + "y" => 2.1, + "extra_field1" => "value1", + "extra_field2" => { + "extra_field3" => 3 + } + } + } + + record = preparer.prepare_for_index("Component", record) + + expect(record).to eq({ + "id" => "1", + "name" => "my_component", + "created_at" => "2019-06-01T12:00:00Z", + "position" => { + "x" => 1.1, + "y" => 2.1 + } + }) + end + + it "ignores excess fields defined in the schema that are missing from the record" do + preparer = build_preparer do |s| + s.object_type "Position" do |t| + t.field "x", "Float!" + t.field "y", "Float!" + end + + s.object_type "Component" do |t| + t.field "created_at", "String!" + t.field "id", "ID!" + t.field "name", "String!" + t.field "position", "Position!" + t.index "components" + end + end + + record = { + "id" => "1", + "name" => "my_component", + "position" => { + "x" => 1.1 + } + } + + record = preparer.prepare_for_index("Component", record) + + expect(record).to eq({ + "id" => "1", + "name" => "my_component", + "position" => { + "x" => 1.1 + } + }) + end + + it "handles abstract types (like type unions) stored in separate indexes, properly omitting `__typename`" do + preparer = build_preparer do |s| + s.object_type "TypeA" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index "type_a" + end + + s.object_type "TypeB" do |t| + t.field "id", "ID!" + t.field "size", "Int" + t.index "type_b" + end + + s.union_type "TypeAOrB" do |t| + t.subtype "TypeA" + t.subtype "TypeB" + end + end + + record = preparer.prepare_for_index("TypeB", {"id" => "1", "size" => 3, "__typename" => "TypeB"}) + + expect(record).to eq({"id" => "1", "size" => 3}) + end + + it "handles abstract types (like type unions) stored in a single index, properly including `__typename`" do + preparer = build_preparer do |s| + s.object_type "TypeA" do |t| + t.field "id", "ID!" + t.field "name", "String" + end + + s.object_type "TypeB" do |t| + t.field "id", "ID!" + t.field "size", "Int" + end + + s.union_type "TypeAOrB" do |t| + t.subtype "TypeA" + t.subtype "TypeB" + t.index "type_a_or_b" + end + end + + record = preparer.prepare_for_index("TypeAOrB", {"id" => "1", "size" => 3, "__typename" => "TypeB"}) + + expect(record).to eq({"id" => "1", "size" => 3, "__typename" => "TypeB"}) + end + + it "handles nested abstract types, properly including `__typename` on them" do + preparer = build_preparer do |s| + s.object_type "Person" do |t| + t.field "name", "String" + t.field "nationality", "String" + end + + s.object_type "Company" do |t| + t.field "name", "String" + t.field "stock_ticker", "String" + end + + s.union_type "Inventor" do |t| + t.subtypes "Person", "Company" + end + + s.object_type "Invention" do |t| + t.field "id", "ID" + t.field "inventor", "Inventor" + t.index "inventions" + end + end + + record = preparer.prepare_for_index("Invention", { + "id" => "1", + "inventor" => { + "name" => "Block", + "stock_ticker" => "SQ", + "__typename" => "Company" + } + }) + + expect(record).to eq({ + "id" => "1", + "inventor" => { + "name" => "Block", + "stock_ticker" => "SQ", + "__typename" => "Company" + } + }) + end + + it "renames fields from the public name to the internal index field name when they differ" do + preparer = build_preparer do |s| + s.object_type "WidgetOptions" do |t| + t.field "color", "String", name_in_index: "clr" + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "options", "WidgetOptions" + t.field "name", "String", name_in_index: "name2" + t.index "my_type" + end + end + + record = preparer.prepare_for_index("MyType", {"id" => "1", "options" => {"color" => "RED"}, "name" => "Winston"}) + + expect(record).to eq({"id" => "1", "options" => {"clr" => "RED"}, "name2" => "Winston"}) + end + end + + context "when working with events for an old JSON schema version" do + include_context "SchemaDefinitionHelpers" + + it "handles events for old versions before a field was deleted" do + preparer = build_preparer_for_old_json_schema_version( + v1_def: ->(schema) { + schema.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index "my_type" + end + }, + + v2_def: ->(schema) { + schema.object_type "MyType" do |t| + t.field "id", "ID!" + t.deleted_field "name" + t.index "my_type" + end + } + ) + + record = preparer.prepare_for_index("MyType", {"id" => "1", "name" => "Winston"}) + + expect(record).to eq({"id" => "1"}) + end + + it "properly omits `__typename` under an embedded field for a non-abstract type, even when the type has been renamed" do + preparer = build_preparer_for_old_json_schema_version( + v1_def: ->(schema) { + schema.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "cost", "Money" + t.index "my_type" + end + + schema.object_type "Money" do |t| + t.field "amount", "Int" + end + }, + + v2_def: ->(schema) { + schema.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "cost", "Money2" + t.index "my_type" + end + + schema.object_type "Money2" do |t| + t.field "amount", "Int" + t.renamed_from "Money" + end + } + ) + + record = preparer.prepare_for_index("MyType", {"id" => "1", "cost" => {"amount" => 10, "__typename" => "Money"}}) + + expect(record).to eq({"id" => "1", "cost" => {"amount" => 10}}) + end + + def build_preparer_for_old_json_schema_version(v1_def:, v2_def:) + v1_results = define_schema do |schema| + schema.json_schema_version 1 + v1_def.call(schema) + end + + v2_results = define_schema do |schema| + schema.json_schema_version 2 + v2_def.call(schema) + end + + v1_merge_result = v2_results.merge_field_metadata_into_json_schema(v1_results.current_public_json_schema) + + expect(v1_merge_result.missing_fields).to be_empty + expect(v1_merge_result.missing_types).to be_empty + + allow(v2_results).to receive(:json_schemas_for).with(1).and_return(v1_merge_result.json_schema) + + RecordPreparer::Factory.new(v2_results).for_json_schema_version(1) + end + + def define_schema(&schema_definition) + super(schema_element_name_form: "snake_case", &schema_definition) + end + end + + def build_preparer(**config_overrides, &schema_definition) + build_indexer(schema_definition: schema_definition, **config_overrides) + .record_preparer_factory + .for_latest_json_schema_version + end + end + end +end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/spec_support/event_matcher_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/spec_support/event_matcher_spec.rb new file mode 100644 index 00000000..8a74ab09 --- /dev/null +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/spec_support/event_matcher_spec.rb @@ -0,0 +1,68 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer/spec_support/event_matcher" +require "rspec/matchers/fail_matchers" + +RSpec.describe "The `be_a_valid_elastic_graph_event` matcher", :builds_indexer, :factories, aggregate_failures: false do + include ::RSpec::Matchers::FailMatchers + + let(:valid_event) { build_upsert_event(:widget) } + let(:invalid_event) { valid_event.merge("type" => "Unknown") } + let(:event_with_extra_field) { build_upsert_event(:widget, extra1: 3) } + + shared_examples "common matcher examples" do + it "passes when positively matched against a valid event" do + expect(valid_event).to be_a_valid_elastic_graph_event + end + + it "passes when negatively matched against an invalid event" do + expect(invalid_event).not_to be_a_valid_elastic_graph_event + end + + it "fails when positively matched against an invalid event" do + expect { + expect(invalid_event).to be_a_valid_elastic_graph_event + }.to fail_including("expected the event[1] to be a valid ElasticGraph event", "type") + end + + it "fails when negatively matched against a valid event" do + expect { + expect(valid_event).not_to be_a_valid_elastic_graph_event + }.to fail_including("expected the event[1] not to be a valid ElasticGraph event", "type") + end + end + + context "with no block" do + include_examples "common matcher examples" + + it "allows extra properties" do + expect(event_with_extra_field).to be_a_valid_elastic_graph_event + end + end + + context "with a block that calls `with_unknown_properties_disallowed`" do + include_examples "common matcher examples" + + it "disallows extra properties" do + expect(event_with_extra_field).not_to be_a_valid_elastic_graph_event + + expect { + expect(event_with_extra_field).to be_a_valid_elastic_graph_event + }.to fail_including("extra1") + end + + def be_a_valid_elastic_graph_event + super(&:with_unknown_properties_disallowed) + end + end + + def be_a_valid_elastic_graph_event + super(for_indexer: build_indexer) + end +end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/test_support/converters_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/test_support/converters_spec.rb new file mode 100644 index 00000000..a95ee0e4 --- /dev/null +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/test_support/converters_spec.rb @@ -0,0 +1,82 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/indexer/test_support/converters" + +module ElasticGraph + class Indexer + module TestSupport + RSpec.describe Converters, :factories do + describe ".upsert_event_for" do + it "can build an `upsert` operation based on a factory-produced record" do + factory_record = { + "id" => "1", + "__version" => 1, + "__typename" => "Widget", + "__json_schema_version" => 1, + "field1" => "value1", + "field2" => "value2" + } + + expect(TestSupport::Converters.upsert_event_for(factory_record)).to eq( + "op" => "upsert", + "id" => "1", + "version" => 1, + "type" => "Widget", + "record" => {"id" => "1", "field1" => "value1", "field2" => "value2"}, + JSON_SCHEMA_VERSION_KEY => 1 + ) + end + end + + describe ".upsert_events_for_records" do + it "converts an array of factory-produced records into an array of operations for upserting" do + record1 = { + "id" => "1", + "__typename" => "Widget", + "__version" => 1, + "__json_schema_version" => 1, + "field1" => "value1", + "field2" => "value2" + } + + record2 = { + "id" => "2", + "__typename" => "Address", + "__version" => 5, + "__json_schema_version" => 1, + "field3" => "value5" + } + + event = TestSupport::Converters.upsert_events_for_records([record1, record2]) + + expect(event).to eq([ + { + "op" => "upsert", + "id" => "1", + "version" => 1, + "type" => "Widget", + "record" => {"id" => "1", "field1" => "value1", "field2" => "value2"}, + JSON_SCHEMA_VERSION_KEY => 1 + }, + { + "op" => "upsert", + "id" => "2", + "version" => 5, + "type" => "Address", + "record" => {"id" => "2", "field3" => "value5"}, + JSON_SCHEMA_VERSION_KEY => 1 + } + ]) + end + end + end + end + end +end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer_spec.rb new file mode 100644 index 00000000..c1037d79 --- /dev/null +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer_spec.rb @@ -0,0 +1,27 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer" + +module ElasticGraph + RSpec.describe Indexer do + it "returns non-nil values from each attribute" do + expect_to_return_non_nil_values_from_all_attributes(build_indexer) + end + + describe ".from_parsed_yaml" do + it "builds an Indexer instance from the contents of a YAML settings file" do + customization_block = lambda { |conn| } + indexer = Indexer.from_parsed_yaml(parsed_test_settings_yaml, &customization_block) + + expect(indexer).to be_a(Indexer) + expect(indexer.datastore_core.client_customization_block).to be(customization_block) + end + end + end +end diff --git a/elasticgraph-indexer_autoscaler_lambda/.rspec b/elasticgraph-indexer_autoscaler_lambda/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-indexer_autoscaler_lambda/.yardopts b/elasticgraph-indexer_autoscaler_lambda/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-indexer_autoscaler_lambda/Gemfile b/elasticgraph-indexer_autoscaler_lambda/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-indexer_autoscaler_lambda/LICENSE.txt b/elasticgraph-indexer_autoscaler_lambda/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-indexer_autoscaler_lambda/README.md b/elasticgraph-indexer_autoscaler_lambda/README.md new file mode 100644 index 00000000..40e8e874 --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/README.md @@ -0,0 +1 @@ +# ElasticGraph::IndexerAutoscalerLambda diff --git a/elasticgraph-indexer_autoscaler_lambda/elasticgraph-indexer_autoscaler_lambda.gemspec b/elasticgraph-indexer_autoscaler_lambda/elasticgraph-indexer_autoscaler_lambda.gemspec new file mode 100644 index 00000000..2d19185f --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/elasticgraph-indexer_autoscaler_lambda.gemspec @@ -0,0 +1,27 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :lambda) do |spec, eg_version| + spec.summary = "ElasticGraph gem that monitors OpenSearch CPU utilization to autoscale indexer lambda concurrency." + + spec.add_dependency "elasticgraph-datastore_core", eg_version + spec.add_dependency "elasticgraph-lambda_support", eg_version + + spec.add_dependency "aws-sdk-lambda", "~> 1.125" + spec.add_dependency "aws-sdk-sqs", "~> 1.80" + + # aws-sdk-sqs requires an XML library be available. On Ruby < 3 it'll use rexml from the standard library but on Ruby 3.0+ + # we have to add an explicit dependency. It supports ox, oga, libxml, nokogiri or rexml, and of those, ox seems to be the + # best choice: it leads benchmarks, is well-maintained, has no dependencies, and is MIT-licensed. + spec.add_dependency "ox", "~> 2.14" + + spec.add_development_dependency "elasticgraph-elasticsearch", eg_version + spec.add_development_dependency "elasticgraph-opensearch", eg_version +end diff --git a/elasticgraph-indexer_autoscaler_lambda/lib/elastic_graph/indexer_autoscaler_lambda.rb b/elasticgraph-indexer_autoscaler_lambda/lib/elastic_graph/indexer_autoscaler_lambda.rb new file mode 100644 index 00000000..50bc9a58 --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/lib/elastic_graph/indexer_autoscaler_lambda.rb @@ -0,0 +1,67 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core" +require "elastic_graph/lambda_support" +require "elastic_graph/support/from_yaml_file" + +module ElasticGraph + # @private + class IndexerAutoscalerLambda + extend Support::FromYamlFile + + # Builds an `ElasticGraph::IndexerAutoscalerLambda` instance from our lambda ENV vars. + def self.from_env + LambdaSupport.build_from_env(self) + end + + # A factory method that builds a IndexerAutoscalerLambda instance from the given parsed YAML config. + # `from_yaml_file(file_name, &block)` is also available (via `Support::FromYamlFile`). + def self.from_parsed_yaml(parsed_yaml, &datastore_client_customization_block) + new(datastore_core: DatastoreCore.from_parsed_yaml(parsed_yaml, for_context: :indexer_autoscaler_lambda, &datastore_client_customization_block)) + end + + # @dynamic datastore_core + attr_reader :datastore_core + + def initialize( + datastore_core:, + sqs_client: nil, + lambda_client: nil + ) + @datastore_core = datastore_core + @sqs_client = sqs_client + @lambda_client = lambda_client + end + + def sqs_client + @sqs_client ||= begin + require "aws-sdk-sqs" + Aws::SQS::Client.new + end + end + + def lambda_client + @lambda_client ||= begin + require "aws-sdk-lambda" + Aws::Lambda::Client.new + end + end + + def concurrency_scaler + @concurrency_scaler ||= begin + require "elastic_graph/indexer_autoscaler_lambda/concurrency_scaler" + ConcurrencyScaler.new( + datastore_core: @datastore_core, + sqs_client: sqs_client, + lambda_client: lambda_client + ) + end + end + end +end diff --git a/elasticgraph-indexer_autoscaler_lambda/lib/elastic_graph/indexer_autoscaler_lambda/concurrency_scaler.rb b/elasticgraph-indexer_autoscaler_lambda/lib/elastic_graph/indexer_autoscaler_lambda/concurrency_scaler.rb new file mode 100644 index 00000000..ae519377 --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/lib/elastic_graph/indexer_autoscaler_lambda/concurrency_scaler.rb @@ -0,0 +1,145 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer_autoscaler_lambda/details_logger" + +module ElasticGraph + class IndexerAutoscalerLambda + # @private + class ConcurrencyScaler + def initialize(datastore_core:, sqs_client:, lambda_client:) + @logger = datastore_core.logger + @datastore_core = datastore_core + @sqs_client = sqs_client + @lambda_client = lambda_client + end + + # AWS requires the value be in this range: + # https://docs.aws.amazon.com/lambda/latest/api/API_ScalingConfig.html#API_ScalingConfig_Contents + MAXIMUM_CONCURRENCY = 1000 + MINIMUM_CONCURRENCY = 2 + + def tune_indexer_concurrency(queue_urls:, min_cpu_target:, max_cpu_target:, event_source_mapping_uuids:) + queue_attributes = get_queue_attributes(queue_urls) + queue_arns = queue_attributes.fetch(:queue_arns) + num_messages = queue_attributes.fetch(:total_messages) + + details_logger = DetailsLogger.new( + logger: @logger, + queue_arns: queue_arns, + queue_urls: queue_urls, + min_cpu_target: min_cpu_target, + max_cpu_target: max_cpu_target, + num_messages: num_messages + ) + + new_target_concurrency = + if num_messages.positive? + cpu_utilization = get_max_cpu_utilization + cpu_midpoint = (max_cpu_target + min_cpu_target) / 2.0 + + current_concurrency = get_total_concurrency(event_source_mapping_uuids) + + if current_concurrency.nil? + details_logger.log_unset + nil + elsif cpu_utilization < min_cpu_target + increase_factor = (cpu_midpoint / cpu_utilization).clamp(0.0, 1.5) + (current_concurrency * increase_factor).round.tap do |new_concurrency| + details_logger.log_increase( + cpu_utilization: cpu_utilization, + current_concurrency: current_concurrency, + new_concurrency: new_concurrency + ) + end + elsif cpu_utilization > max_cpu_target + decrease_factor = cpu_utilization / cpu_midpoint - 1 + (current_concurrency - (current_concurrency * decrease_factor)).round.tap do |new_concurrency| + details_logger.log_decrease( + cpu_utilization: cpu_utilization, + current_concurrency: current_concurrency, + new_concurrency: new_concurrency + ) + end + else + details_logger.log_no_change( + cpu_utilization: cpu_utilization, + current_concurrency: current_concurrency + ) + current_concurrency + end + else + details_logger.log_reset + 0 + end + + if new_target_concurrency && new_target_concurrency != current_concurrency + update_event_source_mapping( + event_source_mapping_uuids: event_source_mapping_uuids, + concurrency: new_target_concurrency + ) + end + end + + private + + def get_max_cpu_utilization + @datastore_core.clients_by_name.values.flat_map do |client| + client.get_node_os_stats.fetch("nodes").values.map do |node| + node.dig("os", "cpu", "percent") + end + end.max.to_f + end + + def get_queue_attributes(queue_urls) + attributes_per_queue = queue_urls.map do |queue_url| + @sqs_client.get_queue_attributes( + queue_url: queue_url, + attribute_names: ["QueueArn", "ApproximateNumberOfMessages"] + ).attributes + end + + total_messages = attributes_per_queue + .map { |attr| Integer(attr.fetch("ApproximateNumberOfMessages")) } + .sum + + queue_arns = attributes_per_queue.map { |attr| attr.fetch("QueueArn") } + + { + total_messages: total_messages, + queue_arns: queue_arns + } + end + + def get_total_concurrency(event_source_mapping_uuids) + maximum_concurrencies = event_source_mapping_uuids.map do |event_source_mapping_uuid| + @lambda_client.get_event_source_mapping( + uuid: event_source_mapping_uuid + ).scaling_config&.maximum_concurrency + end.compact + maximum_concurrencies.empty? ? nil : maximum_concurrencies.sum + end + + def update_event_source_mapping(concurrency:, event_source_mapping_uuids:) + concurrency_per_queue = concurrency / event_source_mapping_uuids.length + + target_concurrency = + concurrency_per_queue.clamp(MINIMUM_CONCURRENCY, MAXIMUM_CONCURRENCY) + + event_source_mapping_uuids.map do |event_source_mapping_uuid| + @lambda_client.update_event_source_mapping( + uuid: event_source_mapping_uuid, + scaling_config: { + maximum_concurrency: target_concurrency + } + ) + end + end + end + end +end diff --git a/elasticgraph-indexer_autoscaler_lambda/lib/elastic_graph/indexer_autoscaler_lambda/details_logger.rb b/elasticgraph-indexer_autoscaler_lambda/lib/elastic_graph/indexer_autoscaler_lambda/details_logger.rb new file mode 100644 index 00000000..94936583 --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/lib/elastic_graph/indexer_autoscaler_lambda/details_logger.rb @@ -0,0 +1,74 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + class IndexerAutoscalerLambda + # @private + class DetailsLogger + def initialize( + logger:, + queue_arns:, + queue_urls:, + min_cpu_target:, + max_cpu_target:, + num_messages: + ) + @logger = logger + + @log_data = { + "message_type" => "ConcurrencyScalerResult", + "queue_arns" => queue_arns, + "queue_urls" => queue_urls, + "min_cpu_target" => min_cpu_target, + "max_cpu_target" => max_cpu_target, + "num_messages" => num_messages + } + end + + def log_increase(cpu_utilization:, current_concurrency:, new_concurrency:) + log_result({ + "action" => "increase", + "cpu_utilization" => cpu_utilization, + "current_concurrency" => current_concurrency, + "new_concurrency" => new_concurrency + }) + end + + def log_decrease(cpu_utilization:, current_concurrency:, new_concurrency:) + log_result({ + "action" => "decrease", + "cpu_utilization" => cpu_utilization, + "current_concurrency" => current_concurrency, + "new_concurrency" => new_concurrency + }) + end + + def log_no_change(cpu_utilization:, current_concurrency:) + log_result({ + "action" => "no_change", + "cpu_utilization" => cpu_utilization, + "current_concurrency" => current_concurrency + }) + end + + def log_reset + log_result({"action" => "reset"}) + end + + def log_unset + log_result({"action" => "unset"}) + end + + private + + def log_result(data) + @logger.info(@log_data.merge(data)) + end + end + end +end diff --git a/elasticgraph-indexer_autoscaler_lambda/lib/elastic_graph/indexer_autoscaler_lambda/lambda_function.rb b/elasticgraph-indexer_autoscaler_lambda/lib/elastic_graph/indexer_autoscaler_lambda/lambda_function.rb new file mode 100644 index 00000000..d6164277 --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/lib/elastic_graph/indexer_autoscaler_lambda/lambda_function.rb @@ -0,0 +1,36 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/lambda_support/lambda_function" + +module ElasticGraph + class IndexerAutoscalerLambda + # @private + class LambdaFunction + prepend LambdaSupport::LambdaFunction + + def initialize + require "elastic_graph/indexer_autoscaler_lambda" + + @concurrency_scaler = ElasticGraph::IndexerAutoscalerLambda.from_env.concurrency_scaler + end + + def handle_request(event:, context:) + @concurrency_scaler.tune_indexer_concurrency( + queue_urls: event.fetch("queue_urls"), + min_cpu_target: event.fetch("min_cpu_target"), + max_cpu_target: event.fetch("max_cpu_target"), + event_source_mapping_uuids: event.fetch("event_source_mapping_uuids") + ) + end + end + end +end + +# Lambda handler for `elasticgraph-indexer_autoscaler_lambda`. +AutoscaleIndexer = ElasticGraph::IndexerAutoscalerLambda::LambdaFunction.new diff --git a/elasticgraph-indexer_autoscaler_lambda/sig/elastic_graph/idexer_autoscaler_lambda/concurrency_scaler.rbs b/elasticgraph-indexer_autoscaler_lambda/sig/elastic_graph/idexer_autoscaler_lambda/concurrency_scaler.rbs new file mode 100644 index 00000000..87af4f97 --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/sig/elastic_graph/idexer_autoscaler_lambda/concurrency_scaler.rbs @@ -0,0 +1,37 @@ +module ElasticGraph + class IndexerAutoscalerLambda + class ConcurrencyScaler + def initialize: ( + datastore_core: DatastoreCore, + sqs_client: Aws::SQS::Client, + lambda_client: Aws::Lambda::Client + ) -> void + + MAXIMUM_CONCURRENCY: ::Integer + MINIMUM_CONCURRENCY: ::Integer + + def tune_indexer_concurrency: ( + queue_urls: ::Array[::String], + min_cpu_target: ::Integer, + max_cpu_target: ::Integer, + event_source_mapping_uuids: ::Array[::String] + ) -> void + + private + + @logger: ::Logger + @datastore_core: DatastoreCore + @sqs_client: Aws::SQS::Client + @lambda_client: Aws::Lambda::Client + + def get_max_cpu_utilization: () -> ::Float + def get_queue_attributes: (::Array[::String]) -> { total_messages: ::Integer, queue_arns: ::Array[::String] } + def get_total_concurrency: (::Array[::String]) -> ::Integer? + + def update_event_source_mapping: ( + concurrency: ::Integer, + event_source_mapping_uuids: ::Array[::String] + ) -> void + end + end +end diff --git a/elasticgraph-indexer_autoscaler_lambda/sig/elastic_graph/idexer_autoscaler_lambda/details_logger.rbs b/elasticgraph-indexer_autoscaler_lambda/sig/elastic_graph/idexer_autoscaler_lambda/details_logger.rbs new file mode 100644 index 00000000..b14e081a --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/sig/elastic_graph/idexer_autoscaler_lambda/details_logger.rbs @@ -0,0 +1,42 @@ +module ElasticGraph + class IndexerAutoscalerLambda + class DetailsLogger + def initialize: ( + logger: ::Logger, + queue_arns: ::Array[::String], + queue_urls: ::Array[::String], + min_cpu_target: ::Integer, + max_cpu_target: ::Integer, + num_messages: ::Integer, + ) -> void + + def log_increase: ( + cpu_utilization: ::Float, + current_concurrency: ::Integer, + new_concurrency: ::Integer + ) -> void + + def log_decrease: ( + cpu_utilization: ::Float, + current_concurrency: ::Integer, + new_concurrency: ::Integer + ) -> void + + def log_no_change: ( + cpu_utilization: ::Float, + current_concurrency: ::Integer + ) -> void + + def log_reset: () -> void + + def log_unset: () -> void + + def log_result: (::Hash[::String, untyped]) -> void + + private + + @logger: ::Logger + @log_data: ::Hash[::String, untyped] + end + end +end diff --git a/elasticgraph-indexer_autoscaler_lambda/sig/elastic_graph/idexer_autoscaler_lambda/lambda_function.rbs b/elasticgraph-indexer_autoscaler_lambda/sig/elastic_graph/idexer_autoscaler_lambda/lambda_function.rbs new file mode 100644 index 00000000..ec1ab125 --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/sig/elastic_graph/idexer_autoscaler_lambda/lambda_function.rbs @@ -0,0 +1,11 @@ +module ElasticGraph + class IndexerAutoscalerLambda + class LambdaFunction + include LambdaSupport::LambdaFunction[void] + include LambdaSupport::_LambdaFunctionClass[void] + @concurrency_scaler: ConcurrencyScaler + end + end +end + +AutoscaleIndexer: ElasticGraph::IndexerAutoscalerLambda::LambdaFunction diff --git a/elasticgraph-indexer_autoscaler_lambda/sig/elastic_graph/indexer_autoscaler_lambda.rbs b/elasticgraph-indexer_autoscaler_lambda/sig/elastic_graph/indexer_autoscaler_lambda.rbs new file mode 100644 index 00000000..0a5260ae --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/sig/elastic_graph/indexer_autoscaler_lambda.rbs @@ -0,0 +1,25 @@ +module ElasticGraph + class IndexerAutoscalerLambda + attr_reader datastore_core: DatastoreCore + + extend _BuildableFromParsedYaml[IndexerAutoscalerLambda] + extend Support::FromYamlFile[IndexerAutoscalerLambda] + + def self.from_env: () -> IndexerAutoscalerLambda + + def initialize: ( + datastore_core: DatastoreCore, + ?sqs_client: Aws::SQS::Client?, + ?lambda_client: Aws::Lambda::Client?, + ) -> void + + @sqs_client: Aws::SQS::Client? + def sqs_client: () -> Aws::SQS::Client + + @lambda_client: Aws::Lambda::Client? + def lambda_client: () -> Aws::Lambda::Client + + @concurrency_scaler: ConcurrencyScaler? + def concurrency_scaler: () -> ConcurrencyScaler + end +end diff --git a/elasticgraph-indexer_autoscaler_lambda/spec/spec_helper.rb b/elasticgraph-indexer_autoscaler_lambda/spec/spec_helper.rb new file mode 100644 index 00000000..9d4a2fa2 --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/spec/spec_helper.rb @@ -0,0 +1,11 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-indexer_autoscaler_lambda`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. diff --git a/elasticgraph-indexer_autoscaler_lambda/spec/support/builds_indexer_autoscaler.rb b/elasticgraph-indexer_autoscaler_lambda/spec/support/builds_indexer_autoscaler.rb new file mode 100644 index 00000000..11cf7e90 --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/spec/support/builds_indexer_autoscaler.rb @@ -0,0 +1,35 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer_autoscaler_lambda" +require "elastic_graph/spec_support/builds_datastore_core" + +module ElasticGraph + module BuildsIndexerAutoscalerLambda + include BuildsDatastoreCore + + def build_indexer_autoscaler( + sqs_client: nil, + lambda_client: nil, + **datastore_core_options, + &customize_datastore_config + ) + datastore_core = build_datastore_core( + for_context: :autoscaling, + **datastore_core_options, + &customize_datastore_config + ) + + IndexerAutoscalerLambda.new( + sqs_client: sqs_client, + lambda_client: lambda_client, + datastore_core: datastore_core + ) + end + end +end diff --git a/elasticgraph-indexer_autoscaler_lambda/spec/unit/elastic_graph/indexer_autoscaler_lambda/concurrency_scaler_spec.rb b/elasticgraph-indexer_autoscaler_lambda/spec/unit/elastic_graph/indexer_autoscaler_lambda/concurrency_scaler_spec.rb new file mode 100644 index 00000000..1fab6150 --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/spec/unit/elastic_graph/indexer_autoscaler_lambda/concurrency_scaler_spec.rb @@ -0,0 +1,277 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "aws-sdk-lambda" +require "aws-sdk-sqs" +require "elastic_graph/indexer_autoscaler_lambda/concurrency_scaler" +require "support/builds_indexer_autoscaler" + +module ElasticGraph + class IndexerAutoscalerLambda + RSpec.describe ConcurrencyScaler, :capture_logs do + include BuildsIndexerAutoscalerLambda + + describe "#tune_indexer_concurrency" do + let(:event_source_mapping_uuid) { "uuid123" } + let(:min_cpu_target) { 70 } + let(:max_cpu_target) { 80 } + let(:cpu_midpoint) { 75 } + + it "1.5x the concurrency when the CPU usage is significantly below the minimum target" do + lambda_client = lambda_client_with_concurrency(200) + concurrency_scaler = build_concurrency_scaler( + datastore_client: datastore_client_with_cpu_usage(10.0), + sqs_client: sqs_client_with_number_of_messages(1), + lambda_client: lambda_client + ) + + tune_indexer_concurrency(concurrency_scaler) + + expect(updated_concurrencies_requested_from(lambda_client)).to eq [300] # 200 * 1.5 + end + + it "increases concurrency by a factor CPU usage when CPU is slightly below the minimum target" do + # CPU is at 50% and our target range is 70-80. 75 / 50 = 1.5, so increase it by 50%. + lambda_client = lambda_client_with_concurrency(200) + concurrency_scaler = build_concurrency_scaler( + datastore_client: datastore_client_with_cpu_usage(50.0), + sqs_client: sqs_client_with_number_of_messages(1), + lambda_client: lambda_client + ) + + tune_indexer_concurrency(concurrency_scaler) + + expect(updated_concurrencies_requested_from(lambda_client)).to eq [300] # 200 + 50% + end + + it "sets concurrency to the max when it cannot be increased anymore when CPU usage is under the limit" do + current_concurrency = ConcurrencyScaler::MAXIMUM_CONCURRENCY - 1 + lambda_client = lambda_client_with_concurrency(current_concurrency) + concurrency_scaler = build_concurrency_scaler( + datastore_client: datastore_client_with_cpu_usage(10), + sqs_client: sqs_client_with_number_of_messages(1), + lambda_client: lambda_client + ) + + tune_indexer_concurrency(concurrency_scaler) + + expect(updated_concurrencies_requested_from(lambda_client)).to eq [ConcurrencyScaler::MAXIMUM_CONCURRENCY] + end + + it "decreases concurrency by a factor of the CPU when the CPU usage is over the limit" do + # CPU is at 90% and our target range is 70-80. 90 / 75 = 1.2, so decrease it by 20%. + lambda_client = lambda_client_with_concurrency(500) + concurrency_scaler = build_concurrency_scaler( + datastore_client: datastore_client_with_cpu_usage(90.0), + sqs_client: sqs_client_with_number_of_messages(1), + lambda_client: lambda_client + ) + + tune_indexer_concurrency(concurrency_scaler) + + expect(updated_concurrencies_requested_from(lambda_client)).to eq [400] # 500 - 20% + end + + it "sets concurrency to the min when it cannot be decreased anymore when CPU utilization is over the limit" do + current_concurrency = ConcurrencyScaler::MINIMUM_CONCURRENCY + 1 + lambda_client = lambda_client_with_concurrency(current_concurrency) + concurrency_scaler = build_concurrency_scaler( + datastore_client: datastore_client_with_cpu_usage(100), + sqs_client: sqs_client_with_number_of_messages(1), + lambda_client: lambda_client + ) + + tune_indexer_concurrency(concurrency_scaler) + + expect(updated_concurrencies_requested_from(lambda_client)).to eq [ConcurrencyScaler::MINIMUM_CONCURRENCY] + end + + it "does not adjust concurrency when the CPU is within the target range" do + lambda_client = lambda_client_with_concurrency(500) + [min_cpu_target, cpu_midpoint, max_cpu_target].each do |cpu_usage| + concurrency_scaler = build_concurrency_scaler( + datastore_client: datastore_client_with_cpu_usage(cpu_usage), + sqs_client: sqs_client_with_number_of_messages(1), + lambda_client: lambda_client + ) + + tune_indexer_concurrency(concurrency_scaler) + end + + expect(updated_concurrencies_requested_from(lambda_client)).to eq [] + end + + it "decreases the concurrency when at least one of the node's CPU is over the limit" do + current_concurrency = 500 + high_cpu_usage = 81 + expect(high_cpu_usage).to be > max_cpu_target + + lambda_client = lambda_client_with_concurrency(current_concurrency) + concurrency_scaler = build_concurrency_scaler( + datastore_client: datastore_client_with_cpu_usage(min_cpu_target, high_cpu_usage), + sqs_client: sqs_client_with_number_of_messages(1), + lambda_client: lambda_client + ) + + tune_indexer_concurrency(concurrency_scaler) + + expect(high_cpu_usage).to be > max_cpu_target + expect(updated_concurrencies_requested_from(lambda_client)).to eq [460] # 500 - 8% since 81/75 = 1.08 + end + + it "sets concurrency to the min when there are no messages in the queue" do + current_concurrency = 500 + lambda_client = lambda_client_with_concurrency(current_concurrency) + concurrency_scaler = build_concurrency_scaler( + datastore_client: datastore_client_with_cpu_usage(min_cpu_target - 1), + sqs_client: sqs_client_with_number_of_messages(0), + lambda_client: lambda_client + ) + + tune_indexer_concurrency(concurrency_scaler) + + expect(updated_concurrencies_requested_from(lambda_client)).to eq [ConcurrencyScaler::MINIMUM_CONCURRENCY] + end + + it "leaves concurrency unset if it is currently unset" do + lambda_client = lambda_client_without_concurrency + + # CPU is at 50% and our target range is 70-80. + concurrency_scaler = build_concurrency_scaler( + datastore_client: datastore_client_with_cpu_usage(50), + sqs_client: sqs_client_with_number_of_messages(1), + lambda_client: lambda_client + ) + + tune_indexer_concurrency(concurrency_scaler) + + expect(updated_concurrencies_requested_from(lambda_client)).to eq [] + end + + it "supports setting the concurrency on multiple sqs queues" do + current_concurrency = 500 + cpu_usage = 60.0 + lambda_client = lambda_client_with_concurrency(current_concurrency) + + queue_urls = [ + "https://sqs.us-west-2.amazonaws.com/000000000/some-eg-app-queue-name1", + "https://sqs.us-west-2.amazonaws.com/000000000/some-eg-app-queue-name2" + ] + + event_source_mapping_uuids = [ + "event_source_mapping_uuid1", + "event_source_mapping_uuid2" + ] + + concurrency_scaler = build_concurrency_scaler( + datastore_client: datastore_client_with_cpu_usage(cpu_usage), + sqs_client: sqs_client_with_number_of_messages(1), + lambda_client: lambda_client + ) + + tune_indexer_concurrency( + concurrency_scaler, + queue_urls: queue_urls, + event_source_mapping_uuids: event_source_mapping_uuids + ) + + # Each event source mapping started with a concurrency of 500 (for a total of 1000). + # Adding 25% (since the midpoint of our target range is 25% higher than our usage of 60) + # gives us 1250 total concurrency. Dividing evenly across queues gives us 625 each. + expect(updated_concurrencies_requested_from( + lambda_client, + event_source_mapping_uuids: event_source_mapping_uuids + )).to eq [625, 625] + end + end + + def updated_concurrencies_requested_from(lambda_client, event_source_mapping_uuids: [event_source_mapping_uuid]) + lambda_client.api_requests.filter_map do |req| + if req.fetch(:operation_name) == :update_event_source_mapping + expect(event_source_mapping_uuids).to include(req.dig(:params, :uuid)) + req.dig(:params, :scaling_config, :maximum_concurrency) + end + end + end + + def datastore_client_with_cpu_usage(percent, percent2 = percent) + stubbed_datastore_client(get_node_os_stats: { + "nodes" => { + "node1" => { + "os" => { + "cpu" => { + "percent" => percent + } + } + }, + "node2" => { + "os" => { + "cpu" => { + "percent" => percent2 + } + } + } + } + }) + end + + def sqs_client_with_number_of_messages(num_messages) + ::Aws::SQS::Client.new(stub_responses: true).tap do |sqs_client| + sqs_client.stub_responses(:get_queue_attributes, { + attributes: { + "ApproximateNumberOfMessages" => num_messages.to_s, + "QueueArn" => "arn:aws:sqs:us-west-2:000000000:some-eg-app-queue-name" + } + }) + end + end + + def lambda_client_with_concurrency(concurrency) + ::Aws::Lambda::Client.new(stub_responses: true).tap do |lambda_client| + lambda_client.stub_responses(:get_event_source_mapping, { + uuid: event_source_mapping_uuid, + scaling_config: { + maximum_concurrency: concurrency + } + }) + end + end + + # If the concurrency on the event source mapping is not set, the scaling_config on the Lambda client will be nil. + def lambda_client_without_concurrency + ::Aws::Lambda::Client.new(stub_responses: true).tap do |lambda_client| + lambda_client.stub_responses(:get_event_source_mapping, { + uuid: event_source_mapping_uuid, + scaling_config: nil + }) + end + end + + def build_concurrency_scaler(datastore_client:, sqs_client:, lambda_client:) + build_indexer_autoscaler( + clients_by_name: {"main" => datastore_client}, + sqs_client: sqs_client, + lambda_client: lambda_client + ).concurrency_scaler + end + + def tune_indexer_concurrency( + concurrency_scaler, + queue_urls: ["https://sqs.us-west-2.amazonaws.com/000000000/some-eg-app-queue-name"], + event_source_mapping_uuids: [event_source_mapping_uuid] + ) + concurrency_scaler.tune_indexer_concurrency( + queue_urls: queue_urls, + min_cpu_target: min_cpu_target, + max_cpu_target: max_cpu_target, + event_source_mapping_uuids: event_source_mapping_uuids + ) + end + end + end +end diff --git a/elasticgraph-indexer_autoscaler_lambda/spec/unit/elastic_graph/indexer_autoscaler_lambda/lambda_function_spec.rb b/elasticgraph-indexer_autoscaler_lambda/spec/unit/elastic_graph/indexer_autoscaler_lambda/lambda_function_spec.rb new file mode 100644 index 00000000..c32bb5cc --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/spec/unit/elastic_graph/indexer_autoscaler_lambda/lambda_function_spec.rb @@ -0,0 +1,44 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "aws-sdk-lambda" +require "aws-sdk-sqs" +require "elastic_graph/spec_support/lambda_function" + +RSpec.describe "Autoscale indexer lambda function" do + include_context "lambda function" + + it "autscales the concurrency of the lambda" do + sqs_client = ::Aws::SQS::Client.new(stub_responses: true).tap do |sqs_client| + sqs_client.stub_responses(:get_queue_attributes, { + attributes: { + "ApproximateNumberOfMessages" => "0", + "QueueArn" => "arn:aws:sqs:us-west-2:000000000:some-eg-app-queue-name" + } + }) + end + + lambda_client = ::Aws::Lambda::Client.new(stub_responses: true) + + allow(::Aws::SQS::Client).to receive(:new).and_return(sqs_client) + allow(::Aws::Lambda::Client).to receive(:new).and_return(lambda_client) + + expect_loading_lambda_to_define_constant( + lambda: "elastic_graph/indexer_autoscaler_lambda/lambda_function.rb", + const: :AutoscaleIndexer + ) do |lambda_function| + event = { + "queue_urls" => ["https://sqs.us-west-2.amazonaws.com/000000000/some-eg-app-queue-name"], + "min_cpu_target" => 70, + "max_cpu_target" => 80, + "event_source_mapping_uuids" => ["12345678-1234-1234-1234-123456789012"] + } + lambda_function.handle_request(event: event, context: {}) + end + end +end diff --git a/elasticgraph-indexer_autoscaler_lambda/spec/unit/elastic_graph/indexer_autoscaler_lambda_spec.rb b/elasticgraph-indexer_autoscaler_lambda/spec/unit/elastic_graph/indexer_autoscaler_lambda_spec.rb new file mode 100644 index 00000000..324f655b --- /dev/null +++ b/elasticgraph-indexer_autoscaler_lambda/spec/unit/elastic_graph/indexer_autoscaler_lambda_spec.rb @@ -0,0 +1,32 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer_autoscaler_lambda" +require "elastic_graph/spec_support/lambda_function" +require "support/builds_indexer_autoscaler" + +module ElasticGraph + RSpec.describe IndexerAutoscalerLambda do + include BuildsIndexerAutoscalerLambda + include_context "lambda function" + + it "returns non-nil values from each attribute" do + expect_to_return_non_nil_values_from_all_attributes(build_indexer_autoscaler) + end + + describe ".from_parsed_yaml" do + it "builds an IndexerAutoscaler instance from the contents of a YAML settings file" do + customization_block = ->(conn) {} + indexer_autoscaler = IndexerAutoscalerLambda.from_parsed_yaml(parsed_test_settings_yaml, &customization_block) + + expect(indexer_autoscaler).to be_a(IndexerAutoscalerLambda) + expect(indexer_autoscaler.datastore_core.client_customization_block).to be(customization_block) + end + end + end +end diff --git a/elasticgraph-indexer_lambda/.rspec b/elasticgraph-indexer_lambda/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-indexer_lambda/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-indexer_lambda/.yardopts b/elasticgraph-indexer_lambda/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-indexer_lambda/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-indexer_lambda/Gemfile b/elasticgraph-indexer_lambda/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-indexer_lambda/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-indexer_lambda/LICENSE.txt b/elasticgraph-indexer_lambda/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-indexer_lambda/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-indexer_lambda/README.md b/elasticgraph-indexer_lambda/README.md new file mode 100644 index 00000000..80ae84e7 --- /dev/null +++ b/elasticgraph-indexer_lambda/README.md @@ -0,0 +1,31 @@ +# ElasticGraph::IndexerLambda + +Adapts elasticgraph-indexer to run in an AWS Lambda. + +## SQS Message Payload Format + +We use [JSON Lines](http://jsonlines.org/) to encode our indexing events. It is just individual JSON objects +delimited by a newline control character(not the `\n` string sequence), such as: + +```jsonl +{"op": "upsert", "__typename": "Payment", "id": "123", "version": "1", "record": {...} } +{"op": "upsert", "__typename": "Payment", "id": "123", "version": "2", record: {...} } +{"op": "delete", "__typename": "Payment", "id": "123", "version": "3"} +``` + +However, due to SQS message size limit, we have to batch our events carefully so each batch is below the size limit. +This makes payload encoding a bit more complicated on the publisher side because each message has a size limit. +The following code snippet respects the max message size limit and sends JSON Lines payloads with proper size: + +```ruby +def partition_into_acceptably_sized_chunks(batch, max_size_per_chunk) + chunk_size = 0 + batch + .map { |item| JSON.generate(item) } + .slice_before do |json| + chunk_size += (json.bytesize + 1) + (chunk_size > max_size_per_chunk).tap { |chunk_done| chunk_size = 0 if chunk_done } + end + .map { |chunk| chunk.join("\n") } +end +``` diff --git a/elasticgraph-indexer_lambda/elasticgraph-indexer_lambda.gemspec b/elasticgraph-indexer_lambda/elasticgraph-indexer_lambda.gemspec new file mode 100644 index 00000000..2f332272 --- /dev/null +++ b/elasticgraph-indexer_lambda/elasticgraph-indexer_lambda.gemspec @@ -0,0 +1,24 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :lambda) do |spec, eg_version| + spec.summary = "Provides an AWS Lambda interface for an elasticgraph API" + + spec.add_dependency "elasticgraph-indexer", eg_version + spec.add_dependency "elasticgraph-lambda_support", eg_version + spec.add_dependency "aws-sdk-s3", "~> 1.146" + + # aws-sdk-s3 requires an XML library be available. On Ruby < 3 it'll use rexml from the standard library but on Ruby 3.0+ + # we have to add an explicit dependency. It supports ox, oga, libxml, nokogiri or rexml, and of those, ox seems to be the + # best choice: it leads benchmarks, is well-maintained, has no dependencies, and is MIT-licensed. + spec.add_dependency "ox", "~> 2.14" + + spec.add_development_dependency "httpx", "~> 1.3" +end diff --git a/elasticgraph-indexer_lambda/lib/elastic_graph/indexer_lambda.rb b/elasticgraph-indexer_lambda/lib/elastic_graph/indexer_lambda.rb new file mode 100644 index 00000000..74d968ca --- /dev/null +++ b/elasticgraph-indexer_lambda/lib/elastic_graph/indexer_lambda.rb @@ -0,0 +1,20 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer" +require "elastic_graph/lambda_support" + +module ElasticGraph + # @private + module IndexerLambda + # Builds an `ElasticGraph::Indexer` instance from our lambda ENV vars. + def self.indexer_from_env + LambdaSupport.build_from_env(Indexer) + end + end +end diff --git a/elasticgraph-indexer_lambda/lib/elastic_graph/indexer_lambda/lambda_function.rb b/elasticgraph-indexer_lambda/lib/elastic_graph/indexer_lambda/lambda_function.rb new file mode 100644 index 00000000..f7c9187c --- /dev/null +++ b/elasticgraph-indexer_lambda/lib/elastic_graph/indexer_lambda/lambda_function.rb @@ -0,0 +1,37 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/lambda_support/lambda_function" + +module ElasticGraph + module IndexerLambda + # @private + class LambdaFunction + prepend LambdaSupport::LambdaFunction + + def initialize + require "elastic_graph/indexer_lambda" + require "elastic_graph/indexer_lambda/sqs_processor" + + indexer = ElasticGraph::IndexerLambda.indexer_from_env + @sqs_processor = ElasticGraph::IndexerLambda::SqsProcessor.new( + indexer.processor, + logger: indexer.logger, + report_batch_item_failures: ENV["REPORT_BATCH_ITEM_FAILURES"] == "true" + ) + end + + def handle_request(event:, context:) + @sqs_processor.process(event) + end + end + end +end + +# Lambda handler for `elasticgraph-indexer_lambda`. +ProcessEventStreamEvent = ElasticGraph::IndexerLambda::LambdaFunction.new diff --git a/elasticgraph-indexer_lambda/lib/elastic_graph/indexer_lambda/sqs_processor.rb b/elasticgraph-indexer_lambda/lib/elastic_graph/indexer_lambda/sqs_processor.rb new file mode 100644 index 00000000..4c0f4118 --- /dev/null +++ b/elasticgraph-indexer_lambda/lib/elastic_graph/indexer_lambda/sqs_processor.rb @@ -0,0 +1,152 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/indexer/indexing_failures_error" +require "json" + +module ElasticGraph + module IndexerLambda + # Responsible for handling lambda event payloads from an SQS lambda trigger. + # + # @private + class SqsProcessor + def initialize(indexer_processor, report_batch_item_failures:, logger:, s3_client: nil) + @indexer_processor = indexer_processor + @report_batch_item_failures = report_batch_item_failures + @logger = logger + @s3_client = s3_client + end + + # Processes the ElasticGraph events in the given `lambda_event`, indexing the data in the datastore. + def process(lambda_event, refresh_indices: false) + events = events_from(lambda_event) + failures = @indexer_processor.process_returning_failures(events, refresh_indices: refresh_indices) + + if failures.any? + failures_error = Indexer::IndexingFailuresError.for(failures: failures, events: events) + raise failures_error unless @report_batch_item_failures + @logger.error(failures_error.message) + end + + format_response(failures) + end + + private + + # Given a lambda event payload, returns an array of raw operations in JSON format. + # + # The SQS payload is wrapped in the following format already: + # See https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#example-standard-queue-message-event for more details + # { + # Records: { + # [ + # { body: }, + # { body: }, + # ... + # ] + # } + # } + # + # Each entry in "Records" is a SQS entry. Since lambda handles some batching + # for you (with some limits), you can get multiple. + # + # We also want to do our own batching in order to cram more into a given payload + # and issue fewer SQS entries and Lambda invocations when possible. As such, we + # encoded multiple JSON with JSON Lines (http://jsonlines.org/) in record body. + # Each JSON Lines object under 'body' should be of the form: + # + # {op: 'upsert', __typename: 'Payment', id: "123", version: "1", record: {...} } \n + # {op: 'upsert', __typename: 'Payment', id: "123", version: "2", record: {...} } \n + # ... + # Note: "\n" at the end of each line is a single byte newline control character, instead of a string sequence + def events_from(lambda_event) + sqs_received_at_by_message_id = {} # : Hash[String, String] + lambda_event.fetch("Records").flat_map do |record| + sqs_metadata = extract_sqs_metadata(record) + if (message_id = sqs_metadata.fetch("message_id", nil)) + sqs_received_at_by_message_id[message_id] = sqs_metadata.dig("latency_timestamps", "sqs_received_at") + end + parse_jsonl(record.fetch("body")).map do |event| + ElasticGraph::Support::HashUtil.deep_merge(event, sqs_metadata) + end + end.tap do + @logger.info({ + "message_type" => "ReceivedSqsMessages", + "sqs_received_at_by_message_id" => sqs_received_at_by_message_id + }) + end + end + + S3_OFFLOADING_INDICATOR = '["software.amazon.payloadoffloading.PayloadS3Pointer"' + + def parse_jsonl(jsonl_string) + if jsonl_string.start_with?(S3_OFFLOADING_INDICATOR) + jsonl_string = get_payload_from_s3(jsonl_string) + end + jsonl_string.split("\n").map { |event| JSON.parse(event) } + end + + def extract_sqs_metadata(record) + sqs_timestamps = { + "processing_first_attempted_at" => millis_to_iso8601(record.dig("attributes", "ApproximateFirstReceiveTimestamp")), + "sqs_received_at" => millis_to_iso8601(record.dig("attributes", "SentTimestamp")) + }.compact + + { + "latency_timestamps" => (sqs_timestamps unless sqs_timestamps.empty?), + "message_id" => record["messageId"] + }.compact + end + + def millis_to_iso8601(millis) + return unless millis + seconds, millis = millis.to_i.divmod(1000) + Time.at(seconds, millis, :millisecond).getutc.iso8601(3) + end + + def get_payload_from_s3(json_string) + s3_pointer = JSON.parse(json_string)[1] + bucket_name = s3_pointer.fetch("s3BucketName") + object_key = s3_pointer.fetch("s3Key") + + begin + s3_client.get_object(bucket: bucket_name, key: object_key).body.read + rescue Aws::S3::Errors::ServiceError => e + raise Errors::S3OperationFailedError, "Error reading large message from S3. bucket: `#{bucket_name}` key: `#{object_key}` message: `#{e.message}`" + end + end + + # The s3 client is being lazily initialized, as it's slow to import/init and it will only be used + # in rare scenarios where large messages need offloaded from SQS -> S3. + # See: (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-s3-messages.html) + def s3_client + @s3_client ||= begin + require "aws-sdk-s3" + Aws::S3::Client.new + end + end + + # Formats the response, including any failures, based on + # https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting + def format_response(failures) + failure_ids = failures.map do |failure| # $ {"itemIdentifier" => String} + {"itemIdentifier" => failure.event["message_id"]} + end + + if failure_ids.any? { |f| f.fetch("itemIdentifier").nil? } + # If we are not able to identify one or more failed events, then we must raise an exception instead of + # returning `batchItemFailures`. Otherwise, the unidentified failed events will not get retried. + raise Errors::MessageIdsMissingError, "Unexpected: some failures did not have a `message_id`, so we are raising an exception instead of returning `batchItemFailures`." + end + + {"batchItemFailures" => failure_ids} + end + end + end +end diff --git a/elasticgraph-indexer_lambda/sig/elastic_graph/indexer_lambda.rbs b/elasticgraph-indexer_lambda/sig/elastic_graph/indexer_lambda.rbs new file mode 100644 index 00000000..1b6899c2 --- /dev/null +++ b/elasticgraph-indexer_lambda/sig/elastic_graph/indexer_lambda.rbs @@ -0,0 +1,5 @@ +module ElasticGraph + module IndexerLambda + def self.indexer_from_env: () -> Indexer + end +end diff --git a/elasticgraph-indexer_lambda/sig/elastic_graph/indexer_lambda/lambda_function.rbs b/elasticgraph-indexer_lambda/sig/elastic_graph/indexer_lambda/lambda_function.rbs new file mode 100644 index 00000000..1854042a --- /dev/null +++ b/elasticgraph-indexer_lambda/sig/elastic_graph/indexer_lambda/lambda_function.rbs @@ -0,0 +1,11 @@ +module ElasticGraph + module IndexerLambda + class LambdaFunction + include LambdaSupport::LambdaFunction[void] + include LambdaSupport::_LambdaFunctionClass[void] + @sqs_processor: SqsProcessor + end + end +end + +ProcessEventStreamEvent: ElasticGraph::IndexerLambda::LambdaFunction diff --git a/elasticgraph-indexer_lambda/sig/elastic_graph/indexer_lambda/sqs_processor.rbs b/elasticgraph-indexer_lambda/sig/elastic_graph/indexer_lambda/sqs_processor.rbs new file mode 100644 index 00000000..0d3dbaba --- /dev/null +++ b/elasticgraph-indexer_lambda/sig/elastic_graph/indexer_lambda/sqs_processor.rbs @@ -0,0 +1,32 @@ +module ElasticGraph + module IndexerLambda + class SqsProcessor + def initialize: ( + Indexer::Processor, + report_batch_item_failures: bool, + logger: ::Logger, + ?s3_client: Aws::S3::Client? + ) -> void + + def process: (::Hash[::String, untyped], ?refresh_indices: bool) -> void + + private + + @indexer_processor: Indexer::Processor + @report_batch_item_failures: bool + @logger: ::Logger + @s3_client: Aws::S3::Client? + + def events_from: (::Hash[::String, untyped]) -> ::Array[::Hash[::String, untyped]] + S3_OFFLOADING_INDICATOR: String + def extract_sqs_metadata: (::Hash[String, untyped]) -> ::Hash[::String, untyped] + def millis_to_iso8601: (::String) -> ::String? + def parse_jsonl: (::String) -> ::Array[::Hash[::String, untyped]] + def get_payload_from_s3: (::String) -> ::String + def s3_client: () -> Aws::S3::Client + def format_response: ( + ::Array[Indexer::FailedEventError] + ) -> {"batchItemFailures" => ::Array[{"itemIdentifier" => ::String}]} + end + end +end diff --git a/elasticgraph-indexer_lambda/spec/spec_helper.rb b/elasticgraph-indexer_lambda/spec/spec_helper.rb new file mode 100644 index 00000000..65744425 --- /dev/null +++ b/elasticgraph-indexer_lambda/spec/spec_helper.rb @@ -0,0 +1,35 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-indexer_lambda`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. + +RSpec.configure do |config| + # The aws-sdk-s3 gem requires that an XML library be available on the load path. Before we upgraded to + # Ruby 3, that requirement was satisfied by rexml from the standard library. In Ruby 3, it's been moved + # out of the standard library (into a bundled gem), which means that it isn't automatically available. + # However, `rexml` is a transitive dependency of some of the other gems of our bundle (rubocop, webmock) + # and it therefore winds up on our load path. Its presence on the load path allowed our tests to pass + # but failures to occur in production (where rexml was no longer available after we upgraded to Ruby 3.x). + # + # To reproduce that production issue, we want to remove rexml from the load path here. However, we can only + # do so if this spec run is limited to spec files from this gem's spec suite. If any spec files are being + # run from other gems (e.g. for the entire ElasticGraph test suite) we must leave it on the load path so + # that it's available for the other things that need it. + # + # Our CI build runs each gem's test suite individually and also runs the entire test suite all together so + # this should still guard against a regression even though the "standard" way we run our tests (e.g. the + # entire test suite) won't benefit from this. + # + # :nocov: -- on any given test run only one side of this conditional is covered. + if config.files_to_run.all? { |f| f.start_with?(__dir__) } + $LOAD_PATH.delete_if { |path| path.include?("rexml") } + end + # :nocov: +end diff --git a/elasticgraph-indexer_lambda/spec/unit/elastic_graph/indexer_lambda/lambda_function_spec.rb b/elasticgraph-indexer_lambda/spec/unit/elastic_graph/indexer_lambda/lambda_function_spec.rb new file mode 100644 index 00000000..a4ec1de3 --- /dev/null +++ b/elasticgraph-indexer_lambda/spec/unit/elastic_graph/indexer_lambda/lambda_function_spec.rb @@ -0,0 +1,23 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/lambda_function" + +RSpec.describe "Indexer lambda function" do + include_context "lambda function" + + it "processes SQS message payloads" do + expect_loading_lambda_to_define_constant( + lambda: "elastic_graph/indexer_lambda/lambda_function.rb", + const: :ProcessEventStreamEvent + ) do |lambda_function| + response = lambda_function.handle_request(event: {"Records" => []}, context: {}) + expect(response).to eq({"batchItemFailures" => []}) + end + end +end diff --git a/elasticgraph-indexer_lambda/spec/unit/elastic_graph/indexer_lambda/sqs_processor_spec.rb b/elasticgraph-indexer_lambda/spec/unit/elastic_graph/indexer_lambda/sqs_processor_spec.rb new file mode 100644 index 00000000..8216c662 --- /dev/null +++ b/elasticgraph-indexer_lambda/spec/unit/elastic_graph/indexer_lambda/sqs_processor_spec.rb @@ -0,0 +1,374 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/indexer/failed_event_error" +require "elastic_graph/indexer/processor" +require "elastic_graph/indexer_lambda/sqs_processor" +require "elastic_graph/spec_support/lambda_function" +require "json" +require "aws-sdk-s3" + +module ElasticGraph + module IndexerLambda + RSpec.describe SqsProcessor, :capture_logs do + let(:indexer_processor) { instance_double(Indexer::Processor, process_returning_failures: []) } + + describe "#process" do + let(:s3_client) { Aws::S3::Client.new(stub_responses: true) } + let(:sqs_processor) { build_sqs_processor(report_batch_item_failures: false) } + + it "processes a lambda event containing a single SQS message with a single ElasticGraph event" do + lambda_event = { + "Records" => [ + {"messageId" => "a", "body" => jsonl({"field1" => {}})} + ] + } + + sqs_processor.process(lambda_event) + + expect(indexer_processor).to have_received(:process_returning_failures).with([ + {"field1" => {}, "message_id" => "a"} + ], refresh_indices: false) + end + + it "processes a lambda event containing multiple SQS messages" do + lambda_event = { + "Records" => [ + {"messageId" => "a", "body" => jsonl({"field1" => {}})}, + {"messageId" => "b", "body" => jsonl({"field2" => {}})}, + {"messageId" => "c", "body" => jsonl({"field3" => {}})} + ] + } + + sqs_processor.process(lambda_event) + + expect(indexer_processor).to have_received(:process_returning_failures).with([ + {"field1" => {}, "message_id" => "a"}, + {"field2" => {}, "message_id" => "b"}, + {"field3" => {}, "message_id" => "c"} + ], refresh_indices: false) + end + + it "processes a lambda event containing multiple ElasticGraph events in the SQS messages" do + lambda_event = { + "Records" => [ + {"messageId" => "a", "body" => jsonl({"field1" => {}}, {"field2" => {}})}, + {"messageId" => "b", "body" => jsonl({"field3" => {}}, {"field4" => {}}, {"field5" => {}})} + ] + } + + sqs_processor.process(lambda_event) + + expect(indexer_processor).to have_received(:process_returning_failures).with([ + {"field1" => {}, "message_id" => "a"}, + {"field2" => {}, "message_id" => "a"}, + {"field3" => {}, "message_id" => "b"}, + {"field4" => {}, "message_id" => "b"}, + {"field5" => {}, "message_id" => "b"} + ], refresh_indices: false) + end + + it "logs the SQS message ids received in the lambda event and the `sqs_received_at` if available" do + sent_timestamp_millis = "796010423456" + sent_timestamp_iso8601 = "1995-03-24T02:00:23.456Z" + + lambda_event = { + "Records" => [ + { + "messageId" => "a", + "body" => jsonl( + {"field1" => {}}, + {"field2" => {}} + ), + "attributes" => { + "SentTimestamp" => sent_timestamp_millis + } + }, + { + "messageId" => "b", + "body" => jsonl( + {"field3" => {}}, + {"field4" => {}}, + {"field5" => {}} + ) + } + ] + } + + expect { + sqs_processor.process(lambda_event) + }.to log a_string_including( + "message_type", "ReceivedSqsMessages", "sqs_received_at_by_message_id", "a", sent_timestamp_iso8601, "b", "null" + ) + end + + it "raises a clear error if the lambda event does not contain SQS messages under `Records` as expected" do + lambda_event = { + "Rows" => [ + {"messageId" => "a", "body" => jsonl({"field1" => {}})}, + {"messageId" => "b", "body" => jsonl({"field2" => {}})}, + {"messageId" => "c", "body" => jsonl({"field1" => {}})} + ] + } + + expect { + sqs_processor.process(lambda_event) + }.to raise_error(KeyError, a_string_including("Records")) + + expect(indexer_processor).not_to have_received(:process_returning_failures) + end + + it "raises a clear error if the SQS messages lack a `body` as expected" do + lambda_event = { + "Records" => [ + {"messageId" => "a", "data" => jsonl({"field1" => {}})}, + {"messageId" => "b", "data" => jsonl({"field2" => {}})}, + {"messageId" => "c", "data" => jsonl({"field1" => {}})} + ] + } + + expect { + sqs_processor.process(lambda_event) + }.to raise_error(KeyError, a_string_including("body")) + + expect(indexer_processor).not_to have_received(:process_returning_failures) + end + + it "retrieves large messages from s3 when an ElasticGraph event was offloaded there" do + bucket_name = "test-bucket-name" + s3_key = "88680f6d-53d4-4143-b8c7-f5b1189213b6" + event_payload = {"test" => "data"} + + lambda_event = { + "Records" => [ + {"messageId" => "a", "body" => JSON.generate([ + "software.amazon.payloadoffloading.PayloadS3Pointer", + {"s3BucketName" => bucket_name, "s3Key" => s3_key} + ])} + ] + } + + s3_client.stub_responses(:get_object, ->(context) { + expect(context.params).to include(bucket: bucket_name, key: s3_key) + {body: jsonl(event_payload)} + }) + + sqs_processor.process(lambda_event) + + expect(indexer_processor).to have_received(:process_returning_failures).with( + [event_payload.merge("message_id" => "a")], + refresh_indices: false + ) + end + + it "throws a detailed error when fetching from s3 fails" do + bucket_name = "test-bucket-name" + s3_key = "88680f6d-53d4-4143-b8c7-f5b1189213b6" + + lambda_event = { + "Records" => [ + {"messageId" => "a", "body" => JSON.generate([ + "software.amazon.payloadoffloading.PayloadS3Pointer", + {"s3BucketName" => bucket_name, "s3Key" => s3_key} + ])} + ] + } + + s3_client.stub_responses(:get_object, "NoSuchkey") + + expect { + sqs_processor.process(lambda_event) + }.to raise_error Errors::S3OperationFailedError, a_string_including( + "Error reading large message from S3. bucket: `#{bucket_name}` key: `#{s3_key}` message: `stubbed-response-error-message`" + ) + end + + it "parses and merges SQS timestamps into non-existing `latency_timestamps` field" do + approximate_first_receive_timestamp_millis = "1696334412345" + sent_timestamp_millis = "796010423456" + + approximate_first_receive_timestamp_iso8601 = "2023-10-03T12:00:12.345Z" + sent_timestamp_iso8601 = "1995-03-24T02:00:23.456Z" + + lambda_event = { + "Records" => [ + { + "messageId" => "a", + "body" => jsonl({"field1" => {}}), + "attributes" => { + "ApproximateFirstReceiveTimestamp" => approximate_first_receive_timestamp_millis, + "SentTimestamp" => sent_timestamp_millis + } + } + ] + } + + sqs_processor.process(lambda_event) + + expect(indexer_processor).to have_received(:process_returning_failures) do |events| + expect(events.first["latency_timestamps"].size).to eq(2) + expect( + events.first["latency_timestamps"]["processing_first_attempted_at"] + ).to eq(approximate_first_receive_timestamp_iso8601) + expect( + events.first["latency_timestamps"]["sqs_received_at"] + ).to eq(sent_timestamp_iso8601) + end + end + + it "parses and merges SQS timestamps into existing `latency_timestamps` field" do + approximate_first_receive_timestamp_millis = "1696334412345" + sent_timestamp_millis = "796010423456" + + approximate_first_receive_timestamp_iso8601 = "2023-10-03T12:00:12.345Z" + sent_timestamp_iso8601 = "1995-03-24T02:00:23.456Z" + + lambda_event = { + "Records" => [ + { + "messageId" => "a", + "body" => jsonl({"latency_timestamps" => {"field1" => "value1"}}), + "attributes" => { + "ApproximateFirstReceiveTimestamp" => approximate_first_receive_timestamp_millis, + "SentTimestamp" => sent_timestamp_millis + } + } + ] + } + + sqs_processor.process(lambda_event) + + expect(indexer_processor).to have_received(:process_returning_failures) do |events| + expect(events.first["latency_timestamps"].size).to eq(3) + expect( + events.first["latency_timestamps"]["processing_first_attempted_at"] + ).to eq(approximate_first_receive_timestamp_iso8601) + expect( + events.first["latency_timestamps"]["sqs_received_at"] + ).to eq(sent_timestamp_iso8601) + expect( + events.first["latency_timestamps"]["field1"] + ).to eq("value1") + end + end + + context "when one or more events fail to process" do + context "when `report_batch_item_failures` is false" do + let(:sqs_processor) { build_sqs_processor(report_batch_item_failures: false) } + + it "raises an `IndexingFailuresError` to notify about the failure and force all SQS messages to be retried" do + allow(indexer_processor).to receive(:process_returning_failures).and_return([ + failure_of("id1", message: "boom1", event: {"id" => "id1", "message_id" => "12"}), + failure_of("id7", message: "boom7", event: {"id" => "id7", "message_id" => "67"}) + ]) + + lambda_event = { + "Records" => [ + {"messageId" => "12", "body" => jsonl({"id" => "id1"}, {"id" => "id2"})}, + {"messageId" => "67", "body" => jsonl({"id" => "id6"}, {"id" => "id7"})} + ] + } + + expect { + sqs_processor.process(lambda_event) + }.to raise_error Indexer::IndexingFailuresError, a_string_including( + "Got 2 failure(s) from 4 event(s)", "boom1", "boom7", + "These failures came from 2 message(s): 12, 67." + ) + end + end + + context "when `report_batch_item_failures` is true" do + let(:sqs_processor) { build_sqs_processor(report_batch_item_failures: true) } + + it "indicates which SQS messages had failures in the lambda response so that only those messages are retried (while still logging the errors)" do + allow(indexer_processor).to receive(:process_returning_failures).and_return([ + failure_of("id1", message: "boom1", event: {"id" => "id1", "message_id" => "12"}), + failure_of("id7", message: "boom7", event: {"id" => "id7", "message_id" => "67"}) + ]) + + lambda_event = { + "Records" => [ + {"messageId" => "12", "body" => jsonl({"id" => "id1"}, {"id" => "id2"})}, + {"messageId" => "34", "body" => jsonl({"id" => "id3"}, {"id" => "id4"})}, + {"messageId" => "5", "body" => jsonl({"id" => "id5"})}, + {"messageId" => "67", "body" => jsonl({"id" => "id6"}, {"id" => "id7"})} + ] + } + + response = nil + + expect { + response = sqs_processor.process(lambda_event) + }.to log a_string_including( + "Got 2 failure(s) from 7 event(s):", "boom1", "boom7", + "These failures came from 2 message(s): 12, 67." + ) + + expect(response).to eq({"batchItemFailures" => [ + {"itemIdentifier" => "12"}, + {"itemIdentifier" => "67"} + ]}) + end + + it "falls back to raising an `IndexingFailuresError` if the SQS id of an event cannot be determined" do + allow(indexer_processor).to receive(:process_returning_failures).and_return([ + failure_of("id1", message: "boom1", event: {"id" => "id1"}), + failure_of("id7", message: "boom7", event: {"id" => "id7", "message_id" => "67"}) + ]) + + lambda_event = { + "Records" => [ + {"body" => jsonl({"id" => "id1"}, {"id" => "id2"})}, + {"messageId" => "34", "body" => jsonl({"id" => "id3"}, {"id" => "id4"})}, + {"messageId" => "5", "body" => jsonl({"id" => "id5"})}, + {"messageId" => "67", "body" => jsonl({"id" => "id6"}, {"id" => "id7"})} + ] + } + + expect { + sqs_processor.process(lambda_event) + }.to log(a_string_including( + "Got 2 failure(s) from 7 event(s)", "boom1", "boom7", + "These failures came from 1 message(s): 67." + )).and raise_error( + Errors::MessageIdsMissingError, + a_string_including("Unexpected: some failures did not have a `message_id`") + ) + end + end + + def failure_of(id, message: "boom", event: {}) + instance_double(Indexer::FailedEventError, id: id, message: message, event: event) + end + end + + def build_sqs_processor(report_batch_item_failures:) + super(report_batch_item_failures: report_batch_item_failures, s3_client: s3_client) + end + end + + context "when instantiated without an S3 client injection" do + include_context "lambda function" + + it "lazily creates the S3 client when needed" do + expect(build_sqs_processor(report_batch_item_failures: false).send(:s3_client)).to be_a Aws::S3::Client + end + end + + def jsonl(*items) + items.map { |i| ::JSON.generate(i) }.join("\n") + end + + def build_sqs_processor(report_batch_item_failures:, **options) + SqsProcessor.new(indexer_processor, report_batch_item_failures: report_batch_item_failures, logger: logger, **options) + end + end + end +end diff --git a/elasticgraph-indexer_lambda/spec/unit/elastic_graph/indexer_lambda_spec.rb b/elasticgraph-indexer_lambda/spec/unit/elastic_graph/indexer_lambda_spec.rb new file mode 100644 index 00000000..8e03d404 --- /dev/null +++ b/elasticgraph-indexer_lambda/spec/unit/elastic_graph/indexer_lambda_spec.rb @@ -0,0 +1,24 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer_lambda" +require "elastic_graph/spec_support/lambda_function" + +module ElasticGraph + RSpec.describe IndexerLambda do + describe ".indexer_from_env" do + include_context "lambda function" + + around { |ex| with_lambda_env_vars(&ex) } + + it "builds an indexer instance" do + expect(IndexerLambda.indexer_from_env).to be_a(Indexer) + end + end + end +end diff --git a/elasticgraph-json_schema/.rspec b/elasticgraph-json_schema/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-json_schema/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-json_schema/.yardopts b/elasticgraph-json_schema/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-json_schema/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-json_schema/Gemfile b/elasticgraph-json_schema/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-json_schema/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-json_schema/LICENSE.txt b/elasticgraph-json_schema/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-json_schema/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-json_schema/README.md b/elasticgraph-json_schema/README.md new file mode 100644 index 00000000..cbc85512 --- /dev/null +++ b/elasticgraph-json_schema/README.md @@ -0,0 +1,3 @@ +# ElasticGraph::JSONSchema + +Provides JSON Schema validation for ElasticGraph. diff --git a/elasticgraph-json_schema/elasticgraph-json_schema.gemspec b/elasticgraph-json_schema/elasticgraph-json_schema.gemspec new file mode 100644 index 00000000..992275db --- /dev/null +++ b/elasticgraph-json_schema/elasticgraph-json_schema.gemspec @@ -0,0 +1,16 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :core) do |spec, eg_version| + spec.summary = "ElasticGraph gem that provides JSON Schema validation." + + spec.add_dependency "elasticgraph-support", eg_version + spec.add_dependency "json_schemer", "~> 2.3" +end diff --git a/elasticgraph-json_schema/lib/elastic_graph/json_schema/json_schema_draft_7_schema.json b/elasticgraph-json_schema/lib/elastic_graph/json_schema/json_schema_draft_7_schema.json new file mode 100644 index 00000000..fb92c7f7 --- /dev/null +++ b/elasticgraph-json_schema/lib/elastic_graph/json_schema/json_schema_draft_7_schema.json @@ -0,0 +1,172 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://json-schema.org/draft-07/schema#", + "title": "Core schema meta-schema", + "definitions": { + "schemaArray": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#" } + }, + "nonNegativeInteger": { + "type": "integer", + "minimum": 0 + }, + "nonNegativeIntegerDefault0": { + "allOf": [ + { "$ref": "#/definitions/nonNegativeInteger" }, + { "default": 0 } + ] + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true, + "default": [] + } + }, + "type": ["object", "boolean"], + "properties": { + "$id": { + "type": "string", + "format": "uri-reference" + }, + "$schema": { + "type": "string", + "format": "uri" + }, + "$ref": { + "type": "string", + "format": "uri-reference" + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "default": true, + "readOnly": { + "type": "boolean", + "default": false + }, + "writeOnly": { + "type": "boolean", + "default": false + }, + "examples": { + "type": "array", + "items": true + }, + "multipleOf": { + "type": "number", + "exclusiveMinimum": 0 + }, + "maximum": { + "type": "number" + }, + "exclusiveMaximum": { + "type": "number" + }, + "minimum": { + "type": "number" + }, + "exclusiveMinimum": { + "type": "number" + }, + "maxLength": { "$ref": "#/definitions/nonNegativeInteger" }, + "minLength": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "pattern": { + "type": "string", + "format": "regex" + }, + "additionalItems": { "$ref": "#" }, + "items": { + "anyOf": [ + { "$ref": "#" }, + { "$ref": "#/definitions/schemaArray" } + ], + "default": true + }, + "maxItems": { "$ref": "#/definitions/nonNegativeInteger" }, + "minItems": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "contains": { "$ref": "#" }, + "maxProperties": { "$ref": "#/definitions/nonNegativeInteger" }, + "minProperties": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "required": { "$ref": "#/definitions/stringArray" }, + "additionalProperties": { "$ref": "#" }, + "definitions": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "properties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "patternProperties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "propertyNames": { "format": "regex" }, + "default": {} + }, + "dependencies": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { "$ref": "#" }, + { "$ref": "#/definitions/stringArray" } + ] + } + }, + "propertyNames": { "$ref": "#" }, + "const": true, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + }, + "type": { + "anyOf": [ + { "$ref": "#/definitions/simpleTypes" }, + { + "type": "array", + "items": { "$ref": "#/definitions/simpleTypes" }, + "minItems": 1, + "uniqueItems": true + } + ] + }, + "format": { "type": "string" }, + "contentMediaType": { "type": "string" }, + "contentEncoding": { "type": "string" }, + "if": { "$ref": "#" }, + "then": { "$ref": "#" }, + "else": { "$ref": "#" }, + "allOf": { "$ref": "#/definitions/schemaArray" }, + "anyOf": { "$ref": "#/definitions/schemaArray" }, + "oneOf": { "$ref": "#/definitions/schemaArray" }, + "not": { "$ref": "#" } + }, + "default": true +} diff --git a/elasticgraph-json_schema/lib/elastic_graph/json_schema/meta_schema_validator.rb b/elasticgraph-json_schema/lib/elastic_graph/json_schema/meta_schema_validator.rb new file mode 100644 index 00000000..387ec76b --- /dev/null +++ b/elasticgraph-json_schema/lib/elastic_graph/json_schema/meta_schema_validator.rb @@ -0,0 +1,69 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/json_schema/validator" +require "elastic_graph/json_schema/validator_factory" +require "elastic_graph/support/hash_util" +require "json" + +module ElasticGraph + # Provides [JSON Schema](https://json-schema.org/) validation for ElasticGraph. + module JSONSchema + # Provides a validator to validate a JSON schema definitions according to the JSON schema meta schema. + # The validator is configured to validate strictly, so that non-standard JSON schema properties are disallowed. + # + # @return [Validator] + # @see .elastic_graph_internal_meta_schema_validator + def self.strict_meta_schema_validator + @strict_meta_schema_validator ||= MetaSchemaLoader.load_strict_validator + end + + # Provides a validator to validate a JSON schema definition according to the JSON schema meta schema. + # The validator is configured to validate strictly, so that non-standard JSON schema properties are disallowed, + # except for internal ElasticGraph metadata properties. + # + # @return [Validator] + # @see .strict_meta_schema_validator + def self.elastic_graph_internal_meta_schema_validator + @elastic_graph_internal_meta_schema_validator ||= MetaSchemaLoader.load_strict_validator({ + "properties" => { + "ElasticGraph" => { + "type" => "object", + "required" => ["type", "nameInIndex"], + "properties" => { + "type" => {"type" => "string"}, + "nameInIndex" => {"type" => "string"} + } + } + } + }) + end + + # Responsible for building {Validator}s that can validate JSON schema definitions. + module MetaSchemaLoader + # Builds a validator to validate a JSON schema definition according to the JSON schema meta schema. + # + # @param overrides [Hash] meta schema overrides + def self.load_strict_validator(overrides = {}) + # Downloaded from: https://json-schema.org/draft-07/schema + schema = ::JSON.parse(::File.read(::File.expand_path("../json_schema_draft_7_schema.json", __FILE__))) + schema = Support::HashUtil.deep_merge(schema, overrides) unless overrides.empty? + + # The meta schema allows additionalProperties in nearly every place. While a JSON schema definition + # with additional properties is considered valid, we do not intend to use any additional properties, + # and any usage of an additional property is almost certainly a typo. So here we set + # `with_unknown_properties_disallowed`. + root_schema = ValidatorFactory.new(schema: schema, sanitize_pii: false) # The meta schema has no PII + .with_unknown_properties_disallowed + .root_schema + + Validator.new(schema: root_schema, sanitize_pii: false) + end + end + end +end diff --git a/elasticgraph-json_schema/lib/elastic_graph/json_schema/validator.rb b/elasticgraph-json_schema/lib/elastic_graph/json_schema/validator.rb new file mode 100644 index 00000000..238924b0 --- /dev/null +++ b/elasticgraph-json_schema/lib/elastic_graph/json_schema/validator.rb @@ -0,0 +1,68 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "json" + +module ElasticGraph + module JSONSchema + # Responsible for validating JSON data against the ElasticGraph JSON schema for a particular type. + # + # @!attribute [r] schema + # @return [Hash] a JSON schema + # @!attribute [r] sanitize_pii + # @return [Boolean] whether to omit data that may contain PII from error messages + class Validator < ::Data.define(:schema, :sanitize_pii) + # Validates the given data against the JSON schema, returning true if the data is valid. + # + # @param data [Object] JSON data to validate + # @return [Boolean] true if the data is valid; false if it is invalid + # + # @see #validate + # @see #validate_with_error_message + def valid?(data) + schema.valid?(data) + end + + # Validates the given data against the JSON schema, returning an array of error objects for + # any validation errors. + # + # @param data [Object] JSON data to validate + # @return [Array>] validation errors; will be empty if `data` is valid + # + # @see #valid? + # @see #validate_with_error_message + def validate(data) + schema.validate(data).map do |error| + # The schemas can be very large and make the output very noisy, hiding what matters. So we remove them here. + error.delete("root_schema") + error.delete("schema") + error + end + end + + # Validates the given data against the JSON schema, returning an error message string if it is invalid. + # The error message is intended to be usable to include in a log message or a raised error. + # + # @param data [Object] JSON data to validate + # @return [nil, String] a validation error message, if the data is invalid + # + # @note The returned error message may contain PII unless {#sanitize_pii} has not been set. + # + # @see #valid? + # @see #validate + def validate_with_error_message(data) + errors = validate(data) + return if errors.empty? + + errors.each { |error| error.delete("data") } if sanitize_pii + + "Validation errors:\n\n#{errors.map { |e| ::JSON.pretty_generate(e) }.join("\n\n")}" + end + end + end +end diff --git a/elasticgraph-json_schema/lib/elastic_graph/json_schema/validator_factory.rb b/elasticgraph-json_schema/lib/elastic_graph/json_schema/validator_factory.rb new file mode 100644 index 00000000..83fe31d3 --- /dev/null +++ b/elasticgraph-json_schema/lib/elastic_graph/json_schema/validator_factory.rb @@ -0,0 +1,113 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/json_schema/validator" +require "json_schemer" + +module ElasticGraph + module JSONSchema + # Factory class responsible for creating {Validator}s for particular ElasticGraph types. + class ValidatorFactory + # @dynamic root_schema + # @private + attr_reader :root_schema + + # @param schema [Hash] the JSON schema for an entire ElasticGraph schema + # @param sanitize_pii [Boolean] whether to omit data that may contain PII from error messages + def initialize(schema:, sanitize_pii:) + @raw_schema = schema + @root_schema = ::JSONSchemer.schema( + schema, + meta_schema: schema.fetch("$schema"), + # Here we opt to have regular expressions resolved using an ecmo-compatible resolver, instead of Ruby's. + # + # We do this because regexp patterns in our JSON schema are intended to be used by JSON schema libraries + # in many languages, not just in Ruby, and we want to support the widest compatibility. For example, + # Ruby requires that we use `\A` and `\z` to anchor the start and end of the string (`^` and `$` anchor the + # start and end of a line instead), where as ecmo regexes treat `^` and `$` as the start and end of the string. + # For a pattern to be usable by non-Ruby publishers, we need to use `^/`$` for our start/end anchors, and we + # want our validator to treat it the same way here. + # + # Also, this was the default before json_schemer 1.0 (and we used 0.x versions for a long time...). + # This maintains the historical behavior we've had. + # + # For more info: + # https://github.com/davishmcclurg/json_schemer/blob/v1.0.0/CHANGELOG.md#breaking-changes + regexp_resolver: "ecma" + ) + + @sanitize_pii = sanitize_pii + @validators_by_type_name = ::Hash.new do |hash, key| + hash[key] = Validator.new( + schema: root_schema.ref("#/$defs/#{key}"), + sanitize_pii: sanitize_pii + ) + end + end + + # Gets the {Validator} for a particular ElasticGraph type. + # + # @param type_name [String] name of an ElasticGraph type + # @return [Validator] + def validator_for(type_name) + @validators_by_type_name[type_name] + end + + # Returns a new factory configured to disallow unknown properties. By default, JSON schema + # allows unknown properties (they'll simply be ignored when validating a JSON document). It + # can be useful to validate more strictly, so that a document with properties not defined in + # the JSON schema gets validation errors. + # + # @param except [Array] paths under which unknown properties should still be allowed + # @return [ValidatorFactory] + def with_unknown_properties_disallowed(except: []) + allow_paths = except.map { |p| p.split(".") } + schema_copy = ::Marshal.load(::Marshal.dump(@raw_schema)) # deep copy so our mutations don't affect caller + prevent_unknown_properties!(schema_copy, allow_paths: allow_paths) + + ValidatorFactory.new(schema: schema_copy, sanitize_pii: @sanitize_pii) + end + + private + + # The meta schema allows additionalProperties in nearly every place. While a JSON schema definition + # with additional properties is considered valid, we do not intend to use any additional properties, + # and any usage of an additional property is almost certainly a typo. So here we mutate the meta + # schema to set `additionalProperties: false` everywhere. + def prevent_unknown_properties!(object, allow_paths:, parent_path: []) + case object + when ::Array + object.each { |value| prevent_unknown_properties!(value, allow_paths: allow_paths, parent_path: parent_path) } + when ::Hash + if object["properties"] + object["additionalProperties"] = false + + allowed_extra_props = allow_paths.filter_map do |path| + *prefix, prop_name = path + prop_name if prefix == parent_path + end + + allowed_extra_props.each_with_object(object["properties"]) do |prop_name, props| + # @type var empty_hash: ::Hash[::String, untyped] + empty_hash = {} + props[prop_name] ||= empty_hash + end + + object["properties"].each do |key, value| + prevent_unknown_properties!(value, allow_paths: allow_paths, parent_path: parent_path + [key]) + end + else + object.each do |key, value| + prevent_unknown_properties!(value, allow_paths: allow_paths, parent_path: parent_path) + end + end + end + end + end + end +end diff --git a/elasticgraph-json_schema/sig/elastic_graph/json_schema/meta_schema_validator.rbs b/elasticgraph-json_schema/sig/elastic_graph/json_schema/meta_schema_validator.rbs new file mode 100644 index 00000000..003c9630 --- /dev/null +++ b/elasticgraph-json_schema/sig/elastic_graph/json_schema/meta_schema_validator.rbs @@ -0,0 +1,13 @@ +module ElasticGraph + module JSONSchema + self.@strict_meta_schema_validator: Validator + def self.strict_meta_schema_validator: () -> Validator + + self.@elastic_graph_internal_meta_schema_validator: Validator + def self.elastic_graph_internal_meta_schema_validator: () -> Validator + + module MetaSchemaLoader + def self.load_strict_validator: (?::Hash[::String, untyped]) -> Validator + end + end +end diff --git a/elasticgraph-json_schema/sig/elastic_graph/json_schema/validator.rbs b/elasticgraph-json_schema/sig/elastic_graph/json_schema/validator.rbs new file mode 100644 index 00000000..adeb6ef7 --- /dev/null +++ b/elasticgraph-json_schema/sig/elastic_graph/json_schema/validator.rbs @@ -0,0 +1,18 @@ +module ElasticGraph + module JSONSchema + type dataHash = ::Hash[::String, untyped] + + class ValidatorValueSuperType + attr_reader schema: ::JSONSchemer::Schema + attr_reader sanitize_pii: bool + def initialize: (schema: ::JSONSchemer::Schema, sanitize_pii: bool) -> void + def with: (?schema: ::JSONSchemer::Schema, ?sanitize_pii: bool) -> self + end + + class Validator < ValidatorValueSuperType + def valid?: (dataHash) -> bool + def validate: (dataHash) -> ::Array[JSONSchemer::error] + def validate_with_error_message: (dataHash) -> ::String? + end + end +end diff --git a/elasticgraph-json_schema/sig/elastic_graph/json_schema/validator_factory.rbs b/elasticgraph-json_schema/sig/elastic_graph/json_schema/validator_factory.rbs new file mode 100644 index 00000000..dd60e7e6 --- /dev/null +++ b/elasticgraph-json_schema/sig/elastic_graph/json_schema/validator_factory.rbs @@ -0,0 +1,21 @@ +module ElasticGraph + module JSONSchema + class ValidatorFactory + attr_reader root_schema: ::JSONSchemer::Schema + @raw_schema: ::Hash[::String, untyped] + @sanitize_pii: bool + @validators_by_type_name: ::Hash[::String, Validator] + def initialize: (schema: ::Hash[::String, untyped], sanitize_pii: bool) -> void + def validator_for: (::String) -> Validator + def with_unknown_properties_disallowed: (?except: ::Array[::String]) -> ValidatorFactory + + private + + def prevent_unknown_properties!: ( + untyped, + allow_paths: ::Array[::Array[::String]], + ?parent_path: ::Array[::String] + ) -> void + end + end +end diff --git a/elasticgraph-json_schema/sig/json_schemer.rbs b/elasticgraph-json_schema/sig/json_schemer.rbs new file mode 100644 index 00000000..d4e708cd --- /dev/null +++ b/elasticgraph-json_schema/sig/json_schemer.rbs @@ -0,0 +1,23 @@ +module JSONSchemer + type error = { + "data" => untyped, + "data_pointer" => ::String, + "schema" => ::Hash[::String, untyped], + "schema_pointer" => ::String, + "root_schema" => ::Hash[::String, untyped], + "type" => ::String, + "details" => untyped? + } + + class Schema + def ref: (::String) -> Schema + def valid?: (::Hash[::String, untyped]) -> bool + def validate: (::Hash[::String, untyped]) -> ::Enumerator[error, error] + end + + def self.schema: ( + ::Hash[::String, untyped], + ?meta_schema: ::String, + ?regexp_resolver: "ruby" | "ecma" + ) -> Schema +end diff --git a/elasticgraph-json_schema/spec/spec_helper.rb b/elasticgraph-json_schema/spec/spec_helper.rb new file mode 100644 index 00000000..a816043e --- /dev/null +++ b/elasticgraph-json_schema/spec/spec_helper.rb @@ -0,0 +1,11 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-json_schema`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. diff --git a/elasticgraph-json_schema/spec/unit/elastic_graph/json_schema/meta_schema_validator_spec.rb b/elasticgraph-json_schema/spec/unit/elastic_graph/json_schema/meta_schema_validator_spec.rb new file mode 100644 index 00000000..31561beb --- /dev/null +++ b/elasticgraph-json_schema/spec/unit/elastic_graph/json_schema/meta_schema_validator_spec.rb @@ -0,0 +1,175 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/json_schema/meta_schema_validator" + +module ElasticGraph + module JSONSchema + RSpec.shared_examples_for "a meta schema validator" do + it "indicates a valid JSON schema is valid" do + schema = { + "type" => "object", + "properties" => { + "is_happy" => { + "anyOf" => [ + {"type" => "string"}, + {"type" => "null"} + ] + } + } + } + + expect(validator.valid?(schema)).to be true + expect(validator.validate(schema).to_a).to be_empty + expect(validator.validate_with_error_message(schema)).to eq(nil) + end + + it "indicates a JSON schema with an invalid value is invalid" do + schema = { + "type" => "object", + "properties" => { + "is_happy" => { + "anyOf" => [ + {"type" => 7}, + {"type" => "null"} + ] + } + } + } + + expect(validator.valid?(schema)).to be false + expect(validator.validate(schema).to_a).to include( + a_hash_including("data" => 7, "data_pointer" => "/properties/is_happy/anyOf/0/type") + ) + expect(validator.validate_with_error_message(schema)).to include( + "/properties/is_happy/anyOf/0/type" + ) + end + + it "indicates a JSON schema with an unknown field is invalid" do + schema = { + "type" => "object", + "properties" => { + "is_happy" => { + "anyOf" => [ + {"type" => "boolean", "foo" => 3}, + {"type" => "null"} + ] + } + } + } + + expect(validator.valid?(schema)).to be false + expect(validator.validate(schema).to_a).to include( + a_hash_including("data" => 3, "data_pointer" => "/properties/is_happy/anyOf/0/foo") + ) + expect(validator.validate_with_error_message(schema)).to include( + "/properties/is_happy/anyOf/0/foo" + ) + end + end + + RSpec.describe "JSONSchema.strict_meta_schema_validator" do + let(:validator) { JSONSchema.strict_meta_schema_validator } + include_examples "a meta schema validator" + + it "does not allow extra `ElasticGraph` metadata alongside object property subschemas" do + schema = { + "type" => "object", + "properties" => { + "is_happy" => { + "type" => "string", + "ElasticGraph" => { + "type" => "String", + "nameInIndex" => "is_happy" + } + } + } + } + + expect(validator.validate_with_error_message(schema)).to include("/properties/is_happy/ElasticGraph") + end + end + + RSpec.describe "JSONSchema.elastic_graph_internal_meta_schema_validator" do + let(:validator) { JSONSchema.elastic_graph_internal_meta_schema_validator } + include_examples "a meta schema validator" + + it "allows extra `ElasticGraph` metadata alongside object property subschemas" do + schema = { + "type" => "object", + "properties" => { + "is_happy" => { + "type" => "string", + "ElasticGraph" => { + "type" => "String", + "nameInIndex" => "is_happy" + } + } + } + } + + expect(validator.valid?(schema)).to be true + expect(validator.validate(schema).to_a).to be_empty + expect(validator.validate_with_error_message(schema)).to eq(nil) + end + + it "requires all `ElasticGraph` metadata properties" do + schema = { + "type" => "object", + "properties" => { + "is_happy" => { + "type" => "string", + "ElasticGraph" => { + "nameInIndex" => "is_happy" + } + } + } + } + expect(validator.validate_with_error_message(schema)).to include( + "/properties/is_happy/ElasticGraph", "type" + ) + + schema = { + "type" => "object", + "properties" => { + "is_happy" => { + "type" => "string", + "ElasticGraph" => { + "type" => "String" + } + } + } + } + expect(validator.validate_with_error_message(schema)).to include( + "/properties/is_happy/ElasticGraph", "nameInIndex" + ) + end + + it "validates the type of `ElasticGraph` metadata properties" do + schema = { + "type" => "object", + "properties" => { + "is_happy" => { + "type" => "string", + "ElasticGraph" => { + "type" => 7, + "nameInIndex" => false + } + } + } + } + + expect(validator.validate_with_error_message(schema)).to include( + "/properties/is_happy/ElasticGraph/type", + "/properties/is_happy/ElasticGraph/nameInIndex" + ) + end + end + end +end diff --git a/elasticgraph-json_schema/spec/unit/elastic_graph/json_schema/validator_factory_spec.rb b/elasticgraph-json_schema/spec/unit/elastic_graph/json_schema/validator_factory_spec.rb new file mode 100644 index 00000000..4b81c08d --- /dev/null +++ b/elasticgraph-json_schema/spec/unit/elastic_graph/json_schema/validator_factory_spec.rb @@ -0,0 +1,41 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/json_schema/validator_factory" + +module ElasticGraph + module JSONSchema + RSpec.describe ValidatorFactory do + it "caches Validator instances" do + type_name = "MyType" + factory = ValidatorFactory.new( + schema: { + "$schema" => JSON_META_SCHEMA, + "$defs" => { + type_name => { + "type" => "object", + "properties" => { + "id" => { + "type" => "string", + "maxLength" => 10 + } + } + } + } + }, + sanitize_pii: false + ) + + # standard:disable RSpec/IdenticalEqualityAssertion + expect(factory.validator_for(type_name)).to be factory.validator_for(type_name) + # standard:enable RSpec/IdenticalEqualityAssertion + end + end + end +end diff --git a/elasticgraph-json_schema/spec/unit/elastic_graph/json_schema/validator_spec.rb b/elasticgraph-json_schema/spec/unit/elastic_graph/json_schema/validator_spec.rb new file mode 100644 index 00000000..17ff72fe --- /dev/null +++ b/elasticgraph-json_schema/spec/unit/elastic_graph/json_schema/validator_spec.rb @@ -0,0 +1,209 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/json_schema/validator" +require "elastic_graph/json_schema/validator_factory" + +module ElasticGraph + module JSONSchema + RSpec.describe Validator do + it "does not touch `additionalProperties` by default" do + validator = validator_for({ + "type" => "object", + "properties" => { + "is_happy" => { + "anyOf" => [ + {"type" => "string"}, + {"type" => "null"} + ] + } + } + }) + + data = {"is_happy" => "yes", "another_property" => 3} + + expect(validator.valid?(data)).to be true + expect(validator.validate(data).to_a).to be_empty + expect(validator.validate_with_error_message(data)).to eq nil + end + + it "can be configured to fail when there are extra properties" do + validator = validator_for({ + "type" => "object", + "properties" => { + "is_happy" => { + "anyOf" => [ + {"type" => "string"}, + {"type" => "null"} + ] + } + } + }, &:with_unknown_properties_disallowed) + + data = {"is_happy" => "yes", "another_property" => 3} + + expect(validator.valid?(data)).to be false + expect(validator.validate(data).to_a).to include( + a_hash_including("data" => 3, "data_pointer" => "/another_property") + ) + expect(validator.validate_with_error_message(data)).to include("another_property") + end + + it "can be configured to fail on extra properties while allowing specific extra properties" do + validator = validator_for({ + "type" => "object", + "properties" => { + "is_happy" => {"type" => "string"}, + "sub_object" => { + "properties" => { + "foo" => {"type" => "string"} + } + } + } + }) do |factory| + factory.with_unknown_properties_disallowed(except: ["extra1", "sub_object.extra2"]) + end + + valid1 = {"is_happy" => "yes", "sub_object" => {"foo" => "abc"}} + valid2 = {"is_happy" => "yes", "extra1" => 1, "sub_object" => {"foo" => "abc"}} + valid3 = {"is_happy" => "yes", "extra1" => 1, "sub_object" => {"foo" => "abc", "extra2" => 1}} + + expect(validator.validate_with_error_message(valid1)).to eq nil + expect(validator.validate_with_error_message(valid2)).to eq nil + expect(validator.validate_with_error_message(valid3)).to eq nil + + invalid1 = {"is_happy" => "yes", "extra2" => 1, "sub_object" => {"foo" => "abc"}} + invalid2 = {"is_happy" => "yes", "sub_object" => {"foo" => "abc", "extra1" => 2}} + + expect(validator.validate_with_error_message(invalid1)).to include("extra2") + expect(validator.validate_with_error_message(invalid2)).to include("extra1") + end + + it "does not mutate the schema when applying `additionalProperties: false`" do + schema = { + "type" => "object", + "properties" => { + "is_happy" => {"type" => "string"}, + "sub_object" => { + "properties" => { + "foo" => {"type" => "string"} + } + } + } + } + + expect { + validator_for(schema) do |factory| + factory.with_unknown_properties_disallowed(except: ["extra1", "sub_object.extra2"]) + end + }.not_to change { schema } + end + + it "ignores specified extra fields that are already defined in the schema" do + validator = validator_for({ + "type" => "object", + "properties" => { + "is_happy" => {"type" => "string"} + } + }) do |factory| + factory.with_unknown_properties_disallowed(except: ["is_happy"]) + end + + expect(validator.validate_with_error_message({"is_happy" => "yes"})).to eq nil + expect(validator.validate_with_error_message({"is_happy" => 3})).to include("is_happy") + end + + it "excludes the given data from validation errors when `sanitize_pii` is `true`" do + unsanitized_validator = validator_for({ + "type" => "object", + "properties" => { + "is_happy" => {"type" => "integer"} + } + }, sanitize_pii: false) + + sanitized_validator = unsanitized_validator.with(sanitize_pii: true) + + expect(sanitized_validator.validate_with_error_message({"is_happy" => "pii_value"})).to exclude("pii_value") + expect(unsanitized_validator.validate_with_error_message({"is_happy" => "pii_value"})).to include("pii_value") + end + + it "can validate with schemas using type references" do + validator = validator_for({ + "$defs" => { + "ID" => { + "type" => "string", + "maxLength" => 10 + }, + "MyType" => { + "type" => "object", + "properties" => { + "id" => { + "$ref" => "#/$defs/ID" + } + } + } + } + }, type_name: "MyType") + + valid_data = {"id" => "my_type_id"} + expect(validator.valid?(valid_data)).to be true + expect(validator.validate(valid_data).to_a).to be_empty + expect(validator.validate_with_error_message(valid_data)).to eq nil + + invalid_data = {"id" => "my_type_id_that_is_way_too_long"} + expect(validator.valid?(invalid_data)).to be false + expect(validator.validate(invalid_data).to_a) + .to contain_exactly( + { + "data" => "my_type_id_that_is_way_too_long", + "data_pointer" => "/id", + "error" => "string length at `/id` is greater than: 10", + "schema_pointer" => "/$defs/ID", + "type" => "maxLength" + } + ) + expect(validator.validate_with_error_message(invalid_data)).to include("my_type_id_that_is_way_too_long") + end + + it "treats regex patterns as ecma does to more closely match standard JSON schema behavior" do + pattern = /^foo$/ + expect(pattern).to match("before\nfoo\nafter") + + validator = validator_for({ + "$defs" => { + "MyType" => { + "type" => "object", + "properties" => { + "word" => { + "pattern" => pattern.source + } + } + } + } + }, type_name: "MyType") + + expect(validator.validate_with_error_message({"word" => "foo"})).to eq nil + expect(validator.validate_with_error_message({"word" => "before\nfoo\nafter"})).to include('"schema_pointer": "/$defs/MyType/properties/word"') + end + + def validator_for(schema, type_name: nil, sanitize_pii: false) + if type_name.nil? + schema = {"$defs" => {"SomeTypeName" => schema}} + type_name = "SomeTypeName" + end + + schema = {"$schema" => JSON_META_SCHEMA}.merge(schema) + + factory = ValidatorFactory.new(schema: schema, sanitize_pii: sanitize_pii) + factory = yield factory if block_given? + factory.validator_for(type_name) + end + end + end +end diff --git a/elasticgraph-lambda_support/.rspec b/elasticgraph-lambda_support/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-lambda_support/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-lambda_support/.yardopts b/elasticgraph-lambda_support/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-lambda_support/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-lambda_support/Gemfile b/elasticgraph-lambda_support/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-lambda_support/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-lambda_support/LICENSE.txt b/elasticgraph-lambda_support/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-lambda_support/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-lambda_support/README.md b/elasticgraph-lambda_support/README.md new file mode 100644 index 00000000..c4d6ca97 --- /dev/null +++ b/elasticgraph-lambda_support/README.md @@ -0,0 +1,6 @@ +# ElasticGraph::LambdaSupport + +This gem contains common lambda support logic that is used by all ElasticGraph +lambdas, such as lambda logging and OpenSearch connection support. + +It is not meant to be used directly by end users of ElasticGraph. diff --git a/elasticgraph-lambda_support/elasticgraph-lambda_support.gemspec b/elasticgraph-lambda_support/elasticgraph-lambda_support.gemspec new file mode 100644 index 00000000..fffa9f6d --- /dev/null +++ b/elasticgraph-lambda_support/elasticgraph-lambda_support.gemspec @@ -0,0 +1,22 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :lambda) do |spec, eg_version| + spec.summary = "ElasticGraph gem that supports running ElasticGraph using AWS Lambda." + + spec.add_dependency "elasticgraph-opensearch", eg_version + spec.add_dependency "faraday_middleware-aws-sigv4", "~> 1.0" + + spec.add_development_dependency "elasticgraph-admin", eg_version + spec.add_development_dependency "elasticgraph-graphql", eg_version + spec.add_development_dependency "elasticgraph-indexer", eg_version + spec.add_development_dependency "elasticgraph-indexer_autoscaler_lambda", eg_version + spec.add_development_dependency "httpx", "~> 1.3" +end diff --git a/elasticgraph-lambda_support/lib/elastic_graph/lambda_support.rb b/elasticgraph-lambda_support/lib/elastic_graph/lambda_support.rb new file mode 100644 index 00000000..e0f2a183 --- /dev/null +++ b/elasticgraph-lambda_support/lib/elastic_graph/lambda_support.rb @@ -0,0 +1,74 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/lambda_support/json_aware_lambda_log_formatter" +require "faraday_middleware/aws_sigv4" +require "json" + +module ElasticGraph + module LambdaSupport + # Helper method for building ElasticGraph components from our lambda ENV vars. + # `klass` is expected to be `ElasticGraph::Admin`, `ElasticGraph::GraphQL`, or `ElasticGraph::Indexer`. + # + # This is meant to only deal with ENV vars and config that are common across all ElasticGraph + # components (e.g. logging and OpenSearch clients). ENV vars that are specific to one component + # should be handled elsewhere. This accepts a block which can further customize configuration as + # needed. + def self.build_from_env(klass) + klass.from_yaml_file( + ENV.fetch("ELASTICGRAPH_YAML_CONFIG"), + datastore_client_customization_block: ->(faraday) { configure_datastore_client(faraday) } + ) do |settings| + settings = settings.merge( + "logger" => override_logger_config(settings.fetch("logger")), + "datastore" => override_datastore_config(settings.fetch("datastore")) + ) + + settings = yield(settings) if block_given? + settings + end + end + + private + + def self.override_datastore_config(datastore_config) + env_urls_by_cluster = ::JSON.parse(ENV.fetch("OPENSEARCH_CLUSTER_URLS")) + file_settings_by_cluster = datastore_config.fetch("clusters").transform_values { |v| v["settings"] } + + datastore_config.merge( + "clusters" => env_urls_by_cluster.to_h do |cluster_name, url| + cluster_def = { + "url" => url, + "backend" => "opensearch", + "settings" => file_settings_by_cluster[cluster_name] || {} + } + + [cluster_name, cluster_def] + end + ) + end + + def self.override_logger_config(logger_config) + logger_config.merge({ + "level" => ENV["ELASTICGRAPH_LOG_LEVEL"], + "formatter" => JSONAwareLambdaLogFormatter.name + }.compact) + end + + def self.configure_datastore_client(faraday) + faraday.request :aws_sigv4, + service: "es", + region: ENV.fetch("AWS_REGION"), # assumes the lambda and OpenSearch cluster live in the same region. + access_key_id: ENV.fetch("AWS_ACCESS_KEY_ID"), + secret_access_key: ENV.fetch("AWS_SECRET_ACCESS_KEY"), + session_token: ENV["AWS_SESSION_TOKEN"] # optional + end + + private_class_method :override_datastore_config, :override_logger_config, :configure_datastore_client + end +end diff --git a/elasticgraph-lambda_support/lib/elastic_graph/lambda_support/json_aware_lambda_log_formatter.rb b/elasticgraph-lambda_support/lib/elastic_graph/lambda_support/json_aware_lambda_log_formatter.rb new file mode 100644 index 00000000..b06ee4e1 --- /dev/null +++ b/elasticgraph-lambda_support/lib/elastic_graph/lambda_support/json_aware_lambda_log_formatter.rb @@ -0,0 +1,51 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "json" +require "logger" + +module ElasticGraph + module LambdaSupport + # A log formatter that supports JSON logging, without requiring _all_ logs to be emitted as JSON. + # + # If the `message` is a hash of JSON data, it will produce a JSON-formatted log message combining the + # standard bits of metadata the AWS Lambda logger already includes in every log message with the passed data. + # + # If it is not a hash of JSON data, it will just delegate to the default formatter used by AWS Lambda. + # + # This is particularly useful to support cloudwatch metric filtering: + # https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/FilterAndPatternSyntax.html#metric-filters-extract-json + class JSONAwareLambdaLogFormatter < ::Logger::Formatter + # Copied from: + # https://github.com/aws/aws-lambda-ruby-runtime-interface-client/blob/2.0.0/lib/aws_lambda_ric/lambda_log_formatter.rb#L8 + FORMAT = "%s, [%s #%d] %5s %s -- %s: %s" + + def call(severity, time, progname, msg) + metadata = { + # These bits of metadata come from the standard AWS Lambda log formatter: + # https://github.com/aws/aws-lambda-ruby-runtime-interface-client/blob/2.0.0/lib/aws_lambda_ric/lambda_log_formatter.rb#L11-L12 + sev: severity[0..0], + datetime: format_datetime(time), + process: $$, + severity: severity, + # standard:disable Style/GlobalVars -- don't have a choice here; this is what the AWS Lambda runtime sets. + request_id: $_global_aws_request_id, + # standard:enable Style/GlobalVars + progname: progname + } + + if msg.is_a?(::Hash) + ::JSON.generate(msg.merge(metadata), space: " ") + else + # See https://github.com/aws/aws-lambda-ruby-runtime-interface-client/blob/2.0.0/lib/aws_lambda_ric/lambda_log_formatter.rb + (FORMAT % metadata.merge({msg: msg2str(msg)})).encode!("UTF-8") + end + end + end + end +end diff --git a/elasticgraph-lambda_support/lib/elastic_graph/lambda_support/lambda_function.rb b/elasticgraph-lambda_support/lib/elastic_graph/lambda_support/lambda_function.rb new file mode 100644 index 00000000..6d4fa8d0 --- /dev/null +++ b/elasticgraph-lambda_support/lib/elastic_graph/lambda_support/lambda_function.rb @@ -0,0 +1,69 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/monotonic_clock" + +module ElasticGraph + module LambdaSupport + # Mixin that can be used to define a lambda function, with common cross-cutting concerns + # handled automatically for you: + # + # - The amount of time it takes to boot the lambda is logged. + # - An error handling hook is provided which applies both to boot-time logic and request handling. + # + # It is designed to be prepending onto a class, like so: + # + # class DoSomething + # prepend LambdaFunction + # + # def initialize + # require 'my_application' + # @application = MyApplication.new(ENV[...]) + # end + # + # def handle_request(event:, context:) + # @application.handle_request(event: event, context: context) + # end + # end + # + # Using `prepend` is necessary so that it can wrap `initialize` and `handle_request` with error handling. + # It is recommended that `require`s be put in `initialize` instead of at the top of the lambda function + # file so that the error handler can handle any errors that happen while loading dependencies. + # + # `handle_exceptions` can also be overridden in order to provide error handling. + module LambdaFunction + def initialize(output: $stdout, monotonic_clock: Support::MonotonicClock.new) + handle_exceptions do + log_duration(output, monotonic_clock, "Booting the lambda function") do + super() + end + end + end + + def handle_request(event:, context:) + handle_exceptions { super } + end + + private + + # By default we just allow exceptions to bubble up. This is provided so that there is an exception handling hook that can be overridden. + def handle_exceptions + yield + end + + def log_duration(output, monotonic_clock, description) + start_ms = monotonic_clock.now_in_ms + yield + stop_ms = monotonic_clock.now_in_ms + duration_ms = stop_ms - start_ms + + output.puts "#{description} took #{duration_ms} milliseconds." + end + end + end +end diff --git a/elasticgraph-lambda_support/sig/aws_lambda_runtime.rbs b/elasticgraph-lambda_support/sig/aws_lambda_runtime.rbs new file mode 100644 index 00000000..e796d604 --- /dev/null +++ b/elasticgraph-lambda_support/sig/aws_lambda_runtime.rbs @@ -0,0 +1 @@ +$_global_aws_request_id: ::String diff --git a/elasticgraph-lambda_support/sig/elastic_graph/lambda_support.rbs b/elasticgraph-lambda_support/sig/elastic_graph/lambda_support.rbs new file mode 100644 index 00000000..acf4b7f9 --- /dev/null +++ b/elasticgraph-lambda_support/sig/elastic_graph/lambda_support.rbs @@ -0,0 +1,13 @@ +module ElasticGraph + module LambdaSupport + def self.build_from_env: [T] ( + Support::FromYamlFile[T] + ) ?{ (parsedYamlSettings) -> parsedYamlSettings } -> T + + private + + def self.override_datastore_config: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + def self.override_logger_config: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + def self.configure_datastore_client: (untyped) -> void + end +end diff --git a/elasticgraph-lambda_support/sig/elastic_graph/lambda_support/json_aware_lambda_log_formatter.rbs b/elasticgraph-lambda_support/sig/elastic_graph/lambda_support/json_aware_lambda_log_formatter.rbs new file mode 100644 index 00000000..89d0871c --- /dev/null +++ b/elasticgraph-lambda_support/sig/elastic_graph/lambda_support/json_aware_lambda_log_formatter.rbs @@ -0,0 +1,7 @@ +module ElasticGraph + module LambdaSupport + class JSONAwareLambdaLogFormatter < ::Logger::Formatter + FORMAT: String + end + end +end diff --git a/elasticgraph-lambda_support/sig/elastic_graph/lambda_support/lambda_function.rbs b/elasticgraph-lambda_support/sig/elastic_graph/lambda_support/lambda_function.rbs new file mode 100644 index 00000000..40007641 --- /dev/null +++ b/elasticgraph-lambda_support/sig/elastic_graph/lambda_support/lambda_function.rbs @@ -0,0 +1,18 @@ +module ElasticGraph + module LambdaSupport + interface _LambdaFunctionClass[T] + def initialize: () -> void + def handle_request: (event: ::Hash[::String, untyped], context: untyped) -> T + end + + module LambdaFunction[T]: _LambdaFunctionClass[T] + def initialize: (?output: io, ?monotonic_clock: Support::MonotonicClock) -> void + def handle_request: (event: ::Hash[::String, untyped], context: untyped) -> T + + private + + def handle_exceptions: [T] () { () -> T } -> T + def log_duration: (io, Support::MonotonicClock, ::String) { () -> void } -> void + end + end +end diff --git a/elasticgraph-lambda_support/spec/spec_helper.rb b/elasticgraph-lambda_support/spec/spec_helper.rb new file mode 100644 index 00000000..4b793e3f --- /dev/null +++ b/elasticgraph-lambda_support/spec/spec_helper.rb @@ -0,0 +1,11 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-lambda_support`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. diff --git a/elasticgraph-lambda_support/spec/unit/elastic_graph/lambda_support/json_aware_lambda_log_formatter_spec.rb b/elasticgraph-lambda_support/spec/unit/elastic_graph/lambda_support/json_aware_lambda_log_formatter_spec.rb new file mode 100644 index 00000000..a91a491c --- /dev/null +++ b/elasticgraph-lambda_support/spec/unit/elastic_graph/lambda_support/json_aware_lambda_log_formatter_spec.rb @@ -0,0 +1,38 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/lambda_support/json_aware_lambda_log_formatter" +require "logger" +require "stringio" + +module ElasticGraph + module LambdaSupport + RSpec.describe JSONAwareLambdaLogFormatter do + let(:logger_formatter) { JSONAwareLambdaLogFormatter.new } + let(:our_logger_output) { ::StringIO.new } + let(:our_logger) { ::Logger.new(our_logger_output, formatter: logger_formatter) } + + it "logs string with expected format and data" do + logger_formatter.datetime_format = "static_for_tests" + our_logger.info "some message" + + expect(our_logger_output.string).to eq("I, [static_for_tests ##{Process.pid}] INFO -- : some message") + end + + it "logs hashes of data as well-formed JSON so that we can apply cloudwatch metric filters to it" do + our_logger.info(some: "data") + json_data = ::JSON.parse(our_logger_output.string) + expect(json_data).to include( + "process" => $$, + "severity" => "INFO", + "some" => "data" + ) + end + end + end +end diff --git a/elasticgraph-lambda_support/spec/unit/elastic_graph/lambda_support/lambda_function_spec.rb b/elasticgraph-lambda_support/spec/unit/elastic_graph/lambda_support/lambda_function_spec.rb new file mode 100644 index 00000000..39bd51b5 --- /dev/null +++ b/elasticgraph-lambda_support/spec/unit/elastic_graph/lambda_support/lambda_function_spec.rb @@ -0,0 +1,125 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/lambda_support/lambda_function" +require "stringio" + +module ElasticGraph + module LambdaSupport + RSpec.describe LambdaFunction do + let(:output) { StringIO.new } + let(:monotonic_clock) { instance_double(Support::MonotonicClock, now_in_ms: 100) } + + it "calls `initialize` and `handle_request` at the expected times" do + lambda_function = new_lambda_function do + def initialize + @boot_state ||= [] + @boot_state << "booted" + end + + def handle_request(event:, context:) + { + event: event, + context: context, + boot_state: @boot_state + } + end + end + + response = lambda_function.handle_request(event: :my_event1, context: :my_context1) + expect(response).to eq({ + event: :my_event1, + context: :my_context1, + boot_state: ["booted"] + }) + + response = lambda_function.handle_request(event: :my_event2, context: :my_context2) + expect(response).to eq({ + event: :my_event2, + context: :my_context2, + boot_state: ["booted"] # verify that boot_state hasn't grown (since on_boot should not be called on every request) + }) + end + + it "allows the `handle_exceptions` hook to be overridden to handle boot-time errors" do + handled_errors = [] + + error_handler = Module.new do + define_method :handle_exceptions do |&block| + block.call + rescue => e + handled_errors << [e.class, e.message] + end + end + + new_lambda_function do + prepend error_handler + + def initialize + raise "Something went wrong during boot" + end + end + + expect(handled_errors).to eq [ + [RuntimeError, "Something went wrong during boot"] + ] + end + + it "allows the `handle_exceptions` hook to be overridden to handle request-time errors" do + handled_errors = [] + + error_handler = Module.new do + define_method :handle_exceptions do |&block| + block.call + rescue => e + handled_errors << [e.class, e.message] + end + end + + lambda_function = new_lambda_function do + prepend error_handler + + def handle_request(event:, context:) + raise "Invalid event: #{event}" + end + end + + lambda_function.handle_request(event: :some_event, context: :some_context) + + expect(handled_errors).to eq [ + [RuntimeError, "Invalid event: some_event"] + ] + end + + it "logs how long the lambda function takes to boot" do + now_in_ms = 250 + allow(monotonic_clock).to receive(:now_in_ms) { now_in_ms } + + new_lambda_function do + define_method :initialize do + now_in_ms = 500 + end + + def handle_request(event:, context:) + end + end + + expect(output.string).to include("Booting the lambda function took 250 milliseconds.") + end + + def new_lambda_function(&definition) + klass = ::Class.new do + prepend LambdaFunction + class_exec(&definition) + end + + klass.new(output: output, monotonic_clock: monotonic_clock) + end + end + end +end diff --git a/elasticgraph-lambda_support/spec/unit/elastic_graph/lambda_support_spec.rb b/elasticgraph-lambda_support/spec/unit/elastic_graph/lambda_support_spec.rb new file mode 100644 index 00000000..dab80e42 --- /dev/null +++ b/elasticgraph-lambda_support/spec/unit/elastic_graph/lambda_support_spec.rb @@ -0,0 +1,138 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin" +require "elastic_graph/graphql" +require "elastic_graph/indexer" +require "elastic_graph/indexer_autoscaler_lambda" +require "elastic_graph/lambda_support" +require "elastic_graph/spec_support/lambda_function" + +module ElasticGraph + RSpec.describe LambdaSupport, ".build_from_env" do + include_context "lambda function" + + shared_examples_for "building an ElasticGraph component" do |klass| + around { |ex| with_lambda_env_vars(&ex) } + + it "builds an instance of the provided class" do + instance = LambdaSupport.build_from_env(klass) + + expect(instance).to be_a klass + end + + it "allows the caller to further customize the settings" do + component = LambdaSupport.build_from_env(klass) do |settings| + settings.merge("logger" => settings.fetch("logger").merge( + "level" => "error" + )) + end + + expect(component.datastore_core.logger.level).to eq ::Logger::ERROR + end + + it "sets `backend` to OpenSearch for each cluster" do + component = LambdaSupport.build_from_env(klass) + backend_by_cluster = component.datastore_core.config.clusters.transform_values(&:backend_client_class) + + expect(backend_by_cluster).to eq({ + "main" => OpenSearch::Client, + "other1" => OpenSearch::Client, + "other2" => OpenSearch::Client, + "other3" => OpenSearch::Client + }) + end + + it "sets the log level based on `ELASTICGRAPH_LOG_LEVEL`" do + with_env "ELASTICGRAPH_LOG_LEVEL" => "ERROR" do + component = LambdaSupport.build_from_env(klass) + + expect(component.datastore_core.logger.level).to eq ::Logger::ERROR + end + end + + it "allows `ELASTICGRAPH_LOG_LEVEL` to be unset" do + expect(ENV["ELASTICGRAPH_LOG_LEVEL"]).to be nil + + component = LambdaSupport.build_from_env(klass) + + expect(component.datastore_core.logger.level).to eq ::Logger::INFO + end + + it "uses our custom log formatter so we can emit JSON logs" do + component = LambdaSupport.build_from_env(klass) + + expect(component.datastore_core.logger.formatter).to be_a(LambdaSupport::JSONAwareLambdaLogFormatter) + end + + context "with a cluster defined in the `OPENSEARCH_CLUSTER_URLS` that is not present in the config" do + around do |ex| + with_lambda_env_vars(cluster_urls: { + "main" => "main_url", + "other1" => "other_1_url", + "not_defined" => "not_defined_url" + }, &ex) + end + + it "creates a datastore ClusterDefinition for that cluster with empty settings" do + component = LambdaSupport.build_from_env(klass) + + clusters = component.datastore_core.config.clusters + expect(clusters.keys).to contain_exactly("main", "other1", "not_defined") + expect(clusters["not_defined"].url).to eq("not_defined_url") + expect(clusters["not_defined"].settings).to eq({}) + end + end + + context "with a subset of opensearch clusters having defined urls" do + around do |ex| + with_lambda_env_vars(cluster_urls: { + "main" => "main_url", + "other1" => "other_1_url" + }, &ex) + end + + it "removes clusters that don't have a defined url in `OPENSEARCH_CLUSTER_URLS`." do + component = LambdaSupport.build_from_env(klass) + + clusters = component.datastore_core.config.clusters + expect(clusters.keys).to contain_exactly("main", "other1") + expect(clusters["main"].url).to eq("main_url") + expect(clusters["main"].settings).to eq({"cluster.max_shards_per_node" => 16000}) + expect(clusters["other1"].url).to eq("other_1_url") + expect(clusters["other1"].settings).to eq({"cluster.max_shards_per_node" => 16001}) + end + end + + it "configures the datastore faraday client" do + allow(FaradayMiddleware::AwsSigV4).to receive(:new).and_call_original + + component = LambdaSupport.build_from_env(klass) + component.datastore_core.clients_by_name.fetch("main") + + expect(FaradayMiddleware::AwsSigV4).to have_received(:new).at_least(:once) + end + end + + context "when passed `ElasticGraph::Admin`" do + include_examples "building an ElasticGraph component", Admin + end + + context "when passed `ElasticGraph::GraphQL`" do + include_examples "building an ElasticGraph component", GraphQL + end + + context "when passed `ElasticGraph::Indexer`" do + include_examples "building an ElasticGraph component", Indexer + end + + context "when passed `ElasticGraph::IndexerAutoscalerLambda`" do + include_examples "building an ElasticGraph component", IndexerAutoscalerLambda + end + end +end diff --git a/elasticgraph-local/.rspec b/elasticgraph-local/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-local/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-local/.yardopts b/elasticgraph-local/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-local/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-local/Gemfile b/elasticgraph-local/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-local/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-local/LICENSE.txt b/elasticgraph-local/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-local/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-local/README.md b/elasticgraph-local/README.md new file mode 100644 index 00000000..0c0405bc --- /dev/null +++ b/elasticgraph-local/README.md @@ -0,0 +1,80 @@ +# ElasticGraph::Local + +Provides support for developing and running ElasticGraph applications locally. +These locally running ElasticGraph applications use 100% fake generated data +so as not to require a publisher of real data to be implemented. + +## Installation + +Add `elasticgraph-local` to a new project `Gemfile`: + +```ruby +source "https://rubygems.org" + +group :development do + gem "factory_bot" + gem "faker" + gem "elasticgraph-local" +end +``` + +As shown above, you can also pull in any gems that will help you +generate fake data. We tend to use [factory_bot](https://github.com/thoughtbot/factory_bot) +and [faker](https://github.com/faker-ruby/faker). `elasticgraph-local` should be defined +in the `development` group (you don't want to include it in any staging or production +deployment). + +Next, install the `elasticgraph-local` rake tasks in your `Rakefile`, with code like: + +``` ruby +require 'elastic_graph/local/rake_tasks' + +ElasticGraph::Local::RakeTasks.new( + local_config_yaml: "config/settings/development.yaml", + path_to_schema: "config/schema.rb" +) do |tasks| + tasks.define_fake_data_batch_for :widgets do |batch| + # Use faker/factory_bot etc here to generate fake data + # and add it to the `batch` array. + # You'll probably want to put that logic in another file + # and load it from here. + end +end +``` + +## Usage + +Everything you need is provided by rake tasks. Run the following to see what they are: + +```bash +$ bundle exec rake -T +``` + +At a high level, this provides tasks that help you to: + +1. Boot Elasticsearch/OpenSearch (+ their corresponding dashboards) locally using the `opensearch:*`/`elasticsearch:*` tasks. +2. Generate and validate ElasticGraph schema artifacts using the `schema_artifacts:*` tasks. +3. Configure your locally running Elasticsearch/OpenSearch using the `clusters:configure:perform` task. +4. Index fake data into Elasticsearch/OpenSearch (either running locally or on AWS) using the `index_fake_data:*` tasks. +5. Boot the ElasticGraph GraphQL endpoint and GraphiQL in-browser UI using the `boot_graphiql` task. + +If you just want to boot ElasticGraph locally without worrying about any of the details, run: + +``` +$ bundle exec rake boot_locally +``` + +That sequences each of the other tasks so that, with a single command, you can go from nothing to a +locally running ElasticGraph instance with data that you can query from your browser. + +### Managing Elasticsearch/Opensearch + +The `opensearch:`/`elasticsearch:` tasks will boot the desired Elasticsearch or OpenSearch version using docker +along with the corresponding dashboards (Kibana for Elasticsearch, OpenSearch Dashboards for OpenSearch). You can +use either the `:boot` or `:daemon` tasks: + +* The `:boot` task will keep Elasticsearch/Opensearch in the foreground, allowing you to see the logs. +* The `:daemon` task runs Elasticsearch/Opensearch as a background daemon task. Notably, it waits to return + until Elasticsearch/Opensearch are ready to receive traffic. + +If you use a `:daemon` task, you can later use the corresponding `:halt` task to stop the daemon. diff --git a/elasticgraph-local/elasticgraph-local.gemspec b/elasticgraph-local/elasticgraph-local.gemspec new file mode 100644 index 00000000..661aacf3 --- /dev/null +++ b/elasticgraph-local/elasticgraph-local.gemspec @@ -0,0 +1,25 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :local) do |spec, eg_version| + spec.summary = "Provides support for developing and running ElasticGraph applications locally." + + spec.add_dependency "elasticgraph-admin", eg_version + spec.add_dependency "elasticgraph-graphql", eg_version + spec.add_dependency "elasticgraph-indexer", eg_version + spec.add_dependency "elasticgraph-rack", eg_version + spec.add_dependency "elasticgraph-schema_definition", eg_version + spec.add_dependency "rackup", "~> 2.1" + spec.add_dependency "rake", "~> 13.2" + + spec.add_development_dependency "elasticgraph-elasticsearch", eg_version + spec.add_development_dependency "elasticgraph-opensearch", eg_version + spec.add_development_dependency "httpx", "~> 1.3" +end diff --git a/elasticgraph-local/lib/elastic_graph/local/config.ru b/elasticgraph-local/lib/elastic_graph/local/config.ru new file mode 100644 index 00000000..25499bab --- /dev/null +++ b/elasticgraph-local/lib/elastic_graph/local/config.ru @@ -0,0 +1,15 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This `config.ru` file is used by the `rake boot_graphiql` task. + +require "elastic_graph/graphql" +require "elastic_graph/rack/graphiql" + +graphql = ElasticGraph::GraphQL.from_yaml_file(ENV.fetch("ELASTICGRAPH_YAML_FILE")) +run ElasticGraph::Rack::GraphiQL.new(graphql) diff --git a/elasticgraph-local/lib/elastic_graph/local/docker_runner.rb b/elasticgraph-local/lib/elastic_graph/local/docker_runner.rb new file mode 100644 index 00000000..d910d070 --- /dev/null +++ b/elasticgraph-local/lib/elastic_graph/local/docker_runner.rb @@ -0,0 +1,117 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "timeout" + +module ElasticGraph + module Local + # @private + class DockerRunner + def initialize(variant, port:, ui_port:, version:, env:, ready_log_line:, daemon_timeout:, output:) + @variant = variant + @port = port + @ui_port = ui_port + @version = version + @env = env + @ready_log_line = ready_log_line + @daemon_timeout = daemon_timeout + @output = output + end + + def boot + # :nocov: -- this is actually covered via a call from `boot_as_daemon` but it happens in a forked process so simplecov doesn't see it. + halt + + prepare_docker_compose_run "up" do |command| + exec(command) # we use `exec` so that our process is replaced with that one. + end + # :nocov: + end + + def halt + prepare_docker_compose_run "down --volumes" do |command| + system(command) + end + end + + def boot_as_daemon(halt_command:) + with_pipe do |read_io, write_io| + fork do + # :nocov: -- simplecov can't track coverage that happens in another process + read_io.close + Process.daemon + pid = Process.pid + $stdout.reopen(write_io) + $stderr.reopen(write_io) + puts pid + boot + write_io.close + # :nocov: + end + + # The `Process.daemon` call in the subprocess changes the pid so we have to capture it this way instead of using + # the return value of `fork`. + pid = read_io.gets.to_i + + @output.puts "Booting #{@variant}; monitoring logs for readiness..." + + ::Timeout.timeout( + @daemon_timeout, + ::Timeout::Error, + <<~EOS + Timed out after #{@daemon_timeout} seconds. The expected "ready" log line[1] was not found in the logs. + + [1] #{@ready_log_line.inspect} + EOS + ) do + loop do + sleep 0.01 + line = read_io.gets + @output.puts line + break if @ready_log_line.match?(line.to_s) + end + end + + @output.puts + @output.puts + @output.puts <<~EOS + Success! #{@variant} #{@version} (pid: #{pid}) has been booted for the #{@env} environment on port #{@port}. + It will continue to run in the background as a daemon. To halt it, run: + + #{halt_command} + EOS + end + end + + private + + def prepare_docker_compose_run(*commands) + name = "#{@env}-#{@version.tr(".", "_")}" + + full_command = commands.map do |command| + "VERSION=#{@version} PORT=#{@port} UI_PORT=#{@ui_port} ENV=#{@env} docker-compose --project-name #{name} #{command}" + end.join(" && ") + + ::Dir.chdir(::File.join(__dir__.to_s, @variant.to_s)) do + yield full_command + end + end + + def with_pipe + read_io, write_io = ::IO.pipe + + begin + yield read_io, write_io + ensure + read_io.close + write_io.close + end + end + end + end +end diff --git a/elasticgraph-local/lib/elastic_graph/local/elasticsearch/Dockerfile b/elasticgraph-local/lib/elastic_graph/local/elasticsearch/Dockerfile new file mode 100644 index 00000000..5810adf0 --- /dev/null +++ b/elasticgraph-local/lib/elastic_graph/local/elasticsearch/Dockerfile @@ -0,0 +1,3 @@ +ARG VERSION=latest +FROM elasticsearch:${VERSION} +RUN bin/elasticsearch-plugin install mapper-size diff --git a/elasticgraph-local/lib/elastic_graph/local/elasticsearch/UI-Dockerfile b/elasticgraph-local/lib/elastic_graph/local/elasticsearch/UI-Dockerfile new file mode 100644 index 00000000..85c9ec4b --- /dev/null +++ b/elasticgraph-local/lib/elastic_graph/local/elasticsearch/UI-Dockerfile @@ -0,0 +1,2 @@ +ARG VERSION=latest +FROM kibana:${VERSION} diff --git a/elasticgraph-local/lib/elastic_graph/local/elasticsearch/docker-compose.yaml b/elasticgraph-local/lib/elastic_graph/local/elasticsearch/docker-compose.yaml new file mode 100644 index 00000000..1b7724ec --- /dev/null +++ b/elasticgraph-local/lib/elastic_graph/local/elasticsearch/docker-compose.yaml @@ -0,0 +1,74 @@ +--- +networks: + default: + name: elastic + external: false +services: + elasticsearch: + build: + context: . + dockerfile: Dockerfile + args: + VERSION: ${VERSION} + container_name: elasticsearch-${VERSION}-${ENV} + environment: + # Note: we use `discovery.type=single-node` to ensure that the Elasticsearch node does not + # try to join a cluster (or let another node join it). This prevents problems when you + # have multiple projects using elasticgraph-local at the same time. You do not want + # their Elasticsearch nodes to try to join into a single cluster. + - discovery.type=single-node + # Note: we use `xpack.security.enabled=false` to silence an annoying warning Elasticsearch 7.13 has + # started spewing (as in hundreds of times!) as we run our test suite: + # + # > warning: 299 Elasticsearch-7.13.0-5ca8591c6fcdb1260ce95b08a8e023559635c6f3 "Elasticsearch built-in + # > security features are not enabled. Without authentication, your cluster could be accessible to anyone. + # > See https://www.elastic.co/guide/en/elasticsearch/reference/7.13/security-minimal-setup.html to enable + # > security." + # + # Since this is only used in local dev/test environments where the added security would make things harder + # (we'd have to setup credentials in our tests), it's simpler/better just to explicitly disable the security, + # which silences the warning. + - xpack.security.enabled=false + # We disable `xpack.ml` because it's not compatible with the `darwin-aarch64` distribution we use on M1 Macs. + # Without that flag, we get this error: + # + # > [2022-01-20T10:06:54,582][ERROR][o.e.b.ElasticsearchUncaughtExceptionHandler] [myron-macbookpro.local] uncaught exception in thread [main] + # > org.elasticsearch.bootstrap.StartupException: ElasticsearchException[Failure running machine learning native code. This could be due to running + # > on an unsupported OS or distribution, missing OS libraries, or a problem with the temp directory. To bypass this problem by running Elasticsearch + # > without machine learning functionality set [xpack.ml.enabled: false].] + # + # See also this github issue: https://github.com/elastic/elasticsearch/pull/68068 + - xpack.ml.enabled=false + # We don't want Elasticsearch to block writes when the disk allocation passes a threshold for our local/test + # Elasticsearch we run using this docker setup. + # https://stackoverflow.com/a/75962819 + # + # Without this, I frequently get `FORBIDDEN/10/cluster create-index blocked (api)` errors when running tests. + - cluster.routing.allocation.disk.threshold_enabled=false + # Necessary on Elasticsearch 8 since our test suites indiscriminately deletes all documents + # between tests to sandbox the state of each test. Without this setting, we get errors like: + # + # > illegal_argument_exception: Wildcard expressions or all indices are not allowed + - action.destructive_requires_name=false + - ES_JAVA_OPTS=-Xms4g -Xmx4g + ulimits: + nofile: + soft: 65536 + hard: 65536 + volumes: + - elasticsearch:/usr/share/elasticsearch/data + ports: + - ${PORT:-9200}:9200 + kibana: + build: + context: . + dockerfile: UI-Dockerfile + args: + VERSION: ${VERSION} + container_name: kibana-${VERSION}-${ENV} + environment: + - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + ports: + - ${UI_PORT:-5601}:5601 +volumes: + elasticsearch: diff --git a/elasticgraph-local/lib/elastic_graph/local/indexing_coordinator.rb b/elasticgraph-local/lib/elastic_graph/local/indexing_coordinator.rb new file mode 100644 index 00000000..fc349538 --- /dev/null +++ b/elasticgraph-local/lib/elastic_graph/local/indexing_coordinator.rb @@ -0,0 +1,58 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer/test_support/converters" + +module ElasticGraph + module Local + # Responsible for coordinating the generation and indexing of fake data batches. + # Designed to be pluggable with different publishing strategies. + # + # @private + class IndexingCoordinator + PARALLELISM = 8 + + def initialize(fake_data_batch_generator, output: $stdout, &publish_batch) + @fake_data_batch_generator = fake_data_batch_generator + @publish_batch = publish_batch + @output = output + end + + def index_fake_data(num_batches) + batch_queue = ::Thread::Queue.new + + publishing_threads = Array.new(PARALLELISM) { new_publishing_thread(batch_queue) } + + num_batches.times do + batch = [] # : ::Array[::Hash[::String, untyped]] + @fake_data_batch_generator.call(batch) + @output.puts "Generated batch of #{batch.size} documents..." + batch_queue << batch + end + + publishing_threads.map { batch_queue << :done } + publishing_threads.each(&:join) + + @output.puts "...done." + end + + private + + def new_publishing_thread(batch_queue) + ::Thread.new do + loop do + batch = batch_queue.pop + break if batch == :done + @publish_batch.call(ElasticGraph::Indexer::TestSupport::Converters.upsert_events_for_records(batch)) + @output.puts "Published batch of #{batch.size} documents..." + end + end + end + end + end +end diff --git a/elasticgraph-local/lib/elastic_graph/local/local_indexer.rb b/elasticgraph-local/lib/elastic_graph/local/local_indexer.rb new file mode 100644 index 00000000..3feec664 --- /dev/null +++ b/elasticgraph-local/lib/elastic_graph/local/local_indexer.rb @@ -0,0 +1,28 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer" +require "elastic_graph/local/indexing_coordinator" + +module ElasticGraph + module Local + # @private + class LocalIndexer + def initialize(local_config_yaml, fake_data_batch_generator, output:) + @local_indexer = ElasticGraph::Indexer.from_yaml_file(local_config_yaml) + @indexing_coordinator = IndexingCoordinator.new(fake_data_batch_generator, output: output) do |batch| + @local_indexer.processor.process(batch) + end + end + + def index_fake_data(num_batches) + @indexing_coordinator.index_fake_data(num_batches) + end + end + end +end diff --git a/elasticgraph-local/lib/elastic_graph/local/opensearch/Dockerfile b/elasticgraph-local/lib/elastic_graph/local/opensearch/Dockerfile new file mode 100644 index 00000000..578c72a0 --- /dev/null +++ b/elasticgraph-local/lib/elastic_graph/local/opensearch/Dockerfile @@ -0,0 +1,4 @@ +ARG VERSION=latest +FROM opensearchproject/opensearch:${VERSION} +RUN /usr/share/opensearch/bin/opensearch-plugin remove opensearch-security +RUN /usr/share/opensearch/bin/opensearch-plugin install --batch mapper-size diff --git a/elasticgraph-local/lib/elastic_graph/local/opensearch/UI-Dockerfile b/elasticgraph-local/lib/elastic_graph/local/opensearch/UI-Dockerfile new file mode 100644 index 00000000..7900f084 --- /dev/null +++ b/elasticgraph-local/lib/elastic_graph/local/opensearch/UI-Dockerfile @@ -0,0 +1,2 @@ +ARG VERSION=latest +FROM opensearchproject/opensearch-dashboards:${VERSION} diff --git a/elasticgraph-local/lib/elastic_graph/local/opensearch/docker-compose.yaml b/elasticgraph-local/lib/elastic_graph/local/opensearch/docker-compose.yaml new file mode 100644 index 00000000..a7d0dd27 --- /dev/null +++ b/elasticgraph-local/lib/elastic_graph/local/opensearch/docker-compose.yaml @@ -0,0 +1,50 @@ +--- +networks: + default: + name: opensearch + external: false +services: + opensearch: + build: + context: . + dockerfile: Dockerfile + args: + VERSION: ${VERSION} + container_name: opensearch-${VERSION}-${ENV} + environment: + # Note: we use `discovery.type=single-node` to ensure that the OpenSearch node does not + # try to join a cluster (or let another node join it). This prevents problems when you + # have multiple projects using elasticgraph-local at the same time. You do not want + # their OpenSearch nodes to try to join into a single cluster. + - discovery.type=single-node + # recommended by https://opensearch.org/downloads.html#minimal + - bootstrap.memory_lock=true + # We don't want OpenSearch to block writes when the disk allocation passes a threshold for our local/test + # OpenSearch we run using this docker setup. + # https://stackoverflow.com/a/75962819 + # + # Without this, I frequently get `FORBIDDEN/10/cluster create-index blocked (api)` errors when running tests. + - cluster.routing.allocation.disk.threshold_enabled=false + - OPENSEARCH_JAVA_OPTS=-Xms4g -Xmx4g + ulimits: + nofile: + soft: 65536 + hard: 65536 + volumes: + - opensearch:/usr/share/opensearch/data + ports: + - ${PORT}:9200 + dashboards: + build: + context: . + dockerfile: UI-Dockerfile + args: + VERSION: ${VERSION} + container_name: dashboards-${VERSION}-${ENV} + environment: + - OPENSEARCH_HOSTS=http://opensearch:9200 + - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true + ports: + - ${UI_PORT}:5601 +volumes: + opensearch: diff --git a/elasticgraph-local/lib/elastic_graph/local/rake_tasks.rb b/elasticgraph-local/lib/elastic_graph/local/rake_tasks.rb new file mode 100644 index 00000000..d6f69924 --- /dev/null +++ b/elasticgraph-local/lib/elastic_graph/local/rake_tasks.rb @@ -0,0 +1,548 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin/rake_tasks" +require "elastic_graph/local/docker_runner" +require "elastic_graph/schema_definition/rake_tasks" +require "rake/tasklib" +require "shellwords" + +module ElasticGraph + # Provides support for developing and running ElasticGraph applications locally. + module Local + # Defines tasks for local development. These tasks include: + # + # - Running OpenSearch and/or Elasticsearch locally (`(elasticsearch|opensearch):[env]:(boot|daemon|halt)`) + # - Managing schema artifacts (`schema_artifacts:(check|dump)`) + # - Configuring OpenSearch/Elasticsearch locally (`clusters:configure:(dry_run|perform)`) + # - Indexing fake data (`index_fake_data:[type]`) + # - Booting an ElasticGraph application locally (`boot_locally`) + # + # @note All tasks (besides the `schema_artifacts` tasks) require `docker` and `docker-compose` to be available on your machine. + class RakeTasks < ::Rake::TaskLib + # When enabled, ElasticGraph will configure the index mappings so that the datastore indexes a `_size` field in each index document. + # ElasticGraph itself does not do anything with this field, but it will be available for your use in any direct queries (e.g. via + # Kibana). + # + # Defaults to `false` since it requires a plugin. + # + # @note Enabling this requires the [mapper-size plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/8.15/mapper-size.html) + # to be installed on your datastore cluster. You are responsible for ensuring that is installed if you enable this feature. If you + # enable this and the plugin is not installed, you will get errors! + # + # @return [Boolean] whether or not the `_size` field should be indexed on each indexed type + # + # @example Enable indexing document sizes + # ElasticGraph::Local::RakeTasks.new( + # local_config_yaml: "config/settings/local.yaml", + # path_to_schema: "config/schema.rb" + # ) do |tasks| + # tasks.index_document_sizes = true + # end + # + # @dynamic index_document_sizes, index_document_sizes= + attr_accessor :index_document_sizes + + # The form of names for schema elements (fields, arguments, directives) generated by ElasticGraph, either `:snake_case` or + # `:camelCase`. For example, if set to `:camelCase`, ElasticGraph will generate a `groupedBy` field, but if set to `:snake_case`, + # ElasticGraph will generate a `grouped_by` field. + # + # Defaults to `:camelCase` since most GraphQL schemas use that casing. + # + # @return [:camelCase, :snake_case] which form to use + # + # @example Use `snake_case` names instead of `camelCase` + # ElasticGraph::Local::RakeTasks.new( + # local_config_yaml: "config/settings/local.yaml", + # path_to_schema: "config/schema.rb" + # ) do |tasks| + # tasks.schema_element_name_form = :snake_case + # end + # + # @dynamic schema_element_name_form, schema_element_name_form= + attr_accessor :schema_element_name_form + + # Overrides for specific names of schema elements (fields, arguments, directives) generated by ElasticGraph. For example, to rename + # the `gt` filter field to `greaterThan`, set to `{gt: "greaterThan"}`. + # + # Defaults to an empty hash. + # + # @return [Hash] overrides for specific field, argument, or directive names + # + # @example Spell out comparison operators instead of using shortened forms + # ElasticGraph::Local::RakeTasks.new( + # local_config_yaml: "config/settings/local.yaml", + # path_to_schema: "config/schema.rb" + # ) do |tasks| + # tasks.schema_element_name_overrides = { + # gt: "greaterThan", + # gte: "greaterThanOrEqualTo", + # lt: "lessThan", + # lte: "lessThanOrEqualTo" + # } + # end + # + # @dynamic schema_element_name_overrides, schema_element_name_overrides= + attr_accessor :schema_element_name_overrides + + # Overrides for the naming formats used by ElasticGraph for derived GraphQL type names. For example, to use `Metrics` instead of + # `AggregatedValues` as the suffix for the generated types supporting getting aggregated metrid values, set to + # `{AggregatedValues: "%{base}Metrics"}`. See {SchemaDefinition::SchemaElements::TypeNamer::DEFAULT_FORMATS} for the available + # formats. + # + # Defaults to an empty hash. + # + # @example Change the `AggregatedValues` type suffix to `Metrics` + # ElasticGraph::Local::RakeTasks.new( + # local_config_yaml: "config/settings/local.yaml", + # path_to_schema: "config/schema.rb" + # ) do |tasks| + # tasks.derived_type_name_formats = {AggregatedValues: "Metrics"} + # end + # + # @dynamic derived_type_name_formats, derived_type_name_formats= + attr_accessor :derived_type_name_formats + + # Overrides for the names of specific GraphQL types. For example, to rename the `JsonSafeLong` scalar to `BigInt`, set to + # `{JsonSafeLong: "BigInt}`. + # + # Defaults to an empty hash. + # + # @return [Hash] overrides for specific type names + # + # @example Rename `JsonSafeLong` to `BigInt` + # ElasticGraph::Local::RakeTasks.new( + # local_config_yaml: "config/settings/local.yaml", + # path_to_schema: "config/schema.rb" + # ) do |tasks| + # tasks.type_name_overrides = {JsonSafeLong: "BigInt"} + # end + # + # @dynamic type_name_overrides, type_name_overrides= + attr_accessor :type_name_overrides + + # Overrides for the names of specific GraphQL enum values for specific enum types. For example, to rename the `DayOfWeek.MONDAY` + # enum to `DayOfWeek.MON`, set to `{DayOfWeek: {MONDAY: "MON"}}`. + # + # Defaults to an empty hash. + # + # @return [Hash>] overrides for the names of specific enum values for specific enum types + # + # @example Shorten the names of the `DayOfWeek` enum values + # ElasticGraph::Local::RakeTasks.new( + # local_config_yaml: "config/settings/local.yaml", + # path_to_schema: "config/schema.rb" + # ) do |tasks| + # tasks.enum_value_overrides_by_type = { + # DayOfWeek: { + # MONDAY: "MON", + # TUESDAY: "TUE", + # WEDNESDAY: "WED", + # THURSDAY: "THU", + # FRIDAY: "FRI", + # SATURDAY: "SAT", + # SUNDAY: "SUN" + # } + # } + # end + # + # @dynamic enum_value_overrides_by_type, enum_value_overrides_by_type= + attr_accessor :enum_value_overrides_by_type + + # List of Ruby modules to extend onto the {SchemaDefinition::API} instance. Designed to support ElasticGraph extensions (such as + # {Apollo::SchemaDefinition::APIExtension}). Defaults to an empty list. + # + # @return [Array] list of extension modules + # + # @example Use `elasticgraph-apollo` + # require "elastic_graph/apollo/schema_definition/api_extension" + # + # ElasticGraph::Local::RakeTasks.new( + # local_config_yaml: "config/settings/local.yaml", + # path_to_schema: "config/schema.rb" + # ) do |tasks| + # tasks.schema_definition_extension_modules = [ElasticGraph::Apollo::SchemaDefinition::APIExtension] + # end + # + # @example Extension that defines a `@since` directive and offers a `since` API on fields + # module SinceExtension + # # `self.extended` is a standard Ruby hook that gets called when a module is extended onto an object. + # # The argument is the object the module was extended onto (a `SchemaDefinition::API` instance in this case). + # def self.extended(api) + # # Define our `@since` directive + # api.raw_sdl "directive @since(date: Date!) on FIELD_DEFINITION" + # + # # In order to hook into fields, extend the `SchemaDefinition::Factory` with a module. The factory is used + # # for creation of all schema definition objects. + # api.factory.extend FactoryExtension + # end + # + # module FactoryExtension + # # Hook into the creation of all `SchemaDefinition::Field` objects so that we can extend each field + # # instance with our `FieldExtension` module. + # def new_field(*args, **options) + # super(*args, **options) do |field| + # field.extend FieldExtension + # yield field if block_given? + # end + # end + # end + # + # # Offer a `f.since date` API on fields. + # module FieldExtension + # def since(date) + # directive "since", date: date + # end + # end + # end + # + # ElasticGraph::Local::RakeTasks.new( + # local_config_yaml: "config/settings/local.yaml", + # path_to_schema: "config/schema.rb" + # ) do |tasks| + # tasks.schema_definition_extension_modules = [SinceExtension] + # end + # + # @dynamic schema_definition_extension_modules, schema_definition_extension_modules= + attr_accessor :schema_definition_extension_modules + + # Whether or not to enforce the requirement that the JSON schema version is incremented every time + # dumping the JSON schemas results in a changed artifact. Defaults to `true`. + # + # @note Generally speaking, you will want this to be `true` for any ElasticGraph application that is in + # production as the versioning of JSON schemas is what supports safe schema evolution as it allows + # ElasticGraph to identify which version of the JSON schema the publishing system was operating on + # when it published an event. + # + # It can be useful to set it to `false` before your application is in production, as you do not want + # to be forced to bump the version after every single schema change while you are building an initial + # prototype. + # + # @return [Boolean] whether to require `json_schema_version` to be incremented on changes that impact `json_schemas.yaml` + # @see SchemaDefinition::API#json_schema_version + # + # @example Disable enforcement during initial prototyping + # ElasticGraph::Local::RakeTasks.new( + # local_config_yaml: "config/settings/local.yaml", + # path_to_schema: "config/schema.rb" + # ) do |tasks| + # # TODO: remove this once we're past the prototyping stage + # tasks.enforce_json_schema_version = false + # end + # + # @dynamic enforce_json_schema_version, enforce_json_schema_version= + attr_accessor :enforce_json_schema_version + + # List of Elasticsearch versions you want to be able to boot. Rake tasks will be defined for each version to support booting and + # halting Elasticsearch locally. Defaults to the versions of Elasticsearch that are exercised by the ElasticGraph test suite, as + # defined by `lib/elastic_graph/local/tested_datastore_versions.yaml`: + # + # {include:file:elasticgraph-local/lib/elastic_graph/local/tested_datastore_versions.yaml} + # + # @return [Array] list of Elasticsearch versions + # @see #opensearch_versions + # + # @example Disable Elasticsearch tasks for a project that uses OpenSearch + # ElasticGraph::Local::RakeTasks.new( + # local_config_yaml: "config/settings/local.yaml", + # path_to_schema: "config/schema.rb" + # ) do |tasks| + # tasks.elasticsearch_versions = [] + # end + # + # @dynamic elasticsearch_versions, elasticsearch_versions= + attr_accessor :elasticsearch_versions + + # List of OpenSearch versions you want to be able to boot. Rake tasks will be defined for each version to support booting and + # halting OpenSearch locally. Defaults to the versions of OpenSearch that are exercised by the ElasticGraph test suite, as + # defined by `lib/elastic_graph/local/tested_datastore_versions.yaml`: + # + # {include:file:elasticgraph-local/lib/elastic_graph/local/tested_datastore_versions.yaml} + # + # @return [Array] list of OpenSearch versions + # @see #elasticsearch_versions + # + # @example Disable OpenSearch tasks for a project that uses Elasticsearch + # ElasticGraph::Local::RakeTasks.new( + # local_config_yaml: "config/settings/local.yaml", + # path_to_schema: "config/schema.rb" + # ) do |tasks| + # tasks.opensearch_versions = [] + # end + # + # @dynamic opensearch_versions, opensearch_versions= + attr_accessor :opensearch_versions + + # Hash mapping environments (e.g. `:test`, `:dev`, etc) to port numbers for use when booting Elasticsearch or OpenSearch. The hash + # automatically includes an entry for the `:local` environment, using a port number extracted from `local_config_yaml`. + # + # @note When booting Elasticsearch/OpenSearch, Kibana (or its OpenSearch equivalent, "OpenSearch Dashboards") will also get booted, + # selecting the port by adding `10000` to the configured port. + # + # @return [Hash] mapping from environment name to port number + # + # @example Define what port to use to boot the datastore for the `:test` environment + # ElasticGraph::Local::RakeTasks.new( + # local_config_yaml: "config/settings/local.yaml", + # path_to_schema: "config/schema.rb" + # ) do |tasks| + # tasks.env_port_mapping = {test: 9999} + # end + # + # @dynamic env_port_mapping, env_port_mapping= + attr_accessor :env_port_mapping + + # IO for printing output (defaults to stdout). + # + # @return [IO] IO object used for printing output. + # + # @dynamic output, output= + attr_accessor :output + + # Maximum time (in seconds) to wait for the datastore to boot when booting it as a daemon. Defaults to 120. + # + # @return [Integer] maximum time in seconds to wait when booting Elasticsearch/OpenSearch as a daemon + # + # @dynamic daemon_timeout, daemon_timeout= + attr_accessor :daemon_timeout + + # Offset we add to a port number for the UI (e.g. Kibana or OpenSearch Dashboards). + # + # Example: if Elasticsearch/OpenSearch is running on port 9876, the UI for it will run on port 19876. + UI_PORT_OFFSET = 10_000 + + # As per https://en.wikipedia.org/wiki/Registered_port, valid user port numbers are 1024 to 49151, but + # with our UI offset we need to truncate the range further. + # + # @private + VALID_PORT_RANGE = 1024..(49151 - UI_PORT_OFFSET) + + # Register a callback for use when indexing a batch fake data. An `index_fake_data:[type]` rake task will be generated for each + # registered callback. + # + # @param type [Symbol] type of data batch. Can be the name of a GraphQL type or any other name you want to give a batch of fake data + # @yield [Array>, Array>] list the block should append to when generating data + # @yieldreturn [void] + # + # @example Register a callback to generate fake `campaigns` data + # ElasticGraph::Local::RakeTasks.new( + # local_config_yaml: "config/settings/local.yaml", + # path_to_schema: "config/schema.rb" + # ) do |tasks| + # tasks.define_fake_data_batch_for :campaigns do |batch| + # batch.concat(FactoryBot.build_list(:campaigns)) + # end + # end + def define_fake_data_batch_for(type, &block) + @fake_data_batch_generator_by_type[type] = block + end + + # @note This method uses keyword args for all required arguments. Optional task settings are instead specified using the block. + # @param local_config_yaml [String, Pathname] path to the settings YAML file for the local/development environment + # @param path_to_schema [String, Pathname] path to the Ruby schema definition file--either the only file that defines the schema + # (using `ElasticGraph.define_schema`) or the "main" schema definition file, which loads other files which further define parts of + # the schema. + # @yield [RakeTasks] instance for further configuration + # @yieldreturn [void] + def initialize(local_config_yaml:, path_to_schema:) + @local_config_yaml = local_config_yaml.to_s + + self.index_document_sizes = false + self.schema_element_name_form = :camelCase + self.schema_element_name_overrides = {} + self.derived_type_name_formats = {} + self.type_name_overrides = {} + self.enum_value_overrides_by_type = {} + self.schema_definition_extension_modules = [] + self.enforce_json_schema_version = true + self.env_port_mapping = {} + self.output = $stdout + self.daemon_timeout = 120 + + datastore_versions = ::YAML.load_file("#{__dir__}/tested_datastore_versions.yaml") + self.elasticsearch_versions = datastore_versions.fetch("elasticsearch") + self.opensearch_versions = datastore_versions.fetch("opensearch") + + @fake_data_batch_generator_by_type = {} + + yield self if block_given? + + # Default the local port from the local_config_yaml file. + self.env_port_mapping = {"local" => local_datastore_port}.merge(env_port_mapping || {}) + if (invalid_port_mapping = env_port_mapping.reject { |env, port| VALID_PORT_RANGE.cover?(port) }).any? + raise "`env_port_mapping` has invalid ports: #{invalid_port_mapping.inspect}. Valid ports must be in the #{VALID_PORT_RANGE} range." + end + + # Load admin and schema def rake tasks... + Admin::RakeTasks.from_yaml_file(local_config_yaml, output: output) + SchemaDefinition::RakeTasks.new( + index_document_sizes: index_document_sizes, + path_to_schema: path_to_schema, + schema_artifacts_directory: local_config.fetch("schema_artifacts").fetch("directory"), + schema_element_name_form: schema_element_name_form, + schema_element_name_overrides: schema_element_name_overrides, + derived_type_name_formats: derived_type_name_formats, + type_name_overrides: type_name_overrides, + enum_value_overrides_by_type: enum_value_overrides_by_type, + extension_modules: schema_definition_extension_modules, + enforce_json_schema_version: enforce_json_schema_version, + output: output + ) + + # ...then define a bunch of our own. + define_docker_tasks("Elasticsearch", "Kibana", elasticsearch_versions, /license \[[^\]]+\] mode \[[^\]]+\] - valid/) + define_docker_tasks("OpenSearch", "OpenSearch Dashboards", opensearch_versions, /o\.o\.n\.Node.+started/) + define_other_tasks + end + + private + + def define_docker_tasks(description, ui_variant, versions, ready_log_line) + variant = description.downcase.to_sym + namespace variant do + env_port_mapping.each do |env, port| + namespace env do + versions.each do |version| + namespace version do + define_docker_tasks_for_version(description, variant, ui_variant, port: port, version: version, env: env, ready_log_line: ready_log_line) + end + end + + if (max_version = versions.max_by { |v| Gem::Version.create(v) }) + define_docker_tasks_for_version(description, variant, ui_variant, port: port, version: max_version, env: env, ready_log_line: ready_log_line) + end + end + end + end + end + + def define_docker_tasks_for_version(description, variant, ui_variant, port:, version:, env:, ready_log_line:) + ui_port = port + UI_PORT_OFFSET + + docker_runner = DockerRunner.new( + variant, + port: port, + ui_port: port + UI_PORT_OFFSET, + version: version, + env: env, + ready_log_line: ready_log_line, + output: output, + daemon_timeout: daemon_timeout + ) + + desc "Boots #{description} #{version} for the #{env} environment on port #{port} (and #{ui_variant} on port #{ui_port})" + task(:boot) { docker_runner.boot } + + desc "Boots #{description} #{version} as a background daemon for the #{env} environment on port #{port} (and #{ui_variant} on port #{ui_port})" + task(:daemon) do |t| + docker_runner.boot_as_daemon(halt_command: "rake #{t.name.sub(/:\w+\z/, ":halt")}") + end + + desc "Halts the #{description} #{version} daemon for the #{env} environment" + task(:halt) { docker_runner.halt } + end + + def define_other_tasks + index_fake_data_tasks = @fake_data_batch_generator_by_type.keys.map do |type| + "index_fake_data:#{type}" + end + + datastore_to_boot = + if elasticsearch_versions.empty? && opensearch_versions.empty? + raise "Both `elasticsearch_versions` and `opensearch_versions` are empty, but we need at least one of them to have a version in order to provide the boot tasks." + elsif elasticsearch_versions.empty? + "OpenSearch" + else + "Elasticsearch" + end + + desc "Boots ElasticGraph locally from scratch: boots #{datastore_to_boot}, configures it, indexes fake data, and boots GraphiQL" + task :boot_locally, [:port, :rackup_args, :no_open] => ["#{datastore_to_boot.downcase}:local:daemon", *index_fake_data_tasks, "boot_graphiql"] + + desc "Boots ElasticGraph locally with the GraphiQL UI, and opens it in a browser." + task :boot_graphiql, [:port, :rackup_args, :no_open] => :ensure_datastore_ready_for_indexing_and_querying do |task, args| + args.with_defaults(port: 9393, rackup_args: "", no_open: false) + port = args.fetch(:port) + + # :nocov: -- we can't test `open` behavior through a test + unless args.fetch(:no_open) + fork do + sleep 3 # give the app a bit of time to boot before we try to open it. + sh "open http://localhost:#{port}/" + end + end + # :nocov: + + sh "ELASTICGRAPH_YAML_FILE=#{@local_config_yaml.shellescape} bundle exec rackup #{::File.join(__dir__.to_s, "config.ru").shellescape} --port #{port} #{args.fetch(:rackup_args)}" + end + + namespace :index_fake_data do + @fake_data_batch_generator_by_type.each do |type, generator| + desc "Indexes num_batches of #{type} fake data into the local datastore" + task type, [:num_batches] => :ensure_datastore_ready_for_indexing_and_querying do |task, args| + require "elastic_graph/local/local_indexer" + args.with_defaults(num_batches: 1) + LocalIndexer.new(@local_config_yaml, generator, output: output).index_fake_data(Integer(args[:num_batches])) + end + end + end + + task :ensure_local_datastore_running do + unless /200 OK/.match?(`curl -is localhost:#{local_datastore_port}`) + if elasticsearch_versions.empty? + raise <<~EOS + OpenSearch is not running locally. You need to start it in another terminal using this command: + + bundle exec rake opensearch:local:boot + EOS + elsif opensearch_versions.empty? + raise <<~EOS + Elasticsearch is not running locally. You need to start it in another terminal using this command: + + bundle exec rake elasticsearch:local:boot + EOS + else + raise <<~EOS + Neither Elasticsearch nor OpenSearch are running locally. You need to start one of them in another terminal using one of these commands: + + bundle exec rake elasticsearch:local:boot + bundle exec rake opensearch:local:boot + EOS + end + end + end + + task ensure_datastore_ready_for_indexing_and_querying: [ + :ensure_local_datastore_running, + "schema_artifacts:dump", + "clusters:configure:perform" + ] + + namespace "clusters:configure" do + %i[dry_run perform].each do |subtask| + desc "(after first dumping the schema artifacts)" + task subtask => [:ensure_local_datastore_running, "schema_artifacts:dump"] + end + end + end + + def local_datastore_port + @local_datastore_port ||= local_config + .fetch("datastore") + .fetch("clusters") + .fetch("main") + .fetch("url")[/localhost:(\d+)$/, 1] + .then { |port_str| Integer(port_str) } + end + + def local_config + @local_config ||= ::YAML.safe_load_file(@local_config_yaml, aliases: true) + end + end + end +end diff --git a/elasticgraph-local/lib/elastic_graph/local/tested_datastore_versions.yaml b/elasticgraph-local/lib/elastic_graph/local/tested_datastore_versions.yaml new file mode 100644 index 00000000..b8ec1a63 --- /dev/null +++ b/elasticgraph-local/lib/elastic_graph/local/tested_datastore_versions.yaml @@ -0,0 +1,8 @@ +# @markup text +# This file determines the versions the ElasticGraph CI build tests against, and is also +# used to provide the default versions offered by the `elasticgraph-local` rake tasks. +elasticsearch: +- 8.15.1 # latest version as of 2024-09-06. +opensearch: +- 2.16.0 # latest version as of 2024-09-06. +- 2.7.0 # lowest version ElasticGraph currently supports diff --git a/elasticgraph-local/sig/elastic_graph/local/docker_runner.rbs b/elasticgraph-local/sig/elastic_graph/local/docker_runner.rbs new file mode 100644 index 00000000..ce9c997e --- /dev/null +++ b/elasticgraph-local/sig/elastic_graph/local/docker_runner.rbs @@ -0,0 +1,34 @@ +module ElasticGraph + module Local + class DockerRunner + def initialize: ( + ::Symbol, + port: ::Integer, + ui_port: ::Integer, + version: ::String, + env: ::String, + ready_log_line: ::Regexp, + daemon_timeout: ::Integer, + output: io + ) -> void + + @variant: ::Symbol + @port: ::Integer + @ui_port: ::Integer + @version: ::String + @env: ::String + @ready_log_line: ::Regexp + @daemon_timeout: ::Integer + @output: io + + def boot: () -> void + def halt: () -> void + def boot_as_daemon: (halt_command: ::String) -> void + + private + + def prepare_docker_compose_run: (*::String) { (::String) -> void } -> void + def with_pipe: [T] () { (::IO, ::IO) -> T } -> T + end + end +end diff --git a/elasticgraph-local/sig/elastic_graph/local/indexing_coordinator.rbs b/elasticgraph-local/sig/elastic_graph/local/indexing_coordinator.rbs new file mode 100644 index 00000000..89211a70 --- /dev/null +++ b/elasticgraph-local/sig/elastic_graph/local/indexing_coordinator.rbs @@ -0,0 +1,21 @@ +module ElasticGraph + module Local + class IndexingCoordinator + PARALLELISM: ::Integer + + @fake_data_batch_generator: fakeDataBatchGenerator + @publish_batch: ^(::Array[::Hash[::String, untyped]]) -> void + @output: io + + def initialize: (fakeDataBatchGenerator, ?output: io) { + (::Array[::Hash[::String, untyped]]) -> void + } -> void + + def index_fake_data: (::Integer) -> void + + private + + def new_publishing_thread: (::Thread::Queue) -> ::Thread + end + end +end diff --git a/elasticgraph-local/sig/elastic_graph/local/local_indexer.rbs b/elasticgraph-local/sig/elastic_graph/local/local_indexer.rbs new file mode 100644 index 00000000..a80f4065 --- /dev/null +++ b/elasticgraph-local/sig/elastic_graph/local/local_indexer.rbs @@ -0,0 +1,10 @@ +module ElasticGraph + module Local + class LocalIndexer + @local_indexer: Indexer + @indexing_coordinator: IndexingCoordinator + def initialize: (::String, fakeDataBatchGenerator, output: io) -> void + def index_fake_data: (::Integer) -> void + end + end +end diff --git a/elasticgraph-local/sig/elastic_graph/local/rake_tasks.rbs b/elasticgraph-local/sig/elastic_graph/local/rake_tasks.rbs new file mode 100644 index 00000000..c2bfeb23 --- /dev/null +++ b/elasticgraph-local/sig/elastic_graph/local/rake_tasks.rbs @@ -0,0 +1,44 @@ +module ElasticGraph + module Local + class RakeTasks < ::Rake::TaskLib + attr_accessor index_document_sizes: bool + attr_accessor schema_element_name_form: :snake_case | :camelCase + attr_accessor schema_element_name_overrides: ::Hash[::Symbol, ::String] + attr_accessor derived_type_name_formats: ::Hash[::Symbol, ::String] + attr_accessor type_name_overrides: ::Hash[::Symbol, ::String] + attr_accessor enum_value_overrides_by_type: ::Hash[::Symbol, ::Hash[::Symbol, ::String]] + attr_accessor schema_definition_extension_modules: ::Array[::Module] + attr_accessor enforce_json_schema_version: bool + attr_accessor elasticsearch_versions: ::Array[::String] + attr_accessor opensearch_versions: ::Array[::String] + attr_accessor env_port_mapping: ::Hash[::String, ::Integer] + attr_accessor output: io + attr_accessor daemon_timeout: ::Integer + + UI_PORT_OFFSET: ::Integer + VALID_PORT_RANGE: ::Range[::Integer] + + def define_fake_data_batch_for: (::Symbol) { (::Array[::Hash[::String, untyped]]) -> void } -> void + + def initialize: ( + local_config_yaml: ::String | ::Pathname, + path_to_schema: ::String | ::Pathname + ) ?{ (RakeTasks) -> void } -> void + + @local_config_yaml: ::String + @fake_data_batch_generator_by_type: ::Hash[::Symbol, ^(::Array[::Hash[::String, untyped]]) -> void] + + private + + def define_docker_tasks: (::String, ::String, ::Array[::String], ::Regexp) -> void + def define_docker_tasks_for_version: (::String, ::Symbol, ::String, port: ::Integer, version: ::String, env: ::String, ready_log_line: ::Regexp) -> void + def define_other_tasks: () -> void + + @local_datastore_port: ::Integer? + def local_datastore_port: () -> ::Integer + + @local_config: ::Hash[::String, untyped]? + def local_config: () -> ::Hash[::String, untyped] + end + end +end diff --git a/elasticgraph-local/sig/elastic_graph/local/types.rbs b/elasticgraph-local/sig/elastic_graph/local/types.rbs new file mode 100644 index 00000000..2e8c06e5 --- /dev/null +++ b/elasticgraph-local/sig/elastic_graph/local/types.rbs @@ -0,0 +1,5 @@ +module ElasticGraph + module Local + type fakeDataBatchGenerator = ^(::Array[::Hash[::String, untyped]]) -> void + end +end diff --git a/elasticgraph-local/spec/acceptance/rake_tasks_spec.rb b/elasticgraph-local/spec/acceptance/rake_tasks_spec.rb new file mode 100644 index 00000000..90d1266b --- /dev/null +++ b/elasticgraph-local/spec/acceptance/rake_tasks_spec.rb @@ -0,0 +1,207 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/local/rake_tasks" +require "elastic_graph/schema_definition/rake_tasks" +require "json" +require "net/http" +require "pathname" +require "tmpdir" + +module ElasticGraph + module Local + RSpec.describe RakeTasks, :rake_task, :factories do + # Some tests here rely on being run from the repo root, due to paths from our + # config files being relative to the root. + around { |ex| Dir.chdir(CommonSpecHelpers::REPO_ROOT, &ex) } + + it "supports fully booting from scratch via a single `boot_locally` rake task" do + rack_port = 9620 + kill_daemon_after("rackup.pid") do |pid_file| + output = run_rake "boot_locally[#{rack_port}, --daemonize --pid #{pid_file}, no_open]", port: 9612 + + expect(output).to include( + # It boots Elasticsearch... + "Success! elasticsearch", "has been booted for the local environment", + # ...dumps schema artifacts... + "datastore_config.yaml` is already up to date", + # ...configures Elasticsearch... + "Updated index template: `widgets`", + # ...indexes a document... + "Published batch of 1 document" + ) + + # ...and then boots GraphiQL, but given how it boots with Rake's `sh`, it's not captured in `output`. + + wait_for_server_readiness(rack_port, path: "/graphql") + + # Validate that we can query the booted server! + response = query_server_on(rack_port, path: "/graphql?query=#{::CGI.escape(<<~EOS)}") + query { + widgets { + total_edge_count + } + } + EOS + + expect(response).to eq({"data" => {"widgets" => {"total_edge_count" => 1}}}) + ensure + # Ensure this doesn't "leak" the running Elasticsearch server. + run_rake "elasticsearch:local:halt", port: 9612 + end + end + + context "when the datastore is not running" do + it "gives a clear error when booting GraphiQL is attempted" do + expect { + kill_daemon_after("rackup.pid") do |pid_file| + run_rake "boot_graphiql[9621, --daemonize --pid #{pid_file}, no_open]" + end + }.to raise_error a_string_including("Neither Elasticsearch nor OpenSearch are running locally") + end + + it "gives a clear error when configuring the datastore is attempted" do + expect { + run_rake "clusters:configure:dry_run" do |t| + t.opensearch_versions = [] + end + }.to raise_error a_string_including("Elasticsearch is not running locally") + + expect { + run_rake "clusters:configure:perform" do |t| + t.opensearch_versions = [] + end + }.to raise_error a_string_including("Elasticsearch is not running locally") + end + + it "gives a clear error when indexing data locally is attempted" do + expect { + run_rake "index_fake_data:widgets[1]" do |t| + t.elasticsearch_versions = [] + end + }.to raise_error a_string_including("OpenSearch is not running locally") + end + + def run_rake(command) + super(command, port: 9617) + end + end + + describe "elasticsearch/opensearch tasks" do + it "times out if booting takes too long" do + expect { + run_rake "elasticsearch:example:8.8.1:daemon", daemon_timeout: 0.1, port: 9615 + }.to raise_error a_string_including("Timed out after 0.1 seconds.") + end + end + + def run_rake(*cli_args, port:, daemon_timeout: nil, batch_size: 1) + outer_output = nil + + config_dir = ::Pathname.new(::File.join(CommonSpecHelpers::REPO_ROOT, "config")) + + # Give a longer timeout to CI than we tolerate locally. + # :nocov: -- only one of the two sides of the ternary gets covered. + daemon_timeout ||= ENV["CI"] ? 120 : 30 + # :nocov: + + # We need to run without bundler because some tasks shell out and run `bundle exec` and + # the local bundler we're running within could interfere. + without_bundler do + super(*cli_args) do |output| + outer_output = output + + RakeTasks.new( + local_config_yaml: config_dir / "settings" / "development.yaml", + path_to_schema: config_dir / "schema.rb" + ) do |t| + t.index_document_sizes = true + t.schema_element_name_form = :snake_case + t.env_port_mapping = {"example" => port} + t.elasticsearch_versions = ["8.7.1", "8.8.1"] + t.opensearch_versions = ["2.7.0"] + t.output = output + t.daemon_timeout = daemon_timeout + + yield t if block_given? + + t.define_fake_data_batch_for(:widgets) do |batch| + batch.concat(Array.new(batch_size) { build(:widget) }) + end + end + end + end + rescue ::Timeout::Error => e + raise ::Timeout::Error.new("#{outer_output.string}\n\n#{e.message}") + end + + def without_bundler + # :nocov: -- Bundler doesn't have to be used to run our test suite, so we handle both cases here + # But only one branch is taken on a given run of the test suite. + return yield unless defined?(::Bundler) + ::Bundler.with_original_env { yield } + # :nocov: + end + + def query_server_on(port, path: "/", parse_json: true) + response = ::Net::HTTP.start("localhost", port) do |http| + http.read_timeout = 2 + http.get(path) + end + + parse_json ? ::JSON.parse(response.body) : response.body + end + + def kill_daemon_after(pid_name) + ::Dir.mktmpdir do |dir| + pid_file = "#{dir}/#{pid_name}" + + begin + yield pid_file + ensure + # :nocov: -- under normal conditions some branches here aren't used + pid = begin + Integer(::File.read(pid_file)) + rescue + nil + end + ::Process.kill(9, pid) if pid + # :nocov: + end + end + end + + def wait_for_server_readiness(port, path:) + started_waiting_at = ::Time.now + last_error = nil + + # Wait up to 30 seconds on CI or 5 seconds locally. (We give CI more time because we have occasionally seen it + # fail at 10 seconds there, and can tolerate it taking longer. Locally you want quick feedback when you run these + # tests, you want to know if a server didn't boot right away, and it doesn't need to need more than 5 seconds). + # :nocov: -- we give CI more time than we do locally, so only one branch will be covered. + iterations = ENV["CI"] ? 300 : 50 + # :nocov: + + iterations.times do + query_server_on(port, path: path, parse_json: false) + rescue Errno::ECONNREFUSED, EOFError => e + # :nocov: -- not always covered (depends on if the rack server is ready). + last_error = e + sleep 0.1 + # :nocov: + else + return + end + + # :nocov: -- only hit when the server fails to boot (which doesn't happen on a successful test run) + raise "Server on port #{port} failed to boot in #{::Time.now - started_waiting_at} seconds; Last error from #{path} was: #{last_error.class}: #{last_error.message}" + # :nocov: + end + end + end +end diff --git a/elasticgraph-local/spec/spec_helper.rb b/elasticgraph-local/spec/spec_helper.rb new file mode 100644 index 00000000..f3db6e64 --- /dev/null +++ b/elasticgraph-local/spec/spec_helper.rb @@ -0,0 +1,11 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-local`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. diff --git a/elasticgraph-local/spec/unit/elastic_graph/local/rake_tasks_spec.rb b/elasticgraph-local/spec/unit/elastic_graph/local/rake_tasks_spec.rb new file mode 100644 index 00000000..7b20ad0b --- /dev/null +++ b/elasticgraph-local/spec/unit/elastic_graph/local/rake_tasks_spec.rb @@ -0,0 +1,228 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/local/rake_tasks" +require "elastic_graph/schema_definition/rake_tasks" +require "pathname" + +module ElasticGraph + module Local + RSpec.describe RakeTasks, :rake_task do + let(:config_dir) { ::Pathname.new(::File.join(CommonSpecHelpers::REPO_ROOT, "config")) } + + it "generates a complete set of tasks when no block is provided" do + output = run_rake "-T" do + RakeTasks.new( + local_config_yaml: config_dir / "settings" / "development.yaml", + path_to_schema: config_dir / "schema.rb" + ) + end + + expected_snippet_1 = <<~EOS + rake boot_graphiql[port,rackup_args,no_open] # Boots ElasticGraph locally with the GraphiQL UI, and opens it in a browser + rake boot_locally[port,rackup_args,no_open] # Boots ElasticGraph locally from scratch: boots Elasticsearch, configures it, indexes fake data, and boots GraphiQL + rake clusters:configure:dry_run # Dry-runs the configuration of datastore clusters, including indices, settings, and scripts / (after first dumping the schema artifacts) + rake clusters:configure:perform # Performs the configuration of datastore clusters, including indices, settings, and scripts / (after first dumping the schema artifacts) + EOS + + expected_snippet_2 = <<~EOS + rake indices:drop[index_def_name,cluster_name] # Drops the specified index definition on the specified datastore cluster + rake indices:drop_prototypes # Drops all prototype index definitions on all datastore clusters + EOS + + expected_snippet_3 = <<~EOS + rake schema_artifacts:check # Checks the artifacts to make sure they are up-to-date, raising an exception if not + rake schema_artifacts:dump # Dumps all schema artifacts based on the current ElasticGraph schema definition + EOS + + # Note: we are careful to avoid asserting on exact versions here, since we don't specify any in this test and + # we want to be able to update `tested_datastore_versions.yaml` without breaking this test. + expect(output).to include(expected_snippet_1, expected_snippet_2, expected_snippet_3, *%w[ + rake elasticsearch:local:boot + rake elasticsearch:local:daemon + rake elasticsearch:local:halt + rake opensearch:local:boot + rake opensearch:local:daemon + rake opensearch:local:halt + ]) + end + + it "defines a task which indexes locally" do + output = run_rake_with_overrides "-T", "index_fake_data:" do |t| + t.define_fake_data_batch_for(:widgets) {} + end + + expect(output).to eq(<<~EOS) + rake index_fake_data:widgets[num_batches] # Indexes num_batches of widgets fake data into the local datastore + EOS + end + + describe "elasticsearch/opensearch tasks" do + it "generates a rake task for each combination of environment and elasticsearch/opensearch version" do + output = list_datastore_management_tasks + + expect(output).to eq(<<~EOS) + rake elasticsearch:example:8.7.1:boot # Boots Elasticsearch 8.7.1 for the example environment on port 9612 (and Kibana on port 19612) + rake elasticsearch:example:8.7.1:daemon # Boots Elasticsearch 8.7.1 as a background daemon for the example environment on port 9612 (and Kibana on port 19612) + rake elasticsearch:example:8.7.1:halt # Halts the Elasticsearch 8.7.1 daemon for the example environment + rake elasticsearch:example:8.8.1:boot # Boots Elasticsearch 8.8.1 for the example environment on port 9612 (and Kibana on port 19612) + rake elasticsearch:example:8.8.1:daemon # Boots Elasticsearch 8.8.1 as a background daemon for the example environment on port 9612 (and Kibana on port 19612) + rake elasticsearch:example:8.8.1:halt # Halts the Elasticsearch 8.8.1 daemon for the example environment + rake elasticsearch:example:boot # Boots Elasticsearch 8.8.1 for the example environment on port 9612 (and Kibana on port 19612) + rake elasticsearch:example:daemon # Boots Elasticsearch 8.8.1 as a background daemon for the example environment on port 9612 (and Kibana on port 19612) + rake elasticsearch:example:halt # Halts the Elasticsearch 8.8.1 daemon for the example environment + rake elasticsearch:local:8.7.1:boot # Boots Elasticsearch 8.7.1 for the local environment on port 9334 (and Kibana on port 19334) + rake elasticsearch:local:8.7.1:daemon # Boots Elasticsearch 8.7.1 as a background daemon for the local environment on port 9334 (and Kibana on port 19334) + rake elasticsearch:local:8.7.1:halt # Halts the Elasticsearch 8.7.1 daemon for the local environment + rake elasticsearch:local:8.8.1:boot # Boots Elasticsearch 8.8.1 for the local environment on port 9334 (and Kibana on port 19334) + rake elasticsearch:local:8.8.1:daemon # Boots Elasticsearch 8.8.1 as a background daemon for the local environment on port 9334 (and Kibana on port 19334) + rake elasticsearch:local:8.8.1:halt # Halts the Elasticsearch 8.8.1 daemon for the local environment + rake elasticsearch:local:boot # Boots Elasticsearch 8.8.1 for the local environment on port 9334 (and Kibana on port 19334) + rake elasticsearch:local:daemon # Boots Elasticsearch 8.8.1 as a background daemon for the local environment on port 9334 (and Kibana on port 19334) + rake elasticsearch:local:halt # Halts the Elasticsearch 8.8.1 daemon for the local environment + rake opensearch:example:2.7.0:boot # Boots OpenSearch 2.7.0 for the example environment on port 9612 (and OpenSearch Dashboards on port 19612) + rake opensearch:example:2.7.0:daemon # Boots OpenSearch 2.7.0 as a background daemon for the example environment on port 9612 (and OpenSearch Dashboards on port 19612) + rake opensearch:example:2.7.0:halt # Halts the OpenSearch 2.7.0 daemon for the example environment + rake opensearch:example:boot # Boots OpenSearch 2.7.0 for the example environment on port 9612 (and OpenSearch Dashboards on port 19612) + rake opensearch:example:daemon # Boots OpenSearch 2.7.0 as a background daemon for the example environment on port 9612 (and OpenSearch Dashboards on port 19612) + rake opensearch:example:halt # Halts the OpenSearch 2.7.0 daemon for the example environment + rake opensearch:local:2.7.0:boot # Boots OpenSearch 2.7.0 for the local environment on port 9334 (and OpenSearch Dashboards on port 19334) + rake opensearch:local:2.7.0:daemon # Boots OpenSearch 2.7.0 as a background daemon for the local environment on port 9334 (and OpenSearch Dashboards on port 19334) + rake opensearch:local:2.7.0:halt # Halts the OpenSearch 2.7.0 daemon for the local environment + rake opensearch:local:boot # Boots OpenSearch 2.7.0 for the local environment on port 9334 (and OpenSearch Dashboards on port 19334) + rake opensearch:local:daemon # Boots OpenSearch 2.7.0 as a background daemon for the local environment on port 9334 (and OpenSearch Dashboards on port 19334) + rake opensearch:local:halt # Halts the OpenSearch 2.7.0 daemon for the local environment + EOS + end + + it "raises an error when a port number is too low" do + expect { + list_datastore_management_tasks do |t| + t.env_port_mapping = {local: "123"} + end + }.to raise_error a_string_including('`env_port_mapping` has invalid ports: {:local=>"123"}') + end + + it "raises an error when a port number is too high" do + expect { + list_datastore_management_tasks do |t| + t.env_port_mapping = {local: "45000"} + end + }.to raise_error a_string_including('`env_port_mapping` has invalid ports: {:local=>"45000"}') + end + + context "when `opensearch_versions` is empty" do + def run_rake_with_overrides(*cli_args) + super do |t| + t.opensearch_versions = [] + end + end + + it "omits the `opensearch:` tasks" do + output = list_datastore_management_tasks + + expect(output).to eq(<<~EOS) + rake elasticsearch:example:8.7.1:boot # Boots Elasticsearch 8.7.1 for the example environment on port 9612 (and Kibana on port 19612) + rake elasticsearch:example:8.7.1:daemon # Boots Elasticsearch 8.7.1 as a background daemon for the example environment on port 9612 (and Kibana on port 19612) + rake elasticsearch:example:8.7.1:halt # Halts the Elasticsearch 8.7.1 daemon for the example environment + rake elasticsearch:example:8.8.1:boot # Boots Elasticsearch 8.8.1 for the example environment on port 9612 (and Kibana on port 19612) + rake elasticsearch:example:8.8.1:daemon # Boots Elasticsearch 8.8.1 as a background daemon for the example environment on port 9612 (and Kibana on port 19612) + rake elasticsearch:example:8.8.1:halt # Halts the Elasticsearch 8.8.1 daemon for the example environment + rake elasticsearch:example:boot # Boots Elasticsearch 8.8.1 for the example environment on port 9612 (and Kibana on port 19612) + rake elasticsearch:example:daemon # Boots Elasticsearch 8.8.1 as a background daemon for the example environment on port 9612 (and Kibana on port 19612) + rake elasticsearch:example:halt # Halts the Elasticsearch 8.8.1 daemon for the example environment + rake elasticsearch:local:8.7.1:boot # Boots Elasticsearch 8.7.1 for the local environment on port 9334 (and Kibana on port 19334) + rake elasticsearch:local:8.7.1:daemon # Boots Elasticsearch 8.7.1 as a background daemon for the local environment on port 9334 (and Kibana on port 19334) + rake elasticsearch:local:8.7.1:halt # Halts the Elasticsearch 8.7.1 daemon for the local environment + rake elasticsearch:local:8.8.1:boot # Boots Elasticsearch 8.8.1 for the local environment on port 9334 (and Kibana on port 19334) + rake elasticsearch:local:8.8.1:daemon # Boots Elasticsearch 8.8.1 as a background daemon for the local environment on port 9334 (and Kibana on port 19334) + rake elasticsearch:local:8.8.1:halt # Halts the Elasticsearch 8.8.1 daemon for the local environment + rake elasticsearch:local:boot # Boots Elasticsearch 8.8.1 for the local environment on port 9334 (and Kibana on port 19334) + rake elasticsearch:local:daemon # Boots Elasticsearch 8.8.1 as a background daemon for the local environment on port 9334 (and Kibana on port 19334) + rake elasticsearch:local:halt # Halts the Elasticsearch 8.8.1 daemon for the local environment + EOS + end + + it "uses an `elasticsearch:` task for `boot_locally`" do + expect { + run_rake_with_overrides("boot_locally", "--dry-run") + }.to output(a_string_including("Invoke elasticsearch:local:daemon").and(excluding("opensearch"))).to_stderr + end + end + + context "when `elasticsearch_versions` is empty" do + def run_rake_with_overrides(*cli_args) + super do |t| + t.elasticsearch_versions = [] + end + end + + it "omits the `elasticsearch:` tasks" do + output = list_datastore_management_tasks + + expect(output).to eq(<<~EOS) + rake opensearch:example:2.7.0:boot # Boots OpenSearch 2.7.0 for the example environment on port 9612 (and OpenSearch Dashboards on port 19612) + rake opensearch:example:2.7.0:daemon # Boots OpenSearch 2.7.0 as a background daemon for the example environment on port 9612 (and OpenSearch Dashboards on port 19612) + rake opensearch:example:2.7.0:halt # Halts the OpenSearch 2.7.0 daemon for the example environment + rake opensearch:example:boot # Boots OpenSearch 2.7.0 for the example environment on port 9612 (and OpenSearch Dashboards on port 19612) + rake opensearch:example:daemon # Boots OpenSearch 2.7.0 as a background daemon for the example environment on port 9612 (and OpenSearch Dashboards on port 19612) + rake opensearch:example:halt # Halts the OpenSearch 2.7.0 daemon for the example environment + rake opensearch:local:2.7.0:boot # Boots OpenSearch 2.7.0 for the local environment on port 9334 (and OpenSearch Dashboards on port 19334) + rake opensearch:local:2.7.0:daemon # Boots OpenSearch 2.7.0 as a background daemon for the local environment on port 9334 (and OpenSearch Dashboards on port 19334) + rake opensearch:local:2.7.0:halt # Halts the OpenSearch 2.7.0 daemon for the local environment + rake opensearch:local:boot # Boots OpenSearch 2.7.0 for the local environment on port 9334 (and OpenSearch Dashboards on port 19334) + rake opensearch:local:daemon # Boots OpenSearch 2.7.0 as a background daemon for the local environment on port 9334 (and OpenSearch Dashboards on port 19334) + rake opensearch:local:halt # Halts the OpenSearch 2.7.0 daemon for the local environment + EOS + end + + it "uses an `opensearch:` task for `boot_locally`" do + expect { + run_rake_with_overrides("boot_locally", "--dry-run") + }.to output(a_string_including("Invoke opensearch:local:daemon").and(excluding("elasticsearch"))).to_stderr + end + end + + context "when both `opensearch_versions` and `elasticsearch_versions` are empty" do + def run_rake_with_overrides(*cli_args) + super do |t| + t.elasticsearch_versions = [] + t.opensearch_versions = [] + end + end + + it "raises an error to indicate that the user needs to select some versions" do + expect { + list_datastore_management_tasks + }.to raise_error a_string_including("Both `elasticsearch_versions` and `opensearch_versions` are empty") + end + end + + def list_datastore_management_tasks(&block) + run_rake_with_overrides("-T", "search:", &block) + end + end + + def run_rake_with_overrides(*cli_args) + run_rake(*cli_args) do |output| + RakeTasks.new( + local_config_yaml: config_dir / "settings" / "development.yaml", + path_to_schema: config_dir / "schema.rb" + ) do |t| + t.env_port_mapping = {"example" => 9612} + t.elasticsearch_versions = ["8.7.1", "8.8.1"] + t.opensearch_versions = ["2.7.0"] + t.output = output + + yield t if block_given? + end + end + end + end + end +end diff --git a/elasticgraph-opensearch/.rspec b/elasticgraph-opensearch/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-opensearch/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-opensearch/.yardopts b/elasticgraph-opensearch/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-opensearch/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-opensearch/Gemfile b/elasticgraph-opensearch/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-opensearch/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-opensearch/LICENSE.txt b/elasticgraph-opensearch/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-opensearch/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-opensearch/README.md b/elasticgraph-opensearch/README.md new file mode 100644 index 00000000..033fdd65 --- /dev/null +++ b/elasticgraph-opensearch/README.md @@ -0,0 +1,3 @@ +# ElasticGraph::OpenSearch + +Wraps the official OpenSearch client for use by ElasticGraph. diff --git a/elasticgraph-opensearch/elasticgraph-opensearch.gemspec b/elasticgraph-opensearch/elasticgraph-opensearch.gemspec new file mode 100644 index 00000000..cb6893b0 --- /dev/null +++ b/elasticgraph-opensearch/elasticgraph-opensearch.gemspec @@ -0,0 +1,20 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :datastore_adapter) do |spec, eg_version| + spec.summary = "Wraps the OpenSearch client for use by ElasticGraph." + + spec.add_dependency "elasticgraph-support", eg_version + spec.add_dependency "faraday", "~> 2.12" + spec.add_dependency "faraday-retry", "~> 2.2" + spec.add_dependency "opensearch-ruby", "~> 3.4" + + spec.add_development_dependency "httpx", "~> 1.3" +end diff --git a/elasticgraph-opensearch/lib/elastic_graph/opensearch/client.rb b/elasticgraph-opensearch/lib/elastic_graph/opensearch/client.rb new file mode 100644 index 00000000..b86fc1bb --- /dev/null +++ b/elasticgraph-opensearch/lib/elastic_graph/opensearch/client.rb @@ -0,0 +1,220 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/errors" +require "elastic_graph/support/faraday_middleware/msearch_using_get_instead_of_post" +require "elastic_graph/support/faraday_middleware/support_timeouts" +require "elastic_graph/support/hash_util" +require "faraday" +require "faraday/retry" +require "opensearch" + +module ElasticGraph + # @private + module OpenSearch + # @private + class Client + # @dynamic cluster_name + attr_reader :cluster_name + + def initialize(cluster_name, url:, faraday_adapter: nil, retry_on_failure: 3, logger: nil) + @cluster_name = cluster_name + + @raw_client = ::OpenSearch::Client.new( + adapter: faraday_adapter, + url: url, + retry_on_failure: retry_on_failure, + # We use `logger` for both the tracer and logger to log everything we can. While the trace and log output do overlap, one is + # not a strict superset of the other (for example, warnings go to `logger`, while full request bodies go to `tracer`). + logger: logger, + tracer: logger + ) do |faraday| + faraday.use Support::FaradayMiddleware::MSearchUsingGetInsteadOfPost + faraday.use Support::FaradayMiddleware::SupportTimeouts + + # Note: this overrides the default retry exceptions, which includes `Faraday::TimeoutError`. + # That's important because we do NOT want a retry on timeout -- a timeout indicates a slow, + # expensive query, and is transformed to a `Errors::RequestExceededDeadlineError` by `SupportTimeouts`, + # anyway. + # + # In addition, it's worth noting that the retry middleware ONLY retries known idempotent HTTP + # methods (e.g. get/put/delete/head/options). POST requests will not be retried. We could + # configure it to make it retry POSTs but we'd need to do an analysis of all ElasticGraph requests to + # make sure all POST requests are truly idempotent, and at least for now, it's sufficient to skip + # any POST requests we make. + faraday.request :retry, + exceptions: [::Faraday::ConnectionFailed, ::Faraday::RetriableResponse], + max: retry_on_failure, + retry_statuses: [500, 502, 503] # Internal Server Error, Bad Gateway, Service Unavailable + + yield faraday if block_given? + end + + # Here we call `app` on each Faraday connection as a way to force it to resolve + # all configured middlewares and adapters. If it cannot load a required dependency + # (e.g. `httpx`), it'll fail fast with a clear error. + # + # Without this, we would instead get an error when the client was used to make + # a request for the first time, which isn't as ideal. + @raw_client.transport.transport.connections.each { |c| c.connection.app } + end + + # Cluster APIs + + def get_cluster_health + transform_errors { |c| c.cluster.health } + end + + def get_node_os_stats + transform_errors { |c| c.nodes.stats(metric: "os") } + end + + def get_flat_cluster_settings + transform_errors { |c| c.cluster.get_settings(flat_settings: true) } + end + + # We only support persistent settings here because the Elasticsearch docs recommend against using transient settings: + # https://www.elastic.co/guide/en/elasticsearch/reference/8.13/cluster-update-settings.html + # + # > We no longer recommend using transient cluster settings. Use persistent cluster settings instead. If a cluster becomes unstable, + # > transient settings can clear unexpectedly, resulting in a potentially undesired cluster configuration. + # + # The OpenSearch documentation doesn't specifically mention this, but the same principle applies. + def put_persistent_cluster_settings(settings) + transform_errors { |c| c.cluster.put_settings(body: {persistent: settings}) } + end + + # Script APIs + + # Gets the script with the given ID. Returns `nil` if the script does not exist. + def get_script(id:) + transform_errors { |c| c.get_script(id: id) } + rescue ::OpenSearch::Transport::Transport::Errors::NotFound + nil + end + + def put_script(id:, body:, context:) + transform_errors { |c| c.put_script(id: id, body: body, context: context) } + end + + def delete_script(id:) + transform_errors { |c| c.delete_script(id: id) } + rescue ::OpenSearch::Transport::Transport::Errors::NotFound + # it's ok if it's already not there. + end + + # Index Template APIs + + def get_index_template(index_template_name) + transform_errors do |client| + client.indices.get_index_template(name: index_template_name) + .fetch("index_templates").to_h do |entry| + index_template = entry.fetch("index_template") + + # OpenSearch ignores `flat_settings` on the `/_index_template` API (but _only_ returns flattened settings from the index + # API). Here we flatten the settings to align with the flattened form ElasticGraph expects and uses everywhere. + flattened_settings = Support::HashUtil.flatten_and_stringify_keys(index_template.fetch("template").fetch("settings")) + + index_template = index_template.merge({ + "template" => index_template.fetch("template").merge({ + "settings" => flattened_settings + }) + }) + + [entry.fetch("name"), index_template] + end.dig(index_template_name) || {} + end + rescue ::OpenSearch::Transport::Transport::Errors::NotFound + {} + end + + def put_index_template(name:, body:) + transform_errors { |c| c.indices.put_index_template(name: name, body: body) } + end + + def delete_index_template(index_template_name) + transform_errors { |c| c.indices.delete_index_template(name: [index_template_name], ignore: [404]) } + end + + # Index APIs + + def get_index(index_name) + transform_errors do |client| + client.indices.get( + index: index_name, + ignore_unavailable: true, + flat_settings: true + )[index_name] || {} + end + end + + def list_indices_matching(index_expression) + transform_errors do |client| + client + .cat + .indices(index: index_expression, format: "json", h: ["index"]) + .map { |index_hash| index_hash.fetch("index") } + end + end + + def create_index(index:, body:) + transform_errors { |c| c.indices.create(index: index, body: body) } + end + + def put_index_mapping(index:, body:) + transform_errors { |c| c.indices.put_mapping(index: index, body: body) } + end + + def put_index_settings(index:, body:) + transform_errors { |c| c.indices.put_settings(index: index, body: body) } + end + + def delete_indices(*index_names) + # `allow_no_indices: true` is needed when we attempt to delete a non-existing index to avoid errors. For rollover indices, + # when we delete the actual indices, we will always perform a wildcard deletion, and `allow_no_indices: true` is needed. + # + # Note that the Elasticsearch API documentation[^1] says that `allow_no_indices` defaults to `true` but a Elasticsearch Ruby + # client code comment[^2] says it defaults to `false`. Regardless, we don't want to rely on the default behavior that could change. + # + # [^1]: https://www.elastic.co/guide/en/elasticsearch/reference/8.12/indices-delete-index.html#delete-index-api-query-params + # [^2]: https://github.com/elastic/elasticsearch-ruby/blob/8.12/elasticsearch-api/lib/elasticsearch/api/actions/indices/delete.rb#L31 + transform_errors do |client| + client.indices.delete(index: index_names, ignore_unavailable: true, allow_no_indices: true) + end + end + + # Document APIs + + def msearch(body:, headers: nil) + transform_errors { |c| c.msearch(body: body, headers: headers) } + end + + def bulk(body:, refresh: false) + transform_errors { |c| c.bulk(body: body, filter_path: DATASTORE_BULK_FILTER_PATH, refresh: refresh) } + end + + # Synchronously deletes all documents in the cluster. Intended for tests to give ourselves a clean slate. + # Supports an `index` argument so the caller can limit the deletion to a specific "scope" (e.g. a set of indices with a common prefix). + # + # Overrides `scroll` to `10s` to avoid getting a "Trying to create too many scroll contexts" error, as discussed here: + # https://discuss.elastic.co/t/too-many-scroll-contexts-with-update-by-query-and-or-delete-by-query/282325/1 + def delete_all_documents(index: "_all") + transform_errors { |c| c.delete_by_query(index: index, body: {query: {match_all: _ = {}}}, refresh: true, scroll: "10s") } + end + + private + + def transform_errors + yield @raw_client + rescue ::OpenSearch::Transport::Transport::Errors::BadRequest => ex + raise Errors::BadDatastoreRequest, ex.message + end + end + end +end diff --git a/elasticgraph-opensearch/sig/elastic_graph/opensearch/client.rbs b/elasticgraph-opensearch/sig/elastic_graph/opensearch/client.rbs new file mode 100644 index 00000000..599911b3 --- /dev/null +++ b/elasticgraph-opensearch/sig/elastic_graph/opensearch/client.rbs @@ -0,0 +1,23 @@ +module ElasticGraph + module OpenSearch + class Client + include DatastoreCore::_Client + extend DatastoreCore::_ClientClass + + def initialize: ( + ::String, + url: ::String, + ?faraday_adapter: ::Symbol?, + ?retry_on_failure: ::Integer, + ?logger: ::Logger? + ) ?{ (::Faraday::RackBuilder) -> void } -> void + + private + + @cluster_name: ::String + @raw_client: ::OpenSearch::Client + + def transform_errors: [T] () { (::OpenSearch::Client) -> T } -> T + end + end +end diff --git a/elasticgraph-opensearch/sig/opensearch.rbs b/elasticgraph-opensearch/sig/opensearch.rbs new file mode 100644 index 00000000..4ec9baa9 --- /dev/null +++ b/elasticgraph-opensearch/sig/opensearch.rbs @@ -0,0 +1,72 @@ +module OpenSearch + type stringOrSymbolHash = ::Hash[(::String | ::Symbol), untyped] + + module API + module Cat + class CatClient + def indices: (?index: ::String, ?format: ::String, ?h: ::Array[::String]) -> ::Array[::Hash[::String, untyped]] + end + end + + module Cluster + class ClusterClient + def health: () -> ::Hash[::String, untyped] + def get_settings: (?flat_settings: bool) -> ::Hash[::String, untyped] + def put_settings: (body: stringOrSymbolHash) -> void + end + end + + module Indices + class IndicesClient + def get_index_template: (name: ::String, ?flat_settings: bool) -> ::Hash[::String, untyped] + def put_index_template: (name: ::String, body: stringOrSymbolHash) -> void + def delete_index_template: (name: ::Array[::String], ?ignore: [::Integer]) -> void + def get: (index: ::String, ?ignore_unavailable: bool, ?flat_settings: bool) -> ::Hash[::String, untyped] + def create: (index: ::String, body: stringOrSymbolHash) -> void + def put_mapping: (index: ::String, body: stringOrSymbolHash) -> void + def put_settings: (index: ::String, body: stringOrSymbolHash) -> void + def delete: (index: ::Array[::String], ?ignore_unavailable: bool, ?allow_no_indices: bool) -> void + end + end + + module Nodes + class NodesClient + def stats: (metric: ::String) -> ::Hash[::String, untyped] + end + end + end + + class Client + def initialize: ( + url: ::String, + ?retry_on_failure: ::Integer, + ?adapter: ::Symbol?, + ?logger: ::Logger?, + ?tracer: ::Logger? + ) { (::Faraday::RackBuilder) -> void } -> void + + def transport: () -> untyped + def get_script: (id: ::String) -> ::Hash[::String, untyped] + def put_script: (id: ::String, body: stringOrSymbolHash, ?context: (::Symbol | ::String)) -> ::Hash[::String, untyped] + def delete_script: (id: ::String) -> ::Hash[::String, untyped] + def msearch: (body: ::Array[stringOrSymbolHash], ?headers: ::Hash[::String, untyped]?) -> ::Hash[::String, untyped] + def bulk: (body: ::Array[stringOrSymbolHash], ?filter_path: ::String, ?refresh: bool) -> ::Hash[::String, untyped] + def delete_by_query: (index: ::String, ?body: stringOrSymbolHash, ?refresh: bool, ?scroll: ::String) -> void + def cat: () -> API::Cat::CatClient + def cluster: () -> API::Cluster::ClusterClient + def indices: () -> API::Indices::IndicesClient + def nodes: () -> API::Nodes::NodesClient + end + + module Transport + module Transport + module Errors + class BadRequest < StandardError + end + + class NotFound < StandardError + end + end + end + end +end diff --git a/elasticgraph-opensearch/spec/spec_helper.rb b/elasticgraph-opensearch/spec/spec_helper.rb new file mode 100644 index 00000000..3e2aeb65 --- /dev/null +++ b/elasticgraph-opensearch/spec/spec_helper.rb @@ -0,0 +1,11 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-opensearch`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. diff --git a/elasticgraph-opensearch/spec/unit/elastic_graph/opensearch/client_spec.rb b/elasticgraph-opensearch/spec/unit/elastic_graph/opensearch/client_spec.rb new file mode 100644 index 00000000..f0e130c5 --- /dev/null +++ b/elasticgraph-opensearch/spec/unit/elastic_graph/opensearch/client_spec.rb @@ -0,0 +1,109 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/opensearch/client" +require "elastic_graph/spec_support/datastore_client_shared_examples" + +module ElasticGraph + module OpenSearch + RSpec.describe Client do + it_behaves_like "a datastore client" do + it "flattens the settings from `get_index_template` because OpenSearch appears to ignore the `flat_settings` argument" do + client = build_client({get_index_template_my_template: {"index_templates" => [{ + "name" => "my_template", + "index_template" => { + "template" => { + "mapping" => "the_mapping", + "settings" => {"index" => {"foo" => 1, "bar" => {"bazz" => 2}}} + }, + "index_patterns" => ["foo*"] + } + }]}}) + + expect(client.get_index_template("my_template")).to eq({ + "index_patterns" => ["foo*"], + "template" => { + "settings" => {"index.foo" => 1, "index.bar.bazz" => 2}, + "mapping" => "the_mapping" + } + }) + end + + def define_stubs(stub, requested_stubs) + stub.get("/") do |env| + response_for({version: {number: "2.12.0", distribution: "opensearch"}}, env) + end + + requested_stubs.each do |stub_name, body| + case stub_name + + # Cluster APIs + in :get_cluster_health + stub.get("/_cluster/health") { |env| response_for(body, env) } + in :get_node_os_stats + stub.get("/_nodes/stats/os") { |env| response_for(body, env) } + in :get_flat_cluster_settings + stub.get("/_cluster/settings?flat_settings=true") { |env| response_for(body, env) } + in :put_persistent_cluster_settings + stub.put("/_cluster/settings") { |env| response_for(body, env) } + + # Script APIs + in :get_script_123 + stub.get("/_scripts/123") { |env| response_for(body, env) } + in :put_script_123 + stub.put("/_scripts/123/update") { |env| response_for(body, env) } + in :delete_script_123 + stub.delete("/_scripts/123") { |env| response_for(body, env) } + + # Index Template APIs + in :get_index_template_my_template + stub.get("/_index_template/my_template") { |env| response_for(body, env) } + in :put_index_template_my_template + stub.put("/_index_template/my_template") { |env| response_for(body, env) } + in :delete_index_template_my_template + stub.delete("/_index_template/my_template") { |env| response_for(body, env) } + + # Index APIs + in :get_index_my_index + stub.get("/my_index?flat_settings=true&ignore_unavailable=true") { |env| response_for(body, env) } + in :list_indices_matching_foo + stub.get("/_cat/indices/foo%2A?format=json&h=index") { |env| response_for(body, env) } + in :create_index_my_index + stub.put("/my_index") { |env| response_for(body, env) } + in :put_index_mapping_my_index + stub.put("/my_index/_mappings") { |env| response_for(body, env) } + in :put_index_settings_my_index + stub.put("/my_index/_settings") { |env| response_for(body, env) } + in :delete_indices_ind1_ind2 + stub.delete("/ind1,ind2?allow_no_indices=true&ignore_unavailable=true") { |env| response_for(body, env) } + + # Document APIs + in :get_msearch + stub.get("/_msearch") do |env| + env.request.timeout ? raise(::Faraday::TimeoutError) : response_for(body, env) + end + in :post_bulk + stub.post("/_bulk?filter_path=items.%2A.status%2Citems.%2A.result%2Citems.%2A.error&refresh=false") do |env| + response_for(body, env) + end + in :delete_all_documents + stub.post("/_all/_delete_by_query?refresh=true&scroll=10s") { |env| response_for(body, env) } + in :delete_test_env_7_documents + stub.post("/test_env_7_%2A/_delete_by_query?refresh=true&scroll=10s") { |env| response_for(body, env) } + + else + # :nocov: -- none of our current tests hit this case + raise "Unexpected stub tag: #{stub_name.inspect}" + # :nocov: + end + end + end + end + end + end +end diff --git a/elasticgraph-query_interceptor/.rspec b/elasticgraph-query_interceptor/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-query_interceptor/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-query_interceptor/.yardopts b/elasticgraph-query_interceptor/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-query_interceptor/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-query_interceptor/Gemfile b/elasticgraph-query_interceptor/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-query_interceptor/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-query_interceptor/LICENSE.txt b/elasticgraph-query_interceptor/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-query_interceptor/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-query_interceptor/README.md b/elasticgraph-query_interceptor/README.md new file mode 100644 index 00000000..1a3ad5e8 --- /dev/null +++ b/elasticgraph-query_interceptor/README.md @@ -0,0 +1,48 @@ +# ElasticGraph::QueryInterceptor + +An ElasticGraph extension library that lets you intercept datastore +queries before they are submitted in order to customize/modify them some +how. + +## Setup + +First, add `elasticgraph-query_interceptor` to your `Gemfile`: + +``` ruby +gem "elasticgraph-query_interceptor" +``` + +Next, configure this library in your ElasticGraph config YAML files. +An optional "config" dictionary can be provided to pass in values to +your interceptor when it is initialized. + +``` yaml +extension_modules: +- require_path: elastic_graph/query_interceptor/graphql_extension + extension_name: ElasticGraph::QueryInterceptor::GraphQLExtension +query_interceptor: + interceptors: + - require_path: ./my_app/example_interceptor + extension_name: MyApp::ExampleInterceptor + config: # Optional + foo: bar +``` + +Define your interceptors at the configured paths. Each interceptor must +implement this interface: + +``` ruby +module YourApp + class ExampleInterceptor + def initialize(elasticgraph_graphql:, config:) + # elasticgraph_graphql is the `ElasticGraph::GraphQL` instance and has access + # to things like the datastore client in case you need it in your interceptor. + end + + def intercept(query, field:, args:, http_request:, context:) + # Call `query.merge_with(...)` as desired to merge in query overrides like filters. + # This method must return a query. + end + end +end +``` diff --git a/elasticgraph-query_interceptor/elasticgraph-query_interceptor.gemspec b/elasticgraph-query_interceptor/elasticgraph-query_interceptor.gemspec new file mode 100644 index 00000000..b74cd3e9 --- /dev/null +++ b/elasticgraph-query_interceptor/elasticgraph-query_interceptor.gemspec @@ -0,0 +1,20 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :extension) do |spec, eg_version| + spec.summary = "An ElasticGraph extension for intercepting datastore queries." + + spec.add_dependency "elasticgraph-graphql", eg_version + spec.add_dependency "elasticgraph-schema_artifacts", eg_version + + spec.add_development_dependency "elasticgraph-elasticsearch", eg_version + spec.add_development_dependency "elasticgraph-opensearch", eg_version + spec.add_development_dependency "elasticgraph-schema_definition", eg_version +end diff --git a/elasticgraph-query_interceptor/lib/elastic_graph/query_interceptor/config.rb b/elasticgraph-query_interceptor/lib/elastic_graph/query_interceptor/config.rb new file mode 100644 index 00000000..c9292fc5 --- /dev/null +++ b/elasticgraph-query_interceptor/lib/elastic_graph/query_interceptor/config.rb @@ -0,0 +1,61 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/extension_loader" + +module ElasticGraph + module QueryInterceptor + # Defines configuration for elasticgraph-query_interceptor + class Config < ::Data.define(:interceptors) + # Builds Config from parsed YAML config. + def self.from_parsed_yaml(parsed_config_hash, parsed_runtime_metadata_hashes: []) + interceptor_hashes = parsed_runtime_metadata_hashes.flat_map { |h| h["interceptors"] || [] } + + if (extension_config = parsed_config_hash["query_interceptor"]) + extra_keys = extension_config.keys - EXPECTED_KEYS + + unless extra_keys.empty? + raise Errors::ConfigError, "Unknown `query_interceptor` config settings: #{extra_keys.join(", ")}" + end + + interceptor_hashes += extension_config.fetch("interceptors") + end + + loader = SchemaArtifacts::RuntimeMetadata::ExtensionLoader.new(InterceptorInterface) + + interceptors = interceptor_hashes.map do |hash| + empty_config = {} # : ::Hash[::Symbol, untyped] + ext = loader.load(hash.fetch("extension_name"), from: hash.fetch("require_path"), config: empty_config) + config = hash["config"] || {} # : ::Hash[::String, untyped] + InterceptorData.new(klass: ext.extension_class, config: config) + end + + new(interceptors) + end + + DEFAULT = new([]) + EXPECTED_KEYS = members.map(&:to_s) + + # Defines a data structure to hold interceptor klass and config + InterceptorData = ::Data.define(:klass, :config) + + # Defines the interceptor interface, which our extension loader will validate against. + class InterceptorInterface + def initialize(elasticgraph_graphql:, config:) + # must be defined, but nothing to do + end + + def intercept(query, field:, args:, http_request:, context:) + # :nocov: -- must return a query to satisfy Steep type checking but never called + query + # :nocov: + end + end + end + end +end diff --git a/elasticgraph-query_interceptor/lib/elastic_graph/query_interceptor/datastore_query_adapter.rb b/elasticgraph-query_interceptor/lib/elastic_graph/query_interceptor/datastore_query_adapter.rb new file mode 100644 index 00000000..cf97bd01 --- /dev/null +++ b/elasticgraph-query_interceptor/lib/elastic_graph/query_interceptor/datastore_query_adapter.rb @@ -0,0 +1,28 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module QueryInterceptor + class DatastoreQueryAdapter + # @dynamic interceptors + attr_reader :interceptors + + def initialize(interceptors) + @interceptors = interceptors + end + + def call(query:, args:, lookahead:, field:, context:) + http_request = context[:http_request] + + interceptors.reduce(query) do |accum, interceptor| + interceptor.intercept(accum, field: field, args: args, http_request: http_request, context: context) + end + end + end + end +end diff --git a/elasticgraph-query_interceptor/lib/elastic_graph/query_interceptor/graphql_extension.rb b/elasticgraph-query_interceptor/lib/elastic_graph/query_interceptor/graphql_extension.rb new file mode 100644 index 00000000..af476ae4 --- /dev/null +++ b/elasticgraph-query_interceptor/lib/elastic_graph/query_interceptor/graphql_extension.rb @@ -0,0 +1,32 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/query_interceptor/config" +require "elastic_graph/query_interceptor/datastore_query_adapter" +require "elastic_graph/support/hash_util" + +module ElasticGraph + module QueryInterceptor + module GraphQLExtension + def datastore_query_adapters + @datastore_query_adapters ||= begin + runtime_metadata_configs = runtime_metadata.graphql_extension_modules.filter_map do |ext_mod| + Support::HashUtil.stringify_keys(ext_mod.extension_config) if ext_mod.extension_class == GraphQLExtension + end + + interceptors = Config + .from_parsed_yaml(config.extension_settings, parsed_runtime_metadata_hashes: runtime_metadata_configs) + .interceptors + .map { |data| data.klass.new(elasticgraph_graphql: self, config: data.config) } + + super + [DatastoreQueryAdapter.new(interceptors)] + end + end + end + end +end diff --git a/elasticgraph-query_interceptor/sig/elastic_graph/query_interceptor/config.rbs b/elasticgraph-query_interceptor/sig/elastic_graph/query_interceptor/config.rbs new file mode 100644 index 00000000..74b7472f --- /dev/null +++ b/elasticgraph-query_interceptor/sig/elastic_graph/query_interceptor/config.rbs @@ -0,0 +1,52 @@ +module ElasticGraph + module QueryInterceptor + type interceptorClass = Class & _InterceptorFactory + + class ConfigSupertype + attr_reader interceptors: ::Array[InterceptorDataSuperType] + + def self.new: (::Array[InterceptorDataSuperType]) -> Config + def with: (?interceptors: ::Array[InterceptorDataSuperType]) -> Config + def self.members: () -> ::Array[::Symbol] + end + + class Config < ConfigSupertype + def self.from_parsed_yaml: (parsedYamlSettings, ?parsed_runtime_metadata_hashes: ::Array[::Hash[::String, untyped]]) -> Config + + DEFAULT: Config + EXPECTED_KEYS: ::Array[::String] + + class InterceptorData < InterceptorDataSuperType + end + + class InterceptorInterface + include _Interceptor + end + end + + class InterceptorDataSuperType + attr_reader klass: interceptorClass + attr_reader config: ::Hash[::String, untyped] + + def initialize: ( + klass: ::ElasticGraph::SchemaArtifacts::RuntimeMetadata::extensionClass, + config: ::Hash[::String, untyped]) -> void + end + + interface _InterceptorFactory + def new: (elasticgraph_graphql: ElasticGraph::GraphQL, config: ::Hash[::String, untyped]) -> _Interceptor + end + + interface _Interceptor + def initialize: (elasticgraph_graphql: ElasticGraph::GraphQL, config: ::Hash[::String, untyped]) -> void + + def intercept: ( + GraphQL::DatastoreQuery, + field: GraphQL::Schema::Field, + args: ::Hash[::String, untyped], + http_request: GraphQL::HTTPRequest, + context: ::GraphQL::Query::Context + ) -> GraphQL::DatastoreQuery + end + end +end diff --git a/elasticgraph-query_interceptor/sig/elastic_graph/query_interceptor/datastore_query_adapter.rbs b/elasticgraph-query_interceptor/sig/elastic_graph/query_interceptor/datastore_query_adapter.rbs new file mode 100644 index 00000000..ce296dc5 --- /dev/null +++ b/elasticgraph-query_interceptor/sig/elastic_graph/query_interceptor/datastore_query_adapter.rbs @@ -0,0 +1,10 @@ +module ElasticGraph + module QueryInterceptor + class DatastoreQueryAdapter + include GraphQL::_QueryAdapter + + attr_reader interceptors: ::Array[_Interceptor] + def initialize: (::Array[_Interceptor]) -> void + end + end +end diff --git a/elasticgraph-query_interceptor/sig/elastic_graph/query_interceptor/graphql_extension.rbs b/elasticgraph-query_interceptor/sig/elastic_graph/query_interceptor/graphql_extension.rbs new file mode 100644 index 00000000..c75b0183 --- /dev/null +++ b/elasticgraph-query_interceptor/sig/elastic_graph/query_interceptor/graphql_extension.rbs @@ -0,0 +1,6 @@ +module ElasticGraph + module QueryInterceptor + module GraphQLExtension: GraphQL + end + end +end diff --git a/elasticgraph-query_interceptor/spec/spec_helper.rb b/elasticgraph-query_interceptor/spec/spec_helper.rb new file mode 100644 index 00000000..0b2d2793 --- /dev/null +++ b/elasticgraph-query_interceptor/spec/spec_helper.rb @@ -0,0 +1,11 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-query_interceptor`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. diff --git a/elasticgraph-query_interceptor/spec/unit/elastic_graph/query_interceptor/config_spec.rb b/elasticgraph-query_interceptor/spec/unit/elastic_graph/query_interceptor/config_spec.rb new file mode 100644 index 00000000..d7ed176b --- /dev/null +++ b/elasticgraph-query_interceptor/spec/unit/elastic_graph/query_interceptor/config_spec.rb @@ -0,0 +1,74 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/query_interceptor/config" + +module ElasticGraph + module QueryInterceptor + RSpec.describe Config, :in_temp_dir do + it "returns a config instance with no interceptors if extension settings has no `query_interceptor` key" do + config = Config.from_parsed_yaml({}) + + expect(config.interceptors).to eq [] + end + + it "raises an error if configured with unknown keys" do + expect { + Config.from_parsed_yaml({"query_interceptor" => {"interceptors" => [], "other_key" => 3}}) + }.to raise_error Errors::ConfigError, a_string_including("other_key") + end + + it "raises an error if configured with no `interceptors` key" do + expect { + Config.from_parsed_yaml({"query_interceptor" => {}}) + }.to raise_error KeyError, a_string_including("interceptors") + end + + it "loads interceptors from disk based on config settings" do + 1.upto(3) do |i| + ::File.write("interceptor#{i}.rb", <<~EOS) + class Interceptor#{i} + def initialize(elasticgraph_graphql:, config:) + end + + def intercept(query, field:, args:, http_request:, context:) + query + end + end + EOS + end + + config = Config.from_parsed_yaml({"query_interceptor" => {"interceptors" => [ + {"extension_name" => "Interceptor1", "require_path" => "./interceptor1"}, + {"extension_name" => "Interceptor2", "require_path" => "./interceptor2", "config" => {"foo" => "bar"}}, + {"extension_name" => "Interceptor3", "require_path" => "./interceptor3"} + ]}}) + + expect(config.interceptors).to eq [ + Config::InterceptorData.new(Interceptor1, {}), + Config::InterceptorData.new(Interceptor2, {"foo" => "bar"}), + Config::InterceptorData.new(Interceptor3, {}) + ] + end + + it "validates the observable interface of the interceptors, reporting an issue if they are invalid" do + ::File.write("invalid_interceptor.rb", <<~EOS) + class InvalidInterceptor + end + EOS + + expect { + Config.from_parsed_yaml({"query_interceptor" => {"interceptors" => [ + {"extension_name" => "InvalidInterceptor", "require_path" => "./invalid_interceptor"} + ]}}) + }.to raise_error Errors::InvalidExtensionError, a_string_including("Missing instance methods:", "intercept") + end + end + end +end diff --git a/elasticgraph-query_interceptor/spec/unit/elastic_graph/query_interceptor/graphql_extension_spec.rb b/elasticgraph-query_interceptor/spec/unit/elastic_graph/query_interceptor/graphql_extension_spec.rb new file mode 100644 index 00000000..38510dc9 --- /dev/null +++ b/elasticgraph-query_interceptor/spec/unit/elastic_graph/query_interceptor/graphql_extension_spec.rb @@ -0,0 +1,168 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/query_interceptor/graphql_extension" +require "elastic_graph/graphql/http_endpoint" + +module ElasticGraph + module QueryInterceptor + RSpec.describe GraphQLExtension, :in_temp_dir, :builds_graphql do + let(:performed_datastore_queries) { [] } + let(:router) { instance_double("ElasticGraph::GraphQL::DatastoreSearchRouter") } + let(:interceptors) do + [ + { + "extension_name" => "MyApp::HideNonPublicThings", + "require_path" => "./hide_non_public_things" + }, + { + "extension_name" => "MyApp::FilterOnUser", + "require_path" => "./filter_on_user", + "config" => {"header" => "USER-NAME", "key" => "user"} + } + ] + end + + before do + ::File.write("hide_non_public_things.rb", <<~EOS) + module MyApp + class HideNonPublicThings + attr_reader :elasticgraph_graphql + + def initialize(elasticgraph_graphql:, config:) + @elasticgraph_graphql = elasticgraph_graphql + end + + def intercept(query, field:, args:, http_request:, context:) + query.merge_with(filter: {"public" => {"equal_to_any_of" => [false]}}) + end + end + end + EOS + + ::File.write("filter_on_user.rb", <<~EOS) + module MyApp + class FilterOnUser + attr_reader :elasticgraph_graphql + + def initialize(elasticgraph_graphql:, config:) + @elasticgraph_graphql = elasticgraph_graphql + @config = config + end + + def intercept(query, field:, args:, http_request:, context:) + user_name = http_request.normalized_headers[@config.fetch("header")] + query.merge_with(filter: {@config.fetch("key") => {"equal_to_any_of" => [user_name]}}) + end + end + end + EOS + + allow(router).to receive(:msearch) do |queries| + performed_datastore_queries.concat(queries) + queries.to_h { |query| [query, GraphQL::DatastoreResponse::SearchResponse::EMPTY] } + end + end + + it "works end-to-end to allow the interceptors to add extra filters to the datastore query" do + graphql = build_graphql( + extension_modules: [GraphQLExtension], + extension_settings: {"query_interceptor" => {"interceptors" => interceptors}}, + datastore_search_router: router + ) + + process(<<~EOS, graphql: graphql, headers: {"USER-NAME" => "yoda"}) + query { + addresses { + total_edge_count + } + } + EOS + + expect(performed_datastore_queries.size).to eq(1) + expect(performed_datastore_queries.first.filters).to contain_exactly( + {"public" => {"equal_to_any_of" => [false]}}, + {"user" => {"equal_to_any_of" => ["yoda"]}} + ) + + expect_configured_interceptors(graphql) do + [MyApp::HideNonPublicThings, MyApp::FilterOnUser] + end + end + + it "adds interceptors defined in runtime metadata" do + schema_artifacts = generate_schema_artifacts do |schema| + schema.register_graphql_extension( + ElasticGraph::QueryInterceptor::GraphQLExtension, + defined_at: "elastic_graph/query_interceptor/graphql_extension", + interceptors: interceptors + ) + end + + graphql = build_graphql( + extension_modules: [GraphQLExtension], + schema_artifacts: schema_artifacts + ) + + expect_configured_interceptors(graphql) do + [MyApp::HideNonPublicThings, MyApp::FilterOnUser] + end + end + + it "does not add interceptors if GraphQLExtension is not used" do + schema_artifacts = generate_schema_artifacts do |schema| + schema.register_graphql_extension( + Module.new, + defined_at: __FILE__, + interceptors: interceptors + ) + end + + graphql = build_graphql( + extension_modules: [GraphQLExtension], + schema_artifacts: schema_artifacts + ) + + expect_configured_interceptors(graphql) { [] } + end + + context "when the GraphQL extension has been registered on the schema with no specific interceptors" do + it "still loads the interceptors from config" do + schema_artifacts = generate_schema_artifacts do |schema| + schema.register_graphql_extension( + ElasticGraph::QueryInterceptor::GraphQLExtension, + defined_at: "elastic_graph/query_interceptor/graphql_extension" + ) + end + + graphql = build_graphql( + schema_artifacts: schema_artifacts, + extension_settings: {"query_interceptor" => {"interceptors" => interceptors}} + ) + + expect_configured_interceptors(graphql) do + [MyApp::HideNonPublicThings, MyApp::FilterOnUser] + end + end + end + + def process(query, graphql:, headers: {}) + headers = headers.merge("Content-Type" => "application/graphql") + request = GraphQL::HTTPRequest.new(url: "http://foo.com/bar", http_method: :post, body: query, headers: headers) + graphql.graphql_http_endpoint.process(request) + end + + def expect_configured_interceptors(graphql) + query_adapter = graphql.datastore_query_adapters.last + expect(query_adapter).to be_a QueryInterceptor::DatastoreQueryAdapter + expect(query_adapter.interceptors).to match_array(yield) # we yield to make it lazy since the interceptors are loaded lazily + expect(query_adapter.interceptors.map(&:elasticgraph_graphql)).to all be graphql + end + end + end +end diff --git a/elasticgraph-query_registry/.rspec b/elasticgraph-query_registry/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-query_registry/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-query_registry/.yardopts b/elasticgraph-query_registry/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-query_registry/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-query_registry/Gemfile b/elasticgraph-query_registry/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-query_registry/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-query_registry/LICENSE.txt b/elasticgraph-query_registry/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-query_registry/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-query_registry/README.md b/elasticgraph-query_registry/README.md new file mode 100644 index 00000000..98c43920 --- /dev/null +++ b/elasticgraph-query_registry/README.md @@ -0,0 +1,139 @@ +# ElasticGraph::QueryRegistry + +`ElasticGraph::QueryRegistry` provides a simple source-controlled query +registry for ElasticGraph applications. This is designed for cases where +the clients of your application are other internal teams in your organization, +who are willing to register their queries before using them. + +Query registration provides a few key benefits: + +* It gives you as the application owner the chance to vet queries and + give feedback to your clients. Queries may not initially be written + in the most optimal way (e.g. leveraging your sharding strategy), so + the review process gives you a chance to provide feedback. +* It allows you to provide stronger guarantees around schema changes. + Tooling is included that will validate each and every registered query + against your schema as part of your CI build, allowing you to quickly + iterate on your schema without needing to check if you'll break clients. +* It allows you to control the data clients have access to. When + a client attempts to register a query accessing fields they aren't + allowed to, you can choose not to approve the query. Once setup and + configured, this library will block clients from submitting queries + that have not been registered. +* Your GraphQL endpoint will be a bit more efficient. Parsing large + GraphQL queries can be a bit slow (in our testing, a 10 KB query + string takes about ~10ms to parse), and the registry will cache and + reuse the parsed form of registered queries. + +Importantly, once installed, registered clients who send unregistered +queries will get errors. Unregistered clients can similarly be blocked +if desired based on a configuration setting. + +## Query Verification Guarantees + +The query verification provided by this library is limited in scope. It +only checks to see if the queries and schema are compatible (in the sense +that the ElasticGraph endpoint will be able to successfully respond to +the queries). It does _not_ give any guarantee that a schema change is +100% safe for clients. For example, if you change a non-null field to be +nullable, it has no impact on ElasticGraph's ability to respond to a query +(and the verification performed by this library will allow it), but it may +break the client (e.g. if the client's usage of the response assumes +non-null field values). + +When changing the GraphQL schema of an ElasticGraph application, you +will still need to consider how it may impact clients, but you won't +need to worry about ElasticGraph beginning to return errors to any +existing queries. + +## Directory Structure + +This library uses a directory as the registry. Conventionally, this +would go in `config/queries` but it can really go anywhere. The directory +structure will look like this: + +``` +config +└── queries + ├── client1 + │ ├── query1.graphql + │ └── query2.graphql + ├── client2 + └── client3 + └── query1.graphql +``` + +Within the registry directory, there is a subdirectory for each +registered client. Each client directory contains that client's +registered queries as a set of `*.graphql` files (the extension is +required). Note that a client can be registered with no +associated queries (such as `client2`, above). This can be important +when you have configured `allow_unregistered_clients: true`. With +this setup, `client2` will not be able to submit any queries, but +a completely unregistered client (say, `client4`) will be able to +execute any query. + +## Setup + +First, add `elasticgraph-query_registry` to your `Gemfile`: + +``` ruby +gem "elasticgraph-query_registry" +``` + +Next, configure this library in your ElasticGraph config YAML files: + +``` yaml +graphql: + extension_modules: + - require_path: elastic_graph/query_registry/graphql_extension + extension_name: ElasticGraph::QueryRegistry::GraphQLExtension +query_registry: + allow_unregistered_clients: false + allow_any_query_for_clients: + - adhoc_client + path_to_registry: config/queries +``` + +Next, load the `ElasticGraph::QueryRegistry` rake tasks in your `Rakefile`: + +``` ruby +require "elastic_graph/query_registry/rake_tasks" + +ElasticGraph::QueryRegistry::RakeTasks.from_yaml_file( + "path/to/settings.yaml", + "config/queries", + require_eg_latency_slo_directive: true +) +``` + +You'll want to add `rake query_registry:validate_queries` to your CI build so +that every registered query is validated as part of every build. + +Finally, your application needs to include a `client:` when submitting +each GraphQL query for execution. The client `name` should match the +name of one of the registry client subdirectories. If you are using +`elasticgraph-lambda`, note that it does this automatically, but you may +need to configure `aws_arn_client_name_extraction_regex` so that it is +able to extract the `client_name` from the IAM ARN correctly. + +Important note: if your application fails to identify clients properly, +and `allow_unregistered_clients` is set to `true`, then _all_ clients +will be allowed to execute _all_ queries! We recommend you set +`allow_unregistered_clients` to `false` unless you specifically need +to allow unregistered clients. For specific clients that need to be +allowed to run any query, you can list them in `allow_any_query_for_clients`. + +## Workflow + +This library also uses some generated artifacts (`*.variables.yaml` files) +so it can detect when a change to the structure or type of a variable is +backward-incompatible. For this to work, it requires that the generated +variables files are kept up-to-date. Any time a change impacts the structure +of any variables used by any queries, you'll need to run a task like +`query_registry:dump_variables[client_name, query_name]` (or +`query_registry:dump_variables:all`) to update the artifacts. + +Don't worry about if you forget this, though--the +`query_registry:validate_queries` task will also fail and give you +instructions anytime a variables file is not up-to-date. diff --git a/elasticgraph-query_registry/elasticgraph-query_registry.gemspec b/elasticgraph-query_registry/elasticgraph-query_registry.gemspec new file mode 100644 index 00000000..ce987cbd --- /dev/null +++ b/elasticgraph-query_registry/elasticgraph-query_registry.gemspec @@ -0,0 +1,22 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :extension) do |spec, eg_version| + spec.summary = "An ElasticGraph extension that supports safer schema evolution by limiting GraphQL queries based on " \ + "a registry and validating registered queries against the schema." + + spec.add_dependency "elasticgraph-graphql", eg_version + spec.add_dependency "elasticgraph-support", eg_version + spec.add_dependency "graphql", "~> 2.3.19" + spec.add_dependency "rake", "~> 13.2" + + spec.add_development_dependency "elasticgraph-elasticsearch", eg_version + spec.add_development_dependency "elasticgraph-opensearch", eg_version +end diff --git a/elasticgraph-query_registry/lib/elastic_graph/query_registry/client_data.rb b/elasticgraph-query_registry/lib/elastic_graph/query_registry/client_data.rb new file mode 100644 index 00000000..bdc9e2aa --- /dev/null +++ b/elasticgraph-query_registry/lib/elastic_graph/query_registry/client_data.rb @@ -0,0 +1,103 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module QueryRegistry + ClientData = ::Data.define(:queries_by_original_string, :queries_by_last_string, :canonical_query_strings, :operation_names, :schema_element_names) do + # @implements ClientData + def self.from(schema, registered_query_strings) + queries_by_original_string = registered_query_strings.to_h do |query_string| + [query_string, ::GraphQL::Query.new(schema.graphql_schema, query_string, validate: false)] + end + + canonical_query_strings = queries_by_original_string.values.map do |q| + canonical_query_string_from(q, schema_element_names: schema.element_names) + end.to_set + + operation_names = queries_by_original_string.values.flat_map { |q| q.operations.keys }.to_set + + new( + queries_by_original_string: queries_by_original_string, + queries_by_last_string: {}, + canonical_query_strings: canonical_query_strings, + operation_names: operation_names, + schema_element_names: schema.element_names + ) + end + + def cached_query_for(query_string) + queries_by_original_string[query_string] || queries_by_last_string[query_string] + end + + def with_updated_last_query(query_string, query) + canonical_string = canonical_query_string_from(query) + + # We normally expect to only see one alternate query form from a client. However, a misbehaving + # client could send us a slightly different query string on each request (imagine if the query + # had a dynamically generated comment with a timestamp). Here we guard against that case by + # pruning out the previous hash entry that resolves to the same registered query, ensuring + # we only cache the most recently seen query string. Note that this operation is unfortunately + # O(N) instead of O(1) but we expect this operation to happen rarely (and we don't expect many + # entries in the `queries_by_last_string` hash). We could maintain a 2nd parallel data structure + # allowing an `O(1)` lookup here but I'd rather not introduce that added complexity for marginal + # benefit. + updated_queries_by_last_string = queries_by_last_string.reject do |_, cached_query| + canonical_query_string_from(cached_query) == canonical_string + end.merge(query_string => query) + + with(queries_by_last_string: updated_queries_by_last_string) + end + + def unregistered_query_error_for(query, client) + if operation_names.include?(query.operation_name.to_s) + "Query #{fingerprint_for(query)} differs from the registered form of `#{query.operation_name}` " \ + "for client #{client.description}." + else + "Query #{fingerprint_for(query)} is unregistered; client #{client.description} has no " \ + "registered query with a `#{query.operation_name}` operation." + end + end + + private + + def fingerprint_for(query) + # `query.fingerprint` raises an error if the query string is nil: + # https://github.com/rmosolgo/graphql-ruby/issues/4942 + query.query_string ? query.fingerprint : "(no query string)" + end + + def canonical_query_string_from(query) + ClientData.canonical_query_string_from(query, schema_element_names: schema_element_names) + end + + def self.canonical_query_string_from(query, schema_element_names:) + return "" unless (document = query.document) + + canonicalized_definitions = document.definitions.map do |definition| + if definition.directives.empty? + definition + else + # Ignore the `@egLatencySlo` directive if it is present. We want to allow it to be included (or not) + # and potentially have different values from the registered query so that clients don't have to register + # a new version of their query just to change the latency SLO value. + # + # Note: we don't ignore _all_ directives here because other directives might cause significant behavioral + # changes that should be enforced by the registry query approval process. + directives = definition.directives.reject do |dir| + dir.name == schema_element_names.eg_latency_slo + end + + definition.merge(directives: directives) + end + end + + document.merge(definitions: canonicalized_definitions).to_query_string + end + end + end +end diff --git a/elasticgraph-query_registry/lib/elastic_graph/query_registry/graphql_extension.rb b/elasticgraph-query_registry/lib/elastic_graph/query_registry/graphql_extension.rb new file mode 100644 index 00000000..7b211438 --- /dev/null +++ b/elasticgraph-query_registry/lib/elastic_graph/query_registry/graphql_extension.rb @@ -0,0 +1,104 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/query_executor" +require "elastic_graph/query_registry/registry" +require "graphql/query/result" +require "pathname" + +module ElasticGraph + module QueryRegistry + module GraphQLExtension + def graphql_query_executor + @graphql_query_executor ||= begin + registry_config = QueryRegistry::Config.from_parsed_yaml(config.extension_settings) + + RegistryAwareQueryExecutor.new( + schema: schema, + monotonic_clock: monotonic_clock, + logger: logger, + slow_query_threshold_ms: config.slow_query_latency_warning_threshold_in_ms, + datastore_search_router: datastore_search_router, + registry_directory: registry_config.path_to_registry, + allow_unregistered_clients: registry_config.allow_unregistered_clients, + allow_any_query_for_clients: registry_config.allow_any_query_for_clients + ) + end + end + end + + class RegistryAwareQueryExecutor < GraphQL::QueryExecutor + def initialize( + registry_directory:, + allow_unregistered_clients:, + allow_any_query_for_clients:, + schema:, + monotonic_clock:, + logger:, + slow_query_threshold_ms:, + datastore_search_router: + ) + super( + schema: schema, + monotonic_clock: monotonic_clock, + logger: logger, + slow_query_threshold_ms: slow_query_threshold_ms, + datastore_search_router: datastore_search_router + ) + + @registry = Registry.build_from_directory( + schema, + registry_directory, + allow_unregistered_clients: allow_unregistered_clients, + allow_any_query_for_clients: allow_any_query_for_clients + ) + end + + private + + def build_and_execute_query(query_string:, variables:, operation_name:, context:, client:) + query, errors = @registry.build_and_validate_query( + query_string, + variables: variables, + operation_name: operation_name, + context: context, + client: client + ) + + if errors.empty? + [query, execute_query(query, client: client)] + else + result = ::GraphQL::Query::Result.new( + query: nil, + values: {"errors" => errors.map { |e| {"message" => e} }} + ) + + [query, result] + end + end + end + + class Config < ::Data.define(:path_to_registry, :allow_unregistered_clients, :allow_any_query_for_clients) + def self.from_parsed_yaml(hash) + hash = hash.fetch("query_registry") { return DEFAULT } + + new( + path_to_registry: hash.fetch("path_to_registry"), + allow_unregistered_clients: hash.fetch("allow_unregistered_clients"), + allow_any_query_for_clients: hash.fetch("allow_any_query_for_clients") + ) + end + + DEFAULT = new( + path_to_registry: (_ = __dir__), + allow_unregistered_clients: true, + allow_any_query_for_clients: [] + ) + end + end +end diff --git a/elasticgraph-query_registry/lib/elastic_graph/query_registry/query_validator.rb b/elasticgraph-query_registry/lib/elastic_graph/query_registry/query_validator.rb new file mode 100644 index 00000000..0f7f7047 --- /dev/null +++ b/elasticgraph-query_registry/lib/elastic_graph/query_registry/query_validator.rb @@ -0,0 +1,98 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/query_registry/variable_backward_incompatibility_detector" +require "elastic_graph/query_registry/variable_dumper" +require "graphql" + +module ElasticGraph + module QueryRegistry + class QueryValidator + def initialize(schema, require_eg_latency_slo_directive:) + @graphql_schema = schema.graphql_schema + @schema_element_names = schema.element_names + @var_dumper = VariableDumper.new(@graphql_schema) + @var_incompat_detector = VariableBackwardIncompatibilityDetector.new + @require_eg_latency_slo_directive = require_eg_latency_slo_directive + end + + def validate(query_string, previously_dumped_variables:, client_name:, query_name:) + # We pass `validate: false` since we do query validation on the operation level down below. + query = ::GraphQL::Query.new(@graphql_schema, query_string, validate: false) + + if query.document.nil? + {nil => query.static_errors.map(&:to_h)} + else + # @type var fragments: ::Array[::GraphQL::Language::Nodes::FragmentDefinition] + # @type var operations: ::Array[::GraphQL::Language::Nodes::OperationDefinition] + fragments, operations = _ = query.document.definitions.partition do |definition| + definition.is_a?(::GraphQL::Language::Nodes::FragmentDefinition) + end + + newly_dumped_variables = @var_dumper.dump_variables_for_operations(operations) + + operations.to_h do |operation| + errors = if operation.name.nil? + [{"message" => "The query has no named operations. We require all registered queries to be named for more useful logging."}] + else + variables_errors = variables_errors_for(_ = operation.name, previously_dumped_variables, newly_dumped_variables, client_name, query_name) + directive_errors = directive_errors_for(operation) + + static_validation_errors_for(query, operation, fragments) + variables_errors + directive_errors + end + + [operation.name, errors] + end + end + end + + private + + def variables_errors_for(operation_name, old_dumped_variables, new_dumped_variables, client_name, query_name) + rake_task = "rake \"query_registry:dump_variables[#{client_name}, #{query_name}]\"" + + if old_dumped_variables.nil? || old_dumped_variables[operation_name].nil? + return [{"message" => "No dumped variables for this operation exist. Correct by running: `#{rake_task}`"}] + end + + old_op_vars = old_dumped_variables[operation_name] + new_op_vars = new_dumped_variables[operation_name] + + if old_op_vars == new_op_vars + # The previously dumped variables are up-to-date. No errors in this case. + [] + elsif (incompatibilities = @var_incompat_detector.detect(old_op_vars: old_op_vars, new_op_vars: new_op_vars)).any? + # The structure of variables has changed in a way that may break the client. Tell the user to verify with them. + descriptions = incompatibilities.map(&:description).join(", ") + [{ + "message" => "The structure of the query variables have had backwards-incompatible changes that may break `#{client_name}`: #{descriptions}. " \ + "To proceed, check with the client to see if this change is compatible with their logic, then run `#{rake_task}` to update the dumped info." + }] + else + # The change to the variables shouldn't break the client, but we still need to keep the file up-to-date. + [{"message" => "The variables file is out-of-date, but the changes to them should not impact `#{client_name}`. Run `#{rake_task}` to update the file."}] + end + end + + def directive_errors_for(operation) + if @require_eg_latency_slo_directive && operation.directives.none? { |dir| dir.name == @schema_element_names.eg_latency_slo } + [{"message" => "Your `#{operation.name}` operation is missing the required `@#{@schema_element_names.eg_latency_slo}(#{@schema_element_names.ms}: Int!)` directive."}] + else + [] + end + end + + def static_validation_errors_for(query, operation, fragments) + # Build a document with just this operation so that we can validate it in isolation, apart from the other operations. + document = query.document.merge(definitions: [operation] + fragments) + query = ::GraphQL::Query.new(@graphql_schema, nil, document: document, validate: false) + @graphql_schema.static_validator.validate(query).fetch(:errors).map(&:to_h) + end + end + end +end diff --git a/elasticgraph-query_registry/lib/elastic_graph/query_registry/query_validators/for_registered_client.rb b/elasticgraph-query_registry/lib/elastic_graph/query_registry/query_validators/for_registered_client.rb new file mode 100644 index 00000000..8f82b86d --- /dev/null +++ b/elasticgraph-query_registry/lib/elastic_graph/query_registry/query_validators/for_registered_client.rb @@ -0,0 +1,124 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/client" +require "elastic_graph/query_registry/client_data" +require "graphql" + +module ElasticGraph + module QueryRegistry + module QueryValidators + # Query validator implementation used for registered clients. + class ForRegisteredClient < ::Data.define( + :schema, + :graphql_schema, + :allow_any_query_for_clients, + :client_data_by_client_name, + :client_cache_mutex, + :provide_query_strings_for_client + ) + def initialize(schema:, client_names:, allow_any_query_for_clients:, provide_query_strings_for_client:) + super( + schema: schema, + graphql_schema: schema.graphql_schema, + allow_any_query_for_clients: allow_any_query_for_clients, + client_cache_mutex: ::Mutex.new, + provide_query_strings_for_client: provide_query_strings_for_client, + client_data_by_client_name: client_names.to_h { |name| [name, nil] }.merge( + # Register a built-in GraphQL query that ElasticGraph itself sometimes has to make. + GraphQL::Client::ELASTICGRAPH_INTERNAL.name => ClientData.from(schema, [GraphQL::EAGER_LOAD_QUERY]) + ) + ) + end + + def applies_to?(client) + return false unless (client_name = client&.name) + client_data_by_client_name.key?(client_name) + end + + def build_and_validate_query(query_string, client:, variables: {}, operation_name: nil, context: {}) + client_data = client_data_for(client.name) + + if (cached_query = client_data.cached_query_for(query_string.to_s)) + prepared_query = prepare_query_for_execution(cached_query, variables: variables, operation_name: operation_name, context: context) + return [prepared_query, []] + end + + query = yield + + # This client allows any query, so we can just return the query with no errors here. + # Note: we could put this at the top of the method, but if the query is registered and matches + # the registered form, the `cached_query` above is more efficient as it avoids unnecessarily + # parsing the query. + return [query, []] if allow_any_query_for_clients.include?(client.name) + + if !client_data.canonical_query_strings.include?(ClientData.canonical_query_string_from(query, schema_element_names: schema.element_names)) + return [query, [client_data.unregistered_query_error_for(query, client)]] + end + + # The query is slightly different from a registered query, but not in any material fashion + # (such as a whitespace or comment difference). Since query parsing can be kinda slow on + # large queries (in our benchmarking, ~10ms on a 10KB query), we want to cache the parsed + # query here. Normally, if a client sends a slightly different form of a query, it's going + # to be in that alternate form every single time, so caching it can be a nice win. + atomically_update_cached_client_data_for(client.name) do |cached_client_data| + # We don't want the cached form of the query to persist the current variables, context, etc being used for this request. + cachable_query = prepare_query_for_execution(query, variables: {}, operation_name: nil, context: {}) + + # We use `_` here because Steep believes `client_data` could be nil. In general, this is + # true; it can be nil, but not at this callsite, because we are in a branch that is only + # executed when `client_data` is _not_ nil. + (_ = cached_client_data).with_updated_last_query(query_string, cachable_query) + end + + [query, []] + end + + private + + def client_data_for(client_name) + if (client_data = client_data_by_client_name[client_name]) + client_data + else + atomically_update_cached_client_data_for(client_name) do |cached_data| + # We expect `cached_data` to be nil if we get here. However, it's technically possible for it + # not to be. If this `client_data_for` method was called with the same client from another thread + # in between the `client_data` fetch above and here, `cached_data` could not be populated. + # In that case, we don't want to pay the expense of re-building `ClientData` for no reason. + cached_data || ClientData.from(schema, provide_query_strings_for_client.call(client_name)) + end + end + end + + # Atomically updates the `ClientData` for the given `client_name`. All updates to our cache MUST go + # through this method to ensure there are no concurrency-related bugs. The caller should pass + # a block which will be yielded the current value in the cache (which can be `nil` initially); the + # block is then responsible for returning an updated copy of `ClientData` in the state that + # should be stored in the cache. + def atomically_update_cached_client_data_for(client_name) + client_cache_mutex.synchronize do + client_data_by_client_name[client_name] = yield client_data_by_client_name[client_name] + end + end + + def prepare_query_for_execution(query, variables:, operation_name:, context:) + ::GraphQL::Query.new( + graphql_schema, + # Here we pass `document` instead of query string, so that we don't have to re-parse the query. + # However, when the document is nil, we still need to pass the query string. + query.document ? nil : query.query_string, + document: query.document, + variables: variables, + operation_name: operation_name, + context: context + ) + end + end + end + end +end diff --git a/elasticgraph-query_registry/lib/elastic_graph/query_registry/query_validators/for_unregistered_client.rb b/elasticgraph-query_registry/lib/elastic_graph/query_registry/query_validators/for_unregistered_client.rb new file mode 100644 index 00000000..5361ab2a --- /dev/null +++ b/elasticgraph-query_registry/lib/elastic_graph/query_registry/query_validators/for_unregistered_client.rb @@ -0,0 +1,31 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module QueryRegistry + module QueryValidators + # Query validator implementation used for unregistered or anonymous clients. + ForUnregisteredClient = ::Data.define(:allow_unregistered_clients, :allow_any_query_for_clients) do + # @implements ForUnregisteredClient + def build_and_validate_query(query_string, client:, variables: {}, operation_name: nil, context: {}) + query = yield + + return [query, []] if allow_unregistered_clients + + client_name = client&.name + return [query, []] if client_name && allow_any_query_for_clients.include?(client_name) + + [query, [ + "Client #{client&.description || "(unknown)"} is not a registered client, it is not in " \ + "`allow_any_query_for_clients` and `allow_unregistered_clients` is false." + ]] + end + end + end + end +end diff --git a/elasticgraph-query_registry/lib/elastic_graph/query_registry/rake_tasks.rb b/elasticgraph-query_registry/lib/elastic_graph/query_registry/rake_tasks.rb new file mode 100644 index 00000000..573a0a66 --- /dev/null +++ b/elasticgraph-query_registry/lib/elastic_graph/query_registry/rake_tasks.rb @@ -0,0 +1,195 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql" +require "elastic_graph/support/from_yaml_file" +require "pathname" +require "rake/tasklib" + +module ElasticGraph + module QueryRegistry + class RakeTasks < ::Rake::TaskLib + # @dynamic self.from_yaml_file + extend Support::FromYamlFile::ForRakeTasks.new(ElasticGraph::GraphQL) + + def initialize(registered_queries_by_client_dir, require_eg_latency_slo_directive: false, output: $stdout, &load_graphql) + @registered_queries_by_client_dir = Pathname.new(registered_queries_by_client_dir) + @require_eg_latency_slo_directive = require_eg_latency_slo_directive + @output = output + @load_graphql = load_graphql + + define_tasks + end + + private + + def define_tasks + namespace :query_registry do + desc "Validates the queries registered in `#{@registered_queries_by_client_dir}`" + task :validate_queries do + perform_query_validation + end + + desc "Updates the registered information about query variables for a specific client (and optionally, a specific query)." + task :dump_variables, :client, :query do |_, args| + dump_variables("#{args.fetch(:client)}/#{args.fetch(:query, "*")}.graphql") + end + + namespace :dump_variables do + desc "Updates the registered information about query variables for all clients." + task :all do + dump_variables("*/*.graphql") + end + end + end + end + + def dump_variables(query_glob) + # We defer the loading of these dependencies until the task is running. As a general rule, + # we want rake tasks to only load their dependencies when they are run--that way, `rake -T` + # stays snappy, and when we run a rake task, only that task's dependencies are loaded + # instead of dependencies for all rake tasks. + require "elastic_graph/query_registry/variable_dumper" + require "yaml" + + variable_dumper = VariableDumper.new(graphql.schema.graphql_schema) + + @registered_queries_by_client_dir.glob(query_glob) do |file| + dumped_variables = variable_dumper.dump_variables_for_query(file.read) + variables_file = variable_file_name_for(file.to_s) + ::File.write(variables_file, variable_file_docs(variables_file) + ::YAML.dump(dumped_variables)) + @output.puts "- Dumped `#{variables_file}`." + end + end + + def variable_file_name_for(query_file_name) + query_file_name.delete_suffix(".graphql") + ".variables.yaml" + end + + def variable_file_docs(file_name) + client_name = file_name[%r{/([^/]+)/[^/]+\.variables\.yaml}, 1] + query_name = file_name[%r{/[^/]+/([^/]+)\.variables\.yaml}, 1] + + <<~EOS + # Generated by `rake "query_registry:dump_variables[#{client_name}, #{query_name}]"`. + # DO NOT EDIT BY HAND. Any edits will be lost the next time the rake task is run. + # + # This file exists to allow `elasticgraph-query_registry` to track the structure of + # the variables for the `#{client_name}/#{query_name}` query, so that we can detect + # when the schema structure of an object or enum variable changes in a way that might + # break the client. + EOS + end + + def perform_query_validation + # We defer the loading of these dependencies until the task is running. As a general rule, + # we want rake tasks to only load their dependencies when they are run--that way, `rake -T` + # stays snappy, and when we run a rake task, only that task's dependencies are loaded + # instead of dependencies for all rake tasks. + require "elastic_graph/query_registry/query_validator" + require "json" + require "yaml" + + validator = QueryValidator.new( + graphql.schema, + require_eg_latency_slo_directive: @require_eg_latency_slo_directive + ) + + all_errors = @registered_queries_by_client_dir.children.sort.flat_map do |client_dir| + @output.puts "For client `#{client_dir.basename}`:" + validate_client_queries(validator, client_dir).tap do + @output.puts + end + end + + unless all_errors.empty? + raise "Found #{count_description(all_errors, "validation error")} total across all queries." + end + end + + def validate_client_queries(validator, client_dir) + # @type var file_name_by_operation_name: ::Hash[::String, ::Pathname] + file_name_by_operation_name = {} + + client_dir.glob("*.graphql").sort.flat_map do |query_file| + previously_dumped_variables = previously_dumped_variables_for(query_file.to_s) + errors_by_operation_name = validator.validate( + query_file.read, + client_name: client_dir.basename.to_s, + query_name: query_file.basename.to_s.delete_suffix(".graphql"), + previously_dumped_variables: previously_dumped_variables + ) + + @output.puts " - #{query_file.basename} (#{count_description(errors_by_operation_name, "operation")}):" + + errors_by_operation_name.flat_map do |op_name, errors| + if (conflicting_file_name = file_name_by_operation_name[op_name.to_s]) + errors += [conflicting_operation_name_error(client_dir, op_name, conflicting_file_name)] + else + file_name_by_operation_name[op_name.to_s] = query_file + end + + op_name ||= "(no operation name)" + if errors.empty? + @output.puts " - #{op_name}: ✅" + else + @output.puts " - #{op_name}: 🛑. Got #{count_description(errors, "validation error")}:\n" + + errors.each_with_index do |error, index| + @output.puts format_error(query_file, index, error) + end + end + + errors + end + end + end + + def previously_dumped_variables_for(query_file_name) + file_name = variable_file_name_for(query_file_name) + return nil unless ::File.exist?(file_name) + ::YAML.safe_load_file(file_name) + end + + def conflicting_operation_name_error(client_dir, operation_name, conflicting_file_name) + message = "A `#{operation_name}` query already exists for `#{client_dir.basename}` in " \ + "`#{conflicting_file_name.basename}`. Each query operation must have a unique name." + + {"message" => message} + end + + def format_error(file_name, index, error_hash) + file_locations = (error_hash["locations"] || []).map do |location| + " source: #{file_name}:#{location["line"]}:#{location["column"]}" + end + + path = error_hash["path"]&.join(".") + + detail_lines = (error_hash["extensions"] || {}) + .merge(error_hash.except("message", "locations", "path", "extensions")) + .map { |key, value| " #{key}: #{value}" } + + [ + " #{index + 1}) #{error_hash["message"]}", + (" path: #{path}" if path), + *file_locations, + *detail_lines + ].compact.join("\n ") + "\n\n" + end + + def count_description(collection, noun) + return "1 #{noun}" if collection.size == 1 + "#{collection.size} #{noun}s" + end + + def graphql + @graphql ||= @load_graphql.call + end + end + end +end diff --git a/elasticgraph-query_registry/lib/elastic_graph/query_registry/registry.rb b/elasticgraph-query_registry/lib/elastic_graph/query_registry/registry.rb new file mode 100644 index 00000000..7face10c --- /dev/null +++ b/elasticgraph-query_registry/lib/elastic_graph/query_registry/registry.rb @@ -0,0 +1,101 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/query_registry/query_validators/for_registered_client" +require "elastic_graph/query_registry/query_validators/for_unregistered_client" +require "graphql" +require "pathname" + +module ElasticGraph + module QueryRegistry + # An abstraction that implements a registry of GraphQL queries. Callers should use + # `build_and_validate_query` to get `GraphQL::Query` objects so that clients are properly + # limited in what queries we execute on their behalf. + # + # Note that this class is designed to be as efficient as possible: + # + # - Registered GraphQL queries are only parsed once, and then the parsed form is used + # each time that query is submitted. In our benchmarking, parsing of large queries + # can be significant, taking ~10ms or so. + # - We delay parsing a registered client's queries until the first time that client + # sends us a query. That way, we don't have to pay any parsing cost for queries + # that were registered by an old client that no longer sends us requests. + # - Likewise, we defer reading a client's registered query strings off of disk until + # the first time it submits a request. + # + # In addition, it's worth noting that we support some basic "fuzzy" matching of query + # strings, based on the query canonicalization performed by the GraphQL gem. Semantically + # insignificant changes to the query string from a registered query (such as whitespace + # differences, or comments) are tolerated. + class Registry + # Public factory method, which builds a `Registry` instance from the given directory. + # Subdirectories are treated as client names, and the files in them are treated as + # individually registered queries. + def self.build_from_directory(schema, directory, allow_unregistered_clients:, allow_any_query_for_clients:) + directory = Pathname.new(directory) + + new( + schema, + client_names: directory.children.map { |client_dir| client_dir.basename.to_s }, + allow_unregistered_clients: allow_unregistered_clients, + allow_any_query_for_clients: allow_any_query_for_clients + ) do |client_name| + # Lazily read queries off of disk when we need to for a given client. + (directory / client_name).glob("*.graphql").map { |file| ::File.read(file.to_s) } + end + end + + # Builds a `GraphQL::Query` object for the given query string, and validates that it is + # an allowed query. Returns a list of registry validation errors in addition to the built + # query object. The list of validation errors will be empty if the query should be allowed. + # A query can be allowed either by virtue of being registered for usage by the given clent, + # or by being for a completely unregistered client (if `allow_unregistered_clients` is `true`). + # + # This is also tolerant of some minimal differences in the query string (such as comments + # and whitespace). If the query differs in a significant way from a registered query, it + # will not be recognized as registered. + def build_and_validate_query(query_string, client:, variables: {}, operation_name: nil, context: {}) + validator = + if @registered_client_validator.applies_to?(client) + @registered_client_validator + else + @unregistered_client_validator + end + + validator.build_and_validate_query(query_string, client: client, variables: variables, operation_name: operation_name, context: context) do + ::GraphQL::Query.new( + @graphql_schema, + query_string, + variables: variables, + operation_name: operation_name, + context: context + ) + end + end + + private + + def initialize(schema, client_names:, allow_unregistered_clients:, allow_any_query_for_clients:, &provide_query_strings_for_client) + @graphql_schema = schema.graphql_schema + allow_any_query_for_clients_set = allow_any_query_for_clients.to_set + + @registered_client_validator = QueryValidators::ForRegisteredClient.new( + schema: schema, + client_names: client_names, + allow_any_query_for_clients: allow_any_query_for_clients_set, + provide_query_strings_for_client: provide_query_strings_for_client + ) + + @unregistered_client_validator = QueryValidators::ForUnregisteredClient.new( + allow_unregistered_clients: allow_unregistered_clients, + allow_any_query_for_clients: allow_any_query_for_clients_set + ) + end + end + end +end diff --git a/elasticgraph-query_registry/lib/elastic_graph/query_registry/variable_backward_incompatibility_detector.rb b/elasticgraph-query_registry/lib/elastic_graph/query_registry/variable_backward_incompatibility_detector.rb new file mode 100644 index 00000000..cb3170ae --- /dev/null +++ b/elasticgraph-query_registry/lib/elastic_graph/query_registry/variable_backward_incompatibility_detector.rb @@ -0,0 +1,104 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module QueryRegistry + # Responsible for comparing old and new variable type info to see if any changes are backwards + # incompatible (and thus my break the client). Incompatibilities are identified by path and described. + class VariableBackwardIncompatibilityDetector + # Entry point. Given the old variables for an operation, and the new variables for it, describes + # any backward incompatibilities in them. + def detect(old_op_vars:, new_op_vars:) + detect_incompatibilities(old_op_vars, new_op_vars, "$", "variable") + end + + private + + # Given an `old` and `new` hash (which could be hashes of variables, or hashes of object fields), + # describes the incompatibities in them. + def detect_incompatibilities(old, new, path, entry_type) + removals = old.keys - new.keys + additions = new.keys - old.keys + commonalities = old.keys & new.keys + + incompatible_removals = removals.map do |name| + # All removals are incompatible, because the client might pass a value for the variable or field. + Incompatibility.new("#{path}#{name}", "removed") + end + + incompatible_commonalities = commonalities.flat_map do |name| + incompatibilities_for("#{path}#{name}", normalize_type_info(old[name]), normalize_type_info(new[name])) + end + + incompatible_additions = additions.filter_map do |name| + # Additions are only incompatible if it's required (non-nullable). + _ = if normalize_type_info(new[name]).fetch("type").end_with?("!") + Incompatibility.new("#{path}#{name}", "new required #{entry_type}") + end + end + + incompatible_removals + incompatible_commonalities + incompatible_additions + end + + # Describes the incompatibilities between the old and new type info. + def incompatibilities_for(path, old_type_info, new_type_info) + type_incompatibilities(path, old_type_info.fetch("type"), new_type_info.fetch("type")) + + enum_value_incompatibilities(path, old_type_info["values"], new_type_info["values"]) + + object_field_incompatibilities(path, old_type_info["fields"], new_type_info["fields"]) + end + + # Describes the incompatibilities between the old and new type names. + def type_incompatibilities(path, old_type, new_type) + if new_type == "#{old_type}!" + # If the variable or field is being required for the first time, the client may not pass a value + # for it and could be broken by this change. + [Incompatibility.new(path, "required for the first time")] + elsif old_type == "#{new_type}!" + [] # nullability was relaxed. That can't break the client so it's fine. + elsif new_type == old_type + [] # the type did not change. + else + # The type name changed. While some type name changes are compatible (e.g. from `ID` to `String`), + # we don't attempt to figure things out at that granularity. + [Incompatibility.new(path, "type changed from `#{old_type}` to `#{new_type}`")] + end + end + + # Describes the incompatibilities between the old and new enum values for a field or variable. + def enum_value_incompatibilities(path, old_enum_values, new_enum_values) + return [] unless old_enum_values && new_enum_values + removed_values = old_enum_values - new_enum_values + return [] if removed_values.empty? + + # Removed enum values could break the client if it ever passes a removed value in a query. + [Incompatibility.new(path, "removed enum values: #{removed_values.join(", ")}")] + end + + # Describes the incompatibilities between old and new object fields via recursion. + def object_field_incompatibilities(path, old_fields, new_fields) + return [] unless old_fields && new_fields + detect_incompatibilities(old_fields, new_fields, "#{path}.", "field") + end + + # Handles the fact that `type_info` can sometimes be a simple string, normalizing + # it to a hash so that we can consistently treat all type infos as hashes with a `type` field. + def normalize_type_info(type_info) + return {"type" => type_info} if type_info.is_a?(::String) + _ = type_info + end + + # Represents a single incompatibility. + Incompatibility = ::Data.define(:path, :explanation) do + # @implements Incompatibility + def description + "#{path} (#{explanation})" + end + end + end + end +end diff --git a/elasticgraph-query_registry/lib/elastic_graph/query_registry/variable_dumper.rb b/elasticgraph-query_registry/lib/elastic_graph/query_registry/variable_dumper.rb new file mode 100644 index 00000000..85c5d8eb --- /dev/null +++ b/elasticgraph-query_registry/lib/elastic_graph/query_registry/variable_dumper.rb @@ -0,0 +1,110 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "graphql" + +module ElasticGraph + module QueryRegistry + # Responsible for dumping structural information about query variables. + # + # This is necessary for the query registry to be able to support object and enum variables. + # To understand why, consider what happens when a field is removed from an input object + # variable used by a client's query. Whether or not it that will break the client depends + # on which fields of the input object the client populates when sending the query to + # ElasticGraph. Similarly, if an enum value is removed from an enum value variable used by + # a client, it could be a breaking change (but only if the client ever passes the removed + # enum value). + # + # To detect this situation, we use this to dump the structural information about all variables. + # When the structure of variables changes, we can then tell the engineer that they need to verify + # that it won't break the client. + class VariableDumper + def initialize(graphql_schema) + @graphql_schema = graphql_schema + end + + # Returns a hash of operations from the given query string. For each operation, the value + # is a hash of variables. + def dump_variables_for_query(query_string) + query = ::GraphQL::Query.new(@graphql_schema, query_string, validate: false) + + if query.document.nil? + # If the query was unparsable, we don't know anything about the variables and must just return an empty hash. + {} + else + # @type var operations: ::Array[::GraphQL::Language::Nodes::OperationDefinition] + operations = _ = query.document.definitions.grep(::GraphQL::Language::Nodes::OperationDefinition) + dump_variables_for_operations(operations) + end + end + + # Returns a hash containing the variables for each operation. + def dump_variables_for_operations(operations) + operations.each_with_index.to_h do |operation, index| + [operation.name || "(Anonymous operation #{index + 1})", variables_for_op(operation)] + end + end + + private + + # Returns a hash of variables for the given GraphQL operation. + def variables_for_op(operation) + operation.variables.sort_by(&:name).to_h do |variable| + type_info = + if (type = @graphql_schema.type_from_ast(variable.type)) + type_info(type) + else + # We should only get here if a variable references a type that is undefined. Since we + # don't know anything about the type other than the name, that's all we can return. + variable.type.to_query_string + end + + [variable.name, type_info] + end + end + + # Returns information about the given type. + # + # Note that this is optimized for human readability over data structure consistency. + # We don't *do* anything with this dumped data (other than comparing its equality + # against the dumped results for the same query in the future), so we don't need + # the sort of data structure consistency we'd normally want. + # + # For scalars (and lists-of-scalars) the *only* meaningful structural information + # is the type signature (e.g. `[ID!]`). On the other hand, we need the `fields` for + # an input object, and the `values` for an enum (along with the type signature for + # those, to distinguish list vs not and nullable vs not). + # + # So, while we return a hash for object/enum variables, for all others we just return + # the type signature string. + def type_info(type) + unwrapped_type = type.unwrap + + if unwrapped_type.kind.input_object? + {"type" => type.to_type_signature, "fields" => fields_for(_ = unwrapped_type)} + elsif unwrapped_type.kind.enum? + {"type" => type.to_type_signature, "values" => (_ = unwrapped_type).values.keys.sort} + else + type.to_type_signature + end + end + + # Returns a hash of input object fields for the given type. + def fields_for(variable_type) + variable_type.arguments.values.sort_by(&:name).to_h do |arg| + if arg.type.unwrap == variable_type + # Don't recurse (it would never terminate); just dump a reference to the type. + [arg.name, arg.type.to_type_signature] + else + [arg.name, type_info(arg.type)] + end + end + end + end + end +end diff --git a/elasticgraph-query_registry/sig/elastic_graph/query_registry/client_data.rbs b/elasticgraph-query_registry/sig/elastic_graph/query_registry/client_data.rbs new file mode 100644 index 00000000..1751d4ef --- /dev/null +++ b/elasticgraph-query_registry/sig/elastic_graph/query_registry/client_data.rbs @@ -0,0 +1,46 @@ +module ElasticGraph + module QueryRegistry + class ClientData + attr_reader queries_by_original_string: ::Hash[::String, ::GraphQL::Query] + attr_reader queries_by_last_string: ::Hash[::String, ::GraphQL::Query] + attr_reader canonical_query_strings: ::Set[::String] + attr_reader operation_names: ::Set[::String] + attr_reader schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + + def self.from: ( + GraphQL::Schema, + ::Array[::String] + ) -> ClientData + + def self.new: ( + queries_by_original_string: ::Hash[::String, ::GraphQL::Query], + queries_by_last_string: ::Hash[::String, ::GraphQL::Query], + canonical_query_strings: ::Set[::String], + operation_names: ::Set[::String], + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + ) -> ClientData + + def with: ( + ?queries_by_original_string: ::Hash[::String, ::GraphQL::Query], + ?queries_by_last_string: ::Hash[::String, ::GraphQL::Query], + ?canonical_query_strings: ::Set[::String], + ?operation_names: ::Set[::String], + ?schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + ) -> ClientData + + def cached_query_for: (::String) -> ::GraphQL::Query? + def with_updated_last_query: (::String, ::GraphQL::Query) -> ClientData + def unregistered_query_error_for: (::GraphQL::Query, GraphQL::Client) -> ::String + + private + + def fingerprint_for: (::GraphQL::Query) -> ::String + def canonical_query_string_from: (::GraphQL::Query) -> ::String + + def self.canonical_query_string_from: ( + ::GraphQL::Query, + schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + ) -> ::String + end + end +end diff --git a/elasticgraph-query_registry/sig/elastic_graph/query_registry/graphql_extension.rbs b/elasticgraph-query_registry/sig/elastic_graph/query_registry/graphql_extension.rbs new file mode 100644 index 00000000..e39bc4f4 --- /dev/null +++ b/elasticgraph-query_registry/sig/elastic_graph/query_registry/graphql_extension.rbs @@ -0,0 +1,51 @@ +module ElasticGraph + module QueryRegistry + module GraphQLExtension: GraphQL + def graphql_query_executor: () -> RegistryAwareQueryExecutor + end + + class RegistryAwareQueryExecutor < GraphQL::QueryExecutor + def initialize: ( + registry_directory: ::String, + allow_unregistered_clients: bool, + allow_any_query_for_clients: ::Array[::String], + schema: GraphQL::Schema, + monotonic_clock: Support::MonotonicClock, + logger: ::Logger, + slow_query_threshold_ms: ::Integer, + datastore_search_router: GraphQL::DatastoreSearchRouter + ) -> void + + private + + @registry: Registry + end + + class ConfigSupertype + attr_reader path_to_registry: ::String + attr_reader allow_unregistered_clients: bool + attr_reader allow_any_query_for_clients: ::Array[::String] + + def self.new: ( + path_to_registry: ::String, + allow_unregistered_clients: bool, + allow_any_query_for_clients: ::Array[::String] + ) -> Config + + def with: ( + ?path_to_registry: ::String, + ?allow_unregistered_clients: bool, + ?allow_any_query_for_clients: ::Array[::String] + ) -> Config + end + + class Config < ConfigSupertype + extend _BuildableFromParsedYaml[Config] + DEFAULT: Config + end + end + + class Config + attr_reader query_registry: QueryRegistry::Config + end +end diff --git a/elasticgraph-query_registry/sig/elastic_graph/query_registry/query_validator.rbs b/elasticgraph-query_registry/sig/elastic_graph/query_registry/query_validator.rbs new file mode 100644 index 00000000..52ae7936 --- /dev/null +++ b/elasticgraph-query_registry/sig/elastic_graph/query_registry/query_validator.rbs @@ -0,0 +1,43 @@ +module ElasticGraph + module QueryRegistry + class QueryValidator + def initialize: ( + GraphQL::Schema, + require_eg_latency_slo_directive: bool + ) -> void + + def validate: ( + ::String, + previously_dumped_variables: ::Hash[::String, ::Hash[::String, VariableDumper::typeInfo]]?, + client_name: ::String, + query_name: ::String + ) -> ::Hash[::String?, ::Array[::GraphQL::validationErrorHash]] + + private + + @graphql_schema: ::GraphQL::Schema + @schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + @var_dumper: VariableDumper + @var_incompat_detector: VariableBackwardIncompatibilityDetector + @require_eg_latency_slo_directive: bool + + def variables_errors_for: ( + ::String, + ::Hash[::String, ::Hash[::String, VariableDumper::typeInfo]]?, + ::Hash[::String, ::Hash[::String, VariableDumper::typeInfo]], + ::String, + ::String + ) -> ::Array[::GraphQL::validationErrorHash] + + def directive_errors_for: ( + ::GraphQL::Language::Nodes::OperationDefinition + ) -> ::Array[::GraphQL::validationErrorHash] + + def static_validation_errors_for: ( + ::GraphQL::Query, + ::GraphQL::Language::Nodes::OperationDefinition, + ::Array[::GraphQL::Language::Nodes::FragmentDefinition] + ) -> ::Array[::GraphQL::validationErrorHash] + end + end +end diff --git a/elasticgraph-query_registry/sig/elastic_graph/query_registry/query_validators/for_registered_client.rbs b/elasticgraph-query_registry/sig/elastic_graph/query_registry/query_validators/for_registered_client.rbs new file mode 100644 index 00000000..f9ad7d9c --- /dev/null +++ b/elasticgraph-query_registry/sig/elastic_graph/query_registry/query_validators/for_registered_client.rbs @@ -0,0 +1,54 @@ +module ElasticGraph + module QueryRegistry + module QueryValidators + class ForRegisteredClientSupertype + attr_reader schema: GraphQL::Schema + attr_reader graphql_schema: ::GraphQL::Schema + attr_reader allow_any_query_for_clients: ::Set[::String] + attr_reader client_cache_mutex: ::Thread::Mutex + attr_reader provide_query_strings_for_client: ^(::String) -> ::Array[::String] + attr_reader client_data_by_client_name: ::Hash[::String, ClientData?] + + def initialize: ( + schema: GraphQL::Schema, + graphql_schema: ::GraphQL::Schema, + allow_any_query_for_clients: ::Set[::String], + client_cache_mutex: ::Thread::Mutex, + provide_query_strings_for_client: ^(::String) -> ::Array[::String], + client_data_by_client_name: ::Hash[::String, ClientData?] + ) -> void + end + + class ForRegisteredClient < ForRegisteredClientSupertype + def initialize: ( + schema: GraphQL::Schema, + client_names: ::Array[::String], + allow_any_query_for_clients: ::Set[::String], + provide_query_strings_for_client: ^(String) -> ::Array[::String] + ) -> void + + def applies_to?: (GraphQL::Client) -> bool + + def build_and_validate_query: ( + ::String?, + client: GraphQL::Client, + ?variables: ::Hash[::String, untyped], + ?operation_name: ::String?, + ?context: ::Hash[::Symbol, untyped] + ) { () -> ::GraphQL::Query } -> [::GraphQL::Query, ::Array[::String]] + + private + + def client_data_for: (::String) -> ClientData + def atomically_update_cached_client_data_for: (::String) { (ClientData?) -> ClientData } -> ClientData + + def prepare_query_for_execution: ( + ::GraphQL::Query, + variables: ::Hash[::String, untyped], + operation_name: ::String?, + context: ::Hash[::Symbol, untyped] + ) -> ::GraphQL::Query + end + end + end +end diff --git a/elasticgraph-query_registry/sig/elastic_graph/query_registry/query_validators/for_unregistered_client.rbs b/elasticgraph-query_registry/sig/elastic_graph/query_registry/query_validators/for_unregistered_client.rbs new file mode 100644 index 00000000..b1c1b10b --- /dev/null +++ b/elasticgraph-query_registry/sig/elastic_graph/query_registry/query_validators/for_unregistered_client.rbs @@ -0,0 +1,23 @@ +module ElasticGraph + module QueryRegistry + module QueryValidators + class ForUnregisteredClient + attr_reader allow_unregistered_clients: bool + attr_reader allow_any_query_for_clients: ::Set[::String] + + def initialize: ( + allow_unregistered_clients: bool, + allow_any_query_for_clients: ::Set[::String] + ) -> void + + def build_and_validate_query: ( + ::String?, + client: GraphQL::Client, + ?variables: ::Hash[::String, untyped], + ?operation_name: ::String?, + ?context: ::Hash[::Symbol, untyped] + ) { () -> ::GraphQL::Query } -> [::GraphQL::Query, ::Array[::String]] + end + end + end +end diff --git a/elasticgraph-query_registry/sig/elastic_graph/query_registry/rake_tasks.rbs b/elasticgraph-query_registry/sig/elastic_graph/query_registry/rake_tasks.rbs new file mode 100644 index 00000000..dfa9f7ea --- /dev/null +++ b/elasticgraph-query_registry/sig/elastic_graph/query_registry/rake_tasks.rbs @@ -0,0 +1,38 @@ +module ElasticGraph + module QueryRegistry + class RakeTasks < ::Rake::TaskLib + def self.from_yaml_file: ( + ::String | ::Pathname, + ::String | ::Pathname, + ?require_eg_latency_slo_directive: bool, + ?output: io + ) -> RakeTasks + + def initialize: ( + ::String | ::Pathname, + ?require_eg_latency_slo_directive: bool, + ?output: io + ) { () -> GraphQL } -> void + + private + + @load_graphql: ^() -> GraphQL + @graphql: GraphQL? + @registered_queries_by_client_dir: ::Pathname + @require_eg_latency_slo_directive: bool + @output: io + + def define_tasks: () -> void + def dump_variables: (::String) -> void + def variable_file_name_for: (::String) -> ::String + def variable_file_docs: (::String) -> ::String + def perform_query_validation: () -> void + def validate_client_queries: (QueryValidator, ::Pathname) -> ::Array[::GraphQL::validationErrorHash] + def previously_dumped_variables_for: (::String) -> ::Hash[::String, ::Hash[::String, VariableDumper::typeInfo]]? + def conflicting_operation_name_error: (::Pathname, ::String?, ::Pathname) -> ::GraphQL::validationErrorHash + def format_error: (::Pathname, ::Integer, ::GraphQL::validationErrorHash) -> ::String + def count_description: (::Array[untyped] | ::Hash[untyped, untyped], ::String) -> ::String + def graphql: () -> GraphQL + end + end +end diff --git a/elasticgraph-query_registry/sig/elastic_graph/query_registry/registry.rbs b/elasticgraph-query_registry/sig/elastic_graph/query_registry/registry.rbs new file mode 100644 index 00000000..a7ece788 --- /dev/null +++ b/elasticgraph-query_registry/sig/elastic_graph/query_registry/registry.rbs @@ -0,0 +1,34 @@ +module ElasticGraph + module QueryRegistry + class Registry + + def self.build_from_directory: ( + GraphQL::Schema, + ::String, + allow_unregistered_clients: bool, + allow_any_query_for_clients: ::Array[::String] + ) -> Registry + + def initialize: ( + GraphQL::Schema, + client_names: ::Array[::String], + allow_unregistered_clients: bool, + allow_any_query_for_clients: ::Array[::String] + ) { (String) -> ::Array[::String] } -> void + + def build_and_validate_query: ( + ::String?, + client: GraphQL::Client, + ?variables: ::Hash[::String, untyped], + ?operation_name: ::String?, + ?context: ::Hash[::Symbol, untyped] + ) -> [::GraphQL::Query, ::Array[::String]] + + private + + @graphql_schema: ::GraphQL::Schema + @registered_client_validator: QueryValidators::ForRegisteredClient + @unregistered_client_validator: QueryValidators::ForUnregisteredClient + end + end +end diff --git a/elasticgraph-query_registry/sig/elastic_graph/query_registry/variable_backward_incompatibility_detector.rbs b/elasticgraph-query_registry/sig/elastic_graph/query_registry/variable_backward_incompatibility_detector.rbs new file mode 100644 index 00000000..bfaa8066 --- /dev/null +++ b/elasticgraph-query_registry/sig/elastic_graph/query_registry/variable_backward_incompatibility_detector.rbs @@ -0,0 +1,52 @@ +module ElasticGraph + module QueryRegistry + class VariableBackwardIncompatibilityDetector + def detect: ( + old_op_vars: ::Hash[::String, VariableDumper::typeInfo], + new_op_vars: ::Hash[::String, VariableDumper::typeInfo] + ) -> ::Array[Incompatibility] + + private + + def detect_incompatibilities: ( + ::Hash[::String, VariableDumper::typeInfo], + ::Hash[::String, VariableDumper::typeInfo], + ::String, + ::String + ) -> ::Array[Incompatibility] + + def incompatibilities_for: ( + ::String, + ::Hash[::String, untyped], + ::Hash[::String, untyped] + ) -> ::Array[Incompatibility] + + def type_incompatibilities: ( + ::String, + ::String, + ::String + ) -> ::Array[Incompatibility] + + def enum_value_incompatibilities: ( + ::String, + ::Array[::String]?, + ::Array[::String]?, + ) -> ::Array[Incompatibility] + + def object_field_incompatibilities: ( + ::String, + ::Hash[::String, VariableDumper::typeInfo]?, + ::Hash[::String, VariableDumper::typeInfo]? + ) -> ::Array[Incompatibility] + + def normalize_type_info: (VariableDumper::typeInfo) -> ::Hash[::String, untyped] + + class Incompatibility + attr_reader path: ::String + attr_reader explanation: ::String + def initialize: (::String, ::String) -> void + def description: () -> ::String + end + end + end +end diff --git a/elasticgraph-query_registry/sig/elastic_graph/query_registry/variable_dumper.rbs b/elasticgraph-query_registry/sig/elastic_graph/query_registry/variable_dumper.rbs new file mode 100644 index 00000000..5fe010bb --- /dev/null +++ b/elasticgraph-query_registry/sig/elastic_graph/query_registry/variable_dumper.rbs @@ -0,0 +1,22 @@ +module ElasticGraph + module QueryRegistry + class VariableDumper + type objectTypeInfo = {"type" => ::String, "fields" => ::Hash[::String, typeInfo]} + type enumTypeInfo = {"type" => ::String, "values" => ::Array[::String]} + type typeInfo = ::String | objectTypeInfo | enumTypeInfo + + def initialize: (::GraphQL::Schema) -> void + def dump_variables_for_query: (::String) -> ::Hash[::String, ::Hash[::String, typeInfo]] + def dump_variables_for_operations: ( + ::Array[::GraphQL::Language::Nodes::OperationDefinition] + ) -> ::Hash[::String, ::Hash[::String, typeInfo]] + + private + + @graphql_schema: ::GraphQL::Schema + def variables_for_op: (::GraphQL::Language::Nodes::OperationDefinition) -> ::Hash[::String, typeInfo] + def type_info: (::GraphQL::Schema::_Type) -> typeInfo + def fields_for: (::GraphQL::Schema::InputObject) -> ::Hash[::String, typeInfo] + end + end +end diff --git a/elasticgraph-query_registry/spec/acceptance/query_registry_spec.rb b/elasticgraph-query_registry/spec/acceptance/query_registry_spec.rb new file mode 100644 index 00000000..eb0b00cd --- /dev/null +++ b/elasticgraph-query_registry/spec/acceptance/query_registry_spec.rb @@ -0,0 +1,152 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/elasticsearch/client" +require "elastic_graph/graphql/client" +require "elastic_graph/query_registry/graphql_extension" +require "elastic_graph/support/hash_util" +require "fileutils" +require "graphql" +require "yaml" + +module ElasticGraph + RSpec.describe "QueryRegistry", :capture_logs, :in_temp_dir do + let(:query_registry_dir) { "query_registry" } + + let(:widget_name_string) do + <<~EOS.strip + query WidgetName { + __type(name: "Widget") { + name + } + } + EOS + end + + let(:part_name_string) do + <<~EOS.strip + query PartName { + __type(name: "Part") { + name + } + } + EOS + end + + it "responds to unregistered queries with a clear error while allowing registered queries" do + register_query "client1", "WidgetName", widget_name_string + register_query "client2", "PartName", part_name_string + + query_executor = build_query_executor_with_registry_options( + allow_unregistered_clients: false, + allow_any_query_for_clients: ["adhoc_client"], + path_to_registry: query_registry_dir + ) + + expect(execute(query_executor, widget_name_string, on_behalf_of: "client1")).to eq("data" => {"__type" => {"name" => "Widget"}}) + expect(execute(query_executor, part_name_string, on_behalf_of: "client2")).to eq("data" => {"__type" => {"name" => "Part"}}) + expect(execute(query_executor, part_name_string, on_behalf_of: "adhoc_client")).to eq("data" => {"__type" => {"name" => "Part"}}) + + expect { + expect(execute(query_executor, widget_name_string, on_behalf_of: "client2")).to match("errors" => [ + {"message" => a_string_including("Query WidgetName", "is unregistered", "client2")} + ]) + expect(execute(query_executor, part_name_string, on_behalf_of: "client1")).to match("errors" => [ + {"message" => a_string_including("Query PartName", "is unregistered", "client1")} + ]) + expect(execute(query_executor, part_name_string, on_behalf_of: "client3")).to match("errors" => [ + {"message" => a_string_including("client3", "is not a registered client")} + ]) + }.to log_warning a_string_including("is unregistered") + end + + it "allows the ElasticGraph eager-loading query to proceed even with restrictive options" do + register_query "client1", "WidgetName", widget_name_string + + graphql = build_graphql_with_registry_options( + allow_unregistered_clients: false, + allow_any_query_for_clients: [], + path_to_registry: query_registry_dir + ) + + executed_query_strings = track_executed_query_strings + graphql.load_dependencies_eagerly + expect(executed_query_strings).to eq [GraphQL::EAGER_LOAD_QUERY] + end + + it "does nothing when loaded but not configured" do + query_executor = build_query_executor_with_registry_options + + expect(execute(query_executor, widget_name_string, on_behalf_of: "client1")).to eq("data" => {"__type" => {"name" => "Widget"}}) + expect(execute(query_executor, widget_name_string, on_behalf_of: "client2")).to eq("data" => {"__type" => {"name" => "Widget"}}) + expect(execute(query_executor, part_name_string, on_behalf_of: "client2")).to eq("data" => {"__type" => {"name" => "Part"}}) + expect(execute(query_executor, part_name_string, on_behalf_of: "adhoc_client")).to eq("data" => {"__type" => {"name" => "Part"}}) + end + + it "handles a request that has no query" do + register_query "client1", "WidgetName", widget_name_string + + query_executor = build_query_executor_with_registry_options( + allow_unregistered_clients: false, + allow_any_query_for_clients: [], + path_to_registry: query_registry_dir + ) + + expect { + expect(execute(query_executor, nil, on_behalf_of: "client1")).to match("errors" => [ + {"message" => a_string_including("Query (no query string) is unregistered", "client1")} + ]) + }.to log_warning a_string_including("is unregistered") + end + + def build_query_executor_with_registry_options(**options) + build_graphql_with_registry_options(**options).graphql_query_executor + end + + def build_graphql_with_registry_options(**options) + extension_settings = + if options.empty? + {} + else + {"query_registry" => Support::HashUtil.stringify_keys(options)} + end + + build_graphql( + extension_settings: extension_settings, + extension_modules: [QueryRegistry::GraphQLExtension], + client_faraday_adapter: DatastoreCore::Configuration::ClientFaradayAdapter.new( + name: :test, + require: nil + ) + ) + end + + def execute(query_executor, query_string, on_behalf_of:) + query_executor.execute(query_string, client: GraphQL::Client.new(name: on_behalf_of, source_description: "some-description")).to_h + end + + def register_query(client_name, query_name, query_string) + query_dir = File.join(query_registry_dir, client_name) + FileUtils.mkdir_p(query_dir) + + full_file_name = File.join(query_dir, query_name) + ".graphql" + File.write(full_file_name, query_string) + end + + def track_executed_query_strings + executed_query_strings = [] + + allow(::GraphQL::Execution::Interpreter).to receive(:run_all).and_wrap_original do |original, schema, queries, **options| + executed_query_strings.concat(queries.map(&:query_string)) + original.call(schema, queries, **options) + end + + executed_query_strings + end + end +end diff --git a/elasticgraph-query_registry/spec/spec_helper.rb b/elasticgraph-query_registry/spec/spec_helper.rb new file mode 100644 index 00000000..23ceffb4 --- /dev/null +++ b/elasticgraph-query_registry/spec/spec_helper.rb @@ -0,0 +1,17 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-query_registry`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. + +RSpec.configure do |config| + config.define_derived_metadata(absolute_file_path: %r{/elasticgraph-query_registry/}) do |meta| + meta[:builds_graphql] = true + end +end diff --git a/elasticgraph-query_registry/spec/unit/elastic_graph/query_registry/query_validator_spec.rb b/elasticgraph-query_registry/spec/unit/elastic_graph/query_registry/query_validator_spec.rb new file mode 100644 index 00000000..cbe8d525 --- /dev/null +++ b/elasticgraph-query_registry/spec/unit/elastic_graph/query_registry/query_validator_spec.rb @@ -0,0 +1,488 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql" +require "elastic_graph/query_registry/query_validator" +require "elastic_graph/query_registry/variable_dumper" + +module ElasticGraph + module QueryRegistry + RSpec.describe QueryValidator do + let(:schema) { build_graphql.schema } + let(:variable_dumper) { VariableDumper.new(schema.graphql_schema) } + + describe "#validate" do + it "returns no errors when given a valid named query with no arguments" do + query_string = <<~EOS + query MyQuery { + widgets { + total_edge_count + } + } + EOS + + expect(validate(query_string)).to eq("MyQuery" => []) + end + + it "allows fragments to be used in queries" do + query_string = <<~EOS + query MyQuery { + widgets { + ...widgetsFields + } + } + + fragment widgetsFields on WidgetConnection { + total_edge_count + } + EOS + + expect(validate(query_string)).to eq("MyQuery" => []) + end + + it "returns errors when a fragment is defined but not used" do + query_string = <<~EOS + query MyQuery { + widgets { + total_edge_count + } + } + + fragment widgetsFields on WidgetConnection { + total_edge_count + } + EOS + + expect(validate(query_string)).to have_errors_for_operations("MyQuery" => [{ + "message" => "Fragment widgetsFields was defined, but not used" + }]) + end + + it "returns errors when given a valid unnamed query with no arguments, since we want all queries to be named for better logging" do + query_string = <<~EOS + query { + widgets { + total_edge_count + } + } + EOS + + expect(validate(query_string)).to have_errors_for_operations(nil => [{ + "message" => a_string_including("no named operations") + }]) + end + + it "returns errors when the query cannot be parsed" do + query_string = <<~EOS + query MyQuery bad syntax { + widgets { + total_edge_count + } + } + EOS + + expect(validate(query_string)).to have_errors_for_operations(nil => [{ + "locations" => [{"line" => 1, "column" => 15}], + "message" => a_string_including("Expected LCURLY, actual: IDENTIFIER (\"bad\")") + }]) + end + + it "returns errors when the query string is empty" do + expect(validate("")).to have_errors_for_operations(nil => [{ + "message" => a_string_including("Unexpected end of document") + }]) + end + + it "returns errors when the query references undefined fields" do + query_string = <<~EOS + query MyQuery { + widgets { + not_a_real_field + } + } + EOS + + expect(validate(query_string)).to have_errors_for_operations("MyQuery" => [{ + "locations" => [{"line" => 3, "column" => 5}], + "message" => a_string_including("not_a_real_field"), + "path" => ["query MyQuery", "widgets", "not_a_real_field"] + }]) + end + + it "returns errors when the query references an undefined field input argument" do + query_string = <<~EOS + query MyQuery { + widgets(filter: {not_a_field: {equal_to_any_of: [1]}}) { + total_edge_count + } + } + EOS + + expect(validate(query_string)).to have_errors_for_operations("MyQuery" => [{ + "locations" => [{"line" => 2, "column" => 20}], + "message" => a_string_including("not_a_field"), + "path" => ["query MyQuery", "widgets", "filter", "not_a_field"] + }]) + end + + context "when the `@eg_latency_slo` directive is required" do + it "returns an error if the query lacks an `@eg_latency_slo` directive" do + query_string = <<~EOS + query MyQuery { + widgets { + total_edge_count + } + } + EOS + + expect(validate(query_string)).to have_errors_for_operations("MyQuery" => [{ + "message" => "Your `MyQuery` operation is missing the required `@eg_latency_slo(ms: Int!)` directive." + }]) + end + + it "does not return an error if the query has an `@eg_latency_slo` directive" do + query_string = <<~EOS + query MyQuery @eg_latency_slo(ms: 5000) { + widgets { + total_edge_count + } + } + EOS + + expect(validate(query_string)).to eq("MyQuery" => []) + end + + def validate(query_string) + super(query_string, require_eg_latency_slo_directive: true) + end + end + + context "when the `@eg_latency_slo` directive is not required" do + it "does still allows the query to have an `@eg_latency_slo` directive" do + query_string = <<~EOS + query MyQuery @eg_latency_slo(ms: 5000) { + widgets { + total_edge_count + } + } + EOS + + expect(validate(query_string)).to eq("MyQuery" => []) + end + + def validate(query_string) + super(query_string, require_eg_latency_slo_directive: false) + end + end + + context "when the query contains multiple operations" do + it "returns no errors if each is individually valid" do + query_string = <<~EOS + query MyQuery1 { + widgets { + total_edge_count + } + } + + query MyQuery2 { + components { + total_edge_count + } + } + EOS + + expect(validate(query_string)).to eq("MyQuery1" => [], "MyQuery2" => []) + end + + it "validates each individual operation and returns the errors for each" do + query_string = <<~EOS + query InvalidField { + widgets { + foo + } + } + + query ValidQuery { + widgets { + total_edge_count + } + } + + query InvalidArg { + components(foo: 1) { + total_edge_count + } + } + EOS + + expect(validate(query_string)).to have_errors_for_operations( + "InvalidField" => [{"message" => a_string_including("foo")}], + "ValidQuery" => [], + "InvalidArg" => [{"message" => a_string_including("foo")}] + ) + end + end + + context "when no `previously_dumped_variables` information is available" do + it "returns an error telling the engineer to run the rake task to dump them" do + query_string = <<~EOS + query MyQuery { + widgets { + total_edge_count + } + } + EOS + + expect(validate(query_string, previously_dumped_variables: nil, client_name: "Bob", query_name: "MyQ")).to have_errors_for_operations("MyQuery" => [{ + "message" => a_string_including("No dumped variables", "query_registry:dump_variables[Bob, MyQ]") + }]) + end + end + + context "when the query has arguments" do + it "still validates correctly in spite of us providing no values for required args" do + query_string = <<~EOS + query FindWidgetGood($id: ID!) { + widgets(filter: {id: {equal_to_any_of: [$id]}}) { + edges { + node { + id + } + } + } + } + + query FindWidgetBad($id: ID!) { + widgets(filter: {id: {equal_to_any_of: [$id]}}) { + edges { + node { + id + not_a_field + } + } + } + } + EOS + + expect(validate(query_string)).to have_errors_for_operations( + "FindWidgetGood" => [], + "FindWidgetBad" => [{"message" => a_string_including("not_a_field")}] + ) + end + + it "returns errors if a variable references an unknown type" do + query_string = <<~EOS + query FindWidget($id: Identifier) { + widgets(filter: {id: {equal_to_any_of: [$id]}}) { + total_edge_count + } + } + EOS + + expect(validate(query_string)).to have_errors_for_operations( + "FindWidget" => [{ + "message" => a_string_including("Identifier isn't a defined input type"), + "path" => ["query FindWidget"], + "extensions" => { + "code" => "variableRequiresValidType", + "variableName" => "id", + "typeName" => "Identifier" + }, + "locations" => [{"line" => 1, "column" => 18}] + }] + ) + end + + it "returns errors if a variable name is malformed" do + query_string = <<~EOS + query FindWidget(id: ID) { + widgets(filter: {id: {equal_to_any_of: [$id]}}) { + total_edge_count + } + } + EOS + + expect(validate(query_string)).to have_errors_for_operations(nil => [{ + "locations" => [{"line" => 1, "column" => 18}], + "message" => a_string_including("Expected VAR_SIGN, actual: IDENTIFIER (\"id\")") + }]) + end + + it "allows variables to be any non-object type (including scalars, lists, enums, lists-of-those, etc)" do + query_string = <<~EOS + query FindWidgets( + $id1: ID!, + $ids2: [ID!], + $ids3: [ID!]!, + $order_by: WidgetSortOrderInput! + ) { + w1: widgets(filter: {id: {equal_to_any_of: [$id1]}}) { + total_edge_count + } + + w2: widgets(filter: {id: {equal_to_any_of: $ids2}}) { + total_edge_count + } + + w3: widgets(filter: {id: {equal_to_any_of: $ids3}}) { + total_edge_count + } + + ordered: widgets(order_by: [$order_by]) { + total_edge_count + } + } + EOS + + expect(validate(query_string)).to eq("FindWidgets" => []) + end + + it "allows all kinds of variables, including objects" do + query_string = <<~EOS + query FindWidgets( + $filter1: WidgetFilterInput, + $filter2: WidgetFilterInput!, + ) { + w1: widgets(filter: $filter1) { + total_edge_count + } + + w2: widgets(filter: $filter2) { + total_edge_count + } + } + EOS + + expect(validate(query_string)).to eq("FindWidgets" => []) + end + + context "when the variable types have changed in a backwards-compatible way" do + it "tells the user to re-run the dump rake task to update the file" do + old_query_string = <<~EOS + query FindWidgets($id: ID!) { + widgets(filter: {id: {equal_to_any_of: [$id]}}) { + edges { + node { + id + } + } + } + } + EOS + previously_dumped_variables = variable_dumper.dump_variables_for_query(old_query_string) + + new_query_string = <<~EOS + query FindWidgets($id: ID!, $first: Int) { + widgets(filter: {id: {equal_to_any_of: [$id]}}, first: $first) { + edges { + node { + id + } + } + } + } + EOS + errors = validate(new_query_string, previously_dumped_variables: previously_dumped_variables, client_name: "C1", query_name: "Q1") + + expect(errors).to have_errors_for_operations("FindWidgets" => [{"message" => a_string_including( + "variables file is out-of-date", + "should not impact `C1`", + 'rake "query_registry:dump_variables[C1, Q1]"' + )}]) + end + end + + context "when the variable types have changed in a backwards-incompatible way" do + it "surfaces the incompatibilities and tells the user to re-run the dump rake task after verifying with the client" do + old_query_string = <<~EOS + query FindWidgets($id: ID!) { + widgets(filter: {id: {equal_to_any_of: [$id]}}) { + edges { + node { + id + } + } + } + } + EOS + previously_dumped_variables = variable_dumper.dump_variables_for_query(old_query_string) + + new_query_string = <<~EOS + query FindWidgets($id: ID!, $first: Int!, $cursor: Cursor!) { + widgets(filter: {id: {equal_to_any_of: [$id]}}, first: $first, after: $cursor) { + edges { + node { + id + } + } + } + } + EOS + errors = validate(new_query_string, previously_dumped_variables: previously_dumped_variables, client_name: "C1", query_name: "Q1") + + expect(errors).to have_errors_for_operations("FindWidgets" => [{"message" => a_string_including( + "backwards-incompatible changes that may break `C1`", + "$cursor (new required variable), $first (new required variable)", + 'rake "query_registry:dump_variables[C1, Q1]"' + )}]) + end + end + + context "when the operation name has changed since the last time variables were dumped" do + it "tells the user to re-dump the variables" do + old_query_string = <<~EOS + query findWidgets($id: ID!) { + widgets(filter: {id: {equal_to_any_of: [$id]}}) { + edges { + node { + id + } + } + } + } + EOS + previously_dumped_variables = variable_dumper.dump_variables_for_query(old_query_string) + + new_query_string = old_query_string.sub("findWidgets", "FindWidgets") + result = validate(new_query_string, previously_dumped_variables: previously_dumped_variables, client_name: "bob", query_name: "FindWidgets") + + expect(result).to have_errors_for_operations("FindWidgets" => [{ + "message" => a_string_including("No dumped variables", "query_registry:dump_variables[bob, FindWidgets]") + }]) + end + end + end + + def validate( + query_string, + previously_dumped_variables: variable_dumper.dump_variables_for_query(query_string), + client_name: "MyClient", + query_name: "MyQuery", + require_eg_latency_slo_directive: false + ) + validator = QueryValidator.new(schema, require_eg_latency_slo_directive: require_eg_latency_slo_directive) + + validator.validate( + query_string, + previously_dumped_variables: previously_dumped_variables, + client_name: client_name, + query_name: query_name + ) + end + + def have_errors_for_operations(errors_by_op_name) + matcher_hash = errors_by_op_name.transform_values do |errors| + a_collection_containing_exactly(*errors.map { |e| a_hash_including(e) }) + end + + match(matcher_hash) + end + end + end + end +end diff --git a/elasticgraph-query_registry/spec/unit/elastic_graph/query_registry/rake_tasks_spec.rb b/elasticgraph-query_registry/spec/unit/elastic_graph/query_registry/rake_tasks_spec.rb new file mode 100644 index 00000000..e8f887b2 --- /dev/null +++ b/elasticgraph-query_registry/spec/unit/elastic_graph/query_registry/rake_tasks_spec.rb @@ -0,0 +1,377 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql" +require "elastic_graph/query_registry/rake_tasks" +require "fileutils" + +module ElasticGraph + module QueryRegistry + RSpec.describe RakeTasks, :rake_task, :in_temp_dir do + attr_reader :last_task_output + let(:query_registry_dir) { "query_registry" } + + it "evaluates the graphql load block lazily so that if loading fails it only interferes with running tasks, not defining them" do + # Simulate schema artifacts being out of date. + load_graphql = lambda { raise "schema artifacts are out of date" } + + # Printing tasks is unaffected... + output = run_rake("--tasks", &load_graphql) + + expect(output).to eq(<<~EOS) + rake query_registry:dump_variables[client,query] # Updates the registered information about query variables for a specific client (and optionally, a specific query) + rake query_registry:dump_variables:all # Updates the registered information about query variables for all clients + rake query_registry:validate_queries # Validates the queries registered in `query_registry` + EOS + + # ...but when you run a task that needs the `GraphQL` instance the failure is surfaced. + expect { + run_rake("query_registry:validate_queries", &load_graphql) + }.to raise_error "schema artifacts are out of date" + end + + describe "query_registry:validate_queries" do + it "validates each query in the subdirectory of the given dir, reporting success for each valid query" do + register_query "client_bob", "CountWidgets.graphql", <<~EOS + query CountWidgets { + widgets { + total_edge_count + } + } + EOS + + register_query "client_bob", "CountComponents.graphql", <<~EOS + query CountComponents { + components { + total_edge_count + } + } + EOS + + register_query "client_jane", "CountParts.graphql", <<~EOS + query CountParts { + parts { + total_edge_count + } + } + EOS + + run_validate_task(after_dumping_variables: true) + + expect(last_task_output.string.strip).to eq(<<~EOS.strip) + For client `client_bob`: + - CountComponents.graphql (1 operation): + - CountComponents: ✅ + - CountWidgets.graphql (1 operation): + - CountWidgets: ✅ + + For client `client_jane`: + - CountParts.graphql (1 operation): + - CountParts: ✅ + EOS + end + + it "reports which queries are invalid" do + register_query "client_bob", "CountWidgets.graphql", <<~EOS + query CountWidgets { + widgets { + total_edge_count + } + } + EOS + + register_query "client_bob", "CountComponents.graphql", <<~EOS + query CountComponents { + components { + total_edge_count2 + } + widgets(foo: 1) { + total_edge_count + } + } + + query AnotherBadQuery { + w1: foo + } + EOS + + register_query "client_jane", "CountParts.graphql", <<~EOS + query CountParts + parts { + total_edge_count2 + } + } + EOS + + expect { + run_validate_task(after_dumping_variables: true) + }.to raise_error(a_string_including("Found 4 validation errors total across all queries.")) + + expect(last_task_output.string.strip).to eq(<<~EOS.strip) + For client `client_bob`: + - CountComponents.graphql (2 operations): + - CountComponents: 🛑. Got 2 validation errors: + 1) Field 'total_edge_count2' doesn't exist on type 'ComponentConnection' + path: query CountComponents.components.total_edge_count2 + source: query_registry/client_bob/CountComponents.graphql:3:5 + code: undefinedField + typeName: ComponentConnection + fieldName: total_edge_count2 + + 2) Field 'widgets' doesn't accept argument 'foo' + path: query CountComponents.widgets.foo + source: query_registry/client_bob/CountComponents.graphql:5:11 + code: argumentNotAccepted + name: widgets + typeName: Field + argumentName: foo + + - AnotherBadQuery: 🛑. Got 1 validation error: + 1) Field 'foo' doesn't exist on type 'Query' + path: query AnotherBadQuery.w1 + source: query_registry/client_bob/CountComponents.graphql:11:3 + code: undefinedField + typeName: Query + fieldName: foo + + - CountWidgets.graphql (1 operation): + - CountWidgets: ✅ + + For client `client_jane`: + - CountParts.graphql (1 operation): + - (no operation name): 🛑. Got 1 validation error: + 1) Expected LCURLY, actual: IDENTIFIER ("parts") at [2, 3] + source: query_registry/client_jane/CountParts.graphql:2:3 + EOS + end + + it "does not allow a single client to use the same operation name on two queries, since we want operation names to be human readable unique identifiers" do + register_query "client_bob", "CountWidgets.graphql", <<~EOS + query CountWidgets { + widgets { + total_edge_count + } + } + EOS + + register_query "client_bob", "CountComponents.graphql", <<~EOS + query CountWidgets { + components { + total_edge_count + } + } + EOS + + expect { + run_validate_task(after_dumping_variables: true) + }.to raise_error(a_string_including("Found 1 validation error total across all queries.")) + + expect(last_task_output.string.strip).to eq(<<~EOS.strip) + For client `client_bob`: + - CountComponents.graphql (1 operation): + - CountWidgets: ✅ + - CountWidgets.graphql (1 operation): + - CountWidgets: 🛑. Got 1 validation error: + 1) A `CountWidgets` query already exists for `client_bob` in `CountComponents.graphql`. Each query operation must have a unique name. + EOS + end + + it "allows different clients to register query operations with the same name, since we don't have a global query operation namespace" do + register_query "client_bob", "CountWidgets.graphql", <<~EOS + query CountWidgets { + widgets { + total_edge_count + } + } + EOS + + register_query "client_jane", "CountWidgets.graphql", <<~EOS + query CountWidgets { + components { + total_edge_count + } + } + EOS + + run_validate_task(after_dumping_variables: true) + + expect(last_task_output.string.strip).to eq(<<~EOS.strip) + For client `client_bob`: + - CountWidgets.graphql (1 operation): + - CountWidgets: ✅ + + For client `client_jane`: + - CountWidgets.graphql (1 operation): + - CountWidgets: ✅ + EOS + end + + it "reports issues with variables" do + register_query "client_bob", "CountWidgets.graphql", <<~EOS + query CountWidgets { + widgets { + total_edge_count + } + } + EOS + + register_query "client_jane", "CountWidgets.graphql", <<~EOS + query CountWidgets { + components { + total_edge_count + } + } + EOS + + expect { + run_validate_task(after_dumping_variables: false) + }.to raise_error(a_string_including("Found 2 validation errors total across all queries.")) + + expect(last_task_output.string.strip).to eq(<<~EOS.strip) + For client `client_bob`: + - CountWidgets.graphql (1 operation): + - CountWidgets: 🛑. Got 1 validation error: + 1) No dumped variables for this operation exist. Correct by running: `rake "query_registry:dump_variables[client_bob, CountWidgets]"` + + + For client `client_jane`: + - CountWidgets.graphql (1 operation): + - CountWidgets: 🛑. Got 1 validation error: + 1) No dumped variables for this operation exist. Correct by running: `rake "query_registry:dump_variables[client_jane, CountWidgets]"` + + EOS + end + + def run_validate_task(after_dumping_variables: false) + if after_dumping_variables + run_rake "query_registry:dump_variables:all" + end + + run_rake "query_registry:validate_queries" + end + end + + describe "query_validator:dump_variables" do + before do + register_query "client_bob", "CountWidgets.graphql", <<~EOS + query CountWidgets($ids: [ID!]) { + widgets(filter: {id: {equal_to_any_of: ids}}) { + total_edge_count + } + } + EOS + + register_query "client_bob", "CountComponents.graphql", <<~EOS + query CountComponents($ids: [ID!]) { + components(filter: {id: {equal_to_any_of: ids}}) { + total_edge_count + } + } + EOS + + register_query "client_jane", "CountWidgets.graphql", <<~EOS + query CountWidgets($ids: [ID!]) { + widgets(filter: {id: {equal_to_any_of: ids}}) { + total_edge_count + } + } + EOS + end + + describe "with client and query args" do + it "dumps the variables for just the one query" do + run_rake "query_registry:dump_variables[client_bob, CountWidgets]" + + expect_dumped_variables_files("client_bob/CountWidgets") + end + end + + describe "with just a client arg" do + it "dumps the variables for all queries for that client" do + run_rake "query_registry:dump_variables[client_bob]" + + expect_dumped_variables_files("client_bob/CountWidgets", "client_bob/CountComponents") + end + end + + describe ":all" do + it "dumps the variables for all queries" do + run_rake "query_registry:dump_variables:all" + + expect_dumped_variables_files( + "client_bob/CountWidgets", + "client_bob/CountComponents", + "client_jane/CountWidgets" + ) + end + end + + def expect_dumped_variables_files(*paths) + expected_files = paths.map { |path| "query_registry/#{path}.variables.yaml" } + expected_output = expected_files.map { |file| "- Dumped `#{file}`." } + + expect(last_task_output.string.split("\n")).to match_array(expected_output) + expect(Dir["**/*.variables.yaml"]).to match_array(expected_files) + + expected_files.each do |file| + query_name = file[/Count\w+/] + client_name = file[/client_\w+/] + + contents = ::File.read(file) + expect(contents).to include("Generated by `rake \"query_registry:dump_variables[#{client_name}, #{query_name}]\"`.") + + dumped_vars = ::YAML.safe_load(contents) + expect(dumped_vars).to eq(expected_dumped_content_for(query_name)) + end + end + + def expected_dumped_content_for(query_name) + {query_name => {"ids" => "[ID!]"}} + end + end + + def register_query(client_name, file_name, query_string) + query_dir = File.join(query_registry_dir, client_name) + FileUtils.mkdir_p(query_dir) + + full_file_name = File.join(query_dir, file_name) + File.write(full_file_name, query_string) + end + + def run_rake(command, &load_graphql) + load_graphql ||= lambda { build_graphql } + + super(command) do |output| + @last_task_output = output + RakeTasks.new(query_registry_dir, output: output, &load_graphql) + end + end + end + + # This is a separate example group because it needs `run_rake` to be defined differently from the group above. + RSpec.describe RakeTasks, ".from_yaml_file", :rake_task do + let(:query_registry_dir) { "query_registry" } + + it "loads the graphql instance from the given yaml file" do + output = run_rake "--tasks" + + expect(output).to include("rake query_registry:validate_queries") + end + + def run_rake(*args) + Dir.chdir CommonSpecHelpers::REPO_ROOT do + super(*args) do |output| + RakeTasks.from_yaml_file(CommonSpecHelpers.test_settings_file, query_registry_dir, output: output).tap do |tasks| + expect(tasks.send(:graphql)).to be_a(GraphQL) + end + end + end + end + end + end +end diff --git a/elasticgraph-query_registry/spec/unit/elastic_graph/query_registry/registry_spec.rb b/elasticgraph-query_registry/spec/unit/elastic_graph/query_registry/registry_spec.rb new file mode 100644 index 00000000..0dbc80f8 --- /dev/null +++ b/elasticgraph-query_registry/spec/unit/elastic_graph/query_registry/registry_spec.rb @@ -0,0 +1,623 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql" +require "elastic_graph/graphql/client" +require "elastic_graph/query_registry/registry" + +module ElasticGraph + module QueryRegistry + RSpec.describe Registry do + let(:schema) { build_graphql.schema } + + let(:widget_name_string) do + <<~EOS.strip + query WidgetName { + __type(name: "Widget") { + name + } + } + EOS + end + + let(:part_name_string) do + <<~EOS.strip + query PartName { + __type(name: "Part") { + name + } + } + EOS + end + + describe "#build_and_validate_query" do + shared_examples_for "any case when a query is returned" do + it "allows the returned query to be executed multiple times with different variables" do + # This query uses a fragment to ensure that our query canonicalization logic (which looks at + # query.document.definitions) can handle multiple kinds of definitions (operations and fragments). + any_name_query = <<~EOS + query GetTypeName($name: String!) { + __type(name: $name) { + ...typeFields + } + } + + fragment typeFields on __Type { + name + } + EOS + + registry = prepare_registry_with(any_name_query) + + result = execute_allowed_query(registry, any_name_query, variables: {"name" => "Address"}) + expect(result.to_h).to eq(result_with_type_name("Address")) + + result = execute_allowed_query(registry, any_name_query, variables: {"name" => "Component"}) + expect(result.to_h).to eq(result_with_type_name("Component")) + + result = execute_allowed_query(registry, any_name_query, variables: {"name" => "Address"}) + expect(result.to_h).to eq(result_with_type_name("Address")) + end + + it "allows the returned query to be executed multiple times with different operations" do + widget_or_part_name_query = <<~EOS + #{widget_name_string} + + #{part_name_string} + EOS + + registry = prepare_registry_with(widget_or_part_name_query) + + result = execute_allowed_query(registry, widget_or_part_name_query, operation_name: "WidgetName") + expect(result.to_h).to eq(result_with_type_name("Widget")) + + result = execute_allowed_query(registry, widget_or_part_name_query, operation_name: "PartName") + expect(result.to_h).to eq(result_with_type_name("Part")) + + result = execute_allowed_query(registry, widget_or_part_name_query, operation_name: "WidgetName") + expect(result.to_h).to eq(result_with_type_name("Widget")) + end + end + + context "for a named client who has registered queries" do + context "when given a query string that is byte-for-byte the same as a registered query" do + include_examples "any case when a query is returned" + + def prepare_registry_with(query_string) + registry_with({"my_client" => [query_string]}) + end + + def execute_allowed_query(registry, query_string, **options) + super(registry, query_string, client: client_named("my_client"), **options) + end + end + + context "when given a query string that lacks the `@egLatencySlo` directive the registered query has" do + include_examples "any case when a query is returned" + + def prepare_registry_with(query_string) + query_string = add_query_directives_to(query_string, "@eg_latency_slo(ms: 2000)") + registry_with({"my_client" => [query_string]}) + end + + def execute_allowed_query(registry, query_string, **options) + super(registry, query_string, client: client_named("my_client"), **options) + end + end + + context "when given a query string that has a `@egLatencySlo` directive the registered query lacks" do + include_examples "any case when a query is returned" + + def prepare_registry_with(query_string) + registry_with({"my_client" => [query_string]}) + end + + def execute_allowed_query(registry, query_string, **options) + query_string = add_query_directives_to(query_string, "@eg_latency_slo(ms: 2000)") + super(registry, query_string, client: client_named("my_client"), **options) + end + end + + context "when given a query string that has a different `@egLatencySlo` directive value compared to the registered query" do + include_examples "any case when a query is returned" + + def prepare_registry_with(query_string) + query_string = add_query_directives_to(query_string, "@eg_latency_slo(ms: 4000)") + registry_with({"my_client" => [query_string]}) + end + + def execute_allowed_query(registry, query_string, **options) + query_string = add_query_directives_to(query_string, "@eg_latency_slo(ms: 2000)") + super(registry, query_string, client: client_named("my_client"), **options) + end + end + + context "when given a query string that has an extra directive (that is not `@egLatencySlo`) not on the registered query" do + it "returns a validation error since the extra directive could be a substantive difference" do + registry = registry_with({"my_client" => [widget_name_string, part_name_string]}) + modified_parts_query = add_query_directives_to(part_name_string, "@some_other_directive") + + query, errors = registry.build_and_validate_query(modified_parts_query, client: client_named("my_client")) + + expect(errors).to contain_exactly(a_string_including( + "Query PartName", "differs from the registered form of `PartName`", "my_client" + )) + expect(query.query_string).to eq(modified_parts_query) + end + end + + context "when given a query string that has lacks a directive (that is not `@egLatencySlo`) present on the registered query" do + it "returns a validation error since the extra directive could be a substantive difference" do + registry = registry_with({"my_client" => [widget_name_string, add_query_directives_to(part_name_string, "@some_other_directive")]}) + + query, errors = registry.build_and_validate_query(part_name_string, client: client_named("my_client")) + + expect(errors).to contain_exactly(a_string_including( + "Query PartName", "differs from the registered form of `PartName`", "my_client" + )) + expect(query.query_string).to eq(part_name_string) + end + end + + context "when the client is also in `allow_any_query_for_clients`" do + context "for a query that is unregistered" do + include_examples "any case when a query is returned" + + def prepare_registry_with(query_string) + some_other_query = <<~EOS + query SomeOtherRegisteredQuery { + __typename + } + EOS + + registry_with({"my_client" => [some_other_query]}, allow_any_query_for_clients: ["my_client"]) + end + + def execute_allowed_query(registry, query_string, **options) + super(registry, query_string, client: client_named("my_client"), **options) + end + end + + context "for a query that is identical to a registered one" do + include_examples "any case when a query is returned" + + def prepare_registry_with(query_string) + registry_with({"my_client" => [query_string]}, allow_any_query_for_clients: ["my_client"]) + end + + def execute_allowed_query(registry, query_string, **options) + super(registry, query_string, client: client_named("my_client"), **options) + end + end + + context "for a query that has the same name as a registered one but differs substantially" do + it "executes it even though it differs from the registered query since the client is in `allow_any_query_for_clients`" do + query1 = "query SomeQuery { __typename }" + query2 = "query SomeQuery { t: __typename }" + + registry = registry_with({"my_client" => [query1]}, allow_any_query_for_clients: ["my_client"]) + + expect(execute_allowed_query(registry, query2, client: client_named("my_client")).to_h).to eq({"data" => {"t" => "Query"}}) + end + end + end + + context "when `allow_unregistered_clients` is `true`" do + it "still validates that the given queries match what has been registered for this client" do + query1 = <<~QUERY + query SomeQuery { + __typename + } + QUERY + + query2 = "query SomeQuery { t: __typename }" + query3 = "query SomeQuery3 { t: __typename }" + + registry = registry_with({"my_client" => [query1]}, allow_unregistered_clients: true) + + expect(execute_allowed_query(registry, query1, client: client_named("my_client")).to_h).to eq({"data" => {"__typename" => "Query"}}) + + query, errors = registry.build_and_validate_query(query2, client: client_named("my_client")) + expect(errors).to contain_exactly(a_string_including( + "Query SomeQuery", "differs from the registered form of `SomeQuery`", "my_client" + )) + expect(query.query_string).to eq(query2) + + query, errors = registry.build_and_validate_query(query3, client: client_named("my_client")) + expect(errors).to contain_exactly(a_string_including( + "Query SomeQuery3", "is unregistered", "my_client" + )) + expect(query.query_string).to eq(query3) + end + end + + def add_query_directives_to(query_string, query_directives_string) + # https://rubular.com/r/T2UXnaXfWr8nIg has examples of this regex + query_string.gsub(/^query (\w+)(?:\([^)]+\))?/) { |it| "#{it} #{query_directives_string}" } + end + + context "when given a query string that has only minor formatting differences compared to a registered query" do + include_examples "any case when a query is returned" + + it "caches the parsed form of the mostly recently seen alternate query string, to avoid unneeded reparsing" do + parse_counts_by_query_string = track_parse_counts + + registry = prepare_registry_with(part_name_string) + get_allowed_query(registry, part_name_string, client: client_named("my_client")) + + expect(parse_counts_by_query_string).to eq(part_name_string => 1) + + modified_query_string1 = modify_query_string(part_name_string) + get_allowed_query(registry, modified_query_string1, client: client_named("my_client")) + expect(parse_counts_by_query_string).to eq(part_name_string => 1, modified_query_string1 => 1) + + # Now that `modified_query_string1` is cached, further requests with that query string should not + # need to be re-parsed. + 3.times do + get_allowed_query(registry, modified_query_string1, client: client_named("my_client")) + end + expect(parse_counts_by_query_string).to eq(part_name_string => 1, modified_query_string1 => 1) + + modified_query_string2 = modify_query_string(modified_query_string1) + get_allowed_query(registry, modified_query_string2, client: client_named("my_client")) + expect(parse_counts_by_query_string).to eq( + part_name_string => 1, + modified_query_string1 => 1, + modified_query_string2 => 1 + ) + + # ...but now that the client submitted a different form of the query, the prior form should have + # been evicted from the cache, and we expect the older form to be need to be re-parsed. + get_allowed_query(registry, modified_query_string1, client: client_named("my_client")) + expect(parse_counts_by_query_string).to eq( + part_name_string => 1, + modified_query_string1 => 2, + modified_query_string2 => 1 + ) + + # ...but so long as `modified_query_string1` keeps getting re-used, no further re-parsings should be required. + 3.times do + get_allowed_query(registry, modified_query_string1, client: client_named("my_client")) + end + expect(parse_counts_by_query_string).to eq( + part_name_string => 1, + modified_query_string1 => 2, + modified_query_string2 => 1 + ) + end + + def prepare_registry_with(query_string) + registry_with({"my_client" => [query_string]}) + end + + def execute_allowed_query(registry, query_string, **options) + modified_query_string = modify_query_string(query_string) + returned_query = get_allowed_query(registry, modified_query_string, client: client_named("my_client"), **options) + returned_query.result + end + + def modify_query_string(query_string) + "# This is a leading comment\n#{query_string}" + end + end + + it "returns a validation error for a query string that has substantive differences compared to a registered query" do + registry = registry_with({"my_client" => [widget_name_string, part_name_string]}) + modified_parts_query = part_name_string.sub("name\n", "the_name: name\n") + + query, errors = registry.build_and_validate_query(modified_parts_query, client: client_named("my_client")) + + expect(errors).to contain_exactly(a_string_including( + "Query PartName", "differs from the registered form of `PartName`", "my_client" + )) + expect(query.query_string).to eq(modified_parts_query) + end + + it "returns a validation error for an unregistered invalid query" do + registry = registry_with({"my_client" => [widget_name_string, part_name_string]}) + + query, errors = registry.build_and_validate_query("not_a_query", client: client_named("my_client")) + + expect(errors).to contain_exactly(a_string_including( + "Query anonymous", "is unregistered", "my_client", + "no registered query with a `` operation" + )) + expect(query.query_string).to eq("not_a_query") + end + + it "echoes no validation errors for a registered invalid query when it is byte-for-byte the same" do + registry = registry_with({"my_client" => ["not_a_query"]}) + + query, errors = registry.build_and_validate_query("not_a_query", client: client_named("my_client")) + + expect(errors).to be_empty + expect(query.result["errors"].to_s).to include("Expected one of", 'actual: IDENTIFIER (\"not_a_query\")') + end + + it "does not consider a query registered by a different client" do + registry = registry_with({"client1" => [widget_name_string], "client2" => [part_name_string]}) + + query, errors = registry.build_and_validate_query(part_name_string, client: client_named("client1")) + expect(errors).to contain_exactly(a_string_including( + "Query PartName", "is unregistered", "client1", + "no registered query with a `PartName` operation" + )) + expect(query.query_string).to eq(part_name_string) + + query, errors = registry.build_and_validate_query(part_name_string, client: client_named("client2")) + expect(errors).to be_empty + expect(query.query_string).to eq(part_name_string) + end + + it "avoids re-parsing the query string when the query string is byte-for-byte identical to a registered one, for efficiency" do + parse_counts_by_query_string = track_parse_counts + registry = registry_with({"client1" => [widget_name_string, part_name_string]}) + + 3.times do + result = execute_allowed_query(registry, widget_name_string, client: client_named("client1")) + expect(result.to_h).to eq(result_with_type_name("Widget")) + + result = execute_allowed_query(registry, part_name_string, client: client_named("client1")) + expect(result.to_h).to eq(result_with_type_name("Part")) + end + + expect(parse_counts_by_query_string).to eq( + widget_name_string => 1, + part_name_string => 1 + ) + end + end + + it "always allows the internal ElasticGraph client to submit the eager load query that happens at boot time" do + registry = registry_with({}, allow_unregistered_clients: false, allow_any_query_for_clients: []) + + result = execute_allowed_query(registry, GraphQL::EAGER_LOAD_QUERY, client: GraphQL::Client::ELASTICGRAPH_INTERNAL) + + expect(result.to_h.dig("data", "__schema", "types")).to include({"kind" => "OBJECT"}) + end + + shared_examples_for "a client not in the registry" do + context "when `allow_unregistered_clients` is `false`" do + it "returns validation errors when given a valid query" do + registry = registry_with({"my_client" => [widget_name_string, part_name_string]}, allow_unregistered_clients: false) + + query, errors = registry.build_and_validate_query(part_name_string, client: client) + + expect(errors).to contain_exactly(a_string_including("not a registered client", client&.name.to_s)) + expect(query.query_string).to eq(part_name_string) + end + + it "returns validation errors when given an invalid query" do + registry = registry_with({"my_client" => [widget_name_string, part_name_string]}, allow_unregistered_clients: false) + + query, errors = registry.build_and_validate_query("not_a_query", client: client) + + expect(errors).to contain_exactly(a_string_including("not a registered client", client&.name.to_s)) + expect(query.query_string).to eq("not_a_query") + end + end + + context "when `allow_unregistered_clients` is `true`" do + include_examples "any case when a query is returned" + + it "returns no validation errors for an invalid query" do + registry = registry_with({}, allow_unregistered_clients: true) + + query, errors = registry.build_and_validate_query("not_a_query", client: client) + + expect(errors).to be_empty + expect(query.result["errors"].to_s).to include("Expected one of", 'actual: IDENTIFIER (\"not_a_query\")') + end + + def prepare_registry_with(query_string) + registry_with({"some_client_these_tests_dont_use" => [query_string]}, allow_unregistered_clients: true) + end + + def execute_allowed_query(registry, query_string, **options) + super(registry, query_string, client: client, **options) + end + end + end + + context "for a named client who has no registered queries" do + include_examples "a client not in the registry" + let(:client) { client_named("unregistered_client") } + + context "when the client is in `allow_any_query_for_clients`" do + include_examples "any case when a query is returned" + + def prepare_registry_with(query_string) + registry_with({}, allow_unregistered_clients: false, allow_any_query_for_clients: [client.name]) + end + + def execute_allowed_query(registry, query_string, **options) + super(registry, query_string, client: client, **options) + end + end + end + + context "for a client with no `name`" do + include_examples "a client not in the registry" + let(:client) { client_named(nil) } + end + + context "for a nil client" do + include_examples "a client not in the registry" + let(:client) { nil } + end + + it "defers parsing queries for a client until the client submits its first query, so that low QPS clients with many large queries are ignored until needed" do + parse_counts_by_query_string = track_parse_counts + + registry = registry_with({"client1" => [widget_name_string], "client2" => [part_name_string]}) + + 3.times do + query = get_allowed_query(registry, widget_name_string, client: client_named("client1")) + expect(query.query_string).to eq(widget_name_string) + end + + expect(parse_counts_by_query_string).to eq(widget_name_string => 1) + + 3.times do + query = get_allowed_query(registry, part_name_string, client: client_named("client2")) + expect(query.query_string).to eq(part_name_string) + end + + expect(parse_counts_by_query_string).to eq(widget_name_string => 1, part_name_string => 1) + end + + def registry_with(queries_by_client_name, allow_unregistered_clients: false, allow_any_query_for_clients: []) + Registry.new( + schema, + client_names: queries_by_client_name.keys, + allow_unregistered_clients: allow_unregistered_clients, + allow_any_query_for_clients: allow_any_query_for_clients + ) do |client_name| + # We use `fetch` here to demonstrate that the registry does not call our block + # with any client names outside of the passed list. + queries_by_client_name.fetch(client_name) + end + end + + def get_allowed_query(registry, query_string, **options) + query, errors = registry.build_and_validate_query(query_string, **options) + expect(errors).to be_empty + query + end + + def execute_allowed_query(registry, query_string, **options) + get_allowed_query(registry, query_string, **options).tap do |query| + # Verify that the executed query is exactly what was submitted rather than what was registered. + # This particularly matters with the `@egLatencySlo` directive. We want the value on the query + # to be used when it differs from the SLO threshold on the registered query. + expect(query.query_string.strip).to eq(query_string.strip) + end.result + end + + def result_with_type_name(name) + {"data" => {"__type" => {"name" => name}}} + end + + def track_parse_counts + parse_counts_by_query_string = ::Hash.new(0) + + allow(::GraphQL).to receive(:parse).and_wrap_original do |original, query_string, **options| + parse_counts_by_query_string[query_string] += 1 unless query_string == GraphQL::EAGER_LOAD_QUERY + original.call(query_string, **options) + end + + parse_counts_by_query_string + end + end + + describe ".build_from_directory", :in_temp_dir do + let(:query_registry_dir) { "registered_queries" } + let(:registry) do + Registry.build_from_directory( + schema, + query_registry_dir, + allow_unregistered_clients: false, + allow_any_query_for_clients: [] + ) + end + + it "builds an instance based on the queries in the given directory" do + register_query("client1", "WidgetName.graphql", widget_name_string) + + query, errors = registry.build_and_validate_query(widget_name_string, client: client_named("client1")) + expect(errors).to be_empty + expect(query.query_string).to eq(widget_name_string) + + query, errors = registry.build_and_validate_query(part_name_string, client: client_named("client1")) + expect(errors).to contain_exactly(a_string_including( + "Query PartName", "is unregistered", "client1", + "no registered query with a `PartName` operation" + )) + expect(query.query_string).to eq(part_name_string) + + query, errors = registry.build_and_validate_query(widget_name_string, client: client_named("other-client")) + expect(errors).to contain_exactly(a_string_including("not a registered client", "other-client")) + expect(query.query_string).to eq(widget_name_string) + end + + it "defers reading from disk until it needs to, and caches the disk results after that" do + file_reads_by_name = track_file_reads + + register_query("client1", "WidgetName.graphql", widget_name_string) + register_query("client2", "WidgetName.graphql", part_name_string) + register_query("client2", "PartName.graphql", part_name_string) + + expect(file_reads_by_name).to be_empty + + registry.build_and_validate_query(widget_name_string, client: client_named("client1")) + expect(file_reads_by_name).to eq({ + "registered_queries/client1/WidgetName.graphql" => 1 + }) + + registry.build_and_validate_query(widget_name_string, client: client_named("client2")) + expect(file_reads_by_name).to eq({ + "registered_queries/client1/WidgetName.graphql" => 1, + "registered_queries/client2/WidgetName.graphql" => 1, + "registered_queries/client2/PartName.graphql" => 1 + }) + + # further uses of the registry should trigger no more reads. + expect { + registry.build_and_validate_query(widget_name_string, client: client_named("client1")) + registry.build_and_validate_query(part_name_string, client: client_named("client1")) + registry.build_and_validate_query(widget_name_string, client: client_named("client2")) + registry.build_and_validate_query(part_name_string, client: client_named("client2")) + }.not_to change { file_reads_by_name } + end + + it "ignores client directory files that do not end in `.graphql`" do + file_reads_by_name = track_file_reads + + register_query("client1", "WidgetName.graphql", widget_name_string) + register_query("client1", "README.md", "## Not a GraphQL query") + + query, errors = registry.build_and_validate_query(widget_name_string, client: client_named("client1")) + expect(errors).to be_empty + expect(query.query_string).to eq(widget_name_string) + + expect(file_reads_by_name.keys).to contain_exactly( + "registered_queries/client1/WidgetName.graphql" + ) + end + + def register_query(client_name, file_name, query_string) + query_dir = File.join(query_registry_dir, client_name) + FileUtils.mkdir_p(query_dir) + + full_file_name = File.join(query_dir, file_name) + File.write(full_file_name, query_string) + end + + def track_file_reads + file_reads_by_name = ::Hash.new(0) + + allow(::File).to receive(:read).and_wrap_original do |original, file_name| + # :nocov: -- the `else` branch here is only covered when the test is run in isolation + # (in that case, there are a bunch of config files that are read off disk). + # When run in a larger test suite those config files have already been read + # and cached in memory. + file_reads_by_name[file_name] += 1 if file_name.include?(query_registry_dir) + # :nocov: + original.call(file_name) + end + + file_reads_by_name + end + end + + def client_named(name) + GraphQL::Client.new(name: name, source_description: "some-description") + end + end + end +end diff --git a/elasticgraph-query_registry/spec/unit/elastic_graph/query_registry/variable_backward_incompatibility_detector_spec.rb b/elasticgraph-query_registry/spec/unit/elastic_graph/query_registry/variable_backward_incompatibility_detector_spec.rb new file mode 100644 index 00000000..c85461d3 --- /dev/null +++ b/elasticgraph-query_registry/spec/unit/elastic_graph/query_registry/variable_backward_incompatibility_detector_spec.rb @@ -0,0 +1,378 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/query_registry/variable_backward_incompatibility_detector" + +module ElasticGraph + module QueryRegistry + RSpec.describe VariableBackwardIncompatibilityDetector, ".detect" do + let(:variable_backward_incompatibility_detector) { VariableBackwardIncompatibilityDetector.new } + + it "returns an empty list when given two empty hashes" do + incompatibilities = detect_incompatibilities( + old: {}, + new: {} + ) + + expect(incompatibilities).to eq([]) + end + + it "returns an empty list when given identical variables" do + incompatibilities = detect_incompatibilities( + old: {"id" => "ID!", "count" => "Int"}, + new: {"id" => "ID!", "count" => "Int"} + ) + + expect(incompatibilities).to eq([]) + end + + it "identifies a removed variable, regardless of nullability, as potentially breaking since the client may pass a value for it" do + incompatibilities = detect_incompatibilities( + old: {"id" => "ID!", "count" => "Int", "foo" => "Int!"}, + new: {"id" => "ID!"} + ) + + expect(incompatibilities).to contain_exactly( + "$count (removed)", + "$foo (removed)" + ) + end + + it "identifies an added variable as potentially breaking only if it is non-null since the client isn't passing a value for it and that causes no problem for nullable vars" do + incompatibilities = detect_incompatibilities( + old: {"id" => "ID!"}, + new: { + "id" => "ID!", + "count" => "Int", + "foo" => "Int!", + "new_enum1" => { + "type" => "Enum1", + "values" => ["A"] + }, + "new_enum2" => { + "type" => "Enum1!", + "values" => ["A"] + }, + "new_object1" => { + "type" => "Object1", + "fields" => { + "foo" => "Int" + } + }, + "new_object2" => { + "type" => "Object1!", + "fields" => { + "foo" => "Int" + } + } + } + ) + + expect(incompatibilities).to contain_exactly( + "$foo (new required variable)", + "$new_enum2 (new required variable)", + "$new_object2 (new required variable)" + ) + end + + it "identifies a variable with a changed type to be potentially breaking unless its only relaxing nullability" do + incompatibilities = detect_incompatibilities( + old: {"id" => "ID!", "count" => "Int", "foo" => "String", "bar" => "Float"}, + new: {"id" => "ID", "count" => "Int!", "foo" => "Int", "bar" => "Int!"} + ) + + expect(incompatibilities).to contain_exactly( + "$count (required for the first time)", + "$foo (type changed from `String` to `Int`)", + "$bar (type changed from `Float` to `Int!`)" + ) + end + + it "handles a variable changing to an enum type or from an enum type" do + incompatibilities = detect_incompatibilities( + old: { + "var1" => { + "type" => "Enum1", + "values" => ["A", "B", "C"] + }, + "var2" => "Int" + }, + new: { + "var1" => "Int", + "var2" => { + "type" => "Enum1", + "values" => ["A", "B", "C"] + } + } + ) + + expect(incompatibilities).to contain_exactly( + "$var1 (type changed from `Enum1` to `Int`)", + "$var2 (type changed from `Int` to `Enum1`)" + ) + end + + it "handles a variable changing to an object type or from an object type" do + incompatibilities = detect_incompatibilities( + old: { + "var1" => { + "type" => "Object1", + "fields" => { + "foo" => "Int" + } + }, + "var2" => "Int" + }, + new: { + "var1" => "Int", + "var2" => { + "type" => "Object1", + "fields" => { + "foo" => "Int" + } + } + } + ) + + expect(incompatibilities).to contain_exactly( + "$var1 (type changed from `Object1` to `Int`)", + "$var2 (type changed from `Int` to `Object1`)" + ) + end + + context "with an enum variable" do + it "does not consider it breaking when the enum values have not changed" do + incompatibilities = detect_incompatibilities( + old: { + "enum1" => { + "type" => "Enum1", + "values" => ["A", "B", "C"] + } + }, + new: { + "enum1" => { + "type" => "Enum1", + "values" => ["A", "B", "C"] + } + } + ) + + expect(incompatibilities).to eq([]) + end + + it "identifies the variable as potentially breaking when it loses an enum value that the client could depend on" do + incompatibilities = detect_incompatibilities( + old: { + "enum1" => { + "type" => "Enum1", + "values" => ["A", "B", "C", "D"] + } + }, + new: { + "enum1" => { + "type" => "Enum1", + "values" => ["A", "C"] + } + } + ) + + expect(incompatibilities).to contain_exactly("$enum1 (removed enum values: B, D)") + end + + it "ignores new enum values since the client can't be broken by an input enum type accepting a new value" do + incompatibilities = detect_incompatibilities( + old: { + "enum1" => { + "type" => "Enum1", + "values" => ["A", "B", "C"] + } + }, + new: { + "enum1" => { + "type" => "Enum1", + "values" => ["A", "B", "C", "D"] + } + } + ) + + expect(incompatibilities).to eq([]) + end + + it "identifies the variable as potentially breaking when it both gains and loses enum values" do + incompatibilities = detect_incompatibilities( + old: { + "enum1" => { + "type" => "Enum1", + "values" => ["A", "B", "C"] + } + }, + new: { + "enum1" => { + "type" => "Enum1", + "values" => ["A", "D", "C"] + } + } + ) + + expect(incompatibilities).to contain_exactly("$enum1 (removed enum values: B)") + end + end + + context "with object variables" do + it "does not consider it breaking when the object fields have not changed" do + incompatibilities = detect_incompatibilities( + old: { + "object1" => { + "type" => "Object1", + "fields" => { + "foo" => "Int", + "bar" => "Int" + } + } + }, + new: { + "object1" => { + "type" => "Object1", + "fields" => { + "foo" => "Int", + "bar" => "Int" + } + } + } + ) + + expect(incompatibilities).to eq([]) + end + + it "identifies the variable as potentially breaking when it loses fields that the client could depend on" do + incompatibilities = detect_incompatibilities( + old: { + "object1" => { + "type" => "Object1", + "fields" => { + "foo" => "Int", + "bar" => "Int" + } + } + }, + new: { + "object1" => { + "type" => "Object1", + "fields" => { + "foo" => "Int" + } + } + } + ) + + expect(incompatibilities).to contain_exactly("$object1.bar (removed)") + end + + it "identifies the variable as potentially breaking when it gains a non-null field, since the client couldn't already be passing values for it" do + incompatibilities = detect_incompatibilities( + old: { + "object1" => { + "type" => "Object1", + "fields" => { + "foo" => "Int", + "bar" => "Int" + } + } + }, + new: { + "object1" => { + "type" => "Object1", + "fields" => { + "foo" => "Int", + "bar" => "Int", + "bazz" => "Int!" + } + } + } + ) + + expect(incompatibilities).to contain_exactly("$object1.bazz (new required field)") + end + + it "ignores new nullable fields since the client can't be broken by the endpoint optionally accepting a field the client doesn't know about" do + incompatibilities = detect_incompatibilities( + old: { + "object1" => { + "type" => "Object1", + "fields" => { + "foo" => "Int", + "bar" => "Int" + } + } + }, + new: { + "object1" => { + "type" => "Object1", + "fields" => { + "foo" => "Int", + "bar" => "Int", + "bazz" => "Int" + } + } + } + ) + + expect(incompatibilities).to eq([]) + end + + it "detects incompatibilities for nested fields" do + incompatibilities = detect_incompatibilities( + old: { + "object1" => { + "type" => "Object1", + "fields" => { + "foo1" => { + "type" => "Object2", + "fields" => { + "foo2" => { + "type" => "Object3", + "fields" => { + "foo3" => "Int", + "foo4" => "Int" + } + } + } + } + } + } + }, + new: { + "object1" => { + "type" => "Object1", + "fields" => { + "foo1" => { + "type" => "Object2", + "fields" => { + "foo2" => { + "type" => "Object3", + "fields" => { + "foo3" => "Int" + } + } + } + } + } + } + } + ) + + expect(incompatibilities).to contain_exactly("$object1.foo1.foo2.foo4 (removed)") + end + end + + def detect_incompatibilities(old:, new:) + variable_backward_incompatibility_detector.detect(old_op_vars: old, new_op_vars: new).map(&:description) + end + end + end +end diff --git a/elasticgraph-query_registry/spec/unit/elastic_graph/query_registry/variable_dumper_spec.rb b/elasticgraph-query_registry/spec/unit/elastic_graph/query_registry/variable_dumper_spec.rb new file mode 100644 index 00000000..391c9565 --- /dev/null +++ b/elasticgraph-query_registry/spec/unit/elastic_graph/query_registry/variable_dumper_spec.rb @@ -0,0 +1,189 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql" +require "elastic_graph/query_registry/variable_dumper" + +module ElasticGraph + module QueryRegistry + RSpec.describe VariableDumper do + let(:dumper) { VariableDumper.new(build_graphql.schema.graphql_schema) } + + describe "#dump_variables_for_query" do + it "returns an empty hash for a valid query that has no variables" do + query_string = <<~EOS + query MyQuery { + widgets { + total_edge_count + } + } + EOS + + expect(dumper.dump_variables_for_query(query_string)).to eq({"MyQuery" => {}}) + end + + it "returns an empty hash for an unparsable query that has variables" do + query_string = <<~EOS + query MyQuery($idFilter: IDFilterInput!) { + widgets(filter: {id: $idFilter}) # missing a curly brace here + total_edge_count + } + } + EOS + + expect(dumper.dump_variables_for_query(query_string)).to eq({}) + end + + it "includes a simple type entry for each scalar or list-of-scalar variable" do + query_string = <<~EOS + query CountWidgetAndAddress($id: ID!, $addressId: ID) { + widgets(filter: {id: {equal_to_any_of: [$id]}}) { + total_edge_count + } + + addresses(filter: {id: {equal_to_any_of: [$addressId]}}){ + total_edge_count + } + } + + query AnotherQuery($ids: [ID!]) { + components(filter: {id: {equal_to_any_of: $ids}}) { + total_edge_count + } + } + EOS + + expect(dumper.dump_variables_for_query(query_string)).to eq({ + "CountWidgetAndAddress" => {"id" => "ID!", "addressId" => "ID"}, + "AnotherQuery" => {"ids" => "[ID!]"} + }) + end + + it "provides a reasonable default name for operations that lack names" do + query_string = <<~EOS + query { + widgets { + total_edge_count + } + } + EOS + + expect(dumper.dump_variables_for_query(query_string)).to eq({ + "(Anonymous operation 1)" => {} + }) + end + + it "includes `values` for enum variables" do + query_string = <<~EOS + query CountWithColorFilterInput($colors: [Color!]) { + widgets(filter: {options: {color: {equal_to_any_of: $colors}}}) { + total_edge_count + } + } + EOS + + expect(dumper.dump_variables_for_query(query_string)).to eq({ + "CountWithColorFilterInput" => { + "colors" => {"type" => "[Color!]", "values" => ["BLUE", "GREEN", "RED"]} + } + }) + end + + it "includes the `fields` for an object variable" do + query_string = <<~EOS + query CountWithIdFilterInput($idFilter: IDFilterInput!) { + widgets(filter: {id: $idFilter}) { + edges { + node { + id + } + } + } + } + EOS + + expect(dumper.dump_variables_for_query(query_string)).to eq({ + "CountWithIdFilterInput" => { + "idFilter" => { + "type" => "IDFilterInput!", + "fields" => {"any_of" => "[IDFilterInput!]", "not" => "IDFilterInput", "equal_to_any_of" => "[ID]"} + } + } + }) + end + + it "just includes the type name for variables that reference undefined types" do + query_string = <<~EOS + query FindWidgets($id1: Identifier!, $id2: ID!) { + widgets(filter: {id: {equal_to_any_of: [$id1, $id2]}}) { + total_edge_count + } + } + EOS + + expect(dumper.dump_variables_for_query(query_string)).to eq({ + "FindWidgets" => { + "id1" => "Identifier!", + "id2" => "ID!" + } + }) + end + + it "handles nested objects" do + query_string = <<~EOS + query CountWithOptions($optionsFilterInput: WidgetOptionsFilterInput!) { + widgets(filter: {options: $optionsFilterInput}) { + total_edge_count + } + } + EOS + + expect(dumper.dump_variables_for_query(query_string)).to eq({"CountWithOptions" => { + "optionsFilterInput" => {"type" => "WidgetOptionsFilterInput!", "fields" => { + "any_of" => "[WidgetOptionsFilterInput!]", + "not" => "WidgetOptionsFilterInput", + "color" => {"type" => "ColorFilterInput", "fields" => { + "any_of" => "[ColorFilterInput!]", + "not" => "ColorFilterInput", + "equal_to_any_of" => {"type" => "[ColorInput]", "values" => ["BLUE", "GREEN", "RED"]} + }}, + "size" => {"type" => "SizeFilterInput", "fields" => { + "any_of" => "[SizeFilterInput!]", + "not" => "SizeFilterInput", + "equal_to_any_of" => {"type" => "[SizeInput]", "values" => ["LARGE", "MEDIUM", "SMALL"]} + }}, + "the_size" => {"type" => "SizeFilterInput", "fields" => { + "any_of" => "[SizeFilterInput!]", + "not" => "SizeFilterInput", + "equal_to_any_of" => {"type" => "[SizeInput]", "values" => ["LARGE", "MEDIUM", "SMALL"]} + }} + }} + }}) + end + + it "ignores fragments" do + query_string = <<~EOS + query CountWidgetAndAddress($id: ID!) { + widgets(filter: {id: {equal_to_any_of: [$id]}}) { + ...widgetsFields + } + } + + fragment widgetsFields on WidgetConnection { + total_edge_count + } + EOS + + expect(dumper.dump_variables_for_query(query_string)).to eq({ + "CountWidgetAndAddress" => {"id" => "ID!"} + }) + end + end + end + end +end diff --git a/elasticgraph-rack/.rspec b/elasticgraph-rack/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-rack/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-rack/.yardopts b/elasticgraph-rack/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-rack/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-rack/Gemfile b/elasticgraph-rack/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-rack/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-rack/LICENSE.txt b/elasticgraph-rack/LICENSE.txt new file mode 100644 index 00000000..c41cd3ef --- /dev/null +++ b/elasticgraph-rack/LICENSE.txt @@ -0,0 +1,24 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +Part of the distributed code (lib/elastic_graph/rack/graphiql/index.html) +comes from the GraphiQL project, licensed under the MIT License. Copyright (c) GraphQL Contributors. diff --git a/elasticgraph-rack/README.md b/elasticgraph-rack/README.md new file mode 100644 index 00000000..2678e9de --- /dev/null +++ b/elasticgraph-rack/README.md @@ -0,0 +1,36 @@ +# ElasticGraph::Rack + +Uses [Rack](https://github.com/rack/rack) to serve an ElasticGraph application. +Intended primarily to make it easy to boot ElasticGraph applications locally, +but could also be used to serve an ElasticGraph application from any Rack +compatible web server. + +## Serving an ElasticGraph GraphQL Endpoint + +`ElasticGraph::Rack::GraphQLEndpoint` is a Rack application. Here's an +example of using it in a Rack `config.ru` file: + +```ruby +require 'elastic_graph/graphql' +require 'elastic_graph/rack/graphql_endpoint' + +graphql = ElasticGraph::GraphQL.from_yaml_file("path/to/config.yaml") +run ElasticGraph::Rack::GraphQLEndpoint.new(graphql) +``` + +Run this with `rackup` (after installing the `rackup` gem) or any other rack-compatible server. + +## Serving a GraphiQL UI + +This gem also provides a simple GraphiQL UI using the CDN-hosted GraphiQL assets. +Here's an example `config.ru` to boot that: + +``` ruby +require 'elastic_graph/graphql' +require 'elastic_graph/rack/graphiql' + +graphql = ElasticGraph::GraphQL.from_yaml_file("path/to/config.yaml") +run ElasticGraph::Rack::GraphiQL.new(graphql) +``` + +Run this with `rackup` (after installing the `rackup` gem) or any other rack-compatible server. diff --git a/elasticgraph-rack/elasticgraph-rack.gemspec b/elasticgraph-rack/elasticgraph-rack.gemspec new file mode 100644 index 00000000..e359c371 --- /dev/null +++ b/elasticgraph-rack/elasticgraph-rack.gemspec @@ -0,0 +1,22 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :local) do |spec, eg_version| + spec.summary = "ElasticGraph gem for serving an ElasticGraph GraphQL endpoint using rack." + + spec.add_dependency "elasticgraph-graphql", eg_version + spec.add_dependency "rack", "~> 3.1" + + spec.add_development_dependency "elasticgraph-admin", eg_version + spec.add_development_dependency "elasticgraph-elasticsearch", eg_version + spec.add_development_dependency "elasticgraph-opensearch", eg_version + spec.add_development_dependency "elasticgraph-indexer", eg_version + spec.add_development_dependency "rack-test", "~> 2.1" +end diff --git a/elasticgraph-rack/lib/elastic_graph/rack.rb b/elasticgraph-rack/lib/elastic_graph/rack.rb new file mode 100644 index 00000000..1eeb5c57 --- /dev/null +++ b/elasticgraph-rack/lib/elastic_graph/rack.rb @@ -0,0 +1,19 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + # Adapts an ElasticGraph GraphQL endpoint to run as a [Rack](https://github.com/rack/rack) application. + # This allows an ElasticGraph GraphQL endpoint to run inside any [Rack-compatible web + # framework](https://github.com/rack/rack#supported-web-frameworks), including [Ruby on Rails](https://rubyonrails.org/), + # or as a stand-alone application. Two configurations are supported: + # + # * Use {Rack::GraphQLEndpoint} to serve a GraphQL endpoint. + # * Use {Rack::GraphiQL} to serve a GraphQL endpoint and the [GraphiQL IDE](https://github.com/graphql/graphiql). + module Rack + end +end diff --git a/elasticgraph-rack/lib/elastic_graph/rack/graphiql.rb b/elasticgraph-rack/lib/elastic_graph/rack/graphiql.rb new file mode 100644 index 00000000..12eaf80f --- /dev/null +++ b/elasticgraph-rack/lib/elastic_graph/rack/graphiql.rb @@ -0,0 +1,44 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/rack/graphql_endpoint" +require "rack/builder" +require "rack/static" + +module ElasticGraph + module Rack + # A [Rack](https://github.com/rack/rack) application that serves both an ElasticGraph GraphQL endpoint + # and a [GraphiQL IDE](https://github.com/graphql/graphiql). This can be used for local development, + # mounted in a [Rails](https://rubyonrails.org/) application, or run in any other Rack-compatible context. + # + # @example Simple config.ru to run GraphiQL as a Rack application, targeting an ElasticGraph endpoint + # require "elastic_graph/graphql" + # require "elastic_graph/rack/graphiql" + # + # graphql = ElasticGraph::GraphQL.from_yaml_file("config/settings/development.yaml") + # run ElasticGraph::Rack::GraphiQL.new(graphql) + module GraphiQL + # Builds a [Rack](https://github.com/rack/rack) application that serves both an ElasticGraph GraphQL endpoint + # and a [GraphiQL IDE](https://github.com/graphql/graphiql). + # + # @param graphql [ElasticGraph::GraphQL] ElasticGraph GraphQL instance + # @return [Rack::Builder] built Rack application + def self.new(graphql) + graphql_endpoint = ElasticGraph::Rack::GraphQLEndpoint.new(graphql) + + ::Rack::Builder.new do + use ::Rack::Static, urls: {"/" => "index.html"}, root: ::File.join(__dir__, "graphiql") + + map "/graphql" do + run graphql_endpoint + end + end + end + end + end +end diff --git a/elasticgraph-rack/lib/elastic_graph/rack/graphiql/README.md b/elasticgraph-rack/lib/elastic_graph/rack/graphiql/README.md new file mode 100644 index 00000000..0382e6b6 --- /dev/null +++ b/elasticgraph-rack/lib/elastic_graph/rack/graphiql/README.md @@ -0,0 +1,38 @@ +## GraphiQL for ElasticGraph + +This directory provides the GraphiQL in browser UI for working with ElasticGraph +applications. The GraphiQL license is included in `LICENSE.txt`, copied verbatim +from: + +https://github.com/graphql/graphiql/blob/graphiql%402.4.0/LICENSE + +The `index.html` file is copied from: + +https://github.com/graphql/graphiql/blob/graphiql%402.4.0/examples/graphiql-cdn/index.html + +However, we've applied some slight changes to make it work for ElasticGraph. + +```diff +diff --git a/elasticgraph-rack/lib/elastic_graph/rack/graphiql/index.html b/elasticgraph-rack/lib/elastic_graph/rack/graphiql/index.html +index 55cf5d05..a672ead9 100644 +--- a/elasticgraph-rack/lib/elastic_graph/rack/graphiql/index.html ++++ b/elasticgraph-rack/lib/elastic_graph/rack/graphiql/index.html +@@ -8,7 +8,7 @@ + + + +- GraphiQL ++ ElasticGraph GraphiQL + + + + + + + + + + + +
Loading...
+ + + + diff --git a/elasticgraph-rack/lib/elastic_graph/rack/graphql_endpoint.rb b/elasticgraph-rack/lib/elastic_graph/rack/graphql_endpoint.rb new file mode 100644 index 00000000..fd0a6aaa --- /dev/null +++ b/elasticgraph-rack/lib/elastic_graph/rack/graphql_endpoint.rb @@ -0,0 +1,65 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/graphql/http_endpoint" +require "rack" + +module ElasticGraph + module Rack + # A simple [Rack](https://github.com/rack/rack) wrapper around an ElasticGraph GraphQL endpoint. + # This can be used for local development, mounted in a [Rails](https://rubyonrails.org/) application, + # or run in any other Rack-compatible context. + # + # @example Simple config.ru to run an ElasticGraph endpoint as a Rack application + # require "elastic_graph/graphql" + # require "elastic_graph/rack/graphql_endpoint" + # + # graphql = ElasticGraph::GraphQL.from_yaml_file("config/settings/development.yaml") + # run ElasticGraph::Rack::GraphQLEndpoint.new(graphql) + class GraphQLEndpoint + # @param graphql [ElasticGraph::GraphQL] ElasticGraph GraphQL instance + def initialize(graphql) + @logger = graphql.logger + @graphql_http_endpoint = graphql.graphql_http_endpoint + end + + # Responds to a Rack request. + # + # @param env [Hash] Rack env + # @return [Array(Integer, Hash, Array)] + def call(env) + rack_request = ::Rack::Request.new(env) + + # Rack doesn't provide a nice method to provide all HTTP headers. In general, + # HTTP headers are prefixed with `HTTP_` as per https://stackoverflow.com/a/6318491/16481862, + # but `Content-Type`, as a "standard" header, isn't exposed that way, sadly. + headers = env + .select { |k, v| k.start_with?("HTTP_") } + .to_h { |k, v| [k.delete_prefix("HTTP_"), v] } + .merge("Content-Type" => rack_request.content_type) + + request = GraphQL::HTTPRequest.new( + http_method: rack_request.request_method.downcase.to_sym, + url: rack_request.url, + headers: headers, + body: rack_request.body&.read + ) + + response = @graphql_http_endpoint.process(request) + + [response.status_code, response.headers.transform_keys(&:downcase), [response.body]] + rescue => e + raise if ENV["RACK_ENV"] == "test" + + @logger.error "Got an exception: #{e.class.name}: #{e.message}\n\n#{e.backtrace.join("\n")}" + error = {message: e.message, exception_class: e.class, backtrace: e.backtrace} + [500, {"content-type" => "application/json"}, [::JSON.generate(errors: [error])]] + end + end + end +end diff --git a/elasticgraph-rack/spec/acceptance/graphiql_spec.rb b/elasticgraph-rack/spec/acceptance/graphiql_spec.rb new file mode 100644 index 00000000..647b5d01 --- /dev/null +++ b/elasticgraph-rack/spec/acceptance/graphiql_spec.rb @@ -0,0 +1,26 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/rack" +require "elastic_graph/rack/graphiql" + +module ElasticGraph::Rack + RSpec.describe GraphiQL, :rack_app do + let(:app_to_test) { GraphiQL.new(build_graphql) } + + it "serves a GraphiQL UI at the root" do + get "/" + + expect(last_response).to be_ok_with_title "ElasticGraph GraphiQL" + end + + def be_ok_with_title(title) + have_attributes(status: 200, body: a_string_including("#{title}")) + end + end +end diff --git a/elasticgraph-rack/spec/acceptance/graphql_endpoint_spec.rb b/elasticgraph-rack/spec/acceptance/graphql_endpoint_spec.rb new file mode 100644 index 00000000..b0bfc94d --- /dev/null +++ b/elasticgraph-rack/spec/acceptance/graphql_endpoint_spec.rb @@ -0,0 +1,111 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/rack/graphql_endpoint" +require "elastic_graph/constants" + +module ElasticGraph::Rack + RSpec.describe GraphQLEndpoint, :rack_app, :uses_datastore do + let(:graphql) { build_graphql } + let(:app_to_test) { GraphQLEndpoint.new(graphql) } + + let(:introspection_query) do + <<~IQUERY + query IntrospectionQuery { + __schema { + types { + kind + name + } + } + } + IQUERY + end + + it "exposes a GraphQL endpoint as a rack app" do + response = call_graphql_query(introspection_query) + + expect(response.dig("data", "__schema", "types").count).to be > 5 + end + + it "supports executing queries submitted as a GET" do + get "/?query=#{::URI.encode_www_form_component(introspection_query)}" + + expect(last_response.status).to eq 200 + expect(::JSON.parse(last_response.body).dig("data", "__schema", "types").count).to be > 5 + end + + it "respects the `#{ElasticGraph::TIMEOUT_MS_HEADER}` header, returning a 504 Gateway Timeout when a datastore query times out" do + with_header ElasticGraph::TIMEOUT_MS_HEADER, "0" do + expect { + post_json "/", JSON.generate(query: <<~QUERY) + query { widgets { edges { node { id } } } } + QUERY + }.to log_warning(a_string_including("ElasticGraph::Errors::RequestExceededDeadlineError")) + end + + expect(last_response.status).to eq 504 + expect_json_error_including("Search exceeded requested timeout.") + end + + it "returns a 400 if the `#{ElasticGraph::TIMEOUT_MS_HEADER}` header value is invalid" do + with_header ElasticGraph::TIMEOUT_MS_HEADER, "zero" do + post_json "/", JSON.generate(query: <<~QUERY) + query { widgets { edges { node { id } } } } + QUERY + end + + expect(last_response.status).to eq 400 + expect_json_error_including(ElasticGraph::TIMEOUT_MS_HEADER, "zero", "invalid") + end + + it "returns a 400 if the request body is not parseable JSON" do + post_json "/", "not json" + + expect(last_response.status).to eq 400 + expect_json_error_including("invalid JSON") + end + + it "responds reasonably if the request lacks a `query` field in the JSON" do + expect { + post_json "/", "{}" + }.to log_warning(a_string_including("No query string")) + + expect(last_response.status).to eq 200 + expect_json_error_including("No query string") + end + + context "when executing the query raises an exception" do + before do + allow(graphql.graphql_query_executor).to receive(:execute).and_raise("boom") + end + + it "allows exceptions to be raised in the test environment" do + expect { + call_graphql_query(introspection_query) + }.to raise_error("boom") + end + + it "renders exceptions (instead of propagating them) in non-test environments" do + with_env "RACK_ENV" => "development" do + expect { + post_json "/", JSON.generate(query: introspection_query) + }.to log_warning(a_string_including("boom")) + + expect(last_response.status).to eq 500 + expect_json_error_including("boom") + end + end + end + + def expect_json_error_including(*parts) + expect(last_response.headers).to include("Content-Type" => "application/json") + expect(::JSON.parse(last_response.body)).to include("errors" => [a_hash_including("message" => a_string_including(*parts))]) + end + end +end diff --git a/elasticgraph-rack/spec/spec_helper.rb b/elasticgraph-rack/spec/spec_helper.rb new file mode 100644 index 00000000..81b786a9 --- /dev/null +++ b/elasticgraph-rack/spec/spec_helper.rb @@ -0,0 +1,21 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-rack`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. + +ENV["RACK_ENV"] = "test" + +RSpec.configure do |config| + config.define_derived_metadata(absolute_file_path: %r{/elasticgraph-rack/}) do |meta| + meta[:builds_graphql] = true + end + + config.when_first_matching_example_defined(:rack_app) { require "support/rack_app" } +end diff --git a/elasticgraph-rack/spec/support/rack_app.rb b/elasticgraph-rack/spec/support/rack_app.rb new file mode 100644 index 00000000..e44af3d4 --- /dev/null +++ b/elasticgraph-rack/spec/support/rack_app.rb @@ -0,0 +1,51 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "rack/test" +require "json" + +RSpec.shared_context "rack app support" do + include Rack::Test::Methods + let(:app) { ::Rack::Lint.new(app_to_test) } + + def last_parsed_response + JSON.parse(last_response.body) + end + + def with_header(name, value) + header name, value + yield + ensure + header name, nil + end + + def call_graphql_query(query) + call_graphql(JSON.generate(query: query)) + end + + def post_json(path, body) + with_header "Content-Type", "application/json" do + with_header "Accept", "application/json" do + post path, body + end + end + end + + def call_graphql(body) + post_json "/", body + expect(last_response).to be_ok + + last_parsed_response.tap do |parsed_response| + expect(parsed_response["errors"]).to eq([]).or eq(nil) + end + end +end + +RSpec.configure do |rspec| + rspec.include_context "rack app support", :rack_app +end diff --git a/elasticgraph-schema_artifacts/.rspec b/elasticgraph-schema_artifacts/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-schema_artifacts/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-schema_artifacts/.yardopts b/elasticgraph-schema_artifacts/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-schema_artifacts/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-schema_artifacts/Gemfile b/elasticgraph-schema_artifacts/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-schema_artifacts/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-schema_artifacts/LICENSE.txt b/elasticgraph-schema_artifacts/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-schema_artifacts/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-schema_artifacts/README.md b/elasticgraph-schema_artifacts/README.md new file mode 100644 index 00000000..7e8569df --- /dev/null +++ b/elasticgraph-schema_artifacts/README.md @@ -0,0 +1,3 @@ +# ElasticGraph::SchemaArtifacts + +Contains code related to ElasticGraph's generated schema artifacts. diff --git a/elasticgraph-schema_artifacts/elasticgraph-schema_artifacts.gemspec b/elasticgraph-schema_artifacts/elasticgraph-schema_artifacts.gemspec new file mode 100644 index 00000000..2c92cb2f --- /dev/null +++ b/elasticgraph-schema_artifacts/elasticgraph-schema_artifacts.gemspec @@ -0,0 +1,21 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :core) do |spec, eg_version| + spec.summary = "ElasticGraph gem containing code related to generated schema artifacts." + + spec.add_dependency "elasticgraph-support", eg_version + + # Necessary since `ScalarType` references coercion adapters defined in the `elasticgraph-graphql` gem. + spec.add_development_dependency "elasticgraph-graphql", eg_version + + # Necessary since `ScalarType` references indexing preparer defined in the `elasticgraph-indexer` gem. + spec.add_development_dependency "elasticgraph-indexer", eg_version +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/artifacts_helper_methods.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/artifacts_helper_methods.rb new file mode 100644 index 00000000..15e73725 --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/artifacts_helper_methods.rb @@ -0,0 +1,34 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaArtifacts + # Mixin that offers convenient helper methods on top of the basic schema artifacts. + # Intended to be mixed into every implementation of the `_SchemaArtifacts` interface. + module ArtifactsHelperMethods + def datastore_scripts + datastore_config.fetch("scripts") + end + + def index_templates + datastore_config.fetch("index_templates") + end + + def indices + datastore_config.fetch("indices") + end + + # Builds a map of index mappings, keyed by index definition name. + def index_mappings_by_index_def_name + @index_mappings_by_index_def_name ||= index_templates + .transform_values { |config| config.fetch("template").fetch("mappings") } + .merge(indices.transform_values { |config| config.fetch("mappings") }) + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/from_disk.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/from_disk.rb new file mode 100644 index 00000000..4917665f --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/from_disk.rb @@ -0,0 +1,112 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/errors" +require "elastic_graph/schema_artifacts/artifacts_helper_methods" +require "elastic_graph/schema_artifacts/runtime_metadata/schema" +require "elastic_graph/support/hash_util" +require "elastic_graph/support/memoizable_data" +require "yaml" + +module ElasticGraph + module SchemaArtifacts + # Builds a `SchemaArtifacts::FromDisk` instance using the provided YAML settings. + def self.from_parsed_yaml(parsed_yaml, for_context:) + schema_artifacts = parsed_yaml.fetch("schema_artifacts") do + raise Errors::ConfigError, "Config is missing required key `schema_artifacts`." + end + + if (extra_keys = schema_artifacts.keys - ["directory"]).any? + raise Errors::ConfigError, "Config has extra `schema_artifacts` keys: #{extra_keys}" + end + + directory = schema_artifacts.fetch("directory") do + raise Errors::ConfigError, "Config is missing required key `schema_artifacts.directory`." + end + + FromDisk.new(directory, for_context) + end + + # Responsible for loading schema artifacts from disk. + class FromDisk < Support::MemoizableData.define(:artifacts_dir, :context) + include ArtifactsHelperMethods + + def graphql_schema_string + @graphql_schema_string ||= read_artifact(GRAPHQL_SCHEMA_FILE) + end + + def json_schemas_for(version) + unless available_json_schema_versions.include?(version) + raise Errors::MissingSchemaArtifactError, "The requested json schema version (#{version}) is not available. " \ + "Available versions: #{available_json_schema_versions.sort.join(", ")}." + end + + json_schemas_by_version[version] + end + + def available_json_schema_versions + @available_json_schema_versions ||= begin + versioned_json_schemas_dir = ::File.join(artifacts_dir, JSON_SCHEMAS_BY_VERSION_DIRECTORY) + if ::Dir.exist?(versioned_json_schemas_dir) + ::Dir.entries(versioned_json_schemas_dir).filter_map { |it| it[/v(\d+)\.yaml/, 1]&.to_i }.to_set + else + ::Set.new + end + end + end + + def latest_json_schema_version + @latest_json_schema_version ||= available_json_schema_versions.max || raise( + Errors::MissingSchemaArtifactError, + "The directory for versioned JSON schemas (#{::File.join(artifacts_dir, JSON_SCHEMAS_BY_VERSION_DIRECTORY)}) could not be found. " \ + "Either the schema artifacts haven't been dumped yet or the schema artifacts directory (#{artifacts_dir}) is misconfigured." + ) + end + + def datastore_config + @datastore_config ||= _ = parsed_yaml_from(DATASTORE_CONFIG_FILE) + end + + def runtime_metadata + @runtime_metadata ||= RuntimeMetadata::Schema.from_hash( + parsed_yaml_from(RUNTIME_METADATA_FILE), + for_context: context + ) + end + + private + + def read_artifact(artifact_name) + file_name = ::File.join(artifacts_dir, artifact_name) + + if ::File.exist?(file_name) + ::File.read(file_name) + else + raise Errors::MissingSchemaArtifactError, "Schema artifact `#{artifact_name}` could not be found. " \ + "Either the schema artifacts haven't been dumped yet or the schema artifacts directory (#{artifacts_dir}) is misconfigured." + end + end + + def parsed_yaml_from(artifact_name) + ::YAML.safe_load(read_artifact(artifact_name)) + end + + def json_schemas_by_version + @json_schemas_by_version ||= ::Hash.new do |hash, json_schema_version| + hash[json_schema_version] = load_json_schema(json_schema_version) + end + end + + # Loads the given JSON schema version from disk. + def load_json_schema(json_schema_version) + parsed_yaml_from(::File.join(JSON_SCHEMAS_BY_VERSION_DIRECTORY, "v#{json_schema_version}.yaml")) + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/computation_detail.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/computation_detail.rb new file mode 100644 index 00000000..21d1732c --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/computation_detail.rb @@ -0,0 +1,34 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + # Details about our aggregation functions. + class ComputationDetail < ::Data.define(:empty_bucket_value, :function) + FUNCTION = "function" + EMPTY_BUCKET_VALUE = "empty_bucket_value" + + def self.from_hash(hash) + new( + empty_bucket_value: hash[EMPTY_BUCKET_VALUE], + function: hash.fetch(FUNCTION).to_sym + ) + end + + def to_dumpable_hash + { + # Keys here are ordered alphabetically; please keep them that way. + EMPTY_BUCKET_VALUE => empty_bucket_value, + FUNCTION => function.to_s + } + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/enum.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/enum.rb new file mode 100644 index 00000000..57c055bb --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/enum.rb @@ -0,0 +1,65 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/hash_dumper" +require "elastic_graph/schema_artifacts/runtime_metadata/sort_field" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + module Enum + # Runtime metadata related to an ElasticGraph enum type. + class Type < ::Data.define(:values_by_name) + VALUES_BY_NAME = "values_by_name" + + def self.from_hash(hash) + values_by_name = hash[VALUES_BY_NAME]&.transform_values do |value_hash| + Value.from_hash(value_hash) + end || {} + + new(values_by_name: values_by_name) + end + + def to_dumpable_hash + { + # Keys here are ordered alphabetically; please keep them that way. + VALUES_BY_NAME => HashDumper.dump_hash(values_by_name, &:to_dumpable_hash) + } + end + end + + # Runtime metadata related to an ElasticGraph enum value. + class Value < ::Data.define(:sort_field, :datastore_value, :datastore_abbreviation, :alternate_original_name) + DATASTORE_VALUE = "datastore_value" + DATASTORE_ABBREVIATION = "datastore_abbreviation" + SORT_FIELD = "sort_field" + ALTERNATE_ORIGINAL_NAME = "alternate_original_name" + + def self.from_hash(hash) + new( + sort_field: hash[SORT_FIELD]&.then { |h| SortField.from_hash(h) }, + datastore_value: hash[DATASTORE_VALUE], + datastore_abbreviation: hash[DATASTORE_ABBREVIATION]&.to_sym, + alternate_original_name: hash[ALTERNATE_ORIGINAL_NAME] + ) + end + + def to_dumpable_hash + { + # Keys here are ordered alphabetically; please keep them that way. + DATASTORE_ABBREVIATION => datastore_abbreviation&.to_s, + DATASTORE_VALUE => datastore_value, + ALTERNATE_ORIGINAL_NAME => alternate_original_name, + SORT_FIELD => sort_field&.to_dumpable_hash + } + end + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/extension.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/extension.rb new file mode 100644 index 00000000..32dde04e --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/extension.rb @@ -0,0 +1,51 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/hash_util" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + # Represents an extension--a class or module (potentially from outside the ElasticGraph + # code base) that implements a standard interface to plug in custom functionality. + # + # Extensions are serialized using two fields: + # - `extension_name`: the Ruby constant of the extension + # - `require_path`: file path to `require` to load the extension + # + # However, an `Extension` instance represents a loaded, resolved extension. + # We eagerly load extensions (and validate them in the `ExtensionLoader`) in + # order to surface any issues with the extension as soon as possible. We don't + # want to defer errors if we can detect any issues with the extension at boot time. + Extension = ::Data.define(:extension_class, :require_path, :extension_config) do + # @implements Extension + + # Loads an extension using a serialized hash, via the provided `ExtensionLoader`. + def self.load_from_hash(hash, via:) + config = Support::HashUtil.symbolize_keys(hash["extension_config"] || {}) # : ::Hash[::Symbol, untyped] + via.load(hash.fetch("extension_name"), from: hash.fetch("require_path"), config: config) + end + + # The name of the extension (based on the name of the extension class). + def extension_name + extension_class.name.to_s + end + + # The serialized form of an extension. + def to_dumpable_hash + # Keys here are ordered alphabetically; please keep them that way. + { + "extension_config" => Support::HashUtil.stringify_keys(extension_config), + "extension_name" => extension_name, + "require_path" => require_path + }.reject { |_, v| v.empty? } + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/extension_loader.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/extension_loader.rb new file mode 100644 index 00000000..39b696a0 --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/extension_loader.rb @@ -0,0 +1,125 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/schema_artifacts/runtime_metadata/extension" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + # Responsible for loading extensions. This loader requires an interface definition + # (a class or module with empty method definitions that just serves to define what + # loaded extensions must implement). That allows us to verify the extension implements + # the interface correctly at load time, rather than deferring exceptions to when the + # extension is later used. + # + # Note, however, that this does not guarantee no runtime exceptions from the use of the + # extension: the extension may return invalid return values, or throw exceptions when + # called. But this verifies the interface to the extent that we can. + class ExtensionLoader + def initialize(interface_def) + @interface_def = interface_def + @loaded_by_name = {} + end + + # Loads the extension using the provided constant name, after requiring the `from` path. + # Memoizes the result. + def load(constant_name, from:, config:) + (@loaded_by_name[constant_name] ||= load_extension(constant_name, from)).tap do |extension| + if extension.require_path != from + raise Errors::InvalidExtensionError, "Extension `#{constant_name}` cannot be loaded from `#{from}`, " \ + "since it has already been loaded from `#{extension.require_path}`." + end + end.with(extension_config: config) + end + + private + + def load_extension(constant_name, require_path) + require require_path + extension_class = ::Object.const_get(constant_name).tap { |ext| verify_interface(constant_name, ext) } + Extension.new(extension_class, require_path, {}) + end + + def verify_interface(constant_name, extension) + # @type var problems: ::Array[::String] + problems = [] + problems.concat(verify_methods("class", extension.singleton_class, @interface_def.singleton_class)) + + if extension.is_a?(::Module) + problems.concat(verify_methods("instance", extension, @interface_def)) + + # We care about the name exactly matching so that we can dump the extension name in a schema + # artifact w/o having to pass around the original constant name. + if extension.name != constant_name.delete_prefix("::") + problems << "- Exposes a name (`#{extension.name}`) that differs from the provided extension name (`#{constant_name}`)" + end + else + problems << "- Is not a class or module as expected" + end + + if problems.any? + raise Errors::InvalidExtensionError, + "Extension `#{constant_name}` does not implement the expected interface correctly. Problems:\n\n" \ + "#{problems.join("\n")}" + end + end + + def verify_methods(type, extension, interface) + interface_methods = list_instance_interface_methods(interface) + extension_methods = list_instance_interface_methods(extension) + + # @type var problems: ::Array[::String] + problems = [] + + if (missing_methods = interface_methods - extension_methods).any? + problems << "- Missing #{type} methods: #{missing_methods.map { |m| "`#{m}`" }.join(", ")}" + end + + interface_methods.intersection(extension_methods).each do |method_name| + unless parameters_match?(extension, interface, method_name) + interface_signature = signature_code_for(interface, method_name) + extension_signature = signature_code_for(extension, method_name) + + problems << "- Method signature for #{type} method `#{method_name}` (`#{extension_signature}`) does not match interface (`#{interface_signature}`)" + end + end + + problems + end + + def list_instance_interface_methods(klass) + # Here we look at more than just the public methods. This is necessary for `initialize`. + # If it's defined on the interface definition, we want to verify it on the extension, + # but Ruby makes `initialize` private by default. + klass.instance_methods(false) + + klass.protected_instance_methods(false) + + klass.private_instance_methods(false) + end + + def parameters_match?(extension, interface, method_name) + interface_parameters = interface.instance_method(method_name).parameters + extension_parameters = extension.instance_method(method_name).parameters + + # Here we compare the parameters for exact equality. This is stricter than we need it + # to be (it doesn't allow the parameters to have different names, for example) but it's + # considerably simpler than us trying to determine what is truly required. For example, + # the name doesn't matter on a positional arg, but would matter on a keyword arg. + interface_parameters == extension_parameters + end + + def signature_code_for(object, method_name) + # @type var file_name: ::String? + # @type var line_number: ::Integer? + file_name, line_number = object.instance_method(method_name).source_location + ::File.read(file_name.to_s).split("\n")[line_number.to_i - 1].strip + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/graphql_field.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/graphql_field.rb new file mode 100644 index 00000000..1f54d600 --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/graphql_field.rb @@ -0,0 +1,54 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/computation_detail" +require "elastic_graph/schema_artifacts/runtime_metadata/relation" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + class GraphQLField < ::Data.define(:name_in_index, :relation, :computation_detail) + EMPTY = new(nil, nil, nil) + NAME_IN_INDEX = "name_in_index" + RELATION = "relation" + AGGREGATION_DETAIL = "computation_detail" + + def self.from_hash(hash) + new( + name_in_index: hash[NAME_IN_INDEX], + relation: hash[RELATION]&.then { |rel_hash| Relation.from_hash(rel_hash) }, + computation_detail: hash[AGGREGATION_DETAIL]&.then { |agg_hash| ComputationDetail.from_hash(agg_hash) } + ) + end + + def to_dumpable_hash + { + # Keys here are ordered alphabetically; please keep them that way. + AGGREGATION_DETAIL => computation_detail&.to_dumpable_hash, + NAME_IN_INDEX => name_in_index, + RELATION => relation&.to_dumpable_hash + } + end + + # Indicates if we need this field in our dumped runtime metadata, when it has the given + # `name_in_graphql`. Fields that have not been customized in some way do not need to be + # included in the dumped runtime metadata. + def needed?(name_in_graphql) + !!relation || !!computation_detail || name_in_index&.!=(name_in_graphql) || false + end + + def with_computation_detail(empty_bucket_value:, function:) + with(computation_detail: ComputationDetail.new( + empty_bucket_value: empty_bucket_value, + function: function + )) + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/hash_dumper.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/hash_dumper.rb new file mode 100644 index 00000000..c01c12d8 --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/hash_dumper.rb @@ -0,0 +1,21 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + module HashDumper + def self.dump_hash(hash) + hash.sort_by(&:first).to_h do |key, value| + [key, yield(value)] + end + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rb new file mode 100644 index 00000000..b8691d3b --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rb @@ -0,0 +1,78 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/hash_dumper" +require "elastic_graph/schema_artifacts/runtime_metadata/index_field" +require "elastic_graph/schema_artifacts/runtime_metadata/sort_field" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + # Runtime metadata related to a datastore index definition. + class IndexDefinition < ::Data.define(:route_with, :rollover, :default_sort_fields, :current_sources, :fields_by_path) + ROUTE_WITH = "route_with" + ROLLOVER = "rollover" + DEFAULT_SORT_FIELDS = "default_sort_fields" + CURRENT_SOURCES = "current_sources" + FIELDS_BY_PATH = "fields_by_path" + + def initialize(route_with:, rollover:, default_sort_fields:, current_sources:, fields_by_path:) + super( + route_with: route_with, + rollover: rollover, + default_sort_fields: default_sort_fields, + current_sources: current_sources.to_set, + fields_by_path: fields_by_path + ) + end + + def self.from_hash(hash) + new( + route_with: hash[ROUTE_WITH], + rollover: hash[ROLLOVER]&.then { |h| Rollover.from_hash(h) }, + default_sort_fields: hash[DEFAULT_SORT_FIELDS]&.map { |h| SortField.from_hash(h) } || [], + current_sources: hash[CURRENT_SOURCES] || [], + fields_by_path: (hash[FIELDS_BY_PATH] || {}).transform_values { |h| IndexField.from_hash(h) } + ) + end + + def to_dumpable_hash + { + # Keys here are ordered alphabetically; please keep them that way. + CURRENT_SOURCES => current_sources.sort, + DEFAULT_SORT_FIELDS => default_sort_fields.map(&:to_dumpable_hash), + FIELDS_BY_PATH => HashDumper.dump_hash(fields_by_path, &:to_dumpable_hash), + ROLLOVER => rollover&.to_dumpable_hash, + ROUTE_WITH => route_with + } + end + + class Rollover < ::Data.define(:frequency, :timestamp_field_path) + FREQUENCY = "frequency" + TIMESTAMP_FIELD_PATH = "timestamp_field_path" + + # @implements Rollover + def self.from_hash(hash) + new( + frequency: hash.fetch(FREQUENCY).to_sym, + timestamp_field_path: hash[TIMESTAMP_FIELD_PATH] + ) + end + + def to_dumpable_hash + { + # Keys here are ordered alphabetically; please keep them that way. + FREQUENCY => frequency.to_s, + TIMESTAMP_FIELD_PATH => timestamp_field_path + } + end + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_field.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_field.rb new file mode 100644 index 00000000..f7a713c2 --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_field.rb @@ -0,0 +1,33 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + # Runtime metadata related to a field on a datastore index definition. + class IndexField < ::Data.define(:source) + SOURCE = "source" + + def self.from_hash(hash) + new( + source: hash[SOURCE] || SELF_RELATIONSHIP_NAME + ) + end + + def to_dumpable_hash + { + # Keys here are ordered alphabetically; please keep them that way. + SOURCE => source + } + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/object_type.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/object_type.rb new file mode 100644 index 00000000..27528162 --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/object_type.rb @@ -0,0 +1,81 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/graphql_field" +require "elastic_graph/schema_artifacts/runtime_metadata/hash_dumper" +require "elastic_graph/schema_artifacts/runtime_metadata/update_target" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + # Provides runtime metadata related to object types. + class ObjectType < ::Data.define( + :update_targets, + :index_definition_names, + :graphql_fields_by_name, + :elasticgraph_category, + # Indicates the name of the GraphQL type from which this type was generated. Note that a `nil` value doesn't + # imply that this type was user-defined; we have recently introduced this metadata and are not yet setting + # it for all generated GraphQL types. For now, we are only setting it for specific cases where we need it. + :source_type, + :graphql_only_return_type + ) + UPDATE_TARGETS = "update_targets" + INDEX_DEFINITION_NAMES = "index_definition_names" + GRAPHQL_FIELDS_BY_NAME = "graphql_fields_by_name" + ELASTICGRAPH_CATEGORY = "elasticgraph_category" + SOURCE_TYPE = "source_type" + GRAPHQL_ONLY_RETURN_TYPE = "graphql_only_return_type" + + def initialize(update_targets:, index_definition_names:, graphql_fields_by_name:, elasticgraph_category:, source_type:, graphql_only_return_type:) + graphql_fields_by_name = graphql_fields_by_name.select { |name, field| field.needed?(name) } + + super( + update_targets: update_targets, + index_definition_names: index_definition_names, + graphql_fields_by_name: graphql_fields_by_name, + elasticgraph_category: elasticgraph_category, + source_type: source_type, + graphql_only_return_type: graphql_only_return_type + ) + end + + def self.from_hash(hash) + update_targets = hash[UPDATE_TARGETS]&.map do |update_target_hash| + UpdateTarget.from_hash(update_target_hash) + end || [] + + graphql_fields_by_name = hash[GRAPHQL_FIELDS_BY_NAME]&.transform_values do |field_hash| + GraphQLField.from_hash(field_hash) + end || {} + + new( + update_targets: update_targets, + index_definition_names: hash[INDEX_DEFINITION_NAMES] || [], + graphql_fields_by_name: graphql_fields_by_name, + elasticgraph_category: hash[ELASTICGRAPH_CATEGORY]&.to_sym || nil, + source_type: hash[SOURCE_TYPE] || nil, + graphql_only_return_type: !!hash[GRAPHQL_ONLY_RETURN_TYPE] + ) + end + + def to_dumpable_hash + { + # Keys here are ordered alphabetically; please keep them that way. + ELASTICGRAPH_CATEGORY => elasticgraph_category&.to_s, + GRAPHQL_FIELDS_BY_NAME => HashDumper.dump_hash(graphql_fields_by_name, &:to_dumpable_hash), + GRAPHQL_ONLY_RETURN_TYPE => graphql_only_return_type ? true : nil, + INDEX_DEFINITION_NAMES => index_definition_names, + SOURCE_TYPE => source_type, + UPDATE_TARGETS => update_targets.map(&:to_dumpable_hash) + } + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/params.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/params.rb new file mode 100644 index 00000000..2eb3377d --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/params.rb @@ -0,0 +1,81 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/hash_dumper" +require "elastic_graph/support/hash_util" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + module Param + def self.dump_params_hash(hash_of_params) + hash_of_params.sort_by(&:first).to_h { |name, param| [name, param.to_dumpable_hash(name)] } + end + + def self.load_params_hash(hash_of_hashes) + hash_of_hashes.to_h { |name, hash| [name, from_hash(hash, name)] } + end + + def self.from_hash(hash, name) + if hash.key?(StaticParam::VALUE) + StaticParam.from_hash(hash) + else + DynamicParam.from_hash(hash, name) + end + end + end + + # Represents metadata about dynamic params we pass to our update scripts. + class DynamicParam < ::Data.define(:source_path, :cardinality) + SOURCE_PATH = "source_path" + CARDINALITY = "cardinality" + + def self.from_hash(hash, name) + new( + source_path: hash[SOURCE_PATH] || name, + cardinality: hash.fetch(CARDINALITY).to_sym + ) + end + + def to_dumpable_hash(param_name) + { + # Keys here are ordered alphabetically; please keep them that way. + CARDINALITY => cardinality.to_s, + SOURCE_PATH => (source_path if source_path != param_name) + } + end + + def value_for(event_or_prepared_record) + case cardinality + when :many then Support::HashUtil.fetch_leaf_values_at_path(event_or_prepared_record, source_path) { [] } + when :one then Support::HashUtil.fetch_value_at_path(event_or_prepared_record, source_path) { nil } + end + end + end + + class StaticParam < ::Data.define(:value) + VALUE = "value" + + def self.from_hash(hash) + new(value: hash.fetch(VALUE)) + end + + def to_dumpable_hash(param_name) + { + # Keys here are ordered alphabetically; please keep them that way. + VALUE => value + } + end + + def value_for(event_or_prepared_record) + value + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/relation.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/relation.rb new file mode 100644 index 00000000..d3762e0c --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/relation.rb @@ -0,0 +1,39 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + class Relation < ::Data.define(:foreign_key, :direction, :additional_filter, :foreign_key_nested_paths) + FOREIGN_KEY = "foreign_key" + DIRECTION = "direction" + ADDITIONAL_FILTER = "additional_filter" + FOREIGN_KEY_NESTED_PATHS = "foreign_key_nested_paths" + + def self.from_hash(hash) + new( + foreign_key: hash[FOREIGN_KEY], + direction: hash.fetch(DIRECTION).to_sym, + additional_filter: hash[ADDITIONAL_FILTER] || {}, + foreign_key_nested_paths: hash[FOREIGN_KEY_NESTED_PATHS] || [] + ) + end + + def to_dumpable_hash + { + # Keys here are ordered alphabetically; please keep them that way. + ADDITIONAL_FILTER => additional_filter, + DIRECTION => direction.to_s, + FOREIGN_KEY => foreign_key, + FOREIGN_KEY_NESTED_PATHS => foreign_key_nested_paths + } + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/scalar_type.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/scalar_type.rb new file mode 100644 index 00000000..0756eed0 --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/scalar_type.rb @@ -0,0 +1,94 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/extension_loader" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + # Provides runtime metadata related to scalar types. + class ScalarType < ::Data.define(:coercion_adapter_ref, :indexing_preparer_ref) + def self.coercion_adapter_extension_loader + @coercion_adapter_extension_loader ||= ExtensionLoader.new(ScalarCoercionAdapterInterface) + end + + def self.indexing_preparer_extension_loader + @indexing_preparer_extension_loader ||= ExtensionLoader.new(ScalarIndexingPreparerInterface) + end + + DEFAULT_COERCION_ADAPTER_REF = { + "extension_name" => "ElasticGraph::GraphQL::ScalarCoercionAdapters::NoOp", + "require_path" => "elastic_graph/graphql/scalar_coercion_adapters/no_op" + } + + DEFAULT_INDEXING_PREPARER_REF = { + "extension_name" => "ElasticGraph::Indexer::IndexingPreparers::NoOp", + "require_path" => "elastic_graph/indexer/indexing_preparers/no_op" + } + + # Loads multiple `ScalarType`s from a hash mapping a scalar type name to its + # serialized hash form (matching what `to_dumpable_hash` returns). We expose a method + # this way because we want to use a single loader for all `ScalarType`s that + # need to get loaded, as it performs some caching for efficiency. + def self.load_many(scalar_type_hashes_by_name) + scalar_type_hashes_by_name.transform_values do |hash| + new( + coercion_adapter_ref: hash.fetch("coercion_adapter"), + # `indexing_preparer` is new as of Q4 2022, and as such is not present in schema artifacts + # dumped before then. Therefore, we allow for the key to not be present in the runtime + # metadata--important so that we don't have a "chicken and egg" problem where the rake tasks + # that need to be loaded to dump new schema artifacts fail at load time due to the missing key. + indexing_preparer_ref: hash.fetch("indexing_preparer", DEFAULT_INDEXING_PREPARER_REF) + ) + end + end + + # Loads the coercion adapter. This is done lazily on first access (rather than eagerly in `load_many`) + # to allow us to remove a runtime dependency of `elasticgraph-schema_artifacts` on `elasticgraph-graphql`. + # The built-in coercion adapters are defined in `elasticgraph-graphql`, and we want to be able to load + # runtime metadata without requiring the `elasticgraph-graphql` gem (and its dependencies) to be available. + # For example, we use runtime metadata from `elasticgraph-indexer` but do not want `elasticgraph-graphql` + # to be loaded as part of that. + # + # elasticgraph-graphql provides the one caller that calls this method, ensuring that the adapters are + # available to be loaded. + def load_coercion_adapter + Extension.load_from_hash(coercion_adapter_ref, via: self.class.coercion_adapter_extension_loader) + end + + def load_indexing_preparer + Extension.load_from_hash(indexing_preparer_ref, via: self.class.indexing_preparer_extension_loader) + end + + def to_dumpable_hash + { + # Keys here are ordered alphabetically; please keep them that way. + "coercion_adapter" => load_coercion_adapter.to_dumpable_hash, + "indexing_preparer" => load_indexing_preparer.to_dumpable_hash + } + end + + # `to_h` is used internally by `Value#with` and we want `#to_dumpable_hash` to be the public API. + private :to_h + end + + class ScalarCoercionAdapterInterface + def self.coerce_input(value, ctx) + end + + def self.coerce_result(value, ctx) + end + end + + class ScalarIndexingPreparerInterface + def self.prepare_for_indexing(value) + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/schema.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/schema.rb new file mode 100644 index 00000000..9e699773 --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/schema.rb @@ -0,0 +1,99 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/enum" +require "elastic_graph/schema_artifacts/runtime_metadata/extension" +require "elastic_graph/schema_artifacts/runtime_metadata/extension_loader" +require "elastic_graph/schema_artifacts/runtime_metadata/hash_dumper" +require "elastic_graph/schema_artifacts/runtime_metadata/index_definition" +require "elastic_graph/schema_artifacts/runtime_metadata/object_type" +require "elastic_graph/schema_artifacts/runtime_metadata/scalar_type" +require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" +require "elastic_graph/support/hash_util" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + # Entry point for runtime metadata for an entire schema. + class Schema < ::Data.define( + :object_types_by_name, + :scalar_types_by_name, + :enum_types_by_name, + :index_definitions_by_name, + :schema_element_names, + :graphql_extension_modules, + :static_script_ids_by_scoped_name + ) + OBJECT_TYPES_BY_NAME = "object_types_by_name" + SCALAR_TYPES_BY_NAME = "scalar_types_by_name" + ENUM_TYPES_BY_NAME = "enum_types_by_name" + INDEX_DEFINITIONS_BY_NAME = "index_definitions_by_name" + SCHEMA_ELEMENT_NAMES = "schema_element_names" + GRAPHQL_EXTENSION_MODULES = "graphql_extension_modules" + STATIC_SCRIPT_IDS_BY_NAME = "static_script_ids_by_scoped_name" + + def self.from_hash(hash, for_context:) + object_types_by_name = hash[OBJECT_TYPES_BY_NAME]&.transform_values do |type_hash| + ObjectType.from_hash(type_hash) + end || {} + + scalar_types_by_name = hash[SCALAR_TYPES_BY_NAME]&.then do |subhash| + ScalarType.load_many(subhash) + end || {} + + enum_types_by_name = hash[ENUM_TYPES_BY_NAME]&.transform_values do |type_hash| + Enum::Type.from_hash(type_hash) + end || {} + + index_definitions_by_name = hash[INDEX_DEFINITIONS_BY_NAME]&.transform_values do |index_hash| + IndexDefinition.from_hash(index_hash) + end || {} + + schema_element_names = SchemaElementNames.from_hash(hash.fetch(SCHEMA_ELEMENT_NAMES)) + + loader = ExtensionLoader.new(Module.new) + graphql_extension_modules = + if for_context == :graphql + hash[GRAPHQL_EXTENSION_MODULES]&.map do |ext_mod_hash| + Extension.load_from_hash(ext_mod_hash, via: loader) + end || [] + else + # Avoid loading GraphQL extrnsion modules if we're not in a GraphQL context. We can't count + # on the extension modules even being available to load in other contexts. + [] # : ::Array[Extension] + end + + static_script_ids_by_scoped_name = hash[STATIC_SCRIPT_IDS_BY_NAME] || {} + + new( + object_types_by_name: object_types_by_name, + scalar_types_by_name: scalar_types_by_name, + enum_types_by_name: enum_types_by_name, + index_definitions_by_name: index_definitions_by_name, + schema_element_names: schema_element_names, + graphql_extension_modules: graphql_extension_modules, + static_script_ids_by_scoped_name: static_script_ids_by_scoped_name + ) + end + + def to_dumpable_hash + Support::HashUtil.recursively_prune_nils_and_empties_from({ + # Keys here are ordered alphabetically; please keep them that way. + ENUM_TYPES_BY_NAME => HashDumper.dump_hash(enum_types_by_name, &:to_dumpable_hash), + GRAPHQL_EXTENSION_MODULES => graphql_extension_modules.map(&:to_dumpable_hash), + INDEX_DEFINITIONS_BY_NAME => HashDumper.dump_hash(index_definitions_by_name, &:to_dumpable_hash), + OBJECT_TYPES_BY_NAME => HashDumper.dump_hash(object_types_by_name, &:to_dumpable_hash), + SCALAR_TYPES_BY_NAME => HashDumper.dump_hash(scalar_types_by_name, &:to_dumpable_hash), + SCHEMA_ELEMENT_NAMES => schema_element_names.to_dumpable_hash, + STATIC_SCRIPT_IDS_BY_NAME => static_script_ids_by_scoped_name + }) + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/schema_element_names.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/schema_element_names.rb new file mode 100644 index 00000000..2f426298 --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/schema_element_names.rb @@ -0,0 +1,165 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/errors" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + # Defines a generic schema element names API. Defined as a separate class to facilitate easy testing. + class SchemaElementNamesDefinition + def self.new(*element_names) + ::Data.define(:form, :overrides, :exposed_name_by_canonical_name, :canonical_name_by_exposed_name) do + const_set(:ELEMENT_NAMES, element_names) + + define_method :initialize do |form:, overrides: {}| + extend(CONVERTERS.fetch(form.to_s) do + raise Errors::SchemaError, + "Invalid schema element name form: #{form.inspect}. " \ + "Only valid values are: #{CONVERTERS.keys.inspect}." + end) + + unused_keys = overrides.keys.map(&:to_s) - element_names.map(&:to_s) + if unused_keys.any? + raise Errors::SchemaError, + "`overrides` contains entries that do not match any schema " \ + "elements: #{unused_keys.to_a.inspect}. Are any misspelled?" + end + + exposed_name_by_canonical_name = element_names.each_with_object({}) do |element, names| + names[element] = overrides.fetch(element) do + overrides.fetch(element.to_s) do + normalize_case(element.to_s) + end + end.to_s + end.freeze + + canonical_name_by_exposed_name = exposed_name_by_canonical_name.invert + validate_no_name_collisions(canonical_name_by_exposed_name, exposed_name_by_canonical_name) + + super( + form: form, + overrides: overrides, + exposed_name_by_canonical_name: exposed_name_by_canonical_name, + canonical_name_by_exposed_name: canonical_name_by_exposed_name + ) + end + + # standard:disable Lint/NestedMethodDefinition + element_names.each do |element| + method_name = SnakeCaseConverter.normalize_case(element.to_s) + define_method(method_name) { exposed_name_by_canonical_name.fetch(element) } + end + + # Returns the _canonical_ name for the given _exposed name_. The canonical name + # is the name we use within the source code of our framework; the exposed name + # is the name exposed in the specific GraphQL schema based on the configuration + # of the project. + def canonical_name_for(exposed_name) + canonical_name_by_exposed_name[exposed_name.to_s] + end + + def self.from_hash(hash) + new( + form: hash.fetch(FORM).to_sym, + overrides: hash[OVERRIDES] || {} + ) + end + + def to_dumpable_hash + { + # Keys here are ordered alphabetically; please keep them that way. + FORM => form.to_s, + OVERRIDES => overrides + } + end + + def to_s + "#<#{self.class.name} form=#{form}, overrides=#{overrides}>" + end + alias_method :inspect, :to_s + + private + + def validate_no_name_collisions(canonical_name_by_exposed_name, exposed_name_by_canonical_name) + return if canonical_name_by_exposed_name.size == exposed_name_by_canonical_name.size + + collisions = exposed_name_by_canonical_name + .group_by { |k, v| v } + .reject { |v, kv_pairs| kv_pairs.size == 1 } + .transform_values { |kv_pairs| kv_pairs.map(&:first) } + .map do |duplicate_exposed_name, canonical_names| + "#{canonical_names.inspect} all map to the same exposed name: #{duplicate_exposed_name}" + end.join(" and ") + + raise Errors::SchemaError, collisions + end + # standard:enable Lint/NestedMethodDefinition + end + end + + FORM = "form" + OVERRIDES = "overrides" + + module SnakeCaseConverter + extend self + + def normalize_case(name) + name.gsub(/([[:upper:]])/) { "_#{$1.downcase}" } + end + end + + module CamelCaseConverter + extend self + + def normalize_case(name) + name.gsub(/_(\w)/) { $1.upcase } + end + end + + CONVERTERS = { + "snake_case" => SnakeCaseConverter, + "camelCase" => CamelCaseConverter + } + end + + SchemaElementNames = SchemaElementNamesDefinition.new( + # Filter arg and operation names: + :filter, + :equal_to_any_of, :gt, :gte, :lt, :lte, :matches, :matches_phrase, :matches_query, :any_of, :all_of, :not, + :time_of_day, :any_satisfy, + # Directives + :eg_latency_slo, :ms, + # For sorting. + :order_by, + # For aggregation + :grouped_by, :count, :count_detail, :aggregated_values, :sub_aggregations, + # Date/time grouping aggregation fields + :as_date_time, :as_date, :as_time_of_day, :as_day_of_week, + # Date/time grouping aggregation arguments + :offset, :amount, :unit, :time_zone, :truncation_unit, + # TODO: Drop support for legacy grouping schema that uses `granularity` and `offset_days` + :granularity, :offset_days, + # For aggregation counts. + :approximate_value, :exact_value, :upper_bound, + # For pagination. + :first, :after, :last, :before, + :edges, :node, :nodes, :cursor, + :page_info, :start_cursor, :end_cursor, :total_edge_count, :has_previous_page, :has_next_page, + # Subfields of `GeoLocation`/`GeoLocationFilterInput`: + :latitude, :longitude, :near, :max_distance, + # Subfields of `MatchesQueryFilterInput`/`MatchesPhraseFilterInput` + :query, :phrase, :allowed_edits_per_term, :require_all_terms, + # Aggregated values field names: + :exact_min, :exact_max, :approximate_min, :approximate_max, :approximate_avg, :approximate_sum, :exact_sum, + :approximate_distinct_value_count + ) + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sort_field.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sort_field.rb new file mode 100644 index 00000000..406ce2ec --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sort_field.rb @@ -0,0 +1,47 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + class SortField < ::Data.define(:field_path, :direction) + def initialize(field_path:, direction:) + unless direction == :asc || direction == :desc + raise Errors::SchemaError, "Sort direction `#{direction.inspect}` is invalid; it must be `:asc` or `:desc`" + end + + super(field_path: field_path, direction: direction) + end + + FIELD_PATH = "field_path" + DIRECTION = "direction" + + def self.from_hash(hash) + new( + field_path: hash[FIELD_PATH], + direction: hash.fetch(DIRECTION).to_sym + ) + end + + def to_dumpable_hash + { + # Keys here are ordered alphabetically; please keep them that way. + DIRECTION => direction.to_s, + FIELD_PATH => field_path + } + end + + def to_query_clause + {field_path => {"order" => direction.to_s}} + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/update_target.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/update_target.rb new file mode 100644 index 00000000..c7d33c14 --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/update_target.rb @@ -0,0 +1,80 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/schema_artifacts/runtime_metadata/params" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + # Provides runtime metadata related to the targets of datastore `update` calls. + class UpdateTarget < ::Data.define( + :type, + :relationship, + :script_id, + :id_source, + :routing_value_source, + :rollover_timestamp_value_source, + :data_params, + :metadata_params + ) + TYPE = "type" + RELATIONSHIP = "relationship" + SCRIPT_ID = "script_id" + ID_SOURCE = "id_source" + ROUTING_VALUE_SOURCE = "routing_value_source" + ROLLOVER_TIMESTAMP_VALUE_SOURCE = "rollover_timestamp_value_source" + DATA_PARAMS = "data_params" + METADATA_PARAMS = "metadata_params" + + def self.from_hash(hash) + new( + type: hash[TYPE], + relationship: hash[RELATIONSHIP], + script_id: hash[SCRIPT_ID], + id_source: hash[ID_SOURCE], + routing_value_source: hash[ROUTING_VALUE_SOURCE], + rollover_timestamp_value_source: hash[ROLLOVER_TIMESTAMP_VALUE_SOURCE], + data_params: Param.load_params_hash(hash[DATA_PARAMS] || {}), + metadata_params: Param.load_params_hash(hash[METADATA_PARAMS] || {}) + ) + end + + def to_dumpable_hash + { + # Keys here are ordered alphabetically; please keep them that way. + DATA_PARAMS => Param.dump_params_hash(data_params), + ID_SOURCE => id_source, + METADATA_PARAMS => Param.dump_params_hash(metadata_params), + RELATIONSHIP => relationship, + ROLLOVER_TIMESTAMP_VALUE_SOURCE => rollover_timestamp_value_source, + ROUTING_VALUE_SOURCE => routing_value_source, + SCRIPT_ID => script_id, + TYPE => type + } + end + + def for_normal_indexing? + script_id == INDEX_DATA_UPDATE_SCRIPT_ID + end + + def params_for(doc_id:, event:, prepared_record:) + data = data_params.to_h do |name, param| + [name, param.value_for(prepared_record)] + end + + meta = metadata_params.to_h do |name, param| + [name, param.value_for(event)] + end + + meta.merge({"id" => doc_id, "data" => data}) + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/artifacts_helper_methods.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/artifacts_helper_methods.rbs new file mode 100644 index 00000000..1d85bc50 --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/artifacts_helper_methods.rbs @@ -0,0 +1,12 @@ +module ElasticGraph + module SchemaArtifacts + module ArtifactsHelperMethods: _SchemaArtifacts + def datastore_scripts: () -> datastoreScriptsByIdHash + def index_templates: () -> ::Hash[::String, untyped] + def indices: () -> ::Hash[::String, untyped] + + @index_mappings_by_index_def_name: ::Hash[::String, untyped]? + def index_mappings_by_index_def_name: () -> ::Hash[::String, untyped] + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/from_disk.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/from_disk.rbs new file mode 100644 index 00000000..6c4f43fe --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/from_disk.rbs @@ -0,0 +1,55 @@ +module ElasticGraph + type datastoreScriptContext = ::String + type datastoreScriptLanguage = "painless" | "expression" | "mustache" | "java" + type datastoreScriptScriptHash = {"lang" => datastoreScriptLanguage, "source" => ::String} + type datastoreScriptPayloadHash = {"context" => datastoreScriptContext, "script" => datastoreScriptScriptHash} + type datastoreScriptsByIdHash = ::Hash[::String, datastoreScriptPayloadHash] + type schemaArtifacts = _SchemaArtifacts & SchemaArtifacts::ArtifactsHelperMethods + + interface _SchemaArtifacts + def graphql_schema_string: () -> ::String + def datastore_config: () -> ::Hash[::String, untyped] + def runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::Schema + def json_schemas_for: (::Integer) -> ::Hash[::String, untyped] + def available_json_schema_versions: () -> ::Set[::Integer] + def latest_json_schema_version: () -> ::Integer + end + + module SchemaArtifacts + def self.from_parsed_yaml: ( + parsedYamlSettings, + for_context: context + ) ?{ (untyped) -> void } -> FromDisk + + class FromDiskSupertype + attr_reader artifacts_dir: ::String + attr_reader context: context + def initialize: (artifacs_dir: ::String, context: context) -> void + end + + class FromDisk < FromDiskSupertype + include _SchemaArtifacts + include ArtifactsHelperMethods + + def self.new: + (artifacs_dir: ::String, context: context) -> instance + | (::String, context) -> instance + + private + + @graphql_schema_string: ::String? + @datastore_config: ::Hash[::String, untyped]? + @runtime_metadata: SchemaArtifacts::RuntimeMetadata::Schema? + @available_json_schema_versions: ::Set[::Integer]? + @latest_json_schema_version: ::Integer? + + def read_artifact: (::String) -> ::String + def parsed_yaml_from: (::String) -> ::Hash[::String, untyped] + + @json_schemas_by_version: ::Hash[::Integer, ::Hash[::String, untyped]] + def json_schemas_by_version: () -> ::Hash[::Integer, ::Hash[::String, untyped]] + + def load_json_schema: (::Integer) -> ::Hash[::String, untyped] + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/interfaces.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/interfaces.rbs new file mode 100644 index 00000000..86986c26 --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/interfaces.rbs @@ -0,0 +1,13 @@ +module ElasticGraph + module SchemaArtifacts + # RubyT is the type in Ruby. JsonT is the type that will be encoded into JSON. + interface _ScalarCoercionAdapter[RubyT, JsonT] + def coerce_input: (untyped, untyped) -> RubyT? + def coerce_result: (untyped, untyped) -> JsonT? + end + + interface _IndexingPreparer[In, Out] + def prepare_for_indexing: (In) -> Out + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/computation_detail.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/computation_detail.rbs new file mode 100644 index 00000000..ef597079 --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/computation_detail.rbs @@ -0,0 +1,32 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + class ComputationDetailSupertype + attr_reader function: ::Symbol + attr_reader empty_bucket_value: ::Numeric? + + def initialize: ( + function: ::Symbol, + empty_bucket_value: ::Numeric? + ) -> void + + def with: ( + ?function: ::Symbol, + ?empty_bucket_value: ::Numeric? + ) -> instance + + def self.new: + (function: ::Symbol, empty_bucket_value: ::Numeric?) -> instance + | (::Symbol, ::Numeric?) -> instance + end + + class ComputationDetail < ComputationDetailSupertype + FUNCTION: "function" + EMPTY_BUCKET_VALUE: "empty_bucket_value" + + def self.from_hash: (::Hash[::String, untyped]) -> ComputationDetail + def to_dumpable_hash: () -> ::Hash[::String, untyped] + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/enum.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/enum.rbs new file mode 100644 index 00000000..9471de78 --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/enum.rbs @@ -0,0 +1,70 @@ +module ElasticGraph + type primitiveValue = ::String | ::Integer + + module SchemaArtifacts + module RuntimeMetadata + module Enum + class TypeSupertype + attr_reader values_by_name: ::Hash[::String, Value] + def initialize: (values_by_name: ::Hash[::String, Value]) -> void + def with: (?values_by_name: ::Hash[::String, Value]) -> Type + end + + class Type < TypeSupertype + VALUES_BY_NAME: "values_by_name" + def self.from_hash: (::Hash[::String, untyped]) -> Type + def to_dumpable_hash: () -> ::Hash[::String, untyped] + end + + class ValueSupertype + attr_reader sort_field: SortField? + attr_reader datastore_value: primitiveValue? + attr_reader datastore_abbreviation: ::Symbol? + attr_reader alternate_original_name: ::String? + + def initialize: ( + sort_field: SortField?, + datastore_value: primitiveValue?, + datastore_abbreviation: ::Symbol?, + alternate_original_name: ::String? + ) -> void + + def with: ( + ?sort_field: SortField?, + ?datastore_value: primitiveValue?, + ?datastore_abbreviation: ::Symbol?, + ?alternate_original_name: ::String? + ) -> Value + + def self.new: ( + sort_field: SortField?, + datastore_value: primitiveValue?, + datastore_abbreviation: ::Symbol?, + alternate_original_name: ::String? + ) -> Value | ( + SortField?, + primitiveValue?, + ::Symbol?, + ::String? + ) -> Value + + def to_h: () -> { + sort_field: SortField?, + datastore_value: primitiveValue?, + datastore_abbreviation: ::Symbol?, + alternate_original_name: ::String? + } + end + + class Value < ValueSupertype + SORT_FIELD: "sort_field" + DATASTORE_VALUE: "datastore_value" + DATASTORE_ABBREVIATION: "datastore_abbreviation" + ALTERNATE_ORIGINAL_NAME: "alternate_original_name" + def self.from_hash: (::Hash[::String, untyped]) -> Value + def to_dumpable_hash: () -> ::Hash[::String, untyped] + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/extension.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/extension.rbs new file mode 100644 index 00000000..dd9bdccb --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/extension.rbs @@ -0,0 +1,33 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + type extensionClass = ::Module | ::Class + + class Extension + attr_reader extension_class: extensionClass + attr_reader require_path: ::String + attr_reader extension_config: ::Hash[::Symbol, untyped] + + def initialize: ( + extension_class: extensionClass, + require_path: ::String, + extension_config: ::Hash[::Symbol, untyped] + ) -> void + + def self.new: + (extension_class: extensionClass, require_path: ::String, extension_config: ::Hash[::Symbol, untyped]) -> Extension + | (extensionClass, ::String, ::Hash[::Symbol, untyped]) -> Extension + + def self.load_from_hash: (::Hash[::String, untyped], via: ExtensionLoader) -> Extension + def extension_name: () -> ::String + def to_dumpable_hash: () -> ::Hash[::String, untyped] + + def with: ( + ?extension_class: extensionClass, + ?require_path: ::String, + ?extension_config: ::Hash[::Symbol, untyped] + ) -> Extension + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/extension_loader.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/extension_loader.rbs new file mode 100644 index 00000000..c5fed3ee --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/extension_loader.rbs @@ -0,0 +1,26 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + class ExtensionLoader + def initialize: (extensionClass) -> void + + def load: (::String, from: ::String, config: ::Hash[::Symbol, untyped]) -> Extension + + private + + @interface_def: extensionClass + @loaded_by_name: ::Hash[::String, Extension] + + type method = ::Method | ::UnboundMethod + type methodFetcher = ^(::Module, ::Symbol) -> method + + def load_extension: (::String, ::String) -> Extension + def verify_interface: (::String, extensionClass) -> void + def verify_methods: (::String, extensionClass, extensionClass) -> ::Array[::String] + def list_instance_interface_methods: (extensionClass) -> ::Array[::Symbol] + def parameters_match?: (extensionClass, extensionClass, ::Symbol) -> bool + def signature_code_for: (extensionClass, ::Symbol) -> ::String + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/graphql_field.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/graphql_field.rbs new file mode 100644 index 00000000..baeba0cb --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/graphql_field.rbs @@ -0,0 +1,42 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + class GraphQLFieldSupertype + attr_reader name_in_index: ::String? + attr_reader relation: Relation? + attr_reader computation_detail: ComputationDetail? + + def initialize: ( + name_in_index: ::String?, + relation: Relation?, + computation_detail: ComputationDetail? + ) -> void + + def with: ( + ?name_in_index: ::String?, + ?relation: Relation?, + ?computation_detail: ComputationDetail? + ) -> instance + + def self.new: + (name_in_index: ::String?, relation: Relation?, computation_detail: ComputationDetail?) -> instance + | (::String?, Relation?, ::Symbol?) -> instance + end + + class GraphQLField < GraphQLFieldSupertype + EMPTY: GraphQLField + NAME_IN_INDEX: "name_in_index" + RELATION: "relation" + AGGREGATION_DETAIL: "computation_detail" + def self.from_hash: (::Hash[::String, untyped]) -> GraphQLField + def to_dumpable_hash: () -> ::Hash[::String, untyped] + def needed?: (::String) -> bool + + def with_computation_detail: ( + empty_bucket_value: ::Numeric?, + function: ::Symbol + ) -> GraphQLField + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/hash_dumper.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/hash_dumper.rbs new file mode 100644 index 00000000..e07a889b --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/hash_dumper.rbs @@ -0,0 +1,9 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + module HashDumper + def self.dump_hash: (::Hash[::String, untyped]) { (untyped) -> untyped } -> ::Hash[::String, untyped] + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rbs new file mode 100644 index 00000000..80c3ccd4 --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rbs @@ -0,0 +1,67 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + class IndexDefinitionSuperType + attr_reader route_with: ::String + attr_reader rollover: IndexDefinition::Rollover? + attr_reader default_sort_fields: ::Array[SortField] + attr_reader current_sources: ::Set[::String] + attr_reader fields_by_path: ::Hash[::String, IndexField] + + def initialize: ( + route_with: ::String, + rollover: IndexDefinition::Rollover?, + default_sort_fields: ::Array[SortField], + current_sources: ::Set[::String], + fields_by_path: ::Hash[::String, IndexField] + ) -> void + + def with: ( + ?route_with: ::String, + ?rollover: IndexDefinition::Rollover?, + ?default_sort_fields: ::Array[SortField], + ?current_sources: ::Enumerable[::String], + ?fields_by_path: ::Hash[::String, IndexField] + ) -> IndexDefinition + end + + class IndexDefinition < IndexDefinitionSuperType + ROUTE_WITH: "route_with" + ROLLOVER: "rollover" + DEFAULT_SORT_FIELDS: "default_sort_fields" + CURRENT_SOURCES: "current_sources" + FIELDS_BY_PATH: "fields_by_path" + + def initialize: ( + route_with: ::String, + rollover: IndexDefinition::Rollover?, + default_sort_fields: ::Array[SortField], + current_sources: ::Enumerable[::String], + fields_by_path: ::Hash[::String, IndexField] + ) -> void + + def self.from_hash: (::Hash[::String, untyped]) -> IndexDefinition + def to_dumpable_hash: () -> ::Hash[::String, untyped] + + class RolloverSupertype + attr_reader frequency: Rollover::frequency + attr_reader timestamp_field_path: ::String + + def initialize: ( + frequency: Rollover::frequency, + timestamp_field_path: ::String) -> void + end + + class Rollover < RolloverSupertype + type frequency = :hourly | :daily | :monthly | :yearly + + FREQUENCY: "frequency" + TIMESTAMP_FIELD_PATH: "timestamp_field_path" + + def self.from_hash: (::Hash[::String, untyped]) -> Rollover + def to_dumpable_hash: () -> ::Hash[::String, untyped] + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_field.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_field.rbs new file mode 100644 index 00000000..dc248c95 --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_field.rbs @@ -0,0 +1,18 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + class IndexFieldSupertype + attr_reader source: ::String + + def initialize: (source: ::String) -> void + def with: (?source: ::String) -> IndexField + end + + class IndexField < IndexFieldSupertype + SOURCE: "source" + def self.from_hash: (::Hash[::String, untyped]) -> IndexField + def to_dumpable_hash: () -> ::Hash[::String, untyped] + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/object_type.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/object_type.rbs new file mode 100644 index 00000000..ba58ebeb --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/object_type.rbs @@ -0,0 +1,45 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + type elasticGraphCategory = :scalar_aggregated_values + + class ObjectTypeSupertype + attr_reader update_targets: ::Array[UpdateTarget] + attr_reader index_definition_names: ::Array[::String] + attr_reader graphql_fields_by_name: ::Hash[::String, GraphQLField] + attr_reader elasticgraph_category: elasticGraphCategory? + attr_reader source_type: ::String? + attr_reader graphql_only_return_type: bool + + def initialize: ( + update_targets: ::Array[UpdateTarget], + index_definition_names: ::Array[::String], + graphql_fields_by_name: ::Hash[::String, GraphQLField], + elasticgraph_category: elasticGraphCategory?, + source_type: ::String?, + graphql_only_return_type: bool + ) -> void + + def with: ( + ?update_targets: ::Array[UpdateTarget], + ?index_definition_names: ::Array[::String], + ?graphql_fields_by_name: ::Hash[::String, GraphQLField], + ?elasticgraph_category: elasticGraphCategory?, + ?source_type: ::String?, + ?graphql_only_return_type: bool + ) -> ObjectType + end + + class ObjectType < ObjectTypeSupertype + UPDATE_TARGETS: "update_targets" + INDEX_DEFINITION_NAMES: "index_definition_names" + GRAPHQL_FIELDS_BY_NAME: "graphql_fields_by_name" + ELASTICGRAPH_CATEGORY: "elasticgraph_category" + SOURCE_TYPE: "source_type" + GRAPHQL_ONLY_RETURN_TYPE: "graphql_only_return_type" + def self.from_hash: (::Hash[::String, untyped]) -> ObjectType + def to_dumpable_hash: () -> ::Hash[::String, untyped] + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/params.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/params.rbs new file mode 100644 index 00000000..5dfa106b --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/params.rbs @@ -0,0 +1,61 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + interface _Param + def to_dumpable_hash: (::String) -> ::Hash[::String, untyped] + def value_for: (::Hash[::String, untyped]) -> untyped + end + + type paramsHash = ::Hash[::String, _Param] + + module Param + def self.dump_params_hash: (paramsHash) -> ::Hash[::String, ::Hash[::String, untyped]] + def self.load_params_hash: (::Hash[::String, ::Hash[::String, untyped]]) -> paramsHash + def self.from_hash: (::Hash[::String, untyped], ::String) -> _Param + end + + type paramCardinality = :one | :many + + class DynamicParamSuperType + attr_reader source_path: ::String + attr_reader cardinality: paramCardinality + + def initialize: ( + source_path: ::String, + cardinality: paramCardinality + ) -> void + + def with: ( + ?source_path: ::String, + ?cardinality: paramCardinality + ) -> DynamicParam + + private + + def to_h: () -> ::Hash[::Symbol, untyped] + end + + class DynamicParam < DynamicParamSuperType + include _Param + + SOURCE_PATH: "source_path" + CARDINALITY: "cardinality" + + def self.from_hash: (::Hash[::String, untyped], ::String) -> DynamicParam + end + + class StaticParamSuperType + attr_reader value: untyped + def initialize: (value: untyped) -> void + def with: (?value: untyped) -> StaticParam + end + + class StaticParam < StaticParamSuperType + include _Param + + VALUE: "value" + def self.from_hash: (::Hash[::String, untyped]) -> StaticParam + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/relation.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/relation.rbs new file mode 100644 index 00000000..c08b3799 --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/relation.rbs @@ -0,0 +1,30 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + class RelationSupertype + attr_reader foreign_key: ::String + attr_reader direction: Relation::direction + attr_reader additional_filter: ::Hash[::String, untyped] + attr_reader foreign_key_nested_paths: ::Array[::String] + + def initialize: ( + foreign_key: ::String, + direction: Relation::direction, + additional_filter: ::Hash[::String, untyped], + foreign_key_nested_paths: ::Array[::String] + ) -> void + end + + class Relation < RelationSupertype + type direction = :in | :out + + FOREIGN_KEY: "foreign_key" + DIRECTION: "direction" + ADDITIONAL_FILTER: "additional_filter" + FOREIGN_KEY_NESTED_PATHS: "foreign_key_nested_paths" + def self.from_hash: (::Hash[::String, untyped]) -> Relation + def to_dumpable_hash: () -> ::Hash[::String, untyped] + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/scalar_type.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/scalar_type.rbs new file mode 100644 index 00000000..76d19aca --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/scalar_type.rbs @@ -0,0 +1,46 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + class ScalarTypeSupertype + attr_reader coercion_adapter_ref: ::Hash[::String, ::String] + attr_reader indexing_preparer_ref: ::Hash[::String, ::String] + + def initialize: ( + coercion_adapter_ref: ::Hash[::String, ::String], + indexing_preparer_ref: ::Hash[::String, ::String] + ) -> void + + def with: ( + ?coercion_adapter_ref: ::Hash[::String, ::String], + ?indexing_preparer_ref: ::Hash[::String, ::String] + ) -> ScalarType + end + + class ScalarType < ScalarTypeSupertype + self.@coercion_adapter_extension_loader: ExtensionLoader? + def self.coercion_adapter_extension_loader: () -> ExtensionLoader + + self.@indexing_preparer_extension_loader: ExtensionLoader? + def self.indexing_preparer_extension_loader: () -> ExtensionLoader + + DEFAULT_COERCION_ADAPTER_REF: ::Hash[::String, ::String] + DEFAULT_INDEXING_PREPARER_REF: ::Hash[::String, ::String] + + def self.load_many: (::Hash[::String, ::Hash[::String, untyped]]) -> ::Hash[::String, ScalarType] + + def load_coercion_adapter: () -> Extension + def load_indexing_preparer: () -> Extension + + def to_dumpable_hash: () -> ::Hash[::String, untyped] + end + + class ScalarCoercionAdapterInterface + extend _ScalarCoercionAdapter[untyped, untyped] + end + + class ScalarIndexingPreparerInterface + extend _IndexingPreparer[untyped, untyped] + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/schema.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/schema.rbs new file mode 100644 index 00000000..a887d2d4 --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/schema.rbs @@ -0,0 +1,48 @@ +module ElasticGraph + module SchemaArtifacts + type context = :admin | :graphql | :indexer | :indexer_autoscaler_lambda + + module RuntimeMetadata + class SchemaSupertype + attr_reader object_types_by_name: ::Hash[::String, ObjectType] + attr_reader scalar_types_by_name: ::Hash[::String, ScalarType] + attr_reader enum_types_by_name: ::Hash[::String, Enum::Type] + attr_reader index_definitions_by_name: ::Hash[::String, IndexDefinition] + attr_reader schema_element_names: SchemaElementNames + attr_reader graphql_extension_modules: ::Array[Extension] + attr_reader static_script_ids_by_scoped_name: ::Hash[::String, ::String] + + def initialize: ( + object_types_by_name: ::Hash[::String, ObjectType], + scalar_types_by_name: ::Hash[::String, ScalarType], + enum_types_by_name: ::Hash[::String, Enum::Type], + index_definitions_by_name: ::Hash[::String, IndexDefinition], + schema_element_names: SchemaElementNames, + graphql_extension_modules: ::Array[Extension], + static_script_ids_by_scoped_name: ::Hash[::String, ::String]) -> void + + def with: ( + ?object_types_by_name: ::Hash[::String, ObjectType], + ?scalar_types_by_name: ::Hash[::String, ScalarType], + ?enum_types_by_name: ::Hash[::String, Enum::Type], + ?index_definitions_by_name: ::Hash[::String, IndexDefinition], + ?schema_element_names: SchemaElementNames, + ?graphql_extension_modules: ::Array[Extension], + ?static_script_ids_by_scoped_name: ::Hash[::String, ::String]) -> Schema + end + + class Schema < SchemaSupertype + OBJECT_TYPES_BY_NAME: "object_types_by_name" + SCALAR_TYPES_BY_NAME: "scalar_types_by_name" + ENUM_TYPES_BY_NAME: "enum_types_by_name" + INDEX_DEFINITIONS_BY_NAME: "index_definitions_by_name" + SCHEMA_ELEMENT_NAMES: "schema_element_names" + GRAPHQL_EXTENSION_MODULES: "graphql_extension_modules" + STATIC_SCRIPT_IDS_BY_NAME: "static_script_ids_by_scoped_name" + + def self.from_hash: (::Hash[::String, untyped], for_context: context) -> Schema + def to_dumpable_hash: () -> ::Hash[::String, untyped] + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/schema_element_names.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/schema_element_names.rbs new file mode 100644 index 00000000..3a70bac5 --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/schema_element_names.rbs @@ -0,0 +1,111 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + # Note: this is a partial signature definition (the ruby file is ignored in `Steepfile`) + class SchemaElementNamesDefinition + module SnakeCaseConverter + def self.normalize_case: (::String) -> ::String + end + + module CamelCaseConverter + def self.normalize_case: (::String) -> ::String + end + end + + class SchemaElementNames + ELEMENT_NAMES: ::Array[::Symbol] + + def canonical_name_for: (::String | ::Symbol) -> ::Symbol + attr_reader filter: ::String + attr_reader equal_to_any_of: ::String + attr_reader gt: ::String + attr_reader gte: ::String + attr_reader lt: ::String + attr_reader lte: ::String + attr_reader matches: ::String + attr_reader matches_query: ::String + attr_reader matches_phrase: ::String + attr_reader all_of: ::String + attr_reader any_of: ::String + attr_reader not: ::String + attr_reader time_of_day: ::String + attr_reader any_satisfy: ::String + + attr_reader eg_latency_slo: ::String + attr_reader ms: ::String + + attr_reader order_by: ::String + attr_reader ordered_by: ::String + + attr_reader grouped_by: ::String + attr_reader count: ::String + attr_reader count_detail: ::String + attr_reader aggregated_values: ::String + attr_reader sub_aggregations: ::String + + attr_reader as_date_time: ::String + attr_reader as_date: ::String + attr_reader as_time_of_day: ::String + attr_reader as_day_of_week: ::String + + attr_reader offset: ::String + attr_reader amount: ::String + attr_reader unit: ::String + attr_reader time_zone: ::String + attr_reader truncation_unit: ::String + + attr_reader offset_days: ::String + attr_reader granularity: ::String + + attr_reader approximate_value: ::String + attr_reader exact_value: ::String + attr_reader upper_bound: ::String + + attr_reader first: ::String + attr_reader after: ::String + attr_reader last: ::String + attr_reader before: ::String + attr_reader edges: ::String + attr_reader nodes: ::String + attr_reader node: ::String + attr_reader cursor: ::String + attr_reader page_info: ::String + attr_reader start_cursor: ::String + attr_reader end_cursor: ::String + attr_reader total_edge_count: ::String + attr_reader has_previous_page: ::String + attr_reader has_next_page: ::String + + attr_reader query: ::String + attr_reader phrase: ::String + attr_reader allowed_edits_per_term: ::Integer + attr_reader require_all_terms: bool + + attr_reader latitude: ::String + attr_reader longitude: ::String + attr_reader near: ::String + attr_reader max_distance: ::String + + attr_reader exact_min: ::String + attr_reader exact_max: ::String + attr_reader approximate_min: ::String + attr_reader approximate_max: ::String + attr_reader approximate_avg: ::String + attr_reader approximate_sum: ::String + attr_reader exact_sum: ::String + attr_reader approximate_distinct_value_count: ::String + + def normalize_case: (::String) -> ::String + def self.from_hash: (::Hash[::String, untyped]) -> SchemaElementNames + def to_dumpable_hash: () -> ::Hash[::String, untyped] + + type form = :snake_case | :camelCase | "snake_case" | "camelCase" + def initialize: (form: form, ?overrides: ::Hash[::Symbol, ::String]) -> void + + def self.new: + (form: form, ?overrides: ::Hash[::Symbol, ::String]) -> instance + | (form, ?::Hash[::Symbol, ::String]) -> instance + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/sort_field.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/sort_field.rbs new file mode 100644 index 00000000..e4f564a5 --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/sort_field.rbs @@ -0,0 +1,24 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + class SortFieldSuperType + attr_reader field_path: ::String + attr_reader direction: SortField::direction + + def initialize: ( + field_path: ::String, + direction: SortField::direction) -> void + end + + class SortField < SortFieldSuperType + type direction = :asc | :desc + + FIELD_PATH: "field_path" + DIRECTION: "direction" + def self.from_hash: (::Hash[::String, untyped]) -> SortField + def to_dumpable_hash: () -> ::Hash[::String, untyped] + def to_query_clause: () -> ::Hash[::String, {"order" => ::String}] + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/update_target.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/update_target.rbs new file mode 100644 index 00000000..eab89105 --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/update_target.rbs @@ -0,0 +1,62 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + class UpdateTargetValueSuperType + attr_reader type: ::String + attr_reader relationship: ::String? + attr_reader script_id: ::String + attr_reader id_source: ::String + attr_reader routing_value_source: ::String? + attr_reader rollover_timestamp_value_source: ::String? + attr_reader data_params: paramsHash + attr_reader metadata_params: paramsHash + + def initialize: ( + type: ::String, + relationship: ::String?, + script_id: ::String, + id_source: ::String, + routing_value_source: ::String?, + rollover_timestamp_value_source: ::String?, + data_params: paramsHash, + metadata_params: paramsHash + ) -> void + + def with: ( + ?type: ::String, + ?relationship: ::String?, + ?script_id: ::String, + ?id_source: ::String, + ?routing_value_source: ::String?, + ?rollover_timestamp_value_source: ::String?, + ?data_params: paramsHash, + ?metadata_params: paramsHash + ) -> UpdateTarget + + private + + def to_h: () -> ::Hash[::Symbol, untyped] + end + + class UpdateTarget < UpdateTargetValueSuperType + TYPE: "type" + RELATIONSHIP: "relationship" + SCRIPT_ID: "script_id" + ID_SOURCE: "id_source" + ROUTING_VALUE_SOURCE: "routing_value_source" + ROLLOVER_TIMESTAMP_VALUE_SOURCE: "rollover_timestamp_value_source" + DATA_PARAMS: "data_params" + METADATA_PARAMS: "metadata_params" + + def self.from_hash: (::Hash[::String, untyped]) -> UpdateTarget + def to_dumpable_hash: () -> ::Hash[::String, untyped] + def for_normal_indexing?: () -> bool + def params_for: ( + doc_id: ::String, + event: ::Hash[::String, untyped], + prepared_record: ::Hash[::String, untyped] + ) -> ::Hash[::String, untyped] + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/spec_helper.rb b/elasticgraph-schema_artifacts/spec/spec_helper.rb new file mode 100644 index 00000000..7ad25d42 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/spec_helper.rb @@ -0,0 +1,11 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-schema_artifacts`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. diff --git a/elasticgraph-schema_artifacts/spec/support/example_extensions/additional_methods.rb b/elasticgraph-schema_artifacts/spec/support/example_extensions/additional_methods.rb new file mode 100644 index 00000000..7ba811ff --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/support/example_extensions/additional_methods.rb @@ -0,0 +1,28 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Extensions + class AdditionalMethods + def self.class_method(a, b) + end + + def self.extra_class_method + end + + def instance_method1 + end + + def instance_method2(foo:) + end + + def extra_instance_method + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/support/example_extensions/args_mismatch.rb b/elasticgraph-schema_artifacts/spec/support/example_extensions/args_mismatch.rb new file mode 100644 index 00000000..3b730486 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/support/example_extensions/args_mismatch.rb @@ -0,0 +1,25 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Extensions + class ArgsMismatch + # missing an arg + def self.class_method(a) + end + + # extra arg + def instance_method1(b) + end + + # positional instead of keyword arg + def instance_method2(foo) + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/support/example_extensions/graphql_extension_modules.rb b/elasticgraph-schema_artifacts/spec/support/example_extensions/graphql_extension_modules.rb new file mode 100644 index 00000000..9876f3ab --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/support/example_extensions/graphql_extension_modules.rb @@ -0,0 +1,14 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaArtifacts + module GraphQLExtensionModule1 + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/support/example_extensions/indexing_preparers.rb b/elasticgraph-schema_artifacts/spec/support/example_extensions/indexing_preparers.rb new file mode 100644 index 00000000..38cce297 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/support/example_extensions/indexing_preparers.rb @@ -0,0 +1,21 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaArtifacts + class IndexingPreparer1 + def self.prepare_for_indexing(value) + end + end + + class IndexingPreparer2 + def self.prepare_for_indexing(value) + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/support/example_extensions/initialize_doesnt_match.rb b/elasticgraph-schema_artifacts/spec/support/example_extensions/initialize_doesnt_match.rb new file mode 100644 index 00000000..66b0ea47 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/support/example_extensions/initialize_doesnt_match.rb @@ -0,0 +1,20 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Extensions + class InitializeDoesntMatch + def initialize(some_arg:, another_arg:) + # No body needed + end + + def do_it + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/support/example_extensions/initialize_missing.rb b/elasticgraph-schema_artifacts/spec/support/example_extensions/initialize_missing.rb new file mode 100644 index 00000000..292ebbe2 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/support/example_extensions/initialize_missing.rb @@ -0,0 +1,16 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Extensions + class InitializeMissing + def do_it + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/support/example_extensions/missing_class_method.rb b/elasticgraph-schema_artifacts/spec/support/example_extensions/missing_class_method.rb new file mode 100644 index 00000000..09119583 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/support/example_extensions/missing_class_method.rb @@ -0,0 +1,22 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Extensions + class MissingClassMethod + # def self.class_method(a, b) + # end + + def instance_method1 + end + + def instance_method2(foo:) + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/support/example_extensions/missing_instance_method.rb b/elasticgraph-schema_artifacts/spec/support/example_extensions/missing_instance_method.rb new file mode 100644 index 00000000..f7628bb1 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/support/example_extensions/missing_instance_method.rb @@ -0,0 +1,22 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Extensions + class MissingInstanceMethod + def self.class_method(a, b) + end + + # def instance_method1 + # end + + def instance_method2(foo:) + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/support/example_extensions/name_mismatch.rb b/elasticgraph-schema_artifacts/spec/support/example_extensions/name_mismatch.rb new file mode 100644 index 00000000..f98a7893 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/support/example_extensions/name_mismatch.rb @@ -0,0 +1,24 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Extensions + class ModuleWithWrongName + def self.class_method(a, b) + end + + def instance_method1 + end + + def instance_method2(foo:) + end + end + + NameMismatch = ModuleWithWrongName + end +end diff --git a/elasticgraph-schema_artifacts/spec/support/example_extensions/not_a_class_or_module.rb b/elasticgraph-schema_artifacts/spec/support/example_extensions/not_a_class_or_module.rb new file mode 100644 index 00000000..e2dd4484 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/support/example_extensions/not_a_class_or_module.rb @@ -0,0 +1,16 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Extensions + NotAClassOrModule = Object.new + + def NotAClassOrModule.class_method(a, b) + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/support/example_extensions/scalar_coercion_adapters.rb b/elasticgraph-schema_artifacts/spec/support/example_extensions/scalar_coercion_adapters.rb new file mode 100644 index 00000000..d2dbcf76 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/support/example_extensions/scalar_coercion_adapters.rb @@ -0,0 +1,27 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaArtifacts + class ScalarCoercionAdapter1 + def self.coerce_input(value, ctx) + end + + def self.coerce_result(value, ctx) + end + end + + class ScalarCoercionAdapter2 + def self.coerce_input(value, ctx) + end + + def self.coerce_result(value, ctx) + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/support/example_extensions/valid.rb b/elasticgraph-schema_artifacts/spec/support/example_extensions/valid.rb new file mode 100644 index 00000000..2872356c --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/support/example_extensions/valid.rb @@ -0,0 +1,22 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Extensions + class Valid + def self.class_method(a, b) + end + + def instance_method1 + end + + def instance_method2(foo:) + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/support/example_extensions/valid_instantiable.rb b/elasticgraph-schema_artifacts/spec/support/example_extensions/valid_instantiable.rb new file mode 100644 index 00000000..0a931537 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/support/example_extensions/valid_instantiable.rb @@ -0,0 +1,20 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Extensions + class ValidInstantiable + def initialize(some_arg:) + # No body needed + end + + def do_it + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/support/example_extensions/valid_module.rb b/elasticgraph-schema_artifacts/spec/support/example_extensions/valid_module.rb new file mode 100644 index 00000000..85d2d69b --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/support/example_extensions/valid_module.rb @@ -0,0 +1,22 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Extensions + module ValidModule + def self.class_method(a, b) + end + + def instance_method1 + end + + def instance_method2(foo:) + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/from_disk_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/from_disk_spec.rb new file mode 100644 index 00000000..9ac3a18c --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/from_disk_spec.rb @@ -0,0 +1,144 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/errors" +require "elastic_graph/schema_artifacts/from_disk" + +module ElasticGraph + module SchemaArtifacts + RSpec.describe SchemaArtifacts, ".from_parsed_yaml" do + it "uses the `schema_artifacts.directory` setting to build a `FromDisk` instance" do + artifacts = from_parsed_yaml({"schema_artifacts" => {"directory" => "some_dir"}}) + + expect(artifacts).to be_a(FromDisk) + expect(artifacts.artifacts_dir).to eq "some_dir" + end + + it "fails with a clear error if the required keys are missing" do + expect { + from_parsed_yaml({}) + }.to raise_error Errors::ConfigError, a_string_including("schema_artifacts") + + expect { + from_parsed_yaml({"schema_artifacts" => {}}) + }.to raise_error Errors::ConfigError, a_string_including("schema_artifacts.directory") + end + + it "fails with a clear error if extra `schema_artifacts` settings are provided" do + expect { + from_parsed_yaml({"schema_artifacts" => {"directory" => "a", "foo" => 3}}) + }.to raise_error Errors::ConfigError, a_string_including("foo") + end + + def from_parsed_yaml(parsed_yaml, for_context: :graphql) + SchemaArtifacts.from_parsed_yaml(parsed_yaml, for_context: for_context) + end + end + + RSpec.describe FromDisk do + it "loads each schema artifact from disk" do + artifacts = FromDisk.new(::File.join(CommonSpecHelpers::REPO_ROOT, "config", "schema", "artifacts"), :graphql) + + expect_artifacts_to_load_and_be_valid(artifacts) + expect(artifacts.datastore_scripts.values.first).to include("context", "script") + end + + context "with multiple json schemas", :in_temp_dir do + let(:artifacts) { FromDisk.new(Dir.pwd, :indexer) } + + before do + ::FileUtils.mkdir_p(JSON_SCHEMAS_BY_VERSION_DIRECTORY) + ::File.write(::File.join(JSON_SCHEMAS_BY_VERSION_DIRECTORY, "v1.yaml"), ::YAML.dump(JSON_SCHEMA_VERSION_KEY => 1)) + ::File.write(::File.join(JSON_SCHEMAS_BY_VERSION_DIRECTORY, "v2.yaml"), ::YAML.dump(JSON_SCHEMA_VERSION_KEY => 2)) + end + + it "retrieves the specified version of the json_schema" do + expect(artifacts.json_schemas_for(1)).to include(JSON_SCHEMA_VERSION_KEY => 1) + expect(artifacts.json_schemas_for(2)).to include(JSON_SCHEMA_VERSION_KEY => 2) + end + + it "lists the available json_schema_versions" do + available_versions = artifacts.available_json_schema_versions + expect(available_versions).to include(1, 2) # We don't want test to keep breaking as new versions are added, so don't assert an exact match. + expect(available_versions).not_to include(nil) # No `nil` values should be present. + end + + it "raises if an unavailable json_schema version is requested" do + expect { + artifacts.json_schemas_for(9999) + }.to raise_error Errors::MissingSchemaArtifactError, a_string_including("is not available", "Available versions: 1, 2") + end + + it "returns the largest JSON schema version as the `latest_json_schema_version`" do + expect(artifacts.latest_json_schema_version).to eq 2 + end + end + + context "before any artifacts have been dumped", :in_temp_dir do + let(:artifacts) { FromDisk.new(Dir.pwd, :graphql) } + + it "raises an error when accessing missing artifacts is attempted" do + expect { artifacts.graphql_schema_string }.to raise_missing_artifacts_error + expect { artifacts.indices }.to raise_missing_artifacts_error + expect { artifacts.datastore_scripts }.to raise_missing_artifacts_error + expect { artifacts.runtime_metadata }.to raise_missing_artifacts_error + end + + it "returns an empty set from `available_json_schema_versions`" do + expect(artifacts.available_json_schema_versions).to eq Set.new + end + + it "raises an error from `latest_json_schema_version`" do + expect { artifacts.latest_json_schema_version }.to raise_missing_artifacts_error + end + + def raise_missing_artifacts_error + raise_error Errors::MissingSchemaArtifactError, a_string_including("could not be found", artifacts.artifacts_dir) + end + end + + describe "#index_mappings_by_index_def_name" do + let(:artifacts) { FromDisk.new(::File.join(CommonSpecHelpers::REPO_ROOT, "config", "schema", "artifacts"), :indexer) } + + it "returns the index mappings" do + mappings = artifacts.index_mappings_by_index_def_name + + expect(mappings.keys).to match_array(artifacts.indices.keys + artifacts.index_templates.keys) + expect(artifacts.indices.dig("addresses", "mappings", "properties", "timestamps", "properties", "created_at")).to eq({ + "type" => "date", + "format" => DATASTORE_DATE_TIME_FORMAT + }) + expect(mappings.dig("addresses", "properties", "timestamps", "properties", "created_at")).to eq({ + "type" => "date", + "format" => DATASTORE_DATE_TIME_FORMAT + }) + end + + it "is memoized to avoid re-computing the mappings" do + mappings1 = artifacts.index_mappings_by_index_def_name + mappings2 = artifacts.index_mappings_by_index_def_name + + expect(mappings1).to be(mappings2) + end + end + + def expect_artifacts_to_load_and_be_valid(artifacts) + expect(artifacts.graphql_schema_string).to include("type Query {") + expect(artifacts.json_schemas_for(1)).to include("$schema" => JSON_META_SCHEMA) + expect(artifacts.datastore_config).to include("indices", "index_templates", "scripts") + expect(artifacts.runtime_metadata).to be_a RuntimeMetadata::Schema + + artifacts.runtime_metadata.scalar_types_by_name.values.each do |scalar_type| + expect(scalar_type.load_coercion_adapter).not_to be_nil + expect(scalar_type.load_indexing_preparer).not_to be_nil + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/enum_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/enum_spec.rb new file mode 100644 index 00000000..e7b587ab --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/enum_spec.rb @@ -0,0 +1,43 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/enum" +require "elastic_graph/spec_support/runtime_metadata_support" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + module Enum + RSpec.describe Type do + include RuntimeMetadataSupport + + it "builds from a minimal hash" do + enum_type = Enum::Type.from_hash({}) + + expect(enum_type).to eq Enum::Type.new(values_by_name: {}) + end + end + + RSpec.describe Value do + include RuntimeMetadataSupport + + it "builds from a minimal hash" do + enum_value = Enum::Value.from_hash({}) + + expect(enum_value).to eq Enum::Value.new( + sort_field: nil, + datastore_value: nil, + datastore_abbreviation: nil, + alternate_original_name: nil + ) + end + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/extension_loader_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/extension_loader_spec.rb new file mode 100644 index 00000000..a07898f5 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/extension_loader_spec.rb @@ -0,0 +1,180 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/schema_artifacts/runtime_metadata/extension" +require "elastic_graph/schema_artifacts/runtime_metadata/extension_loader" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + RSpec.describe ExtensionLoader do + let(:loader) { ExtensionLoader.new(ExampleExtension) } + + it "loads an extension class matching the provided constant" do + extension = loader.load("ElasticGraph::Extensions::Valid", from: "support/example_extensions/valid", config: {}) + + expect(extension).to eq_extension(ElasticGraph::Extensions::Valid, from: "support/example_extensions/valid", config: {}) + end + + it "loads an extension module matching the provided constant" do + extension = loader.load("ElasticGraph::Extensions::ValidModule", from: "support/example_extensions/valid_module", config: {}) + + expect(extension).to eq_extension(ElasticGraph::Extensions::ValidModule, from: "support/example_extensions/valid_module", config: {}) + end + + it "can load an extension when the constant is prefixed with `::`" do + extension = loader.load("::ElasticGraph::Extensions::Valid", from: "support/example_extensions/valid", config: {}) + + expect(extension).to eq_extension(ElasticGraph::Extensions::Valid, from: "support/example_extensions/valid", config: {}) + end + + it "memoizes the loading of the extension, avoiding the cost of re-loading an already loaded extension, while allowing differing config" do + allow(loader).to receive(:require).and_call_original + allow(::Object).to receive(:const_get).and_call_original + + extension1 = loader.load("ElasticGraph::Extensions::Valid", from: "support/example_extensions/valid", config: {"size" => 10}) + extension2 = loader.load("ElasticGraph::Extensions::Valid", from: "support/example_extensions/valid", config: {"size" => 20}) + + expect(extension1).to eq_extension(ElasticGraph::Extensions::Valid, from: "support/example_extensions/valid", config: {"size" => 10}) + expect(extension2).to eq_extension(ElasticGraph::Extensions::Valid, from: "support/example_extensions/valid", config: {"size" => 20}) + + expect(loader).to have_received(:require).once + expect(::Object).to have_received(:const_get).once + end + + it "raises a clear error if a constant is re-loaded but with a different `from` argument" do + loader.load("ElasticGraph::Extensions::Valid", from: "support/example_extensions/valid", config: {}) + + expect { + loader.load("ElasticGraph::Extensions::Valid", from: "support/example_extensions/additional_methods", config: {}) + }.to raise_error Errors::InvalidExtensionError, a_string_including( + "ElasticGraph::Extensions::Valid", + "cannot be loaded from `support/example_extensions/additional_methods`", + "already been loaded from `support/example_extensions/valid`" + ) + end + + it "raises a clear error when the `from:` arg isn't a valid require path" do + expect { + loader.load("ElasticGraph::Extensions::Valid", from: "support/example_extensions/not_a_file_name", config: {}) + }.to raise_error LoadError, a_string_including("support/example_extensions/not_a_file_name") + end + + it "raises a clear error when the constant name is not defined after attempting to load the extension" do + expect { + loader.load("ElasticGraph::Extensions::Typo", from: "support/example_extensions/valid", config: {}) + }.to raise_error NameError, a_string_including("ElasticGraph::Extensions::Typo") + end + + it "verifies the extension matches the interface definition, notifying of missing instance methods" do + expect { + loader.load("ElasticGraph::Extensions::MissingInstanceMethod", from: "support/example_extensions/missing_instance_method", config: {}) + }.to raise_error Errors::InvalidExtensionError, a_string_including( + "ElasticGraph::Extensions::MissingInstanceMethod", + "Missing instance methods", "instance_method1" + ) + end + + it "verifies the extension matches the interface definition, notifying of missing class methods" do + expect { + loader.load("ElasticGraph::Extensions::MissingClassMethod", from: "support/example_extensions/missing_class_method", config: {}) + }.to raise_error Errors::InvalidExtensionError, a_string_including( + "ElasticGraph::Extensions::MissingClassMethod", + "Missing class methods", "class_method" + ) + end + + it "verifies the extension matches the interface definition, notifying of argument mis-matches" do + expect { + loader.load("ElasticGraph::Extensions::ArgsMismatch", from: "support/example_extensions/args_mismatch", config: {}) + }.to raise_error Errors::InvalidExtensionError, a_string_including( + "ElasticGraph::Extensions::ArgsMismatch", + "Method signature", "def self.class_method", "def instance_method1", "def instance_method2" + ) + end + + it "verifies that the extension is a class or module" do + expect { + loader.load("ElasticGraph::Extensions::NotAClassOrModule", from: "support/example_extensions/not_a_class_or_module", config: {}) + }.to raise_error Errors::InvalidExtensionError, a_string_including( + "ElasticGraph::Extensions::NotAClassOrModule", "not a class or module" + ).and(excluding("class_method", "instance_method1", "instance_method2")) + end + + it "verifies that the extension name matches the provided name" do + expect { + loader.load("ElasticGraph::Extensions::NameMismatch", from: "support/example_extensions/name_mismatch", config: {}) + }.to raise_error Errors::InvalidExtensionError, a_string_including( + "ElasticGraph::Extensions::NameMismatch", + "differs from the provided extension name", + "ElasticGraph::Extensions::ModuleWithWrongName" + ) + end + + it "ignores extra methods defined on the extension beyond what the interface requires" do + extension = loader.load("ElasticGraph::Extensions::AdditionalMethods", from: "support/example_extensions/additional_methods", config: {}) + + expect(extension).to eq_extension(ElasticGraph::Extensions::AdditionalMethods, from: "support/example_extensions/additional_methods", config: {}) + end + + context "with an instantiable extension interface" do + let(:loader) { ExtensionLoader.new(ExampleInstantiableExtension) } + + it "raises an exception if the extension is missing the required `initialize` method" do + expect { + loader.load("ElasticGraph::Extensions::InitializeMissing", from: "support/example_extensions/initialize_missing", config: {}) + }.to raise_error Errors::InvalidExtensionError, a_string_including( + "ElasticGraph::Extensions::InitializeMissing", + "Missing instance methods: `initialize`" + ) + end + + it "raises an exception if the extension's `initialize` accepts different arguments" do + expect { + loader.load("ElasticGraph::Extensions::InitializeDoesntMatch", from: "support/example_extensions/initialize_doesnt_match", config: {}) + }.to raise_error Errors::InvalidExtensionError, a_string_including( + "ElasticGraph::Extensions::InitializeDoesntMatch", + "Method signature for instance method `initialize` (`def initialize(some_arg:, another_arg:)`) does not match interface (`def initialize(some_arg:)`)" + ) + end + + it "returns a valid implementation" do + extension = loader.load("ElasticGraph::Extensions::ValidInstantiable", from: "support/example_extensions/valid_instantiable", config: {}) + + expect(extension).to eq_extension(ElasticGraph::Extensions::ValidInstantiable, from: "support/example_extensions/valid_instantiable", config: {}) + end + end + + def eq_extension(extension_class, from:, config:) + eq(Extension.new(extension_class, from, config)) + end + end + + class ExampleExtension + def self.class_method(a, b) + end + + def instance_method1 + end + + def instance_method2(foo:) + end + end + + class ExampleInstantiableExtension + def initialize(some_arg:) + # No body needed + end + + def do_it + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/extension_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/extension_spec.rb new file mode 100644 index 00000000..78a530ef --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/extension_spec.rb @@ -0,0 +1,35 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/extension" +require "elastic_graph/schema_artifacts/runtime_metadata/extension_loader" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + RSpec.describe Extension do + let(:loader) { ExtensionLoader.new(Class.new) } + + it "can roundtrip through a primitive ruby hash for easy serialization and deserialization" do + extension = loader.load("ElasticGraph::Extensions::Valid", from: "support/example_extensions/valid", config: {foo: "bar"}) + hash = extension.to_dumpable_hash + + expect(hash).to eq({ + "extension_config" => {"foo" => "bar"}, + "extension_name" => "ElasticGraph::Extensions::Valid", + "require_path" => "support/example_extensions/valid" + }) + + reloaded_extension = Extension.load_from_hash(hash, via: loader) + + expect(reloaded_extension).to eq(extension) + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/graphql_field_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/graphql_field_spec.rb new file mode 100644 index 00000000..c2561ead --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/graphql_field_spec.rb @@ -0,0 +1,48 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/graphql_field" +require "elastic_graph/spec_support/runtime_metadata_support" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + RSpec.describe GraphQLField do + include RuntimeMetadataSupport + + it "builds from a minimal hash" do + field = GraphQLField.from_hash({}) + + expect(field).to eq GraphQLField.new( + name_in_index: nil, + relation: nil, + computation_detail: nil + ) + end + + it "offers `with_computation_detail` updating aggregation detail" do + field = GraphQLField.new( + name_in_index: nil, + relation: nil, + computation_detail: nil + ) + + updated = field.with_computation_detail( + empty_bucket_value: 0, + function: :sum + ) + + expect(updated.computation_detail).to eq(ComputationDetail.new( + empty_bucket_value: 0, + function: :sum + )) + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_definition_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_definition_spec.rb new file mode 100644 index 00000000..e22a6330 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_definition_spec.rb @@ -0,0 +1,55 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/index_definition" +require "elastic_graph/spec_support/runtime_metadata_support" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + RSpec.describe IndexDefinition do + include RuntimeMetadataSupport + + it "builds from a minimal hash" do + index_def = IndexDefinition.from_hash({}) + + expect(index_def).to eq IndexDefinition.new( + route_with: nil, + rollover: nil, + default_sort_fields: [], + current_sources: Set.new, + fields_by_path: {} + ) + end + + it "includes fields that only have default values when serializing, even though other default runtime metadata elements get dropped, so our GraphQL logic can easily see what all the valid field paths are" do + index_def = index_definition_with(fields_by_path: { + "foo.bar" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "foo.bazz" => index_field_with(source: "other") + }) + + expect(index_def.to_dumpable_hash["fields_by_path"]).to eq({ + "foo.bar" => index_field_with(source: SELF_RELATIONSHIP_NAME).to_dumpable_hash, + "foo.bazz" => index_field_with(source: "other").to_dumpable_hash + }) + end + + describe IndexDefinition::Rollover do + it "builds from a minimal hash" do + rollover = IndexDefinition::Rollover.from_hash({"frequency" => "yearly"}) + + expect(rollover).to eq IndexDefinition::Rollover.new( + frequency: :yearly, + timestamp_field_path: nil + ) + end + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_field_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_field_spec.rb new file mode 100644 index 00000000..173708c6 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_field_spec.rb @@ -0,0 +1,26 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/index_field" +require "elastic_graph/spec_support/runtime_metadata_support" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + RSpec.describe IndexField do + include RuntimeMetadataSupport + + it "builds from a minimal hash" do + field = IndexField.from_hash({}) + + expect(field).to eq IndexField.new(source: SELF_RELATIONSHIP_NAME) + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/object_type_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/object_type_spec.rb new file mode 100644 index 00000000..e19df9ea --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/object_type_spec.rb @@ -0,0 +1,68 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/object_type" +require "elastic_graph/spec_support/runtime_metadata_support" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + RSpec.describe ObjectType do + include RuntimeMetadataSupport + + it "ignores fields that have no meaningful runtime metadata" do + object_type = object_type_with(graphql_fields_by_name: { + "has_relation" => graphql_field_with(name_in_index: nil, relation: relation_with), + "has_computation_detail" => graphql_field_with(computation_detail: :sum), + "has_alternate_index_name" => graphql_field_with(name_in_index: "alternate"), + "has_nil_index_name" => graphql_field_with(name_in_index: nil), + "has_same_index_name" => graphql_field_with(name_in_index: "has_same_index_name") + }) + + expect(object_type.graphql_fields_by_name.keys).to contain_exactly( + "has_relation", + "has_computation_detail", + "has_alternate_index_name" + ) + end + + it "builds from a minimal hash" do + type = ObjectType.from_hash({}) + + expect(type).to eq ObjectType.new( + update_targets: [], + index_definition_names: [], + graphql_fields_by_name: {}, + elasticgraph_category: nil, + source_type: nil, + graphql_only_return_type: false + ) + end + + it "exposes `elasticgraph_category` as a symbol while keeping it as a string in dumped form" do + type = ObjectType.from_hash({"elasticgraph_category" => "scalar_aggregated_values"}) + + expect(type.elasticgraph_category).to eq :scalar_aggregated_values + expect(type.to_dumpable_hash).to include("elasticgraph_category" => "scalar_aggregated_values") + end + + it "models `graphql_only_return_type` as `true` or `nil` so that our runtime metadata pruning can omit nils" do + type = ObjectType.from_hash({}) + + expect(type.graphql_only_return_type).to eq false + expect(type.to_dumpable_hash).to include("graphql_only_return_type" => nil) + + type = ObjectType.from_hash({"graphql_only_return_type" => true}) + + expect(type.graphql_only_return_type).to eq true + expect(type.to_dumpable_hash).to include("graphql_only_return_type" => true) + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/params_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/params_spec.rb new file mode 100644 index 00000000..faf978a6 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/params_spec.rb @@ -0,0 +1,115 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/params" +require "elastic_graph/spec_support/runtime_metadata_support" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + RSpec.describe DynamicParam do + include RuntimeMetadataSupport + + it "coerces the cardinality to a symbol in memory vs a string in dumped form" do + param = dynamic_param_with(cardinality: :one) + + dumped = param.to_dumpable_hash("my_param") + expect(dumped["cardinality"]).to eq("one") + + loaded = DynamicParam.from_hash(dumped, "my_param") + expect(loaded).to eq param + expect(loaded.cardinality).to eq(:one) + end + + it "sets `source_path` in the dumped hashunset if it is different from the param name" do + param = dynamic_param_with(source_path: "foo") + + dumped = param.to_dumpable_hash("bar") + expect(dumped.fetch("source_path")).to eq("foo") + + loaded = DynamicParam.from_hash(dumped, "bar") + expect(loaded).to eq param + expect(loaded.source_path).to eq("foo") + end + + it "leaves `source_path` unset when dumping it if it is the same as the param name to avoid bloating our runtime metadata dumped artifact" do + param = dynamic_param_with(source_path: "foo") + + dumped = param.to_dumpable_hash("foo") + expect(dumped.fetch("source_path")).to eq(nil) + + loaded = DynamicParam.from_hash(dumped, "foo") + expect(loaded).to eq param + expect(loaded.source_path).to eq("foo") + end + + describe "#value_for" do + context "for a param with `:many` cardinality" do + it "fetches multiple values from the given data hash" do + param = dynamic_param_with(source_path: "foo.bar", cardinality: :many) + + value = param.value_for({"foo" => [{"bar" => [2, 3]}, {"bar" => 5}]}) + + expect(value).to eq([2, 3, 5]) + end + + it "returns `[]` if the `source_path` is not found" do + param = dynamic_param_with(source_path: "foo.bar", cardinality: :many) + + value = param.value_for({}) + + expect(value).to eq([]) + end + end + + context "for a param with `:one` cardinality" do + it "fetches a single value from the given data hash" do + param = dynamic_param_with(source_path: "foo.bar", cardinality: :one) + + value = param.value_for({"foo" => {"bar" => 7}}) + + expect(value).to eq(7) + end + + it "returns `nil` if the `source_path` is not found" do + param = dynamic_param_with(source_path: "foo.bar", cardinality: :one) + + value = param.value_for({"foo" => {}}) + + expect(value).to eq(nil) + end + end + + context "on an unrecognized cardinality" do + it "returns `nil`" do + param = dynamic_param_with(source_path: "foo.bar", cardinality: :unsure) + + value = param.value_for({"foo" => {"bar" => 7}}) + + expect(value).to eq(nil) + end + end + end + end + + RSpec.describe StaticParam do + include RuntimeMetadataSupport + + describe "#value_for" do + it "returns the param's static value regardless of the given data hash" do + param = static_param_with(17) + + value = param.value_for({"value" => 12}) + + expect(value).to eq(17) + end + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/relation_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/relation_spec.rb new file mode 100644 index 00000000..7a30764a --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/relation_spec.rb @@ -0,0 +1,31 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/relation" +require "elastic_graph/spec_support/runtime_metadata_support" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + RSpec.describe Relation do + include RuntimeMetadataSupport + + it "builds from a minimal hash" do + relation = Relation.from_hash({"direction" => "in"}) + + expect(relation).to eq Relation.new( + direction: :in, + foreign_key: nil, + additional_filter: {}, + foreign_key_nested_paths: [] + ) + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/scalar_type_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/scalar_type_spec.rb new file mode 100644 index 00000000..2028916f --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/scalar_type_spec.rb @@ -0,0 +1,34 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/scalar_type" +require "elastic_graph/spec_support/runtime_metadata_support" +require "support/example_extensions/indexing_preparers" +require "support/example_extensions/scalar_coercion_adapters" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + RSpec.describe ScalarType do + include RuntimeMetadataSupport + + it "allows `with:` to be used to update a single attribute" do + scalar_type = ScalarType.new( + coercion_adapter_ref: scalar_coercion_adapter1.to_dumpable_hash, + indexing_preparer_ref: indexing_preparer1.to_dumpable_hash + ) + expect(scalar_type.load_coercion_adapter).to eq(scalar_coercion_adapter1) + + scalar_type = scalar_type.with(coercion_adapter_ref: scalar_coercion_adapter2.to_dumpable_hash) + expect(scalar_type.load_coercion_adapter).to eq(scalar_coercion_adapter2) + expect(scalar_type.load_indexing_preparer).to eq(indexing_preparer1) + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_element_names_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_element_names_spec.rb new file mode 100644 index 00000000..2560f027 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_element_names_spec.rb @@ -0,0 +1,137 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + ExampleElementNames = SchemaElementNamesDefinition.new( + :foo, :multi_word_snake, :multiWordCamel + ) + + RSpec.describe SchemaElementNamesDefinition do + it "exposes the set of element names via an `ELEMENT_NAMES` constant" do + expect(ExampleElementNames::ELEMENT_NAMES).to eq [:foo, :multi_word_snake, :multiWordCamel] + end + + it "exposes camelCase element names when so configured, via snake case attributes" do + names = new_with(form: :camelCase) + + expect(names).to have_attributes( + foo: "foo", + multi_word_snake: "multiWordSnake", + multi_word_camel: "multiWordCamel" + ) + end + + it "exposes snake_case element names when so configured, via snake case attributes" do + names = new_with(form: :snake_case) + + expect(names).to have_attributes( + foo: "foo", + multi_word_snake: "multi_word_snake", + multi_word_camel: "multi_word_camel" + ) + end + + it "exposes a `normalize_case` method that converts to snake_case when so configured" do + names = new_with(form: :snake_case) + + expect(names.normalize_case("foo_bar")).to eq "foo_bar" + expect(names.normalize_case("fooBar")).to eq "foo_bar" + expect(names.normalize_case("FooBar")).to eq "_foo_bar" + end + + it "exposes a `normalize_case` method that converts to camelCase when so configured" do + names = new_with(form: :camelCase) + + expect(names.normalize_case("foo_bar")).to eq "fooBar" + expect(names.normalize_case("fooBar")).to eq "fooBar" + expect(names.normalize_case("FooBar")).to eq "FooBar" + end + + it "allows overrides" do + names = new_with(form: :camelCase, overrides: {foo: :bar}) + + expect(names).to have_attributes( + foo: "bar", + multi_word_snake: "multiWordSnake", + multi_word_camel: "multiWordCamel" + ) + end + + it "allows instantiation args to be passed as strings since that is how it is loaded from YAML config" do + names = new_with(form: "camelCase", overrides: {"foo" => "bar"}) + + expect(names).to have_attributes( + foo: "bar", + multi_word_snake: "multiWordSnake", + multi_word_camel: "multiWordCamel" + ) + end + + it "raises a clear error when given an invalid `form` option" do + expect { + new_with(form: "kebab-case") + }.to raise_error(Errors::SchemaError, /kebab-case/) + end + + it "raises a clear error when given an unused override" do + expect { + new_with(overrides: {goo: :bar}) + }.to raise_error(Errors::SchemaError, a_string_including("overrides", "goo")) + end + + it "raises a normal `NoMethodError` when accessing an undefined element name is attempted" do + names = new_with(form: :camelCase) + + expect { + names.not_a_method + }.to raise_error(NoMethodError, /not_a_method/) + end + + it "raises an error if two canonical names resolve to the same exposed name" do + expect { + new_with(overrides: {"foo" => "bar", "multi_word_snake" => "bar"}) + }.to raise_error(Errors::SchemaError, a_string_including("bar", "foo", "multi_word_snake")) + end + + it "inspects nicely" do + names = new_with(form: :camelCase, overrides: {foo: :bar}) + + expect(names.inspect).to eq "#:bar}>" + expect(names.to_s).to eq names.inspect + end + + describe "#canonical_name_for" do + let(:names) { new_with(form: "camelCase", overrides: {"foo" => "bar"}) } + + it "returns the canonical name for a given exposed name" do + expect(names.canonical_name_for("bar")).to eq :foo + expect(names.canonical_name_for("multiWordSnake")).to eq :multi_word_snake + expect(names.canonical_name_for("multiWordCamel")).to eq :multiWordCamel + end + + it "accepts either a string or symbol as an argument" do + expect(names.canonical_name_for("bar")).to eq :foo + expect(names.canonical_name_for(:bar)).to eq :foo + end + + it "returns `nil` if there is no canonical name for the given exposed name" do + expect(names.canonical_name_for("not_a_name")).to be nil + end + end + + def new_with(form: :camelCase, overrides: {}) + ExampleElementNames.new(form: form, overrides: overrides) + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_spec.rb new file mode 100644 index 00000000..4a175042 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_spec.rb @@ -0,0 +1,376 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/update_target" +require "elastic_graph/schema_artifacts/runtime_metadata/graphql_field" +require "elastic_graph/schema_artifacts/runtime_metadata/index_definition" +require "elastic_graph/schema_artifacts/runtime_metadata/index_field" +require "elastic_graph/schema_artifacts/runtime_metadata/object_type" +require "elastic_graph/schema_artifacts/runtime_metadata/scalar_type" +require "elastic_graph/schema_artifacts/runtime_metadata/schema" +require "elastic_graph/spec_support/runtime_metadata_support" +require "support/example_extensions/graphql_extension_modules" +require "support/example_extensions/indexing_preparers" +require "support/example_extensions/scalar_coercion_adapters" +require "yaml" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + RSpec.describe Schema do + include RuntimeMetadataSupport + + it "can roundtrip a schema through a primitive ruby hash for easy serialization and deserialization" do + schema = Schema.new( + object_types_by_name: { + "Widget" => ObjectType.new( + index_definition_names: ["widgets"], + update_targets: [ + UpdateTarget.new( + type: "WidgetCurrency", + relationship: "currency", + script_id: "some_script_id", + id_source: "cost.currency", + routing_value_source: "cost.currency_name", + rollover_timestamp_value_source: "currency_introduced_on", + data_params: {"workspace_id" => DynamicParam.new(source_path: "wid", cardinality: :one)}, + metadata_params: {"relationshipName" => StaticParam.new(value: "currency")} + ), + UpdateTarget.new( + type: nil, + relationship: nil, + script_id: nil, + id_source: "id", + routing_value_source: nil, + rollover_timestamp_value_source: nil, + data_params: {}, + metadata_params: {} + ) + ], + graphql_fields_by_name: { + "name_graphql" => GraphQLField.new( + name_in_index: "name_index", + computation_detail: nil, + relation: nil + ), + "parent" => GraphQLField.new( + name_in_index: nil, + computation_detail: nil, + relation: Relation.new( + foreign_key: "grandparents.parents.some_id", + direction: :out, + additional_filter: {"flag_field" => {"equalToAnyOf" => [true]}}, + foreign_key_nested_paths: ["grandparents", "grandparents.parents"] + ) + ), + "sum" => GraphQLField.new( + name_in_index: nil, + computation_detail: ComputationDetail.new( + empty_bucket_value: 0, + function: :sum + ), + relation: nil + ) + }, + elasticgraph_category: :some_category, + source_type: "SomeType", + graphql_only_return_type: true + ) + }, + scalar_types_by_name: { + "ScalarType1" => ScalarType.new( + scalar_coercion_adapter1.to_dumpable_hash, + indexing_preparer1.to_dumpable_hash + ), + "ScalarType2" => ScalarType.new( + scalar_coercion_adapter2.to_dumpable_hash, + indexing_preparer2.to_dumpable_hash + ) + }, + enum_types_by_name: { + "WidgetSort" => Enum::Type.new({ + "id_ASC" => Enum::Value.new(SortField.new("id", :asc), nil, nil, nil), + "id_DESC" => Enum::Value.new(SortField.new("id", :desc), nil, nil, nil) + }), + "DateGroupingGranularity" => Enum::Type.new({ + "DAY" => Enum::Value.new(nil, "day", nil, "DAILY"), + "YEAR" => Enum::Value.new(nil, "year", nil, "YEARLY") + }), + "DistanceUnit" => Enum::Type.new({ + "MILE" => Enum::Value.new(nil, nil, :mi, nil), + "KILOMETER" => Enum::Value.new(nil, nil, :km, nil) + }) + }, + index_definitions_by_name: { + "widgets" => IndexDefinition.new( + route_with: nil, + rollover: IndexDefinition::Rollover.new(:monthly, "created_at"), + default_sort_fields: [ + SortField.new("path.to.field1", :desc), + SortField.new("path.to.field2", :asc) + ], + current_sources: [SELF_RELATIONSHIP_NAME], + fields_by_path: { + "foo.bar" => IndexField.new(source: "other") + } + ), + "addresses" => IndexDefinition.new( + route_with: nil, + rollover: IndexDefinition::Rollover.new(:yearly, nil), + default_sort_fields: [], + current_sources: [SELF_RELATIONSHIP_NAME], + fields_by_path: {} + ), + "components" => IndexDefinition.new( + route_with: "group_id", + rollover: nil, + default_sort_fields: [], + current_sources: [SELF_RELATIONSHIP_NAME], + fields_by_path: {} + ) + }, + schema_element_names: SchemaElementNames.new( + form: :snake_case, + overrides: {"any_of" => "or"} + ), + graphql_extension_modules: [graphql_extension_module1], + static_script_ids_by_scoped_name: { + "filter/time_of_day" => "time_of_day_4474b200b6a00f385ed49f7c9669cbf3" + } + ) + + hash = schema.to_dumpable_hash + + expect(hash).to eq( + "object_types_by_name" => { + "Widget" => { + "index_definition_names" => ["widgets"], + "update_targets" => [ + { + "type" => "WidgetCurrency", + "relationship" => "currency", + "script_id" => "some_script_id", + "id_source" => "cost.currency", + "routing_value_source" => "cost.currency_name", + "rollover_timestamp_value_source" => "currency_introduced_on", + "data_params" => {"workspace_id" => {"source_path" => "wid", "cardinality" => "one"}}, + "metadata_params" => {"relationshipName" => {"value" => "currency"}} + }, + { + "id_source" => "id" + } + ], + "graphql_fields_by_name" => { + "name_graphql" => { + "name_in_index" => "name_index" + }, + "parent" => { + "relation" => { + "foreign_key" => "grandparents.parents.some_id", + "direction" => "out", + "additional_filter" => {"flag_field" => {"equalToAnyOf" => [true]}}, + "foreign_key_nested_paths" => ["grandparents", "grandparents.parents"] + } + }, + "sum" => { + "computation_detail" => { + "empty_bucket_value" => 0, + "function" => "sum" + } + } + }, + "elasticgraph_category" => "some_category", + "source_type" => "SomeType", + "graphql_only_return_type" => true + } + }, + "scalar_types_by_name" => { + "ScalarType1" => { + "coercion_adapter" => { + "extension_name" => "ElasticGraph::SchemaArtifacts::ScalarCoercionAdapter1", + "require_path" => "support/example_extensions/scalar_coercion_adapters" + }, + "indexing_preparer" => { + "extension_name" => "ElasticGraph::SchemaArtifacts::IndexingPreparer1", + "require_path" => "support/example_extensions/indexing_preparers" + } + }, + "ScalarType2" => { + "coercion_adapter" => { + "extension_name" => "ElasticGraph::SchemaArtifacts::ScalarCoercionAdapter2", + "require_path" => "support/example_extensions/scalar_coercion_adapters" + }, + "indexing_preparer" => { + "extension_name" => "ElasticGraph::SchemaArtifacts::IndexingPreparer2", + "require_path" => "support/example_extensions/indexing_preparers" + } + } + }, + "enum_types_by_name" => { + "WidgetSort" => { + "values_by_name" => { + "id_ASC" => {"sort_field" => { + "field_path" => "id", + "direction" => "asc" + }}, + "id_DESC" => {"sort_field" => { + "field_path" => "id", + "direction" => "desc" + }} + } + }, + "DateGroupingGranularity" => { + "values_by_name" => { + "DAY" => {"datastore_value" => "day", "alternate_original_name" => "DAILY"}, + "YEAR" => {"datastore_value" => "year", "alternate_original_name" => "YEARLY"} + } + }, + "DistanceUnit" => { + "values_by_name" => { + "MILE" => {"datastore_abbreviation" => "mi"}, + "KILOMETER" => {"datastore_abbreviation" => "km"} + } + } + }, + "index_definitions_by_name" => { + "widgets" => { + "rollover" => {"frequency" => "monthly", "timestamp_field_path" => "created_at"}, + "default_sort_fields" => [ + {"field_path" => "path.to.field1", "direction" => "desc"}, + {"field_path" => "path.to.field2", "direction" => "asc"} + ], + "current_sources" => [SELF_RELATIONSHIP_NAME], + "fields_by_path" => { + "foo.bar" => { + "source" => "other" + } + } + }, + "addresses" => { + "rollover" => {"frequency" => "yearly"}, + "current_sources" => [SELF_RELATIONSHIP_NAME] + }, + "components" => { + "route_with" => "group_id", + "current_sources" => [SELF_RELATIONSHIP_NAME] + } + }, + "schema_element_names" => { + "form" => "snake_case", + "overrides" => {"any_of" => "or"} + }, + "graphql_extension_modules" => [{ + "extension_name" => "ElasticGraph::SchemaArtifacts::GraphQLExtensionModule1", + "require_path" => "support/example_extensions/graphql_extension_modules" + }], + "static_script_ids_by_scoped_name" => { + "filter/time_of_day" => "time_of_day_4474b200b6a00f385ed49f7c9669cbf3" + } + ) + + expect(Schema.from_hash(hash, for_context: :graphql)).to eq schema + end + + it "ignores object types that have no meaningful runtime metadata" do + schema = schema_with(object_types_by_name: { + "UpdateTargetsOnly" => object_type_with(update_targets: [UpdateTarget.new( + type: "WidgetCurrency", + relationship: "currency", + script_id: "some_script_id", + id_source: "cost.currency", + routing_value_source: nil, + rollover_timestamp_value_source: nil, + data_params: {"workspace_id" => dynamic_param_with(cardinality: :many)}, + metadata_params: {} + )]), + "IndexDefinitionNamesOnly" => object_type_with(index_definition_names: ["foo", "bar"]), + "FieldsByGraphQLOnly" => object_type_with(graphql_fields_by_name: { + "name_graphql" => GraphQLField.new( + name_in_index: "name_index", + computation_detail: nil, + relation: nil + ) + }), + "NoMetadata" => object_type_with + }) + + schema = Schema.from_hash(schema.to_dumpable_hash, for_context: :graphql) + + expect(schema.object_types_by_name.keys).to contain_exactly( + "UpdateTargetsOnly", + "IndexDefinitionNamesOnly", + "FieldsByGraphQLOnly" + ) + end + + it "ignores enum types that have no meaningful runtime metadata" do + schema = schema_with(enum_types_by_name: { + "HasValues" => enum_type_with(values_by_name: { + "id_ASC" => Enum::Value.new(SortField.new("id", :asc), nil, nil, nil), + "id_DESC" => Enum::Value.new(SortField.new("id", :desc), nil, nil, nil) + }), + "NoValues" => enum_type_with(values_by_name: {}) + }) + + schema = Schema.from_hash(schema.to_dumpable_hash, for_context: :graphql) + + expect(schema.enum_types_by_name.keys).to contain_exactly("HasValues") + end + + it "builds from a minimal hash" do + schema = Schema.from_hash({ + "schema_element_names" => {"form" => "camelCase"} + }, for_context: :graphql) + + expect(schema).to eq Schema.new( + object_types_by_name: {}, + scalar_types_by_name: {}, + enum_types_by_name: {}, + index_definitions_by_name: {}, + schema_element_names: SchemaElementNames.from_hash({"form" => "camelCase"}), + graphql_extension_modules: [], + static_script_ids_by_scoped_name: {} + ) + end + + it "only loads `graphql_extension_modules` for the `:graphql` context since the extension module gems may not be available in other contexts" do + schema = schema_with(graphql_extension_modules: [graphql_extension_module1]) + + expect(Schema.from_hash(schema.to_dumpable_hash, for_context: :admin).graphql_extension_modules).to eq [] + expect(Schema.from_hash(schema.to_dumpable_hash, for_context: :indexer).graphql_extension_modules).to eq [] + expect(Schema.from_hash(schema.to_dumpable_hash, for_context: :graphql).graphql_extension_modules).to eq [graphql_extension_module1] + end + + it "dumps all hashes in alphabetical order for consistency" do + full_dumped_runtime_metadata = ::YAML.safe_load_file(::File.join( + CommonSpecHelpers::REPO_ROOT, "config", "schema", "artifacts", RUNTIME_METADATA_FILE + )) + + expect(paths_to_non_alphabetical_hashes_in(full_dumped_runtime_metadata)).to eq [] + end + + def paths_to_non_alphabetical_hashes_in(hash, parent_path: []) + paths = [] + # :nocov: -- only fully covered when the test above fails + if hash.keys != hash.keys.sort + paths << (parent_path.empty? ? "" : parent_path.join(".")) + end + # :nocov: + + hash.flat_map do |key, value| + if value.is_a?(::Hash) + paths_to_non_alphabetical_hashes_in(value, parent_path: parent_path + [key]) + else + [] + end + end + paths + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/sort_field_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/sort_field_spec.rb new file mode 100644 index 00000000..dffdcceb --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/sort_field_spec.rb @@ -0,0 +1,44 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/sort_field" +require "elastic_graph/spec_support/runtime_metadata_support" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + RSpec.describe SortField do + include RuntimeMetadataSupport + + it "raises a clear error if `direction` is not `:asc` or `:desc`" do + sort_field_with(direction: :asc) + sort_field_with(direction: :desc) + + expect { + sort_field_with(direction: :fesc) + }.to raise_error Errors::SchemaError, a_string_including(":fesc", ":asc", ":desc") + end + + it "can be converted to a datastore sort clause" do + sort_field = sort_field_with( + field_path: "path.to.field", + direction: :desc + ) + + expect(sort_field.to_query_clause).to eq({"path.to.field" => {"order" => "desc"}}) + end + + it "builds from a minimal hash" do + sort_field = SortField.from_hash({"direction" => "asc"}) + + expect(sort_field).to eq SortField.new(direction: :asc, field_path: nil) + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/update_target_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/update_target_spec.rb new file mode 100644 index 00000000..1ec6b9e0 --- /dev/null +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/update_target_spec.rb @@ -0,0 +1,130 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/update_target" +require "elastic_graph/spec_support/runtime_metadata_support" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + RSpec.describe UpdateTarget do + include RuntimeMetadataSupport + + it "builds from a minimal hash" do + update_target = UpdateTarget.from_hash({}) + + expect(update_target).to eq UpdateTarget.new( + type: nil, + relationship: nil, + script_id: nil, + id_source: nil, + routing_value_source: nil, + rollover_timestamp_value_source: nil, + data_params: {}, + metadata_params: {} + ) + end + + it "allows `data_params` to contain both dynamic params and static params" do + update_target = normal_indexing_update_target_with( + data_params: { + "name" => dynamic_param_with(source_path: "some_name", cardinality: :one), + "relationshipName" => static_param_with("__self") + } + ) + + dumped = update_target.to_dumpable_hash + expect(dumped.fetch("data_params")).to eq({ + "name" => {"cardinality" => "one", "source_path" => "some_name"}, + "relationshipName" => {"value" => "__self"} + }) + + reloaded = UpdateTarget.from_hash(dumped) + expect(reloaded).to eq(update_target) + end + + describe "#for_normal_indexing?" do + it "returns `false` for a derived indexing update target" do + update_target = derived_indexing_update_target_with(type: "Type1") + + expect(update_target.for_normal_indexing?).to eq(false) + end + + it "returns `true` for a normal indexing update target" do + update_target = normal_indexing_update_target_with(type: "Type1") + + expect(update_target.for_normal_indexing?).to eq(true) + end + end + + describe "#params_for" do + it "includes the given `doc_id` as `id`" do + params = params_for(doc_id: "abc123") + + expect(params).to include("id" => "abc123") + end + + it "extracts `metadata_params` from `event` and includes them" do + params = params_for( + metadata_params: { + "foo" => static_param_with(43), + "bar" => dynamic_param_with(source_path: "some.nested.field", cardinality: :one), + "bazz" => dynamic_param_with(source_path: "some.other.field", cardinality: :many) + }, + event: { + "some" => { + "nested" => {"field" => "hello"}, + "other" => {"field" => 12} + } + } + ) + + without_id_or_data = params.except("id", "data") + + expect(without_id_or_data).to eq( + "foo" => 43, + "bar" => "hello", + "bazz" => [12] + ) + end + + it "extracts `event_params` from `prepared_record` and include them under `data`" do + params = params_for( + data_params: { + "foo" => static_param_with(43), + "bar" => dynamic_param_with(source_path: "some.nested.field", cardinality: :one), + "bazz" => dynamic_param_with(source_path: "some.other.field", cardinality: :many) + }, + prepared_record: { + "some" => { + "nested" => {"field" => "hello"}, + "other" => {"field" => 12} + } + } + ) + + expect(params.fetch("data")).to eq( + "foo" => 43, + "bar" => "hello", + "bazz" => [12] + ) + end + + def params_for(doc_id: "doc_id", event: {}, prepared_record: {}, data_params: {}, metadata_params: {}) + update_target = normal_indexing_update_target_with( + data_params: data_params, + metadata_params: metadata_params + ) + + update_target.params_for(doc_id: doc_id, event: event, prepared_record: prepared_record) + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/.rspec b/elasticgraph-schema_definition/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-schema_definition/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-schema_definition/.yardopts b/elasticgraph-schema_definition/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-schema_definition/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-schema_definition/Gemfile b/elasticgraph-schema_definition/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-schema_definition/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-schema_definition/LICENSE.txt b/elasticgraph-schema_definition/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-schema_definition/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-schema_definition/README.md b/elasticgraph-schema_definition/README.md new file mode 100644 index 00000000..08035e49 --- /dev/null +++ b/elasticgraph-schema_definition/README.md @@ -0,0 +1,7 @@ +# ElasticGraph::SchemaDefinition + +Provides the ElasticGraph schema definition API, which is used to +generate ElasticGraph's schema artifacts. + +This gem is not intended to be used in production--production should +just use the schema artifacts instead. diff --git a/elasticgraph-schema_definition/elasticgraph-schema_definition.gemspec b/elasticgraph-schema_definition/elasticgraph-schema_definition.gemspec new file mode 100644 index 00000000..76fb2045 --- /dev/null +++ b/elasticgraph-schema_definition/elasticgraph-schema_definition.gemspec @@ -0,0 +1,26 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :local) do |spec, eg_version| + spec.summary = "ElasticGraph gem that provides the schema definition API and generates schema artifacts." + + spec.add_dependency "elasticgraph-graphql", eg_version # needed since we validate that scalar `coerce_with` options are valid (which loads scalar coercion adapters) + spec.add_dependency "elasticgraph-indexer", eg_version # needed since we validate that scalar `prepare_for_indexing_with` options are valid (which loads indexing preparer adapters) + spec.add_dependency "elasticgraph-json_schema", eg_version + spec.add_dependency "elasticgraph-schema_artifacts", eg_version + spec.add_dependency "elasticgraph-support", eg_version + spec.add_dependency "graphql", "~> 2.3.19" + spec.add_dependency "rake", "~> 13.2" + + spec.add_development_dependency "elasticgraph-admin", eg_version + spec.add_development_dependency "elasticgraph-datastore_core", eg_version + spec.add_development_dependency "elasticgraph-elasticsearch", eg_version + spec.add_development_dependency "elasticgraph-opensearch", eg_version +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/api.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/api.rb new file mode 100644 index 00000000..4d626f4c --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/api.rb @@ -0,0 +1,359 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/schema_artifacts/runtime_metadata/extension" +require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" +require "elastic_graph/schema_definition/results" +require "elastic_graph/schema_definition/state" + +module ElasticGraph + # The main entry point for schema definition from ElasticGraph applications. + # + # Call this API from a Ruby file configured as the `path_to_schema` (or from a Ruby file + # `load`ed from the `path_to_schema` file). + # + # @example + # ElasticGraph.define_schema do |schema| + # # The `schema` object provides the schema definition API. Use it in this block. + # end + def self.define_schema + if (api_instance = ::Thread.current[:ElasticGraph_SchemaDefinition_API_instance]) + yield api_instance + else + raise Errors::SchemaError, "No active `SchemaDefinition::API` instance is available. " \ + "Let ElasticGraph load the schema definition files." + end + end + + # Provides the ElasticGraph schema definition API. The primary entry point is {.define_schema}. + module SchemaDefinition + # Root API object that provides the schema definition API. + # + # @example + # ElasticGraph.define_schema do |schema| + # # The `schema` object is an instance of `API` + # end + class API + include Mixins::HasReadableToSAndInspect.new + + # @dynamic state, factory + + # @return [State] object which holds all state for the schema definition + attr_reader :state + + # @return [Factory] object responsible for instantiating all schema element classes + attr_reader :factory + + # @private + def initialize( + schema_elements, + index_document_sizes, + extension_modules: [], + derived_type_name_formats: {}, + type_name_overrides: {}, + enum_value_overrides_by_type: {}, + output: $stdout + ) + @state = State.with( + api: self, + schema_elements: schema_elements, + index_document_sizes: index_document_sizes, + derived_type_name_formats: derived_type_name_formats, + type_name_overrides: type_name_overrides, + enum_value_overrides_by_type: enum_value_overrides_by_type, + output: output + ) + + @factory = @state.factory + + extension_modules.each { |mod| extend(mod) } + + # These lines must come _after_ the extension modules are applied, so that the extension modules + # have a chance to hook into the factory in order to customize built in types if desired. + @factory.new_built_in_types(self).register_built_in_types + @state.initially_registered_built_in_types.merge(@state.types_by_name.keys) + end + + # Defines a raw GraphQL SDL snippet that will be included in the generated `schema.graphql` artifact. Designed to be an escape hatch, + # for when ElasticGraph doesn’t provide another way to write some type of GraphQL SDL element that you need. Currently, the only + # known use case is to define custom GraphQL directives. + # + # @param string [String] Raw snippet of SDL + # @return [void] + # + # @example Define a custom directive and use it + # ElasticGraph.define_schema do |schema| + # # Define a directive we can use to annotate what system a data type comes from. + # schema.raw_sdl "directive @sourcedFrom(system: String!) on OBJECT" + # + # schema.object_type "Transaction" do |t| + # t.directive "sourcedFrom", system: "transaction-processor" + # end + # end + def raw_sdl(string) + @state.sdl_parts << string + nil + end + + # Defines a [GraphQL object type](https://graphql.org/learn/schema/#object-types-and-fields) Use it to define a concrete type that + # has subfields. Object types can either be _indexed_ (e.g. directly indexed in the datastore, and available to query from the + # root `Query` object) or _embedded_ in other indexed types. + # + # @param name [String] name of the object type + # @yield [SchemaElements::ObjectType] object type object + # @return [void] + # + # @example Define embedded and indexed object types + # ElasticGraph.define_schema do |schema| + # # `Money` is an embedded object type + # schema.object_type "Money" do |t| + # t.field "currency", "String" + # t.field "amount", "JsonSafeLong" + # end + # + # # `Transaction` is an indexed object type + # schema.object_type "Transaction" do |t| + # t.root_query_fields plural: "transactions" + # t.field "id", "ID" + # t.field "cost", "Money" + # t.index "transactions" + # end + # end + def object_type(name, &block) + @state.register_object_interface_or_union_type @factory.new_object_type(name.to_s, &block) + nil + end + + # Defines a [GraphQL interface](https://graphql.org/learn/schema/#interfaces). Use it to define an abstract supertype with + # one or more fields that concrete implementations of the interface must also define. Each implementation can be an + # {SchemaElements::ObjectType} or {SchemaElements::InterfaceType}. + # + # @param name [String] name of the interface + # @yield [SchemaElements::InterfaceType] interface type object + # @return [void] + # + # @example Define an interface and implement it + # ElasticGraph.define_schema do |schema| + # schema.interface_type "Athlete" do |t| + # t.field "name", "String" + # t.field "team", "String" + # end + # + # schema.object_type "BaseballPlayer" do |t| + # t.implements "Athlete" + # t.field "name", "String" + # t.field "team", "String" + # t.field "battingAvg", "Float" + # end + # + # schema.object_type "BasketballPlayer" do |t| + # t.implements "Athlete" + # t.field "name", "String" + # t.field "team", "String" + # t.field "pointsPerGame", "Float" + # end + # end + def interface_type(name, &block) + @state.register_object_interface_or_union_type @factory.new_interface_type(name.to_s, &block) + nil + end + + # Defines a [GraphQL enum type](https://graphql.org/learn/schema/#enumeration-types). + # The type is restricted to an enumerated set of values, each with a unique name. + # Use `value` or `values` to define the enum values in the passed block. + # + # Note: if required by your configuration, this may generate a pair of Enum types (an input + # enum and an output enum). + # + # @param name [String] name of the enum type + # @yield [SchemaElements::EnumType] enum type object + # @return [void] + # + # @example Define an enum type + # ElasticGraph.define_schema do |schema| + # schema.enum_type "Currency" do |t| + # t.value "USD" do |v| + # v.documentation "US Dollars." + # end + # + # t.value "JPY" do |v| + # v.documentation "Japanese Yen." + # end + # + # # You can define multiple values in one call if you don't care about their docs or directives. + # t.values "GBP", "AUD" + # end + # end + def enum_type(name, &block) + @state.register_enum_type @factory.new_enum_type(name.to_s, &block) + nil + end + + # Defines a [GraphQL union type](https://graphql.org/learn/schema/#union-types). Use it to define an abstract supertype with one or + # more concrete subtypes. Each subtype must be an {SchemaElements::ObjectType}, but they do not have to share any fields in common. + # + # @param name [String] name of the union type + # @yield [SchemaElements::UnionType] union type object + # @return [void] + # + # @example Define a union type + # ElasticGraph.define_schema do |schema| + # schema.object_type "Card" do |t| + # # ... + # end + # + # schema.object_type "BankAccount" do |t| + # # ... + # end + # + # schema.object_type "BitcoinWallet" do |t| + # # ... + # end + # + # schema.union_type "FundingSource" do |t| + # t.subtype "Card" + # t.subtypes "BankAccount", "BitcoinWallet" + # end + # end + def union_type(name, &block) + @state.register_object_interface_or_union_type @factory.new_union_type(name.to_s, &block) + nil + end + + # Defines a [GraphQL scalar type](https://graphql.org/learn/schema/#scalar-types). ElasticGraph itself uses this to define a few + # common scalar types (e.g. `Date` and `DateTime`), but it is also available to you to use to define your own custom scalar types. + # + # @param name [String] name of the scalar type + # @yield [SchemaElements::ScalarType] scalar type object + # @return [void] + # + # @example Define a scalar type + # ElasticGraph.define_schema do |schema| + # schema.scalar_type "URL" do |t| + # t.mapping type: "keyword" + # t.json_schema type: "string", format: "uri" + # end + # end + def scalar_type(name, &block) + @state.register_scalar_type @factory.new_scalar_type(name.to_s, &block) + nil + end + + # Registers the name of a type that existed in a prior version of the schema but has been deleted. + # + # @note In situations where this API applies, ElasticGraph will give you an error message indicating that you need to use this API + # or {SchemaElements::TypeWithSubfields#renamed_from}. Likewise, when ElasticGraph no longer needs to know about this, it'll give you a warning + # indicating the call to this method can be removed. + # + # @param name [String] name of type that used to exist but has been deleted + # @return [void] + # + # @example Indicate that `Widget` has been deleted + # ElasticGraph.define_schema do |schema| + # schema.deleted_type "Widget" + # end + def deleted_type(name) + @state.register_deleted_type( + name, + defined_at: caller_locations(1, 1).first, # : ::Thread::Backtrace::Location + defined_via: %(schema.deleted_type "#{name}") + ) + nil + end + + # Registers a GraphQL extension module that will be loaded and used by `elasticgraph-graphql`. While such + # extension modules can also be configured in a settings YAML file, it can be useful to register it here + # when you want to ensure that the extension is used in all environments. For example, an extension library + # that defines custom schema elements (such as `elasticgraph-apollo`) may need to ensure its corresponding + # GraphQL extension module is used since the custom schema elements would not work correctly otherwise. + # + # @param extension_module [Module] GraphQL extension module + # @param defined_at [String] the `require` path of the extension module + # @param extension_config [Hash] configuration options for the extension module + # @return [void] + # + # @example Register `elasticgraph-query_registry` extension module + # require(query_registry_require_path = "elastic_graph/query_registry/graphql_extension") + # + # ElasticGraph.define_schema do |schema| + # schema.register_graphql_extension ElasticGraph::QueryRegistry::GraphQLExtension, + # defined_at: query_registry_require_path + # end + def register_graphql_extension(extension_module, defined_at:, **extension_config) + @state.graphql_extension_modules << SchemaArtifacts::RuntimeMetadata::Extension.new(extension_module, defined_at, extension_config) + nil + end + + # @return the results of the schema definition + def results + @results ||= Results.new(@state) + end + + # Defines the version number of the current JSON schema. Importantly, every time a change is made that impacts the JSON schema + # artifact, the version number must be incremented to ensure that each different version of the JSON schema is identified by a unique + # version number. The publisher will then include this version number in published events to identify the version of the schema it + # was using. This avoids the need to deploy the publisher and ElasticGraph indexer at the same time to keep them in sync. + # + # @note While this is an important part of how ElasticGraph is designed to support schema evolution, it can be annoying constantly + # have to increment this while rapidly changing the schema during prototyping. You can disable the requirement to increment this + # on every JSON schema change by setting `enforce_json_schema_version` to `false` in your `Rakefile`. + # + # @param version [Integer] current version number of the JSON schema artifact + # @return [void] + # @see Local::RakeTasks#enforce_json_schema_version + # + # @example Set the JSON schema version to 1 + # ElasticGraph.define_schema do |schema| + # schema.json_schema_version 1 + # end + def json_schema_version(version) + if !version.is_a?(Integer) || version < 1 + raise Errors::SchemaError, "`json_schema_version` must be a positive integer. Specified version: #{version}" + end + + if @state.json_schema_version + raise Errors::SchemaError, "`json_schema_version` can only be set once on a schema. Previously-set version: #{@state.json_schema_version}" + end + + @state.json_schema_version = version + @state.json_schema_version_setter_location = caller_locations(1, 1).to_a.first + nil + end + + # Registers a customization callback that will be applied to every built-in type automatically provided by ElasticGraph. Provides + # an opportunity to customize the built-in types (e.g. to add directives to them or whatever). + # + # @yield [SchemaElements::EnumType, SchemaElements::InputType, SchemaElements::InterfaceType, SchemaElements::ObjectType, SchemaElements::ScalarType, SchemaElements::UnionType] built in type + # @return [void] + # + # @example Customize documentation of built-in types + # ElasticGraph.define_schema do |schema| + # schema.on_built_in_types do |type| + # type.append_to_documentation "This is a built-in ElasticGraph type." + # end + # end + def on_built_in_types(&customization_block) + @state.built_in_types_customization_blocks << customization_block + nil + end + + # While the block executes, makes any `ElasticGraph.define_schema` calls operate on this `API` instance. + # + # @private + def as_active_instance + # @type var old_value: API? + old_value = ::Thread.current[:ElasticGraph_SchemaDefinition_API_instance] + ::Thread.current[:ElasticGraph_SchemaDefinition_API_instance] = self + yield + ensure + ::Thread.current[:ElasticGraph_SchemaDefinition_API_instance] = old_value + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/factory.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/factory.rb new file mode 100644 index 00000000..fa299edb --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/factory.rb @@ -0,0 +1,503 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" +require "elastic_graph/schema_definition/schema_elements/argument" +require "elastic_graph/schema_definition/schema_elements/built_in_types" +require "elastic_graph/schema_definition/schema_elements/deprecated_element" +require "elastic_graph/schema_definition/schema_elements/directive" +require "elastic_graph/schema_definition/schema_elements/enum_type" +require "elastic_graph/schema_definition/schema_elements/enum_value" +require "elastic_graph/schema_definition/schema_elements/enums_for_indexed_types" +require "elastic_graph/schema_definition/schema_elements/field" +require "elastic_graph/schema_definition/schema_elements/field_source" +require "elastic_graph/schema_definition/schema_elements/graphql_sdl_enumerator" +require "elastic_graph/schema_definition/schema_elements/input_field" +require "elastic_graph/schema_definition/schema_elements/input_type" +require "elastic_graph/schema_definition/schema_elements/interface_type" +require "elastic_graph/schema_definition/schema_elements/object_type" +require "elastic_graph/schema_definition/schema_elements/relationship" +require "elastic_graph/schema_definition/schema_elements/scalar_type" +require "elastic_graph/schema_definition/schema_elements/sort_order_enum_value" +require "elastic_graph/schema_definition/schema_elements/type_reference" +require "elastic_graph/schema_definition/schema_elements/type_with_subfields" +require "elastic_graph/schema_definition/schema_elements/union_type" + +module ElasticGraph + module SchemaDefinition + # A class responsible for instantiating all schema elements. We want all schema element instantiation + # to go through this one class to support extension libraries. ElasticGraph supports extension libraries + # that provide modules that get extended onto specific instances of ElasticGraph framework classes. We + # prefer this approach rather than having extension library modules applied via `include` or `prepend`, + # because they _permanently modify_ the host classes. ElasticGraph is designed to avoid all mutable + # global state, and that includes mutations to ElasticGraph class ancestor chains from extension libraries. + # + # Concretely, if we included or prepended extension libraries modules, we'd have a hard time keeping our + # tests order-independent and deterministic while running all the ElasticGraph test suites in the same + # Ruby process. A test using an extension library could cause a core ElasticGraph class to get mutated + # in a way that impacts a test that runs in the same process later. Instead, we expect extension libraries + # to hook into ElasticGraph using `extend` on particular object instances. + # + # But that creates a bit of a problem: how can an extension library extend a module onto every instance + # of a specific type of schema element while it is in use? The answer is this factory class: + # + # - An extension library can extend a module onto `schema.factory`. + # - That module can in turn override any of these factory methods and extend another module onto the schema + # element instances. + # + # @private + class Factory + include Mixins::HasReadableToSAndInspect.new + + def initialize(state) + @state = state + end + + # Helper method to help enforce our desired invariant: we want _every_ instantiation of these schema + # element classes to happen via this factory method provided here. To enforce that, this helper returns + # the `new` method (as a `Method` object) after removing it from the given class. That makes it impossible + # for `new` to be called by anyone except from the factory using the captured method object. + def self.prevent_non_factory_instantiation_of(klass) + klass.method(:new).tap do + klass.singleton_class.undef_method :new + end + end + + def new_deprecated_element(name, defined_at:, defined_via:) + @@deprecated_element_new.call(schema_def_state: @state, name: name, defined_at: defined_at, defined_via: defined_via) + end + @@deprecated_element_new = prevent_non_factory_instantiation_of(SchemaElements::DeprecatedElement) + + def new_argument(field, name, value_type) + @@argument_new.call(@state, field, name, value_type).tap do |argument| + yield argument if block_given? + end + end + @@argument_new = prevent_non_factory_instantiation_of(SchemaElements::Argument) + + def new_built_in_types(api) + @@built_in_types_new.call(api, @state) + end + @@built_in_types_new = prevent_non_factory_instantiation_of(SchemaElements::BuiltInTypes) + + def new_directive(name, arguments) + @@directive_new.call(name, arguments) + end + @@directive_new = prevent_non_factory_instantiation_of(SchemaElements::Directive) + + def new_enum_type(name, &block) + @@enum_type_new.call(@state, name, &(_ = block)) + end + @@enum_type_new = prevent_non_factory_instantiation_of(SchemaElements::EnumType) + + def new_enum_value(name, original_name) + @@enum_value_new.call(@state, name, original_name) do |enum_value| + yield enum_value if block_given? + end + end + @@enum_value_new = prevent_non_factory_instantiation_of(SchemaElements::EnumValue) + + def new_enums_for_indexed_types + @@enums_for_indexed_types_new.call(@state) + end + @@enums_for_indexed_types_new = prevent_non_factory_instantiation_of(SchemaElements::EnumsForIndexedTypes) + + # Hard to type check this. + # @dynamic new_field + __skip__ = def new_field(**kwargs, &block) + @@field_new.call(schema_def_state: @state, **kwargs, &block) + end + @@field_new = prevent_non_factory_instantiation_of(SchemaElements::Field) + + def new_graphql_sdl_enumerator(all_types_except_root_query_type) + @@graphql_sdl_enumerator_new.call(@state, all_types_except_root_query_type) + end + @@graphql_sdl_enumerator_new = prevent_non_factory_instantiation_of(SchemaElements::GraphQLSDLEnumerator) + + # Hard to type check this. + # @dynamic new_input_field + __skip__ = def new_input_field(**kwargs) + input_field = @@input_field_new.call(new_field(as_input: true, **kwargs)) + yield input_field + input_field + end + @@input_field_new = prevent_non_factory_instantiation_of(SchemaElements::InputField) + + def new_input_type(name) + @@input_type_new.call(@state, name) do |input_type| + yield input_type + end + end + @@input_type_new = prevent_non_factory_instantiation_of(SchemaElements::InputType) + + def new_filter_input_type(source_type, name_prefix: source_type, category: :filter_input) + new_input_type(@state.type_ref(name_prefix).as_static_derived_type(category).name) do |t| + t.documentation <<~EOS + Input type used to specify filters on `#{source_type}` fields. + + Will be ignored if passed as an empty object (or as `null`). + EOS + + t.field @state.schema_elements.any_of, "[#{t.name}!]" do |f| + f.documentation <<~EOS + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + EOS + end + + t.field @state.schema_elements.not, t.name do |f| + f.documentation <<~EOS + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + EOS + end + + yield t + end + end + + # Builds the standard set of filter input types for types which are indexing leaf types. + # + # All GraphQL leaf types (enums and scalars) are indexing leaf types, but some GraphQL object types are + # as well. For example, `GeoLocation` is an object type in GraphQL (with separate lat/long fields) but is + # an indexing leaf type because we use the datastore `geo_point` type for it. + def build_standard_filter_input_types_for_index_leaf_type(source_type, name_prefix: source_type, &define_filter_fields) + single_value_filter = new_filter_input_type(source_type, name_prefix: name_prefix, &define_filter_fields) + list_filter = new_list_filter_input_type(source_type, name_prefix: name_prefix, any_satisfy_type_category: :list_element_filter_input) + list_element_filter = new_list_element_filter_input_type(source_type, name_prefix: name_prefix, &define_filter_fields) + + [single_value_filter, list_filter, list_element_filter] + end + + # Builds the standard set of filter input types for types which are indexing object types. + # + # Most GraphQL object types are indexing object types as well, but not all. + # For example, `GeoLocation` is an object type in GraphQL (with separate lat/long fields) but is + # an indexing leaf type because we use the datastore `geo_point` type for it. + def build_standard_filter_input_types_for_index_object_type(source_type, name_prefix: source_type, &define_filter_fields) + single_value_filter = new_filter_input_type(source_type, name_prefix: name_prefix, &define_filter_fields) + list_filter = new_list_filter_input_type(source_type, name_prefix: name_prefix, any_satisfy_type_category: :filter_input) + fields_list_filter = new_fields_list_filter_input_type(source_type, name_prefix: name_prefix) + + [single_value_filter, list_filter, fields_list_filter] + end + + def build_relay_pagination_types(type_name, include_total_edge_count: false, derived_indexed_types: [], support_pagination: true, &customize_connection) + [ + (edge_type_for(type_name) if support_pagination), + connection_type_for(type_name, include_total_edge_count, derived_indexed_types, support_pagination, &customize_connection) + ].compact + end + + def new_interface_type(name) + @@interface_type_new.call(@state, name.to_s) do |interface_type| + yield interface_type + end + end + @@interface_type_new = prevent_non_factory_instantiation_of(SchemaElements::InterfaceType) + + def new_object_type(name) + @@object_type_new.call(@state, name.to_s) do |object_type| + yield object_type if block_given? + end + end + @@object_type_new = prevent_non_factory_instantiation_of(SchemaElements::ObjectType) + + def new_scalar_type(name) + @@scalar_type_new.call(@state, name.to_s) do |scalar_type| + yield scalar_type + end + end + @@scalar_type_new = prevent_non_factory_instantiation_of(SchemaElements::ScalarType) + + def new_sort_order_enum_value(enum_value, sort_order_field_path) + @@sort_order_enum_value_new.call(enum_value, sort_order_field_path) + end + @@sort_order_enum_value_new = prevent_non_factory_instantiation_of(SchemaElements::SortOrderEnumValue) + + def new_type_reference(name) + @@type_reference_new.call(name, @state) + end + @@type_reference_new = prevent_non_factory_instantiation_of(SchemaElements::TypeReference) + + def new_type_with_subfields(schema_kind, name, wrapping_type:, field_factory:) + @@type_with_subfields_new.call(schema_kind, @state, name, wrapping_type: wrapping_type, field_factory: field_factory) do |type_with_subfields| + yield type_with_subfields + end + end + @@type_with_subfields_new = prevent_non_factory_instantiation_of(SchemaElements::TypeWithSubfields) + + def new_union_type(name) + @@union_type_new.call(@state, name.to_s) do |union_type| + yield union_type + end + end + @@union_type_new = prevent_non_factory_instantiation_of(SchemaElements::UnionType) + + def new_field_source(relationship_name:, field_path:) + @@field_source_new.call(relationship_name, field_path) + end + @@field_source_new = prevent_non_factory_instantiation_of(SchemaElements::FieldSource) + + def new_relationship(field, cardinality:, related_type:, foreign_key:, direction:) + @@relationship_new.call( + field, + cardinality: cardinality, + related_type: related_type, + foreign_key: foreign_key, + direction: direction + ) + end + @@relationship_new = prevent_non_factory_instantiation_of(SchemaElements::Relationship) + + # Responsible for creating a new `*AggregatedValues` type for an index leaf type. + # + # An index leaf type is a scalar, enum, object type that is backed by a single, indivisible + # field in the index. All scalar and enum types are index leaf types, and object types + # rarely (but sometimes) are. For example, the `GeoLocation` object type has two subfields + # (`latitude` and `longitude`) but is backed by a single `geo_point` field in the index, + # so it is an index leaf type. + def new_aggregated_values_type_for_index_leaf_type(index_leaf_type) + new_object_type @state.type_ref(index_leaf_type).as_aggregated_values.name do |type| + type.graphql_only true + type.documentation "A return type used from aggregations to provided aggregated values over `#{index_leaf_type}` fields." + type.runtime_metadata_overrides = {elasticgraph_category: :scalar_aggregated_values} + + type.field @state.schema_elements.approximate_distinct_value_count, "JsonSafeLong", graphql_only: true do |f| + # Note: the 1-6% accuracy figure comes from the Elasticsearch docs: + # https://www.elastic.co/guide/en/elasticsearch/reference/8.10/search-aggregations-metrics-cardinality-aggregation.html#_counts_are_approximate + f.documentation <<~EOS + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + EOS + + f.runtime_metadata_computation_detail empty_bucket_value: 0, function: :cardinality + end + + yield type + end + end + + private + + def new_list_filter_input_type(source_type, name_prefix:, any_satisfy_type_category:) + any_satisfy = @state.schema_elements.any_satisfy + all_of = @state.schema_elements.all_of + + new_filter_input_type "[#{source_type}]", name_prefix: name_prefix, category: :list_filter_input do |t| + t.field any_satisfy, @state.type_ref(name_prefix).as_static_derived_type(any_satisfy_type_category).name do |f| + f.documentation <<~EOS + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + EOS + end + + t.field all_of, "[#{t.name}!]" do |f| + f.documentation <<~EOS + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `#{t.name}` input because of collisions between key names. For example, if you want to provide + multiple `#{any_satisfy}: ...` filters, you could do `#{all_of}: [{#{any_satisfy}: ...}, {#{any_satisfy}: ...}]`. + + Will be ignored when `null` or an empty list is passed. + EOS + end + + define_list_counts_filter_field_on(t) + end + end + + # Generates a filter type used on elements of a list. Referenced from a `#{type}ListFilterInput` input + # (which is referenced from `any_satisfy`). + def new_list_element_filter_input_type(source_type, name_prefix:) + new_filter_input_type source_type, name_prefix: name_prefix, category: :list_element_filter_input do |t| + t.documentation <<~EOS + Input type used to specify filters on elements of a `[#{source_type}]` field. + + Will be ignored if passed as an empty object (or as `null`). + EOS + + # While we support `not: {any_satisfy: ...}` we do not support `any_satisfy: {not ...}` at this time. + # Since `any_satisfy` does not have a node in the datastore query expression, the naive way we'd + # generate the datastore filter would be the same for both cases. However, they should have different + # semantics. + # + # For example, if we have these documents: + # + # - d1: {tags: ["a", "b"]} + # - d2: {tags: ["b", "c"]} + # - d3: {tags: []} + # - d4: {tags: ["a"]} + # + # Then `not: {any_satisfy: {equal_to_any_of: ["a"]}}` should (and does) match d2 and d3. + # But `any_satisfy: {not: {equal_to_any_of: ["a"]}}` should match d1 and d2 (both have a tag that is not equal to "a"). + # However, Elasticsearch and OpenSearch do not allow us to express that. + # + # Technically, we could probably get it to work if we implemented negations of all our filter operators. + # For example, `gt` negated is `lte`, `lt` negated is `gte`, etc. But for some operators that's not easy. + # There is no available negation of `equal_to_any_of`, but we could maybe get it to work by using a regex + # operator that matches any term EXCEPT the provided value, but that's non-trivial to implement and could + # be quite expensive. So for now we just don't support this. + # + # ...therefore, we need to omit `not` from the generated filter here. + t.graphql_fields_by_name.delete(@state.schema_elements.not) + + yield t + end + end + + # Generates a filter type used for objects within a list (either at a parent or some ancestor level) + # when the `nested ` type is not used. The datastore indexes each leaf field as its own flattened list + # of values. We mirror that structure with this filter type, only offering `any_satisfy` on leaf fields. + def new_fields_list_filter_input_type(source_type_name, name_prefix:) + source_type = @state.object_types_by_name.fetch(source_type_name) + + new_filter_input_type source_type_name, name_prefix: name_prefix, category: :fields_list_filter_input do |t| + t.documentation <<~EOS + Input type used to specify filters on a `#{source_type_name}` object referenced directly + or transitively from a list field that has been configured to index each leaf field as + its own flattened list of values. + + Will be ignored if passed as an empty object (or as `null`). + EOS + + source_type.graphql_fields_by_name.each do |field_name, field| + next unless field.filterable? + t.graphql_fields_by_name[field_name] = field.to_filter_field( + parent_type: t, + # We are never filtering on single values in this context (since we are already + # within a list that isn't using the `nested` mapping type). + for_single_value: false + ) + end + + # We want to add a `count` field so that clients can filter on the count of elements of this list field. + # However, if the object type of this field has a user-defined `count` field then we cannot do that, as that + # would create a conflict. So we omit it in that case. Users will still be able to filter on the count of + # the leaf fields if they spell out the full filter path to a leaf field. + count_field_name = @state.schema_elements.count + if t.graphql_fields_by_name.key?(count_field_name) + @state.output.puts <<~EOS + WARNING: Since a `#{source_type_name}.#{count_field_name}` field exists, ElasticGraph is not able to + define its typical `#{t.name}.#{count_field_name}` field, which allows clients to filter on the count + of values for a `[#{source_type.name}]` field. Clients will still be able to filter on the `#{count_field_name}` + at a leaf field path. However, there are a couple ways this naming conflict can be avoided if desired: + + 1. Pick a different name for the `#{source_type_name}.#{count_field_name}` field. + 2. Change the name used by ElasticGraph for this field. To do that, pass a + `schema_element_name_overrides: {#{count_field_name.inspect} => "alt_name"}` option alongside + `schema_element_name_form: ...` when defining `ElasticGraph::SchemaDefinition::RakeTasks` + (typically in the `Rakefile`). + EOS + else + define_list_counts_filter_field_on(t) + end + end + end + + def define_list_counts_filter_field_on(type) + # Note: we use `IntFilterInput` (instead of `JsonSafeLongFilterInput` or similar...) to align with the + # `integer` mapping type we use for the `__counts` field. If we ever change that + # in `list_counts_mapping.rb`, we'll want to consider changing this as well. + # + # We use `name_in_index: __counts` because we need to indicate that it's the list `count` operator + # rather than a schema field named "counts". Our filter interpreter logic relies on that name. + # We can count on `__counts` not being used by a real schema field because the GraphQL spec reserves + # the `__` prefix for its own use. + type.field @state.schema_elements.count, @state.type_ref("Int").as_filter_input.name, name_in_index: LIST_COUNTS_FIELD do |f| + f.documentation <<~EOS + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + EOS + end + end + + def edge_type_for(type_name) + type_ref = @state.type_ref(type_name) + new_object_type type_ref.as_edge.name do |t| + t.relay_pagination_type = true + t.runtime_metadata_overrides = {elasticgraph_category: :relay_edge} + + t.documentation <<~EOS + Represents a specific `#{type_name}` in the context of a `#{type_ref.as_connection.name}`, + providing access to both the `#{type_name}` and a pagination `Cursor`. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. + EOS + + t.field @state.schema_elements.node, type_name do |f| + f.documentation "The `#{type_name}` of this edge." + end + + t.field @state.schema_elements.cursor, "Cursor" do |f| + f.documentation <<~EOS + The `Cursor` of this `#{type_name}`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `#{type_name}`. + EOS + end + end + end + + def connection_type_for(type_name, include_total_edge_count, derived_indexed_types, support_pagination) + type_ref = @state.type_ref(type_name) + new_object_type type_ref.as_connection.name do |t| + t.relay_pagination_type = true + t.runtime_metadata_overrides = {elasticgraph_category: :relay_connection} + + if support_pagination + t.documentation <<~EOS + Represents a paginated collection of `#{type_name}` results. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. + EOS + else + t.documentation "Represents a collection of `#{type_name}` results." + end + + if support_pagination + t.field @state.schema_elements.edges, "[#{type_ref.as_edge.name}!]!" do |f| + f.documentation "Wraps a specific `#{type_name}` to pair it with its pagination cursor." + end + end + + t.field @state.schema_elements.nodes, "[#{type_name}!]!" do |f| + f.documentation "The list of `#{type_name}` results." + end + + if support_pagination + t.field @state.schema_elements.page_info, "PageInfo!" do |f| + f.documentation "Provides pagination-related information." + end + end + + if include_total_edge_count + t.field @state.schema_elements.total_edge_count, "JsonSafeLong!" do |f| + f.documentation "The total number of edges available in this connection to paginate over." + end + end + + yield t if block_given? + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_fields/append_only_set.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_fields/append_only_set.rb new file mode 100644 index 00000000..cd2b8278 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_fields/append_only_set.rb @@ -0,0 +1,79 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_definition/indexing/derived_fields/field_initializer_support" + +module ElasticGraph + module SchemaDefinition + module Indexing + module DerivedFields + # Responsible for providing bits of the painless script specific to a {DerivedIndexedType#append_only_set} field. + # + # @api private + class AppendOnlySet < ::Data.define(:destination_field, :source_field) + # `Data.define` provides the following methods: + # @dynamic destination_field, source_field + + # @return [Array] painless functions required by `append_only_set`. + def function_definitions + [IDEMPOTENTLY_INSERT_VALUES, IDEMPOTENTLY_INSERT_VALUE] + end + + # @return [String] a line of painless code to append a value to the set and return a boolean indicating if the set was updated. + def apply_operation_returning_update_status + %{appendOnlySet_idempotentlyInsertValues(data["#{source_field}"], ctx._source.#{destination_field})} + end + + # The statements here initialize the field to an empty list if it is null. This primarily happens when the document + # does not already exist, but can also happen when we add a new derived field to an existing type. + # + # @return [Array] a list of painless statements that must be called at the top of the script to set things up. + def setup_statements + FieldInitializerSupport.build_empty_value_initializers(destination_field, leaf_value: FieldInitializerSupport::EMPTY_PAINLESS_LIST) + end + + private + + IDEMPOTENTLY_INSERT_VALUES = <<~EOS + // Wrapper around `idempotentlyInsertValue` that handles a list of values. + // Returns `true` if the list field was updated. + boolean appendOnlySet_idempotentlyInsertValues(List values, List sortedList) { + boolean listUpdated = false; + + for (def value : values) { + listUpdated = appendOnlySet_idempotentlyInsertValue(value, sortedList) || listUpdated; + } + + return listUpdated; + } + EOS + + IDEMPOTENTLY_INSERT_VALUE = <<~EOS + // Idempotently inserts the given value in the `sortedList`, returning `true` if the list was updated. + boolean appendOnlySet_idempotentlyInsertValue(def value, List sortedList) { + // As per https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/Collections.html#binarySearch(java.util.List,java.lang.Object): + // + // > Returns the index of the search key, if it is contained in the list; otherwise, (-(insertion point) - 1). + // > The insertion point is defined as the point at which the key would be inserted into the list: the index + // > of the first element greater than the key, or list.size() if all elements in the list are less than the + // > specified key. Note that this guarantees that the return value will be >= 0 if and only if the key is found. + int binarySearchResult = Collections.binarySearch(sortedList, value); + + if (binarySearchResult < 0) { + sortedList.add(-binarySearchResult - 1, value); + return true; + } else { + return false; + } + } + EOS + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_fields/field_initializer_support.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_fields/field_initializer_support.rb new file mode 100644 index 00000000..d0a62f34 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_fields/field_initializer_support.rb @@ -0,0 +1,59 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module Indexing + # Contains implementation logic for the different kinds of derived fields. + # + # @api private + module DerivedFields + # Contains helper logic for field initialization applicable to all types of derived fields. + # + # @api private + module FieldInitializerSupport + # Painless literal for an empty list, from [the docs](https://www.elastic.co/guide/en/elasticsearch/painless/8.15/painless-operators-reference.html#list-initialization-operator). + EMPTY_PAINLESS_LIST = "[]" + + # Painless literal for an empty map, from [the docs](https://www.elastic.co/guide/en/elasticsearch/painless/8.15/painless-operators-reference.html#map-initialization-operator). + EMPTY_PAINLESS_MAP = "[:]" + + # @return [Array] a list of painless statements that will initialize a given `destination_field` path to an empty value. + def self.build_empty_value_initializers(destination_field, leaf_value:) + snippets = [] # : ::Array[::String] + path_so_far = [] # : ::Array[::String] + + destination_field.split(".").each do |path_part| + path_to_this_part = (path_so_far + [path_part]).join(".") + is_leaf = path_to_this_part == destination_field + + unless is_leaf && leaf_value == :leave_unset + # The empty value of all parent fields must be an empty painless map, but for a leaf field it can be different. + empty_value = is_leaf ? leaf_value : EMPTY_PAINLESS_MAP + + snippets << default_source_field_to_empty(path_to_this_part, empty_value.to_s) + path_so_far << path_part + end + end + + snippets + end + + # @return [String] a painless statement that will default a single field to an empty value. + def self.default_source_field_to_empty(field_path, empty_value) + <<~EOS.strip + if (ctx._source.#{field_path} == null) { + ctx._source.#{field_path} = #{empty_value}; + } + EOS + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_fields/immutable_value.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_fields/immutable_value.rb new file mode 100644 index 00000000..39a6f3ec --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_fields/immutable_value.rb @@ -0,0 +1,99 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module Indexing + module DerivedFields + # Responsible for providing bits of the painless script specific to a {DerivedIndexedType#immutable_value} field. + # + # @api private + class ImmutableValue < ::Data.define(:destination_field, :source_field, :nullable, :can_change_from_null) + # `Data.define` provides the following methods: + # @dynamic destination_field, source_field + + # @return [String] a line of painless code to manage an immutable value field and return a boolean indicating if it was updated. + def apply_operation_returning_update_status + *parent_parts, field = destination_field.split(".") + parent_parts = ["ctx", "_source"] + parent_parts + + %{immutableValue_idempotentlyUpdateValue(scriptErrors, data["#{source_field}"], #{parent_parts.join(".")}, "#{destination_field}", "#{field}", #{nullable}, #{can_change_from_null})} + end + + # @return [Array] a list of painless statements that must be called at the top of the script to set things up. + def setup_statements + FieldInitializerSupport.build_empty_value_initializers(destination_field, leaf_value: :leave_unset) + end + + # @return [Array] painless functions required by `immutable_value`. + def function_definitions + [IDEMPOTENTLY_SET_VALUE] + end + + private + + # Painless function which manages an `immutable_value` field. + IDEMPOTENTLY_SET_VALUE = <<~EOS + boolean immutableValue_idempotentlyUpdateValue(List scriptErrors, List values, def parentObject, String fullPath, String fieldName, boolean nullable, boolean canChangeFromNull) { + boolean fieldAlreadySet = parentObject.containsKey(fieldName); + + // `values` is always passed to us as a `List` (the indexer normalizes to a list, wrapping single + // values in a list as needed) but we only ever expect at most 1 element. + def newValueCandidate = values.isEmpty() ? null : values[0]; + + if (fieldAlreadySet) { + def currentValue = parentObject[fieldName]; + + // Usually we do not allow `immutable_value` fields to ever change values. However, we make + // a special case for `null`, but only when `can_change_from_null: true` has been configured. + // This can be important when deriving a field that has not always existed on the source events. + // On early events, the value may be `null`, and, when this is enabled, we do not want that to + // interfere with our ability to set the value to the correct non-null value based on a different + // event which has a value for the source field. + if (canChangeFromNull) { + if (currentValue == null) { + parentObject[fieldName] = newValueCandidate; + return true; + } + + // When `can_change_from_null: true` is enabled we also need to ignore NEW `null` values that we + // see _after_ a non-null value. This is necessary because an ElasticGraph invariant is that events + // can be processed in any order. So we might process an old event (predating the existence of the + // source field) after we've already set the field to a non-null value. We must always "converge" + // on the same indexed state regardless, of the order events are seen, so here we just ignore it. + if (newValueCandidate == null) { + return false; + } + } + + // Otherwise, if the values differ, it means we are attempting to mutate the immutable value field, which we cannot allow. + if (currentValue != newValueCandidate) { + if (currentValue == null) { + scriptErrors.add("Field `" + fullPath + "` cannot be changed (" + currentValue + " => " + newValueCandidate + "). Set `can_change_from_null: true` on the `immutable_value` definition to allow this."); + } else { + scriptErrors.add("Field `" + fullPath + "` cannot be changed (" + currentValue + " => " + newValueCandidate + ")."); + } + } + + return false; + } + + if (newValueCandidate == null && !nullable) { + scriptErrors.add("Field `" + fullPath + "` cannot be set to `null`, but the source event contains no value for it. Remove `nullable: false` from the `immutable_value` definition to allow this."); + return false; + } + + parentObject[fieldName] = newValueCandidate; + return true; + } + EOS + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value.rb new file mode 100644 index 00000000..2cf677ff --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value.rb @@ -0,0 +1,62 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module Indexing + module DerivedFields + # Responsible for providing bits of the painless script specific to a {DerivedIndexedType#min_value} or + # {DerivedIndexedType#max_value} field. + # + # @api private + class MinOrMaxValue < ::Data.define(:destination_field, :source_field, :min_or_max) + # `Data.define` provides the following methods: + # @dynamic destination_field, source_field, min_or_max + + # @return [String] a line of painless code to manage a min or max value field and return a boolean indicating if it was updated. + def apply_operation_returning_update_status + *parent_parts, field = destination_field.split(".") + parent_parts = ["ctx", "_source"] + parent_parts + + %{#{min_or_max}Value_idempotentlyUpdateValue(data["#{source_field}"], #{parent_parts.join(".")}, "#{field}")} + end + + # @return [Array] a list of painless statements that must be called at the top of the script to set things up. + def setup_statements + FieldInitializerSupport.build_empty_value_initializers(destination_field, leaf_value: :leave_unset) + end + + # @return [Array] painless functions required by a min or max value field. + def function_definitions + [MinOrMaxValue.function_def(min_or_max)] + end + + # @param min_or_max [:min, :max] which type of function to generate. + # @return [String] painless function for managing a min or max field. + def self.function_def(min_or_max) + operator = (min_or_max == :min) ? "<" : ">" + + <<~EOS + boolean #{min_or_max}Value_idempotentlyUpdateValue(List values, def parentObject, String fieldName) { + def currentFieldValue = parentObject[fieldName]; + def #{min_or_max}NewValue = values.isEmpty() ? null : Collections.#{min_or_max}(values); + + if (currentFieldValue == null || (#{min_or_max}NewValue != null && #{min_or_max}NewValue.compareTo(currentFieldValue) #{operator} 0)) { + parentObject[fieldName] = #{min_or_max}NewValue; + return true; + } + + return false; + } + EOS + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_indexed_type.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_indexed_type.rb new file mode 100644 index 00000000..117438f1 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_indexed_type.rb @@ -0,0 +1,346 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/update_target" +require "elastic_graph/schema_definition/indexing/derived_fields/append_only_set" +require "elastic_graph/schema_definition/indexing/derived_fields/immutable_value" +require "elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value" +require "elastic_graph/schema_definition/scripting/script" + +module ElasticGraph + module SchemaDefinition + module Indexing + # Used to configure the derivation of a derived indexed type from a source type. + # This type is yielded from {Mixins::HasIndices#derive_indexed_type_fields}. + # + # @example Derive a `Course` type from `StudentCourseEnrollment` events + # ElasticGraph.define_schema do |schema| + # # `StudentCourseEnrollment` is a directly indexed type. + # schema.object_type "StudentCourseEnrollment" do |t| + # t.field "id", "ID" + # t.field "courseId", "ID" + # t.field "courseName", "String" + # t.field "studentName", "String" + # t.field "courseStartDate", "Date" + # + # t.index "student_course_enrollments" + # + # # Here we define how the `Course` indexed type is derived when we index `StudentCourseEnrollment` events. + # t.derive_indexed_type_fields "Course", from_id: "courseId" do |derive| + # # `derive` is an instance of `DerivedIndexedType`. + # derive.immutable_value "name", from: "courseName" + # derive.append_only_set "students", from: "studentName" + # derive.min_value "firstOfferedDate", from: "courseStartDate" + # derive.max_value "mostRecentlyOfferedDate", from: "courseStartDate" + # end + # end + # + # # `Course` is an indexed type that is derived entirely from `StudentCourseEnrollment` events. + # schema.object_type "Course" do |t| + # t.field "id", "ID" + # t.field "name", "String" + # t.field "students", "[String!]!" + # t.field "firstOfferedDate", "Date" + # t.field "mostRecentlyOfferedDate", "Date" + # + # t.index "courses" + # end + # end + # + # @!attribute source_type + # @return [SchemaElements::ObjectType] the type used as a source for this derive type + # @!attribute destination_type_ref + # @private + # @!attribute id_source + # @return [String] path to field on the source type used as `id` on the derived type + # @!attribute routing_value_source + # @return [String, nil] path to field on the source type used for shard routing + # @!attribute rollover_timestamp_value_source + # @return [String, nil] path to field on the source type used as the timestamp field for rollover + # @!attribute fields + # @return [Array] derived field definitions + class DerivedIndexedType < ::Struct.new( + :source_type, + :destination_type_ref, + :id_source, + :routing_value_source, + :rollover_timestamp_value_source, + :fields + ) + # @param source_type [SchemaElements::ObjectType] the type used as a source for this derive type + # @param destination_type_ref [SchemaElements::TypeReference] the derived type + # @param id_source [String] path to field on the source type used as `id` on the derived type + # @param routing_value_source [String, nil] path to field on the source type used for shard routing + # @param rollover_timestamp_value_source [String, nil] path to field on the source type used as the timestamp field for rollover + # @yield [DerivedIndexedType] the `DerivedIndexedType` instance + # @api private + def initialize( + source_type:, + destination_type_ref:, + id_source:, + routing_value_source:, + rollover_timestamp_value_source: + ) + fields = [] # : ::Array[_DerivedField] + super( + source_type: source_type, + destination_type_ref: destination_type_ref, + id_source: id_source, + routing_value_source: routing_value_source, + rollover_timestamp_value_source: rollover_timestamp_value_source, + fields: fields + ) + yield self + end + + # Configures `field_name` (on the derived indexing type) to contain the set union of all values from + # the `from` field on the source type. Values are only ever appended to the set, so the field will + # act as an append-only set. + # + # @param field_name [String] name of field on the derived indexing type to store the derived set + # @param from [String] path to field on the source type to source values from + # @return [DerivedIndexedType::AppendOnlySet] + # + # @example + # ElasticGraph.define_schema do |schema| + # schema.object_type "StudentCourseEnrollment" do |t| + # t.field "id", "ID" + # t.field "courseId", "ID" + # t.field "studentName", "String" + # + # t.index "student_course_enrollments" + # + # t.derive_indexed_type_fields "Course", from_id: "courseId" do |derive| + # derive.append_only_set "students", from: "studentName" + # end + # end + # + # schema.object_type "Course" do |t| + # t.field "id", "ID" + # t.field "students", "[String!]!" + # + # t.index "courses" + # end + # end + def append_only_set(field_name, from:) + fields << DerivedFields::AppendOnlySet.new(field_name, from) + end + + # Configures `field_name` (on the derived indexing type) to contain a single immutable value from the + # `from` field on the source type. Immutability is enforced by triggering an indexing failure with a + # clear error if any event's source value is different from the value already indexed on this field. + # + # @param field_name [String] name of field on the derived indexing type to store the derived value + # @param from [String] path to field on the source type to source values from + # @param nullable [Boolean] whether the field is allowed to be set to `null`. When set to false, events + # that contain a `null` value in the `from` field will be rejected instead of setting the field’s value + # to `null`. + # @param can_change_from_null [Boolean] whether a one-time mutation of the field value is allowed from + # `null` to a non-`null` value. This can be useful when dealing with a field that may not have a value + # on all source events. For example, if the source field was not initially part of the schema of your + # source dataset, you may have old records that lack a value for this field. When set, this option + # allows a one-time mutation of the field value from `null` to a non-`null` value. Once set to a + # non-`null` value, any additional `null` values that are encountered will be ignored (ensuring that + # the indexed data converges on the same state regardless of the order the events are ingested in). + # Note: this option cannot be enabled when `nullable: false` has been set. + # @return [DerivedFields::ImmutableValue] + # + # @example + # ElasticGraph.define_schema do |schema| + # schema.object_type "StudentCourseEnrollment" do |t| + # t.field "id", "ID" + # t.field "courseId", "ID" + # t.field "courseName", "String" + # + # t.index "student_course_enrollments" + # + # t.derive_indexed_type_fields "Course", from_id: "courseId" do |derive| + # derive.immutable_value "name", from: "courseName" + # end + # end + # + # schema.object_type "Course" do |t| + # t.field "id", "ID" + # t.field "name", "String" + # + # t.index "courses" + # end + # end + def immutable_value(field_name, from:, nullable: true, can_change_from_null: false) + if !nullable && can_change_from_null + raise Errors::SchemaError, "`can_change_from_null: true` is not allowed with `nullable: false` (as there would be no `null` values to change from)." + end + + fields << DerivedFields::ImmutableValue.new( + destination_field: field_name, + source_field: from, + nullable: nullable, + can_change_from_null: can_change_from_null + ) + end + + # Configures `field_name` (on the derived indexing type) to contain the minimum of all values from the `from` + # field on the source type. + # + # @param field_name [String] name of field on the derived indexing type to store the derived value + # @param from [String] path to field on the source type to source values from + # @return [DerivedIndexedType::MinOrMaxValue] + # + # @example + # ElasticGraph.define_schema do |schema| + # schema.object_type "StudentCourseEnrollment" do |t| + # t.field "id", "ID" + # t.field "courseId", "ID" + # t.field "courseStartDate", "Date" + # + # t.index "student_course_enrollments" + # + # t.derive_indexed_type_fields "Course", from_id: "courseId" do |derive| + # derive.min_value "firstOfferedDate", from: "courseStartDate" + # end + # end + # + # schema.object_type "Course" do |t| + # t.field "id", "ID" + # t.field "firstOfferedDate", "Date" + # + # t.index "courses" + # end + # end + def min_value(field_name, from:) + fields << DerivedFields::MinOrMaxValue.new(field_name, from, :min) + end + + # Configures `field_name` (on the derived indexing type) to contain the maximum of all values from the `from` + # field on the source type. + # + # @param field_name [String] name of field on the derived indexing type to store the derived value + # @param from [String] path to field on the source type to source values from + # @return [DerivedIndexedType::MinOrMaxValue] + # + # @example + # ElasticGraph.define_schema do |schema| + # schema.object_type "StudentCourseEnrollment" do |t| + # t.field "id", "ID" + # t.field "courseId", "ID" + # t.field "courseStartDate", "Date" + # + # t.index "student_course_enrollments" + # + # t.derive_indexed_type_fields "Course", from_id: "courseId" do |derive| + # derive.max_value "mostRecentlyOfferedDate", from: "courseStartDate" + # end + # end + # + # schema.object_type "Course" do |t| + # t.field "id", "ID" + # t.field "mostRecentlyOfferedDate", "Date" + # + # t.index "courses" + # end + # end + def max_value(field_name, from:) + fields << DerivedFields::MinOrMaxValue.new(field_name, from, :max) + end + + # @return [Scripting::Script] Painless script that will maintain the derived fields + # @api private + def painless_script + Scripting::Script.new( + source: generate_script.strip, + name: "#{destination_type_ref}_from_#{source_type.name}", + language: "painless", + context: "update" + ) + end + + # @return [SchemaArtifacts::RuntimeMetadata::UpdateTarget] runtime metadata for the source type + # @api private + def runtime_metadata_for_source_type + SchemaArtifacts::RuntimeMetadata::UpdateTarget.new( + type: destination_type_ref.name, + relationship: nil, + script_id: painless_script.id, + id_source: id_source, + routing_value_source: routing_value_source, + rollover_timestamp_value_source: rollover_timestamp_value_source, + metadata_params: {}, + data_params: fields.map(&:source_field).to_h do |f| + [f, SchemaArtifacts::RuntimeMetadata::DynamicParam.new(source_path: f, cardinality: :many)] + end + ) + end + + private + + def generate_script + if fields.empty? + raise Errors::SchemaError, "`derive_indexed_type_fields` definition for #{destination_type_ref} (from #{source_type.name}) " \ + "has no derived field definitions." + end + + sorted_fields = fields.sort_by(&:destination_field) + + # We use `uniq` here to avoid re-doing the same setup multiple times, since multiple fields can sometimes + # need the same setup (such as initializing a common parent field to an empty map). + function_defs = sorted_fields.flat_map(&:function_definitions).uniq.map(&:strip).sort + + setup_statements = [STATIC_SETUP_STATEMENTS] + sorted_fields.flat_map(&:setup_statements).uniq.map(&:strip) + + apply_update_statements = sorted_fields.map { |f| apply_update_statement(f).strip } + + # Note: comments in the script are effectively "free" since: + # + # - The compiler will strip them out. + # - We only send the script to the datastore once (when configuring the cluster), and later + # reference it only by id--so we don't pay for the larger payload on each indexing request. + <<~EOS + #{function_defs.join("\n\n")} + + #{setup_statements.join("\n")} + + #{apply_update_statements.join("\n")} + + if (!#{SCRIPT_ERRORS_VAR}.isEmpty()) { + throw new IllegalArgumentException("#{DERIVED_INDEX_FAILURE_MESSAGE_PREAMBLE}: " + #{SCRIPT_ERRORS_VAR}.join(" ")); + } + + // For records with no new values to index, only skip the update if the document itself doesn't already exist. + // Otherwise create an (empty) document to reflect the fact that the id has been seen. + if (ctx._source.id != null && #{sorted_fields.map { |f| was_noop_variable(f) }.join(" && ")}) { + ctx.op = 'none'; + } else { + // Here we set `_source.id` because if we don't, it'll never be set, making these docs subtly + // different from docs indexed the normal way. + // + // Note also that we MUST use `params.id` instead of `ctx._id`. The latter works on an update + // of an existing document, but is unavailable when we are inserting the document for the first time. + ctx._source.id = params.id; + } + EOS + end + + def apply_update_statement(field) + "boolean #{was_noop_variable(field)} = !#{field.apply_operation_returning_update_status};" + end + + def was_noop_variable(field) + "#{field.destination_field.gsub(".", "__")}_was_noop" + end + + SCRIPT_ERRORS_VAR = "scriptErrors" + + STATIC_SETUP_STATEMENTS = <<~EOS.strip + Map data = params.data; + // A variable to accumulate script errors so that we can surface _all_ issues and not just the first. + List #{SCRIPT_ERRORS_VAR} = new ArrayList(); + EOS + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/event_envelope.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/event_envelope.rb new file mode 100644 index 00000000..957e7fa3 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/event_envelope.rb @@ -0,0 +1,74 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" + +module ElasticGraph + module SchemaDefinition + module Indexing + # Contains logic related to "event envelope"--the layer of metadata that wraps all indexing events. + # + # @api private + module EventEnvelope + # @param indexed_type_names [Array] names of the indexed types + # @param json_schema_version [Integer] the version of the JSON schema + # @return [Hash] the JSON schema for the ElasticGraph event envelope for the given `indexed_type_names`. + def self.json_schema(indexed_type_names, json_schema_version) + { + "type" => "object", + "properties" => { + "op" => { + "type" => "string", + "enum" => %w[upsert] + }, + "type" => { + "type" => "string", + # Sorting doesn't really matter here, but it's nice for the output in the schema artifact to be consistent. + "enum" => indexed_type_names.sort + }, + "id" => { + "type" => "string", + "maxLength" => DEFAULT_MAX_KEYWORD_LENGTH + }, + "version" => { + "type" => "integer", + "minimum" => 0, + "maximum" => (2**63) - 1 + }, + "record" => { + "type" => "object" + }, + "latency_timestamps" => { + "type" => "object", + "additionalProperties" => false, + "patternProperties" => { + "^\\w+_at$" => {"type" => "string", "format" => "date-time"} + } + }, + JSON_SCHEMA_VERSION_KEY => { + "const" => json_schema_version + }, + "message_id" => { + "type" => "string", + "description" => "The optional ID of the message containing this event from whatever messaging system is being used between the publisher and the ElasticGraph indexer." + } + }, + "additionalProperties" => false, + "required" => ["op", "type", "id", "version", JSON_SCHEMA_VERSION_KEY], + "if" => { + "properties" => { + "op" => {"const" => "upsert"} + } + }, + "then" => {"required" => ["record"]} + } + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb new file mode 100644 index 00000000..a87ebd38 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb @@ -0,0 +1,181 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/schema_definition/indexing/json_schema_field_metadata" +require "elastic_graph/schema_definition/indexing/list_counts_mapping" +require "elastic_graph/support/hash_util" +require "elastic_graph/support/memoizable_data" + +module ElasticGraph + module SchemaDefinition + module Indexing + # Represents a field in a JSON document during indexing. + # + # @api private + class Field < Support::MemoizableData.define( + :name, + :name_in_index, + :type, + :json_schema_layers, + :indexing_field_type, + :accuracy_confidence, + :json_schema_customizations, + :mapping_customizations, + :source, + :runtime_field_script + ) + # JSON schema overrides that automatically apply to specific mapping types so that the JSON schema + # validation will reject values which cannot be indexed into fields of a specific mapping type. + # + # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/number.html Elasticsearch numeric field type documentation + # @note We don't handle `integer` here because it's the default numeric type (handled by our definition of the `Int` scalar type). + # @note Likewise, we don't handle `long` here because a custom scalar type must be used for that since GraphQL's `Int` type can't handle long values. + JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE = { + "byte" => {"minimum" => -(2**7), "maximum" => (2**7) - 1}, + "short" => {"minimum" => -(2**15), "maximum" => (2**15) - 1}, + "keyword" => {"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH}, + "text" => {"maxLength" => DEFAULT_MAX_TEXT_LENGTH} + } + + # @return [Hash] the mapping for this field. The returned hash should be composed entirely + # of Ruby primitives that, when converted to a JSON string, match the structure required by + # [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html). + def mapping + @mapping ||= begin + raw_mapping = indexing_field_type + .to_mapping + .merge(Support::HashUtil.stringify_keys(mapping_customizations)) + + if (object_type = type.fully_unwrapped.as_object_type) && type.list? && mapping_customizations[:type] == "nested" + # If it's an object list field using the `nested` type, we need to add a `__counts` field to + # the mapping for all of its subfields which are lists. + ListCountsMapping.merged_into(raw_mapping, for_type: object_type) + else + raw_mapping + end + end + end + + # @return [Hash] the JSON schema definition for this field. The returned object should + # be composed entirely of Ruby primitives that, when converted to a JSON string, match the + # requirements of [the JSON schema spec](https://json-schema.org/). + def json_schema + json_schema_layers + .reverse # resolve layers from innermost to outermost wrappings + .reduce(inner_json_schema) { |acc, layer| process_layer(layer, acc) } + .merge(outer_json_schema_customizations) + .then { |h| Support::HashUtil.stringify_keys(h) } + end + + # @return [JSONSchemaFieldMetadata] additional ElasticGraph metadata to be stored in the JSON schema for this field. + def json_schema_metadata + JSONSchemaFieldMetadata.new(type: type.name, name_in_index: name_in_index) + end + + # Builds a hash containing the mapping for the provided fields, normalizing it in the same way that the + # datastore does so that consistency checks between our index configuration and what's in the datastore + # work properly. + # + # @param fields [Array] fields to generate a mapping hash from + # @return [Hash] generated mapping hash + def self.normalized_mapping_hash_for(fields) + # When an object field has `properties`, the datastore normalizes the mapping by dropping + # the `type => object` (it's implicit, as `properties` are only valid on an object...). + # OTOH, when there are no properties, the datastore normalizes the mapping by dropping the + # empty `properties` entry and instead returning `type => object`. + return {"type" => "object"} if fields.empty? + + # Partition the fields into runtime fields and normal fields based on the presence of runtime_script + runtime_fields, normal_fields = fields.partition(&:runtime_field_script) + + mapping_hash = { + "properties" => normal_fields.to_h { |f| [f.name_in_index, f.mapping] } + } + unless runtime_fields.empty? + mapping_hash["runtime"] = runtime_fields.to_h do |f| + [f.name_in_index, f.mapping.merge({"script" => {"source" => f.runtime_field_script}})] + end + end + + mapping_hash + end + + private + + def inner_json_schema + user_specified_customizations = + if user_specified_json_schema_customizations_go_on_outside? + {} # : ::Hash[::String, untyped] + else + Support::HashUtil.stringify_keys(json_schema_customizations) + end + + customizations_from_mapping = JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE[mapping["type"]] || {} + customizations = customizations_from_mapping.merge(user_specified_customizations) + customizations = indexing_field_type.format_field_json_schema_customizations(customizations) + + ref = {"$ref" => "#/$defs/#{type.unwrapped_name}"} + return ref if customizations.empty? + + # Combine any customizations with type ref under an "allOf" subschema: + # All of these properties must hold true for the type to be valid. + # + # Note that if we simply combine the customizations with the `$ref` + # at the same level, it will not work, because other subschema + # properties are ignored when they are in the same object as a `$ref`: + # https://github.com/json-schema-org/JSON-Schema-Test-Suite/blob/2.0.0/tests/draft7/ref.json#L165-L168 + {"allOf" => [ref, customizations]} + end + + def outer_json_schema_customizations + return {} unless user_specified_json_schema_customizations_go_on_outside? + Support::HashUtil.stringify_keys(json_schema_customizations) + end + + # Indicates if the user-specified JSON schema customizations should go on the inside + # (where they normally go) or on the outside. They only go on the outside when it's + # an array field, because then they apply to the array itself instead of the items in the + # array. + def user_specified_json_schema_customizations_go_on_outside? + json_schema_layers.include?(:array) + end + + def process_layer(layer, schema) + case layer + when :nullable + make_nullable(schema) + when :array + make_array(schema) + else + # :nocov: - layer is only ever `:nullable` or `:array` so we never get here + schema + # :nocov: + end + end + + def make_nullable(schema) + # Here we use "anyOf" to ensure that JSON can either match the schema OR null. + # + # (Using "oneOf" would mean that if we had a schema that also allowed null, + # null would never be allowed, since "oneOf" must match exactly one subschema). + { + "anyOf" => [ + schema, + {"type" => "null"} + ] + } + end + + def make_array(schema) + {"type" => "array", "items" => schema} + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_reference.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_reference.rb new file mode 100644 index 00000000..671b7f0c --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_reference.rb @@ -0,0 +1,51 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module Indexing + # @!parse class FieldReference < ::Data; end + FieldReference = ::Data.define( + :name, + :name_in_index, + :type, + :mapping_options, + :json_schema_options, + :accuracy_confidence, + :source, + :runtime_field_script + ) + + # A lazy reference to a {Field}. It contains all attributes needed to build a {Field}, but the referenced `type` may not be + # resolvable yet (which is why this exists). + # + # @api private + class FieldReference < ::Data + # @return [Field, nil] the {Field} this reference resolves to (if it can be resolved) + def resolve + return nil unless (resolved_type = type.fully_unwrapped.resolved) + + Indexing::Field.new( + name: name, + name_in_index: name_in_index, + type: type, + json_schema_layers: type.json_schema_layers, + indexing_field_type: resolved_type.to_indexing_field_type, + accuracy_confidence: accuracy_confidence, + json_schema_customizations: json_schema_options, + mapping_customizations: mapping_options, + source: source, + runtime_field_script: runtime_field_script + ) + end + + # @dynamic initialize, with, name, name_in_index, type, mapping_options, json_schema_options, accuracy_confidence, source, runtime_field_script + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/enum.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/enum.rb new file mode 100644 index 00000000..abd0cc4f --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/enum.rb @@ -0,0 +1,65 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module Indexing + # Contains implementation logic for the different types of indexing fields. + # + # @api private + module FieldType + # @!parse class Enum < ::Data; end + Enum = ::Data.define(:enum_value_names) + + # Responsible for the JSON schema and mapping of a {SchemaElements::EnumType}. + # + # @!attribute [r] enum_value_names + # @return [Array] list of names of values in this enum type. + # + # @api private + class Enum < ::Data + # @return [Hash] the JSON schema for this enum type. + def to_json_schema + {"type" => "string", "enum" => enum_value_names} + end + + # @return [Hash] the datastore mapping for this enum type. + def to_mapping + {"type" => "keyword"} + end + + # @return [Hash] additional ElasticGraph metadata to put in the JSON schema for this enum type. + def json_schema_field_metadata_by_field_name + {} + end + + # @param customizations [Hash] JSON schema customizations + # @return [Hash] formatted customizations. + def format_field_json_schema_customizations(customizations) + # Since an enum type already restricts the values to a small set of allowed values, we do not need to keep + # other customizations (such as the `maxLength` field customization EG automatically applies to fields + # indexed as a `keyword`--we don't allow enum values to exceed that length, anyway). + # + # It's desirable to restrict what customizations are applied because when a publisher uses the JSON schema + # to generate code using a library such as https://github.com/pwall567/json-kotlin-schema-codegen, we found + # that the presence of extra field customizations inhibits the library's ability to generate code in the way + # we want (it causes the type of the enum to change since the JSON schema changes from a direct `$ref` to + # being wrapped in an `allOf`). + # + # However, we still want to apply `enum` customizations--this allows a user to "narrow" the set of allowed + # values for a field. For example, a `Currency` enum could contain every currency, and a user may want to + # restrict a specific `currency` field to a subset of currencies (e.g. to just USD, CAD, and EUR). + customizations.slice("enum") + end + + # @dynamic initialize, enum_value_names + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/object.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/object.rb new file mode 100644 index 00000000..18e6788c --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/object.rb @@ -0,0 +1,113 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/support/hash_util" +require "elastic_graph/support/memoizable_data" + +module ElasticGraph + module SchemaDefinition + module Indexing + module FieldType + # Responsible for the JSON schema and mapping of a {SchemaElements::ObjectType}. + # + # @!attribute [r] type_name + # @return [String] name of the object type + # @!attribute [r] subfields + # @return [Array] the subfields of this object type + # @!attribute [r] mapping_options + # @return [Hash] options to be included in the mapping + # @!attribute [r] json_schema_options + # @return [Hash] options to be included in the JSON schema + # + # @api private + class Object < Support::MemoizableData.define(:type_name, :subfields, :mapping_options, :json_schema_options) + # @return [Hash] the datastore mapping for this object type. + def to_mapping + @to_mapping ||= begin + base_mapping = Field.normalized_mapping_hash_for(subfields) + # When a custom mapping type is used, we need to omit `properties`, because custom mapping + # types generally don't use `properties` (and if you need to use `properties` with a custom + # type, you're responsible for defining the properties). + base_mapping = base_mapping.except("properties") if (mapping_options[:type] || "object") != "object" + base_mapping.merge(Support::HashUtil.stringify_keys(mapping_options)) + end + end + + # @return [Hash] the JSON schema for this object type. + def to_json_schema + @to_json_schema ||= + if json_schema_options.empty? + # Fields that are `sourced_from` an alternate type must not be included in this types JSON schema, + # since events of this type won't include them. + other_source_subfields, json_schema_candidate_subfields = subfields.partition(&:source) + validate_sourced_fields_have_no_json_schema_overrides(other_source_subfields) + json_schema_subfields = json_schema_candidate_subfields.reject(&:runtime_field_script) + + { + "type" => "object", + "properties" => json_schema_subfields.to_h { |f| [f.name, f.json_schema] }.merge(json_schema_typename_field), + # Note: `__typename` is intentionally not included in the `required` list. If `__typename` is present + # we want it validated (as we do by merging in `json_schema_typename_field`) but we only want + # to require it in the context of a union type. The union's json schema requires the field. + "required" => json_schema_subfields.map(&:name).freeze + }.freeze + else + Support::HashUtil.stringify_keys(json_schema_options) + end + end + + # @return [Hash] additional ElasticGraph metadata to put in the JSON schema for this object type. + def json_schema_field_metadata_by_field_name + subfields.to_h { |f| [f.name, f.json_schema_metadata] } + end + + # @param customizations [Hash] JSON schema customizations + # @return [Hash] formatted customizations. + def format_field_json_schema_customizations(customizations) + customizations + end + + private + + def after_initialize + subfields.freeze + end + + # Returns a __typename property which we use for union types. + # + # This must always be set to the name of the type (thus the const value). + # + # We also add a "default" value. This does not impact validation, but rather + # aids tools like our kotlin codegen to save publishers from having to set the + # property explicitly when creating events. + def json_schema_typename_field + { + "__typename" => { + "type" => "string", + "const" => type_name, + "default" => type_name + } + } + end + + def validate_sourced_fields_have_no_json_schema_overrides(other_source_subfields) + problem_fields = other_source_subfields.reject { |f| f.json_schema_customizations.empty? } + return if problem_fields.empty? + + field_descriptions = problem_fields.map(&:name).sort.map { |f| "`#{f}`" }.join(", ") + raise Errors::SchemaError, + "`#{type_name}` has #{problem_fields.size} field(s) (#{field_descriptions}) that are `sourced_from` " \ + "another type and also have JSON schema customizations. Instead, put the JSON schema " \ + "customizations on the source type's field definitions." + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/scalar.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/scalar.rb new file mode 100644 index 00000000..d0018e73 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/scalar.rb @@ -0,0 +1,51 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/hash_util" + +module ElasticGraph + module SchemaDefinition + module Indexing + module FieldType + # @!parse class Scalar < ::Data; end + Scalar = ::Data.define(:scalar_type) + + # Responsible for the JSON schema and mapping of a {SchemaElements::ScalarType}. + # + # @!attribute [r] scalar_type + # @return [SchemaElements::ScalarType] the scalar type + # + # @api private + class Scalar < ::Data + # @return [Hash] the datastore mapping for this scalar type. + def to_mapping + Support::HashUtil.stringify_keys(scalar_type.mapping_options) + end + + # @return [Hash] the JSON schema for this scalar type. + def to_json_schema + Support::HashUtil.stringify_keys(scalar_type.json_schema_options) + end + + # @return [Hash] additional ElasticGraph metadata to put in the JSON schema for this scalar type. + def json_schema_field_metadata_by_field_name + {} + end + + # @param customizations [Hash] JSON schema customizations + # @return [Hash] formatted customizations. + def format_field_json_schema_customizations(customizations) + customizations + end + + # @dynamic initialize, scalar_type + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/union.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/union.rb new file mode 100644 index 00000000..839a340b --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/union.rb @@ -0,0 +1,70 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_definition/indexing/field_type/object" +require "elastic_graph/support/hash_util" + +module ElasticGraph + module SchemaDefinition + module Indexing + module FieldType + # Responsible for the JSON schema and mapping of a {SchemaElements::UnionType}. + # + # @note In JSON schema, we model this with a `oneOf`, and a `__typename` field on each subtype. + # @note Within the mapping, we have a single object type that has a set union of the properties + # of the subtypes (and also a `__typename` keyword field). + # + # @!attribute [r] subtypes_by_name + # @return [Hash] the subtypes of the union, keyed by name. + # + # @api private + class Union < ::Data.define(:subtypes_by_name) + # @return [Hash] the JSON schema for this union type. + def to_json_schema + subtype_json_schemas = subtypes_by_name.keys.map { |name| {"$ref" => "#/$defs/#{name}"} } + + # A union type can represent multiple subtypes, referenced by the "anyOf" clause below. + # We also add a requirement for the presence of __typename to indicate which type + # is being referenced (this property is pre-defined on the type itself as a constant). + # + # Note: Although both "oneOf" and "anyOf" keywords are valid for combining schemas + # to form a union, and validate equivalently when no object can satisfy multiple of the + # subschemas (which is the case here given the __typename requirements are mutually + # exclusive), we chose to use "oneOf" here because it works better with this library: + # https://github.com/pwall567/json-kotlin-schema-codegen + { + "required" => %w[__typename], + "oneOf" => subtype_json_schemas + } + end + + # @return [Hash] the datastore mapping for this union type. + def to_mapping + mapping_subfields = subtypes_by_name.values.map(&:subfields).reduce([], :union) + + Support::HashUtil.deep_merge( + Field.normalized_mapping_hash_for(mapping_subfields), + {"properties" => {"__typename" => {"type" => "keyword"}}} + ) + end + + # @return [Hash] additional ElasticGraph metadata to put in the JSON schema for this union type. + def json_schema_field_metadata_by_field_name + {} + end + + # @param customizations [Hash] JSON schema customizations + # @return [Hash] formatted customizations. + def format_field_json_schema_customizations(customizations) + customizations + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb new file mode 100644 index 00000000..5f706080 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb @@ -0,0 +1,318 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/index_definition" +require "elastic_graph/schema_artifacts/runtime_metadata/index_field" +require "elastic_graph/schema_definition/indexing/derived_indexed_type" +require "elastic_graph/schema_definition/indexing/list_counts_mapping" +require "elastic_graph/schema_definition/indexing/rollover_config" +require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" +require "elastic_graph/schema_definition/schema_elements/field_path" +require "elastic_graph/support/hash_util" + +module ElasticGraph + module SchemaDefinition + # Contains schema definition logic specific to indexing (such as JSON schema and mapping generation). + module Indexing + # Represents an index in a datastore. Defined within an indexed type. Modeled as a separate object to facilitate + # further customization of the index. + # + # @!attribute [r] name + # @return [String] name of the index + # @!attribute [r] default_sort_pairs + # @return [Array<(String, Symbol)>] (field name, direction) pairs for the default sort + # @!attribute [r] settings + # @return [Hash<(String, Object)>] datastore settings for the index + # @!attribute [r] schema_def_state + # @return [State] schema definition state + # @!attribute [r] indexed_type + # @return [SchemaElements::ObjectType, SchemaElements::InterfaceType, SchemaElements::UnionType] type backed by this index + # @!attribute [r] routing_field_path + # @return [Array] path to the field used for shard routing + # @!attribute [r] rollover_config + # @return [RolloverConfig, nil] rollover configuration for the index + class Index < Struct.new(:name, :default_sort_pairs, :settings, :schema_def_state, :indexed_type, :routing_field_path, :rollover_config) + include Mixins::HasReadableToSAndInspect.new { |i| i.name } + + # @param name [String] name of the index + # @param settings [Hash<(String, Object)>] datastore settings for the index + # @param schema_def_state [State] schema definition state + # @param indexed_type [SchemaElements::ObjectType, SchemaElements::InterfaceType, SchemaElements::UnionType] type backed by this index + # @yield [Index] the index, for further customization + # @api private + def initialize(name, settings, schema_def_state, indexed_type) + if name.include?(ROLLOVER_INDEX_INFIX_MARKER) + raise Errors::SchemaError, "`#{name}` is an invalid index definition name since it contains " \ + "`#{ROLLOVER_INDEX_INFIX_MARKER}` which ElasticGraph treats as special." + end + + settings = DEFAULT_SETTINGS.merge(Support::HashUtil.flatten_and_stringify_keys(settings, prefix: "index")) + + super(name, [], settings, schema_def_state, indexed_type, [], nil) + + # `id` is the field Elasticsearch/OpenSearch use for routing by default: + # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-routing-field.html + # By using it here, it will cause queries to pass a `routing` parameter when + # searching with id filtering on an index that does not use custom shard routing, giving + # us a nice efficiency boost. + self.routing_field_path = public_field_path("id", explanation: "indexed types must have an `id` field") + + yield self if block_given? + end + + # Specifies how documents in this index should sort by default, when no `orderBy` argument is provided to the GraphQL query. + # + # @note the field name strings can be a dot-separated nested fields, but all referenced + # fields must exist when this is called. + # + # @param field_name_direction_pairs [Array<(String, Symbol)>] pairs of field names and `:asc` or `:desc` + # @return [void] + # + # @example Sort on `name` (ascending) with `createdAt` (descending) as a tie-breaker + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "id", "ID!" + # t.field "name", "String" + # t.field "createdAt", "DateTime" + # + # t.index "campaigns"do |i| + # i.default_sort "name", :asc, "createdAt", :desc + # end + # end + # end + def default_sort(*field_name_direction_pairs) + self.default_sort_pairs = field_name_direction_pairs + end + + # Causes this index to "rollover" at the provided `frequency` based on the value of the provided `timestamp_field_path_name`. + # This is particularly useful for time-series data. Partitioning the data into `hourly`, `daily`, `monthly` or `yearly` buckets + # allows for different index configurations, and can be necessary when a dataset is too large to fit in one dataset given + # Elasticsearch/OpenSearch limitations on the number of shards in one index. In addition, ElasticGraph optimizes queries which + # filter on the timestamp field to target the subset of the indices in which matching documents could reside. + # + # @note the timestamp field specified here **must be immutable**. To understand why, consider a `:yearly` rollover + # index used for data based on `createdAt`; if ElasticGraph ingests record `123` with a createdAt of `2023-12-31T23:59:59Z`, it + # will be indexed in the `2023` index. Later if it receives an update event for record `123` with a `createdAt` of + # `2024-01-01T00:00:00Z` (a mere one second later!), ElasticGraph will store the new version of the payment in the `2024` index, + # and leave the old copy of the payment in the `2023` index unchanged. It’ll have duplicates for that document. + # @note changing the `rollover` configuration on an existing index that already has data will result in duplicate documents + # + # @param frequency [:yearly, :monthly, :daily, :hourly] how often to rollover the index + # @param timestamp_field_path_name [String] dot-separated path to the timestamp field used for rollover. Note: all referenced + # fields must exist when this is called. + # @return [void] + # + # @example Define a `campaigns` index to rollover yearly based on `createdAt` + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "id", "ID!" + # t.field "name", "String" + # t.field "createdAt", "DateTime" + # + # t.index "campaigns"do |i| + # i.rollover :yearly, "createdAt" + # end + # end + # end + def rollover(frequency, timestamp_field_path_name) + timestamp_field_path = public_field_path(timestamp_field_path_name, explanation: "it is referenced as an index `rollover` field") + + unless date_and_datetime_types.include?(timestamp_field_path.type.fully_unwrapped.name) + date_or_datetime_description = date_and_datetime_types.map { |t| "`#{t}`" }.join(" or ") + raise Errors::SchemaError, "rollover field `#{timestamp_field_path.full_description}` cannot be used for rollover since it is not a #{date_or_datetime_description} field." + end + + if timestamp_field_path.type.list? + raise Errors::SchemaError, "rollover field `#{timestamp_field_path.full_description}` cannot be used for rollover since it is a list field." + end + + timestamp_field_path.path_parts.each { |f| f.json_schema nullable: false } + + self.rollover_config = RolloverConfig.new( + frequency: frequency, + timestamp_field_path: timestamp_field_path + ) + end + + # Configures the index to [route documents to shards](https://www.elastic.co/guide/en/elasticsearch/reference/8.15/mapping-routing-field.html) + # based on the specified field. ElasticGraph optimizes queries that filter on the shard routing field so that they only run on a + # subset of nodes instead of all nodes. This can make a big difference in query performance if queries usually filter on a certain + # field. Using an appropriate field for shard routing is often essential for horizontal scaling, as it avoids having every query + # hit every node, allowing additional nodes to increase query throughput. + # + # @note it is essential that the shards are well-balanced. If the data’s distribution is lopsided, using this feature can make + # performance worse. + # @note the routing field specified here **must be immutable**. If ElasticGraph receives an updated version of a document with a + # different routing value, it’ll write the new version of the document to a different shard and leave the copy on the old shard + # unchanged, leading to duplicates. + # @note changing the shard routing configuration on an existing index that already has data will result in duplicate documents + # + # @param routing_field_path_name [String] dot-separated path to the field used for shard routing. Note: all referenced + # fields must exist when this is called. + # @return [void] + # + # @example Define a `campaigns` index to shard on `organizationId` + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "id", "ID!" + # t.field "name", "String" + # t.field "organizationId", "ID" + # + # t.index "campaigns"do |i| + # i.route_with "organizationId" + # end + # end + # end + def route_with(routing_field_path_name) + routing_field_path = public_field_path(routing_field_path_name, explanation: "it is referenced as an index `route_with` field") + + unless routing_field_path.type.leaf? + raise Errors::SchemaError, "shard routing field `#{routing_field_path.full_description}` cannot be used for routing since it is not a leaf field." + end + + self.routing_field_path = routing_field_path + + routing_field_path.path_parts[0..-2].each { |f| f.json_schema nullable: false } + routing_field_path.last_part.json_schema nullable: false, pattern: HAS_NON_WHITE_SPACE_REGEX + indexed_type.append_to_documentation "For more performant queries on this type, please filter on `#{routing_field_path_name}` if possible." + end + + # @see #route_with + # @return [Boolean] whether or not this index uses custom shard routing + def uses_custom_routing? + routing_field_path.path_in_index != "id" + end + + # @return [Hash] datastore configuration for this index for when it does not use rollover + def to_index_config + { + "aliases" => {}, + "mappings" => mappings, + "settings" => settings + }.compact + end + + # @return [Hash] datastore configuration for the index template that will be defined if rollover is used + def to_index_template_config + { + "index_patterns" => ["#{name}#{ROLLOVER_INDEX_INFIX_MARKER}*"], + "template" => { + "aliases" => {}, + "mappings" => mappings, + "settings" => settings + } + } + end + + # @return [SchemaArtifacts::RuntimeMetadata::IndexDefinition] runtime metadata for this index + def runtime_metadata + SchemaArtifacts::RuntimeMetadata::IndexDefinition.new( + route_with: routing_field_path.path_in_index, + rollover: rollover_config&.runtime_metadata, + current_sources: indexed_type.current_sources, + fields_by_path: indexed_type.index_field_runtime_metadata_tuples.to_h, + default_sort_fields: default_sort_pairs.each_slice(2).map do |(graphql_field_path_name, direction)| + SchemaArtifacts::RuntimeMetadata::SortField.new( + field_path: public_field_path(graphql_field_path_name, explanation: "it is referenced as an index `default_sort` field").path_in_index, + direction: direction + ) + end + ) + end + + private + + # A regex that requires at least one non-whitespace character. + # Note: this does not use the `/S` character class because it's recommended to use a small subset + # of Regex syntax: + # + # > The regular expression syntax used is from JavaScript (ECMA 262, specifically). However, that + # > complete syntax is not widely supported, therefore it is recommended that you stick to the subset + # > of that syntax described below. + # + # (From https://json-schema.org/understanding-json-schema/reference/regular_expressions.html) + HAS_NON_WHITE_SPACE_REGEX = "[^ \t\n]+" + + DEFAULT_SETTINGS = { + "index.mapping.ignore_malformed" => false, + "index.mapping.coerce" => false, + "index.number_of_replicas" => 1, + "index.number_of_shards" => 1 + } + + def mappings + field_mappings = indexed_type + .to_indexing_field_type + .to_mapping + .except("type") # `type` is invalid at the mapping root because it always has to be an object. + .then { |mapping| ListCountsMapping.merged_into(mapping, for_type: indexed_type) } + .then do |fm| + Support::HashUtil.deep_merge(fm, {"properties" => { + "__sources" => {"type" => "keyword"}, + "__versions" => { + "type" => "object", + # __versions is map keyed by relationship name, with values that are maps keyed by id. Since it's not + # a static object with known fields, we need to use dynamic here. Passing `false` allows some level + # of dynamicness. As per https://www.elastic.co/guide/en/elasticsearch/reference/8.7/dynamic.html#dynamic-parameters: + # + # > New fields are ignored. These fields will not be indexed or searchable, but will still appear in the _source + # > field of returned hits. These fields will not be added to the mapping, and new fields must be added explicitly. + # + # We need `__versions` to be in `_source` (so that our update scripts can operate on it), but + # have no need for it to be searchable (as it's just an internal data structure used for indexing). + # + # Note: we intentionally set false as a string here, because that's how the datastore echoes it back + # to us when you query the mapping (even if you set it as a boolean). Our checks for index mapping + # consistency fail validation if we set it as a boolean since the datastore doesn't echo it back as + # a boolean. + "dynamic" => "false" + } + }}) + end + + {"dynamic" => "strict"}.merge(field_mappings).tap do |hash| + # If we are using custom shard routing, we want to require a `routing` value to be provided + # in every single index, get, delete or update request; otherwise the request might be + # made against the wrong shard. + hash["_routing"] = {"required" => true} if uses_custom_routing? + hash["_size"] = {"enabled" => true} if schema_def_state.index_document_sizes? + end + end + + def public_field_path(public_path_string, explanation:) + parent_is_not_list = ->(parent_field) { !parent_field.type.list? } + resolver = SchemaElements::FieldPath::Resolver.new + resolved_path = resolver.resolve_public_path(indexed_type, public_path_string, &parent_is_not_list) + return resolved_path if resolved_path + + path_parts = public_path_string.split(".") + error_msg = "Field `#{indexed_type.name}.#{public_path_string}` cannot be resolved, but #{explanation}." + + # If it is a nested field path, the problem could be that a type has been referenced which does not exist, so mention that. + if path_parts.size > 1 + error_msg += " Verify that all fields and types referenced by `#{public_path_string}` are defined." + end + + # If the first part of the path doesn't resolve, the problem could be that the field is defined after the `index` call + # but it needs to be defined before it, so mention that. + if resolver.resolve_public_path(indexed_type, path_parts.first, &parent_is_not_list).nil? + error_msg += " Note: the `#{indexed_type.name}.#{path_parts.first}` definition must come before the `index` call." + end + + raise Errors::SchemaError, error_msg + end + + def date_and_datetime_types + @date_and_datetime_types ||= %w[Date DateTime].map do |type| + schema_def_state.type_namer.name_for(type) + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/json_schema_field_metadata.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/json_schema_field_metadata.rb new file mode 100644 index 00000000..2570ef6f --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/json_schema_field_metadata.rb @@ -0,0 +1,34 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module Indexing + # @!parse class JSONSchemaFieldMetadata; end + JSONSchemaFieldMetadata = ::Data.define(:type, :name_in_index) + + # Metadata about an ElasticGraph field that needs to be stored in our versioned JSON schemas + # alongside the JSON schema fields. + # + # @!attribute [r] type + # @return [String] name of the ElasticGraph type for this field + # @!attribute [r] name_in_index + # @return [String] name of the field in the index + # + # @api private + class JSONSchemaFieldMetadata < ::Data + # @return [Hash] hash form of the metadata that can be dumped in JSON schema + def to_dumpable_hash + {"type" => type, "nameInIndex" => name_in_index} + end + + # @dynamic initialize, type, name_in_index + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/json_schema_with_metadata.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/json_schema_with_metadata.rb new file mode 100644 index 00000000..c1b634ba --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/json_schema_with_metadata.rb @@ -0,0 +1,234 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" + +module ElasticGraph + module SchemaDefinition + module Indexing + # Represents the result of merging a JSON schema with metadata. The result includes both + # the merged JSON schema and a list of `failed_fields` indicating which fields metadata + # could not be determined for. + # + # @private + class JSONSchemaWithMetadata < ::Data.define( + # The JSON schema. + :json_schema, + # A set of fields (in the form `Type.field`) that were needed but not found. + :missing_fields, + # A set of type names that were needed but not found. + :missing_types, + # A set of `DeprecatedElement` objects that create conflicting definitions. + :definition_conflicts, + # A set of fields that have been deleted but that must be retained (e.g. for custom shard routing or rollover) + :missing_necessary_fields + ) + def json_schema_version + json_schema.fetch(JSON_SCHEMA_VERSION_KEY) + end + + # Responsible for building `JSONSchemaWithMetadata` instances. + # + # @private + class Merger + # @dynamic unused_deprecated_elements + attr_reader :unused_deprecated_elements + + def initialize(schema_def_results) + @field_metadata_by_type_and_field_name = schema_def_results.json_schema_field_metadata_by_type_and_field_name + @renamed_types_by_old_name = schema_def_results.state.renamed_types_by_old_name + @deleted_types_by_old_name = schema_def_results.state.deleted_types_by_old_name + @renamed_fields_by_type_name_and_old_field_name = schema_def_results.state.renamed_fields_by_type_name_and_old_field_name + @deleted_fields_by_type_name_and_old_field_name = schema_def_results.state.deleted_fields_by_type_name_and_old_field_name + @state = schema_def_results.state + @derived_indexing_type_names = schema_def_results.derived_indexing_type_names + + @unused_deprecated_elements = ( + @renamed_types_by_old_name.values + + @deleted_types_by_old_name.values + + @renamed_fields_by_type_name_and_old_field_name.values.flat_map(&:values) + + @deleted_fields_by_type_name_and_old_field_name.values.flat_map(&:values) + ).to_set + end + + def merge_metadata_into(json_schema) + missing_fields = ::Set.new + missing_types = ::Set.new + definition_conflicts = ::Set.new + old_type_name_by_current_name = {} # : ::Hash[String, String] + + defs = json_schema.fetch("$defs").to_h do |type_name, type_def| + if type_name != EVENT_ENVELOPE_JSON_SCHEMA_NAME && (properties = type_def["properties"]) + current_type_name = determine_current_type_name( + type_name, + missing_types: missing_types, + definition_conflicts: definition_conflicts + ) + + if current_type_name + old_type_name_by_current_name[current_type_name] = type_name + end + + properties = properties.to_h do |field_name, prop| + unless field_name == "__typename" + field_metadata = current_type_name&.then do |name| + field_metadata_for( + name, + field_name, + missing_fields: missing_fields, + definition_conflicts: definition_conflicts + ) + end + + prop = prop.merge({"ElasticGraph" => field_metadata&.to_dumpable_hash}) + end + + [field_name, prop] + end + + type_def = type_def.merge({"properties" => properties}) + end + + [type_name, type_def] + end + + json_schema = json_schema.merge("$defs" => defs) + + JSONSchemaWithMetadata.new( + json_schema: json_schema, + missing_fields: missing_fields, + missing_types: missing_types, + definition_conflicts: definition_conflicts, + missing_necessary_fields: identify_missing_necessary_fields(json_schema, old_type_name_by_current_name) + ) + end + + private + + # Given a historical `type_name`, determines (and returns) the current name for that type. + def determine_current_type_name(type_name, missing_types:, definition_conflicts:) + exists_currently = @field_metadata_by_type_and_field_name.key?(type_name) + deleted = @deleted_types_by_old_name[type_name]&.tap { |elem| @unused_deprecated_elements.delete(elem) } + renamed = @renamed_types_by_old_name[type_name]&.tap { |elem| @unused_deprecated_elements.delete(elem) } + + if [exists_currently, deleted, renamed].count(&:itself) > 1 + definition_conflicts.merge([deleted, renamed].compact) + end + + return type_name if exists_currently + return nil if deleted + return renamed.name if renamed + + missing_types << type_name + nil + end + + # Given a historical `type_name` and `field_name` determines (and returns) the field metadata for it. + def field_metadata_for(type_name, field_name, missing_fields:, definition_conflicts:) + full_name = "#{type_name}.#{field_name}" + + current_meta = @field_metadata_by_type_and_field_name.dig(type_name, field_name) + deleted = @deleted_fields_by_type_name_and_old_field_name.dig(type_name, field_name)&.tap do |elem| + @unused_deprecated_elements.delete(elem) + end + renamed = @renamed_fields_by_type_name_and_old_field_name.dig(type_name, field_name)&.tap do |elem| + @unused_deprecated_elements.delete(elem) + end + + if [current_meta, deleted, renamed].count(&:itself) > 1 + definition_conflicts.merge([deleted, renamed].compact.map { |elem| elem.with(name: full_name) }) + end + + return current_meta if current_meta + return nil if deleted + return @field_metadata_by_type_and_field_name.dig(type_name, renamed.name) if renamed + + missing_fields << full_name + nil + end + + def identify_missing_necessary_fields(json_schema, old_type_name_by_current_name) + json_schema_resolver = JSONSchemaResolver.new(@state, json_schema, old_type_name_by_current_name) + version = json_schema.fetch(JSON_SCHEMA_VERSION_KEY) + + types_to_check = @state.object_types_by_name.values.select do |type| + type.indexed? && !@derived_indexing_type_names.include?(type.name) + end + + types_to_check.flat_map do |object_type| + object_type.indices.flat_map do |index_def| + identify_missing_necessary_fields_for_index_def(object_type, index_def, json_schema_resolver, version) + end + end + end + + def identify_missing_necessary_fields_for_index_def(object_type, index_def, json_schema_resolver, json_schema_version) + { + "routing" => index_def.routing_field_path, + "rollover" => index_def.rollover_config&.timestamp_field_path + }.compact.filter_map do |field_type, field_path| + if json_schema_resolver.necessary_path_missing?(field_path) + # The JSON schema v # {json_schema_version} artifact has no field that maps to the #{field_type} path of `#{field_path.fully_qualified_path_in_index}`. + + MissingNecessaryField.new( + field_type: field_type, + fully_qualified_path: field_path.fully_qualified_path_in_index + ) + end + end + end + + class JSONSchemaResolver + def initialize(state, json_schema, old_type_name_by_current_name) + @state = state + @old_type_name_by_current_name = old_type_name_by_current_name + @meta_by_old_type_and_name_in_index = ::Hash.new do |hash, type_name| + properties = json_schema.fetch("$defs").fetch(type_name).fetch("properties") + + hash[type_name] = properties.filter_map do |name, prop| + if (metadata = prop["ElasticGraph"]) + [metadata.fetch("nameInIndex"), metadata] + end + end.to_h + end + end + + # Indicates if the given `field_path` is (1) necessary and (2) missing from the JSON schema, indicating a problem. + # + # - Returns `false` is the given `field_path` is present in the JSON schema. + # - Returns `false` is the parent type of `field_path` has not been retained in this JSON schema version + # (in that case, the field path is not necessary). + # - Otherwise, returns `true` since the field path is both necessary and missing. + def necessary_path_missing?(field_path) + parent_type = field_path.first_part.parent_type.name + + field_path.path_parts.any? do |path_part| + necessary_path_part_missing?(parent_type, path_part.name_in_index) do |meta| + parent_type = @state.type_ref(meta.fetch("type")).fully_unwrapped.name + end + end + end + + private + + def necessary_path_part_missing?(parent_type, name_in_index) + old_type_name = @old_type_name_by_current_name[parent_type] + return false unless old_type_name + + meta = @meta_by_old_type_and_name_in_index.dig(old_type_name, name_in_index) + yield meta if meta + !meta + end + end + end + + MissingNecessaryField = ::Data.define(:field_type, :fully_qualified_path) + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/list_counts_mapping.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/list_counts_mapping.rb new file mode 100644 index 00000000..4dc46794 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/list_counts_mapping.rb @@ -0,0 +1,53 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/support/hash_util" + +module ElasticGraph + module SchemaDefinition + module Indexing + # To support filtering on the `count` of a list field, we need to index the counts as we ingest + # events. This is responsible for defining the mapping for the special `__counts` field in which + # we store the list counts. + # + # @private + module ListCountsMapping + # Builds the `__counts` field mapping for the given `for_type`. Returns a new `mapping_hash` with + # the extra `__counts` field merged into it. + def self.merged_into(mapping_hash, for_type:) + counts_properties = for_type.indexing_fields_by_name_in_index.values.flat_map do |field| + field.paths_to_lists_for_count_indexing.map do |path| + # We chose the `integer` type here because: + # + # - While we expect datasets with more documents than the max integer value (~2B), we don't expect + # individual documents to have any list fields with more elements than can fit in an integer. + # - Using `long` would allow for much larger counts, but we don't want to take up double the + # storage space for this. + # + # Note that `new_list_filter_input_type` (in `schema_definition/factory.rb`) relies on this, and + # has chosen to use `IntFilterInput` (rather than `JsonSafeLongFilterInput`) for filtering these count values. + # If we change the mapping type here, we should re-evaluate the filter used there. + [path, {"type" => "integer"}] + end + end.to_h + + return mapping_hash if counts_properties.empty? + + Support::HashUtil.deep_merge(mapping_hash, { + "properties" => { + LIST_COUNTS_FIELD => { + "properties" => counts_properties + } + } + }) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/relationship_resolver.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/relationship_resolver.rb new file mode 100644 index 00000000..8f0f03fb --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/relationship_resolver.rb @@ -0,0 +1,96 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module Indexing + # @private + class RelationshipResolver + def initialize(schema_def_state:, object_type:, relationship_name:, sourced_fields:, field_path_resolver:) + @schema_def_state = schema_def_state + @object_type = object_type + @relationship_name = relationship_name + @sourced_fields = sourced_fields + @field_path_resolver = field_path_resolver + end + + def resolve + relation_field = object_type.graphql_fields_by_name[relationship_name] + + if relation_field.nil? + [nil, "#{relationship_error_prefix} is not defined. Is it misspelled?"] + elsif (relationship = relation_field.relationship).nil? + [nil, "#{relationship_error_prefix} is not a relationship. It must be defined using `relates_to_one` or `relates_to_many`."] + elsif (related_type = schema_def_state.object_types_by_name[relationship.related_type.unwrap_non_null.name]).nil? + issue = + if schema_def_state.types_by_name.key?(relationship.related_type.fully_unwrapped.name) + "references a type which is not an object type: `#{relationship.related_type.name}`. Only object types can be used in relations." + else + "references an unknown type: `#{relationship.related_type.name}`. Is it misspelled?" + end + + [nil, "#{relationship_error_prefix} #{issue}"] + elsif !related_type.indexed? + [nil, "#{relationship_error_prefix} references a type which is not indexed: `#{related_type.name}`. Only indexed types can be used in relations."] + else + relation_metadata = relation_field.runtime_metadata_graphql_field.relation # : SchemaArtifacts::RuntimeMetadata::Relation + foreign_key_parent_type = (relation_metadata.direction == :in) ? related_type : object_type + + if (foreign_key_error = validate_foreign_key(foreign_key_parent_type, relation_metadata)) + [nil, foreign_key_error] + else + [ResolvedRelationship.new(relationship_name, relation_field, relationship, related_type, relation_metadata), nil] + end + end + end + + private + + # @dynamic schema_def_state, object_type, relationship_name, sourced_fields, field_path_resolver + attr_reader :schema_def_state, :object_type, :relationship_name, :sourced_fields, :field_path_resolver + + # Helper method for building the prefix of relationship-related error messages. + def relationship_error_prefix + sourced_fields_description = + if sourced_fields.empty? + "" + else + " (referenced from `sourced_from` on field(s): #{sourced_fields.map { |f| "`#{f.name}`" }.join(", ")})" + end + + "`#{relationship_description}`#{sourced_fields_description}" + end + + def validate_foreign_key(foreign_key_parent_type, relation_metadata) + foreign_key_field = field_path_resolver.resolve_public_path(foreign_key_parent_type, relation_metadata.foreign_key) { true } + # If its an inbound foreign key, verify that the foreign key exists on the related type. + # Note: we don't verify this for outbound foreign keys, because when we define a relationship with an outbound foreign + # key, we automatically define an indexing only field for the foreign key (since it exists on the same type). We don't + # do that for an inbound foreign key, though (since the foreign key exists on another type). Allowing a relationship + # definition on type A to add a field to type B's schema would be weird and surprising. + if relation_metadata.direction == :in && foreign_key_field.nil? + "#{relationship_error_prefix} uses `#{foreign_key_parent_type.name}.#{relation_metadata.foreign_key}` as the foreign key, " \ + "but that field does not exist as an indexing field. To continue, define it, define a relationship on `#{foreign_key_parent_type.name}` " \ + "that uses it as the foreign key, use another field as the foreign key, or remove the `#{relationship_description}` definition." + elsif foreign_key_field && foreign_key_field.type.fully_unwrapped.name != "ID" + "#{relationship_error_prefix} uses `#{foreign_key_field.fully_qualified_path}` as the foreign key, " \ + "but that field is not an `ID` field as expected. To continue, change it's type, use another field " \ + "as the foreign key, or remove the `#{relationship_description}` definition." + end + end + + def relationship_description + "#{object_type.name}.#{relationship_name}" + end + end + + # @private + ResolvedRelationship = ::Data.define(:relationship_name, :relationship_field, :relationship, :related_type, :relation_metadata) + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/rollover_config.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/rollover_config.rb new file mode 100644 index 00000000..099a0222 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/rollover_config.rb @@ -0,0 +1,25 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/index_definition" + +module ElasticGraph + module SchemaDefinition + module Indexing + # @private + class RolloverConfig < ::Data.define(:frequency, :timestamp_field_path) + def runtime_metadata + SchemaArtifacts::RuntimeMetadata::IndexDefinition::Rollover.new( + frequency: frequency, + timestamp_field_path: timestamp_field_path.path_in_index + ) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_factory.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_factory.rb new file mode 100644 index 00000000..dc44510a --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_factory.rb @@ -0,0 +1,54 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module Indexing + # Helper class that contains common logic for instantiating `UpdateTargets`. + # @private + module UpdateTargetFactory + def self.new_normal_indexing_update_target( + type:, + relationship:, + id_source:, + data_params:, + routing_value_source:, + rollover_timestamp_value_source: + ) + SchemaArtifacts::RuntimeMetadata::UpdateTarget.new( + type: type, + relationship: relationship, + script_id: INDEX_DATA_UPDATE_SCRIPT_ID, + id_source: id_source, + metadata_params: standard_metadata_params.merge({ + "relationship" => SchemaArtifacts::RuntimeMetadata::StaticParam.new(value: relationship) + }), + data_params: data_params, + routing_value_source: routing_value_source, + rollover_timestamp_value_source: rollover_timestamp_value_source + ) + end + + private_class_method def self.standard_metadata_params + @standard_metadata_params ||= { + "sourceId" => single_value_param_from("id"), + "sourceType" => single_value_param_from("type"), + "version" => single_value_param_from("version") + } + end + + private_class_method def self.single_value_param_from(source_path) + SchemaArtifacts::RuntimeMetadata::DynamicParam.new( + source_path: source_path, + cardinality: :one + ) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_resolver.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_resolver.rb new file mode 100644 index 00000000..56a2a879 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_resolver.rb @@ -0,0 +1,195 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/params" +require "elastic_graph/schema_definition/indexing/update_target_factory" + +module ElasticGraph + module SchemaDefinition + module Indexing + # Responsible for resolving a relationship and a set of `sourced_from` fields into an `UpdateTarget` + # that contains the instructions for how the primary type should be updated from the related type's + # source events. + # + # @private + class UpdateTargetResolver + def initialize( + object_type:, + resolved_relationship:, + sourced_fields:, + field_path_resolver: + ) + @object_type = object_type + @resolved_relationship = resolved_relationship + @sourced_fields = sourced_fields + @field_path_resolver = field_path_resolver + end + + # Resolves the `object_type`, `resolved_relationship`, and `sourced_fields` into an `UpdateTarget`, validating + # that everything is defined correctly. + # + # Returns a tuple of the `update_target` (if valid), and a list of errors. + def resolve + relationship_errors = validate_relationship + data_params, data_params_errors = resolve_data_params + routing_value_source, routing_error = resolve_field_source(RoutingSourceAdapter) + rollover_timestamp_value_source, rollover_timestamp_error = resolve_field_source(RolloverTimestampSourceAdapter) + equivalent_field_errors = resolved_relationship.relationship.validate_equivalent_fields(field_path_resolver) + + all_errors = relationship_errors + data_params_errors + equivalent_field_errors + [routing_error, rollover_timestamp_error].compact + + if all_errors.empty? + update_target = UpdateTargetFactory.new_normal_indexing_update_target( + type: object_type.name, + relationship: resolved_relationship.relationship_name, + id_source: resolved_relationship.relation_metadata.foreign_key, + data_params: data_params, + routing_value_source: routing_value_source, + rollover_timestamp_value_source: rollover_timestamp_value_source + ) + end + + [update_target, all_errors] + end + + private + + # @dynamic object_type, resolved_relationship, sourced_fields, field_path_resolver + attr_reader :object_type, :resolved_relationship, :sourced_fields, :field_path_resolver + + # Applies additional validations (beyond what `RelationshipResolver` applies) on relationships that are + # used by `sourced_from` fields. + def validate_relationship + errors = [] # : ::Array[::String] + + if resolved_relationship.relationship.many? + errors << "#{relationship_error_prefix} is a `relates_to_many` relationship, but `sourced_from` is only supported on a `relates_to_one` relationship." + end + + relation_metadata = resolved_relationship.relationship_field.runtime_metadata_graphql_field.relation # : SchemaArtifacts::RuntimeMetadata::Relation + if relation_metadata.direction == :out + errors << "#{relationship_error_prefix} has an outbound foreign key (`dir: :out`), but `sourced_from` is only supported via inbound foreign key (`dir: :in`) relationships." + end + + unless relation_metadata.additional_filter.empty? + errors << "#{relationship_error_prefix} is a `relationship` using an `additional_filter` but `sourced_from` is not supported on relationships with `additional_filter`." + end + + errors + end + + # Helper method for building the prefix of relationship-related error messages. + def relationship_error_prefix + sourced_fields_description = "(referenced from `sourced_from` on field(s): #{sourced_fields.map { |f| "`#{f.name}`" }.join(", ")})" + "`#{object_type.name}.#{resolved_relationship.relationship_name}` #{sourced_fields_description}" + end + + # Resolves the `sourced_fields` into a data params map, validating them along the way. + # + # Returns a tuple of the data params and a list of any errors that occurred during resolution. + def resolve_data_params + related_type = resolved_relationship.related_type + errors = [] # : ::Array[::String] + + data_params = sourced_fields.filter_map do |field| + field_source = field.source # : SchemaElements::FieldSource + + referenced_field_path = field_path_resolver.resolve_public_path(related_type, field_source.field_path) do |parent_field| + !parent_field.type.list? + end + + if referenced_field_path.nil? + explanation = + if field_source.field_path.include?(".") + "could not be resolved: some parts do not exist on their respective types as non-list fields" + else + "does not exist as an indexing field" + end + + errors << "`#{object_type.name}.#{field.name}` has an invalid `sourced_from` argument: `#{related_type.name}.#{field_source.field_path}` #{explanation}." + nil + elsif referenced_field_path.type.unwrap_non_null != field.type.unwrap_non_null + errors << "The type of `#{object_type.name}.#{field.name}` is `#{field.type}`, but the type of it's source (`#{related_type.name}.#{field_source.field_path}`) is `#{referenced_field_path.type}`. These must agree to use `sourced_from`." + nil + elsif field.type.non_null? + errors << "The type of `#{object_type.name}.#{field.name}` (`#{field.type}`) is not nullable, but this is not allowed for `sourced_from` fields since the value will be `null` before the related type's event is ingested." + nil + else + param = SchemaArtifacts::RuntimeMetadata::DynamicParam.new( + source_path: referenced_field_path.path_in_index, + cardinality: :one + ) + + [field.name_in_index, param] + end + end.to_h + + [data_params, errors] + end + + # Helper method that assists with resolving `routing_value_source` and `rollover_timestamp_value_source`. + # Uses an `adapter` for the differences in these two cases. + # + # Returns a tuple of the resolved source (if successful) and an error (if invalid). + def resolve_field_source(adapter) + # For now we only support one index (so we can use the first index) but someday we may need to support multiple. + index = object_type.indices.first # : Index + + field_source_graphql_path_string = adapter.get_field_source(resolved_relationship.relationship, index) do |local_need| + relationship_name = resolved_relationship.relationship_name + + error = "Cannot update `#{object_type.name}` documents with data from related `#{relationship_name}` events, " \ + "because #{adapter.cannot_update_reason(object_type, relationship_name)}. To fix it, add a call like this to the " \ + "`#{object_type.name}.#{relationship_name}` relationship definition: `rel.equivalent_field " \ + "\"[#{resolved_relationship.related_type.name} field]\", locally_named: \"#{local_need}\"`." + + return [nil, error] + end + + if field_source_graphql_path_string + field_path = field_path_resolver.resolve_public_path(resolved_relationship.related_type, field_source_graphql_path_string) do |parent_field| + !parent_field.type.list? + end + + [field_path&.path_in_index, nil] + else + [nil, nil] + end + end + + # Adapter for the `routing_value_source` case for use by `resolve_field_source`. + # + # @private + module RoutingSourceAdapter + def self.get_field_source(relationship, index, &block) + relationship.routing_value_source_for_index(index, &block) + end + + def self.cannot_update_reason(object_type, relationship_name) + "`#{object_type.name}` uses custom shard routing but we don't know what `#{relationship_name}` field to use " \ + "to route the `#{object_type.name}` update requests" + end + end + + # Adapter for the `rollover_timestamp_value_source` case for use by `resolve_field_source`. + # + # @private + module RolloverTimestampSourceAdapter + def self.get_field_source(relationship, index, &block) + relationship.rollover_timestamp_value_source_for_index(index, &block) + end + + def self.cannot_update_reason(object_type, relationship_name) + "`#{object_type.name}` uses a rollover index but we don't know what `#{relationship_name}` timestamp field to use " \ + "to select an index for the `#{object_type.name}` update requests" + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/json_schema_pruner.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/json_schema_pruner.rb new file mode 100644 index 00000000..173b5ccc --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/json_schema_pruner.rb @@ -0,0 +1,61 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" + +module ElasticGraph + module SchemaDefinition + # Prunes unused type definitions from a given JSON schema. + # + # @private + class JSONSchemaPruner + def self.prune(original_json_schema) + initial_type_names = [EVENT_ENVELOPE_JSON_SCHEMA_NAME] + original_json_schema + .dig("$defs", EVENT_ENVELOPE_JSON_SCHEMA_NAME, "properties", "type", "enum") + + types_to_keep = referenced_type_names(initial_type_names, original_json_schema["$defs"]) + + # The .select will preserve the sort order of the original hash + pruned_defs = original_json_schema["$defs"].select { |k, _v| types_to_keep.include?(k) } + + original_json_schema.merge("$defs" => pruned_defs) + end + + # Returns a list of type names indicating all types referenced from any type in source_type_names. + private_class_method + def self.referenced_type_names(source_type_names, original_defs) + return Set.new if source_type_names.empty? + + referenced_type_defs = original_defs.select { |k, _| source_type_names.include?(k) } + ref_names = collect_ref_names(referenced_type_defs) + + referenced_type_names(ref_names, original_defs) + source_type_names + end + + private_class_method + def self.collect_ref_names(hash) + hash.flat_map do |key, value| + case value + when ::Hash + collect_ref_names(value) + when ::Array + value.grep(::Hash).flat_map { |subhash| collect_ref_names(subhash) } + when ::String + if key == "$ref" && (type = value[%r{\A#/\$defs/(.+)\z}, 1]) + [type] + else + [] + end + else + [] + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/can_be_graphql_only.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/can_be_graphql_only.rb new file mode 100644 index 00000000..e340b5ea --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/can_be_graphql_only.rb @@ -0,0 +1,31 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + # Namespace for modules that are used as mixins. Mixins are used to offer a consistent API for + # schema definition features that apply to multiple types of schema elements. + module Mixins + # Used to indicate if a type only exists in the GraphQL schema (e.g. it has no indexing component). + module CanBeGraphQLOnly + # Sets whether or not this type only exists in the GraphQL schema + # + # @param value [Boolean] whether or not this type only exists in the GraphQL schema + # @return [void] + def graphql_only(value) + @graphql_only = value + end + + # @return [Boolean] whether or not this type only exists in the GraphQL schema + def graphql_only? + !!@graphql_only + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_derived_graphql_type_customizations.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_derived_graphql_type_customizations.rb new file mode 100644 index 00000000..029c1877 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_derived_graphql_type_customizations.rb @@ -0,0 +1,119 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module Mixins + # Mixin that supports the customization of derived GraphQL types. + # + # For each type you define, ElasticGraph generates a number of derived GraphQL types that are needed to facilitate the ElasticGraph + # Query API. Methods in this module can be used to customize those derived GraphQL types. + module HasDerivedGraphQLTypeCustomizations + # Registers a customization block for the named derived graphql types. The provided block will get run on the named derived GraphQL + # types, allowing them to be customized. + # + # @param type_names [Array] names of the derived types to customize, or `:all` to customize all derived types + # @return [void] + # + # @example Customize named derived GraphQL types + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "id", "ID!" + # t.index "campaigns" + # + # t.customize_derived_types "CampaignFilterInput", "CampaignSortOrderInput" do |dt| + # # Add a `@deprecated` directive to two specific derived types. + # dt.directive "deprecated" + # end + # end + # end + # + # @example Customize all derived GraphQL types + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "id", "ID!" + # t.index "campaigns" + # + # t.customize_derived_types :all do |dt| + # # Add a `@deprecated` directive to all derived types. + # dt.directive "deprecated" + # end + # end + # end + def customize_derived_types(*type_names, &customization_block) + if type_names.include?(:all) + derived_type_customizations_for_all_types << customization_block + else + type_names.each do |t| + derived_type_customizations_by_name[t.to_s] << customization_block + end + end + end + + # Registers a customization block for the named fields on the named derived GraphQL type. The provided block will get run on the + # named fields of the named derived GraphQL type, allowing them to be customized. + # + # @param type_name [String] name of the derived type containing fields you want to customize + # @param field_names [Array] names of the fields on the derived types that you wish to customize + # @return [void] + # + # @example Customize named fields of a derived GraphQL type + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "id", "ID!" + # t.index "campaigns" + # + # t.customize_derived_type_fields "CampaignConnection", "pageInfo", "totalEdgeCount" do |dt| + # # Add a `@deprecated` directive to `CampaignConnection.pageInfo` and `CampaignConnection.totalEdgeCount`. + # dt.directive "deprecated" + # end + # end + # end + def customize_derived_type_fields(type_name, *field_names, &customization_block) + customizations_by_field = derived_field_customizations_by_type_and_field_name[type_name] + + field_names.each do |field_name| + customizations_by_field[field_name] << customization_block + end + end + + # @private + def derived_type_customizations_for_type(type) + derived_type_customizations_by_name[type.name] + derived_type_customizations_for_all_types + end + + # @private + def derived_field_customizations_by_name_for_type(type) + derived_field_customizations_by_type_and_field_name[type.name] + end + + # @private + def derived_type_customizations_by_name + @derived_type_customizations_by_name ||= ::Hash.new do |hash, type_name| + hash[type_name] = [] + end + end + + # @private + def derived_field_customizations_by_type_and_field_name + @derived_field_customizations_by_type_and_field_name ||= ::Hash.new do |outer_hash, type| + outer_hash[type] = ::Hash.new do |inner_hash, field_name| + inner_hash[field_name] = [] + end + end + end + + private + + def derived_type_customizations_for_all_types + @derived_type_customizations_for_all_types ||= [] + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_directives.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_directives.rb new file mode 100644 index 00000000..5096186d --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_directives.rb @@ -0,0 +1,65 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module Mixins + # Provides support for annotating any schema element with a GraphQL directive. + module HasDirectives + # Adds a GraphQL directive to the current schema element. + # + # @note If you’re using a custom directive rather than a standard GraphQL directive like `@deprecated`, you’ll also need to use + # {API#raw_sdl} to define the custom directive. + # + # @param name [String] name of the directive + # @param arguments [Hash] arguments for the directive + # @return [void] + # + # @example Add a standard GraphQL directive to a field + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "id", "ID" do |f| + # f.directive "deprecated" + # end + # end + # end + # + # @example Define a custom GraphQL directive and add it to an object type + # ElasticGraph.define_schema do |schema| + # # Define a directive we can use to annotate what system a data type comes from. + # schema.raw_sdl "directive @sourcedFrom(system: String!) on OBJECT" + # + # schema.object_type "Campaign" do |t| + # t.field "id", "ID" + # t.directive "sourcedFrom", system: "campaigns" + # end + # end + def directive(name, arguments = {}) + directives << schema_def_state.factory.new_directive(name, arguments) + end + + # Helper method designed for use by including classes to get the formatted directive SDL. + # + # @param suffix_with [String] suffix to add on the end of the SDL + # @param prefix_with [String] prefix to add to the beginning of the SDL + # @return [String] SDL string for the directives + # @api private + def directives_sdl(suffix_with: "", prefix_with: "") + sdl = directives.map(&:to_sdl).join(" ") + return sdl if sdl.empty? + prefix_with + sdl + suffix_with + end + + # @return [Array] directives attached to this schema element + def directives + @directives ||= [] + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_documentation.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_documentation.rb new file mode 100644 index 00000000..529b63c4 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_documentation.rb @@ -0,0 +1,74 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module Mixins + # Supports GraphQL documentation. + module HasDocumentation + # @dynamic doc_comment, doc_comment= + # @!attribute doc_comment + # @return [String, nil] current documentation string for the schema element + attr_accessor :doc_comment + + # Sets the documentation of the schema element. + # + # @param comment [String] the documentation string + # @return [void] + # + # @example Define documentation on an object type and on a field + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.documentation "A marketing campaign." + # + # t.field "id", "ID" do |f| + # f.documentation <<~EOS + # The identifier of the campaign. + # + # Note: this is randomly generated. + # EOS + # end + # end + # end + def documentation(comment) + self.doc_comment = comment + end + + # Appends some additional documentation to the existing documentation string. + # + # @param comment [String] additional documentation + # @return [void] + def append_to_documentation(comment) + new_documentation = doc_comment ? "#{doc_comment}\n\n#{comment}" : comment + documentation(new_documentation) + end + + # Formats the documentation using GraphQL SDL syntax. + # + # @return [String] formatted documentation string + def formatted_documentation + return nil unless (comment = doc_comment) + %("""\n#{comment.chomp}\n"""\n) + end + + # Generates a documentation string that is derived from the schema elements existing documentation. + # + # @param intro [String] string that goes before the schema element's existing documentation + # @param outro [String, nil] string that goes after the schema element's existing documentation + # @return [String] + def derived_documentation(intro, outro = nil) + outro &&= "\n\n#{outro}." + return "#{intro}.#{outro}" unless doc_comment + + quoted_doc = doc_comment.split("\n").map { |line| "> #{line}" }.join("\n") + "#{intro}:\n\n#{quoted_doc}#{outro}" + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_indices.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_indices.rb new file mode 100644 index 00000000..8648f95c --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_indices.rb @@ -0,0 +1,281 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/schema_definition/indexing/update_target_factory" + +module ElasticGraph + module SchemaDefinition + module Mixins + # Provides APIs for defining datastore indices. + module HasIndices + # @dynamic runtime_metadata_overrides + # @private + attr_accessor :runtime_metadata_overrides + + # @private + def initialize(*args, **options) + super(*args, **options) + self.runtime_metadata_overrides = {} + yield self + + # Freeze `indices` so that the indexable status of a type does not change after instantiation. + # (That would cause problems.) + indices.freeze + end + + # Converts the current type from being an _embedded_ type (that is, a type that is embedded within another indexed type) to an + # _indexed_ type that resides in the named index definition. Indexed types are directly indexed into the datastore, and will be + # queryable from the root `Query` type. + # + # @note Use {#root_query_fields} on indexed types to name the field that will be exposed on `Query`. + # @note Indexed types must also define an `id` field, which ElasticGraph will use as the primary key. + # @note Datastore index settings can also be defined (or overridden) in an environment-specific settings YAML file. Index settings + # that you want to configure differently for different environments (such as `index.number_of_shards`—-production and staging + # will probably need different numbers!) should be configured in the per-environment YAML configuration files rather than here. + # + # @param name [String] name of the index. See the [Elasticsearch docs](https://www.elastic.co/guide/en/elasticsearch/reference/8.15/indices-create-index.html#indices-create-api-path-params) + # for restrictions. + # @param settings [Hash] datastore index settings you want applied to every environment. See the [Elasticsearch docs](https://www.elastic.co/guide/en/elasticsearch/reference/8.15/index-modules.html#index-modules-settings) + # for a list of valid settings, but be sure to omit the `index.` prefix here. + # @yield [Indexing::Index] the index, so it can be customized further + # @return [void] + # + # @example Define a `campaigns` index + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "id", "ID" + # + # t.index( + # "campaigns", + # # Configure `index.refresh_interval`. + # refresh_interval: "1s", + # # Use `index.search` to log warnings for any search query that take more than five seconds. + # search: {slowlog: {level: "WARN", threshold: {query: {warn: "5s"}}}} + # ) do |i| + # # The index can be customized further here. + # end + # end + # end + def index(name, **settings, &block) + indices.replace([Indexing::Index.new(name, settings, schema_def_state, self, &block)]) + end + + # List of indices. (Currently we only store one but we may support multiple in the future). + # + # @private + def indices + @indices ||= [] + end + + # @return [Boolean] true if this type has an index + def indexed? + indices.any? + end + + # Abstract types are rare, so return false. This can be overridden in the host class. + # + # @private + def abstract? + false + end + + # Configures the ElasticGraph indexer to derive another type from this indexed type, using the `from_id` field as + # the source of the `id` of the derived type, and the provided block for the definitions of the derived fields. + # + # @param name [String] name of the derived type + # @param from_id [String] path to the source type field with `id` values for the derived type + # @param route_with [String, nil] path to the source type field with values for shard routing on the derived type + # @param rollover_with [String, nil] path to the source type field with values for index rollover on the derived type + # @yield [Indexing::DerivedIndexedType] configuration object for field derivations + # @return [void] + # + # @example Derive a `Course` type from `StudentCourseEnrollment` events + # ElasticGraph.define_schema do |schema| + # # `StudentCourseEnrollment` is a directly indexed type. + # schema.object_type "StudentCourseEnrollment" do |t| + # t.field "id", "ID" + # t.field "courseId", "ID" + # t.field "courseName", "String" + # t.field "studentName", "String" + # t.field "courseStartDate", "Date" + # + # t.index "student_course_enrollments" + # + # # Here we define how the `Course` indexed type is derived when we index `StudentCourseEnrollment` events. + # t.derive_indexed_type_fields "Course", from_id: "courseId" do |derive| + # # `derive` is an instance of `DerivedIndexedType`. + # derive.immutable_value "name", from: "courseName" + # derive.append_only_set "students", from: "studentName" + # derive.min_value "firstOfferedDate", from: "courseStartDate" + # derive.max_value "mostRecentlyOfferedDate", from: "courseStartDate" + # end + # end + # + # # `Course` is an indexed type that is derived entirely from `StudentCourseEnrollment` events. + # schema.object_type "Course" do |t| + # t.field "id", "ID" + # t.field "name", "String" + # t.field "students", "[String!]!" + # t.field "firstOfferedDate", "Date" + # t.field "mostRecentlyOfferedDate", "Date" + # + # t.index "courses" + # end + # end + def derive_indexed_type_fields( + name, + from_id:, + route_with: nil, + rollover_with: nil, + &block + ) + Indexing::DerivedIndexedType.new( + source_type: self, + destination_type_ref: schema_def_state.type_ref(name).to_final_form, + id_source: from_id, + routing_value_source: route_with, + rollover_timestamp_value_source: rollover_with, + &block + ).tap { |dit| derived_indexed_types << dit } + end + + # @return [Array] list of derived types for this source type + def derived_indexed_types + @derived_indexed_types ||= [] + end + + # @private + def runtime_metadata(extra_update_targets) + SchemaArtifacts::RuntimeMetadata::ObjectType.new( + update_targets: derived_indexed_types.map(&:runtime_metadata_for_source_type) + [self_update_target].compact + extra_update_targets, + index_definition_names: indices.map(&:name), + graphql_fields_by_name: runtime_metadata_graphql_fields_by_name, + elasticgraph_category: nil, + source_type: nil, + graphql_only_return_type: graphql_only? + ).with(**runtime_metadata_overrides) + end + + # Determines what the root `Query` fields will be to query this indexed type. In addition, this method accepts a block, which you + # can use to customize the root query field (such as adding a GraphQL directive to it). + # + # @param plural [String] the plural name of the entity; used for the root `Query` field that queries documents of this indexed type + # @param singular [String, nil] the singular name of the entity; used for the root `Query` field (with an `Aggregations` suffix) that + # queries aggregations of this indexed type. If not provided, will derive it from the type name (e.g. converting it to `camelCase` + # or `snake_case`, depending on configuration). + # @yield [SchemaElements::Field] field on the root `Query` type used to query this indexed type, to support customization + # @return [void] + # + # @example Set `plural` and `singular` names + # ElasticGraph.define_schema do |schema| + # schema.object_type "Person" do |t| + # t.field "id", "ID" + # + # # Results in `Query.people` and `Query.personAggregations`. + # t.root_query_fields plural: "people", singular: "person" + # + # t.index "people" + # end + # end + # + # @example Customize `Query` fields + # ElasticGraph.define_schema do |schema| + # schema.object_type "Person" do |t| + # t.field "id", "ID" + # + # t.root_query_fields plural: "people", singular: "person" do |f| + # # Marks `Query.people` and `Query.personAggregations` as deprecated. + # f.directive "deprecated" + # end + # + # t.index "people" + # end + # end + def root_query_fields(plural:, singular: nil, &customization_block) + @plural_root_query_field_name = plural + @singular_root_query_field_name = singular + @root_query_fields_customizations = customization_block + end + + # @return [String] the plural name of the entity; used for the root `Query` field that queries documents of this indexed type + def plural_root_query_field_name + @plural_root_query_field_name || naively_pluralize_type_name(name) + end + + # @return [String] the singular name of the entity; used for the root `Query` field (with an `Aggregations` suffix) that queries + # aggregations of this indexed type. If not provided, will derive it from the type name (e.g. converting it to `camelCase` or + # `snake_case`, depending on configuration). + def singular_root_query_field_name + @singular_root_query_field_name || to_field_name(name) + end + + # @private + def root_query_fields_customizations + @root_query_fields_customizations + end + + # @private + def fields_with_sources + indexing_fields_by_name_in_index.values.reject { |f| f.source.nil? } + end + + private + + def self_update_target + return nil if abstract? || !indexed? + + # We exclude `id` from `data_params` because `Indexer::Operator::Update` automatically includes + # `params.id` so we don't want it duplicated at `params.data.id` alongside other data params. + # + # In addition, we exclude fields that have an alternate `source` -- those fields will get populated + # by a different event and we don't want to risk "stomping" their value via this update target. + data_params = indexing_fields_by_name_in_index.select { |name, field| name != "id" && field.source.nil? }.to_h do |field| + [field, SchemaArtifacts::RuntimeMetadata::DynamicParam.new(source_path: field, cardinality: :one)] + end + + index_runtime_metadata = indices.first.runtime_metadata + + Indexing::UpdateTargetFactory.new_normal_indexing_update_target( + type: name, + relationship: SELF_RELATIONSHIP_NAME, + id_source: "id", + data_params: data_params, + # Some day we may want to consider supporting multiple indices. If/when we add support for that, + # we'll need to change the runtime metadata here to have a map of these values, keyed by index + # name. + routing_value_source: index_runtime_metadata.route_with, + rollover_timestamp_value_source: index_runtime_metadata.rollover&.timestamp_field_path + ) + end + + def runtime_metadata_graphql_fields_by_name + graphql_fields_by_name.transform_values(&:runtime_metadata_graphql_field) + end + + # Provides a "best effort" conversion of a type name to the plural form. + # In practice, schema definers should set `root_query_field` on their + # indexed types so we don't have to try to convert the type to its plural + # form. Still, this has value, particularly given our existing tests + # (where I don't want to require that we set this). + # + # Note: we could pull in ActiveSupport to pluralize more accurately, but I + # really don't want to pull in any part of Rails just for that :(. + def naively_pluralize_type_name(type_name) + normalized = to_field_name(type_name) + normalized + (normalized.end_with?("s") ? "es" : "s") + end + + def to_field_name(type_name) + name_without_leading_uppercase = type_name.sub(/^([[:upper:]])/) { $1.downcase } + schema_def_state.schema_elements.normalize_case(name_without_leading_uppercase) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect.rb new file mode 100644 index 00000000..da189962 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect.rb @@ -0,0 +1,46 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module Mixins + # Dynamic mixin that provides readable output from `#to_s` and `#inspect`. The default + # output Ruby prints for these methods is quite unwieldy for all our schema definition + # types, because we have a large interconnected object graph and `Struct` classes print + # all their state. In fact, before implementing this, we observed output of more than + # 5 million characters long! + # + # To use this module include a new instance of it: + # + # include HasReadableToSAndInspect.new + # + # Optionally, provide a block that, given an instance of the class, returns a string description for + # inclusion in the output: + # + # include HasReadableToSAndInspect.new { |obj| obj.name } + # + # @private + class HasReadableToSAndInspect < ::Module + def initialize + if block_given? + define_method :to_s do + "#<#{self.class.name} #{yield self}>" + end + else + # When no block is given, we just want to use the stock `Object#to_s`, which renders the memory address. + define_method :to_s do + ::Object.instance_method(:to_s).bind_call(self) + end + end + + alias_method :inspect, :to_s + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_subtypes.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_subtypes.rb new file mode 100644 index 00000000..59bd3c2c --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_subtypes.rb @@ -0,0 +1,116 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/schema_definition/indexing/field_type/union" +require "elastic_graph/schema_definition/schema_elements/list_counts_state" + +module ElasticGraph + module SchemaDefinition + module Mixins + # Provides common support for abstract GraphQL types that have subtypes (e.g. union and interface types). + # + # @private + module HasSubtypes + def to_indexing_field_type + subtypes_by_name = recursively_resolve_subtypes.to_h do |type| + [type.name, _ = type.to_indexing_field_type] + end + + Indexing::FieldType::Union.new(subtypes_by_name) + end + + def graphql_fields_by_name + merge_fields_by_name_from_subtypes(&:graphql_fields_by_name) + end + + def indexing_fields_by_name_in_index + merge_fields_by_name_from_subtypes(&:indexing_fields_by_name_in_index) + .merge("__typename" => schema_def_state.factory.new_field(name: "__typename", type: "String", parent_type: _ = self)) + end + + def indexed? + super || subtypes_indexed? + end + + def recursively_resolve_subtypes + resolve_subtypes.flat_map do |type| + type.is_a?(HasSubtypes) ? (_ = type).recursively_resolve_subtypes : [type] + end + end + + def abstract? + true + end + + def current_sources + resolve_subtypes.flat_map(&:current_sources) + end + + def index_field_runtime_metadata_tuples( + path_prefix: "", + parent_source: SELF_RELATIONSHIP_NAME, + list_counts_state: SchemaElements::ListCountsState::INITIAL + ) + resolve_subtypes.flat_map do |t| + t.index_field_runtime_metadata_tuples( + path_prefix: path_prefix, + parent_source: parent_source, + list_counts_state: list_counts_state + ) + end + end + + private + + def merge_fields_by_name_from_subtypes + resolved_subtypes = resolve_subtypes + + resolved_subtypes.reduce(_ = {}) do |fields_by_name, subtype| + fields_by_name.merge(yield subtype) do |field_name, def1, def2| + if (def1.name_in_index == def2.name_in_index && def1.resolve_mapping != def2.resolve_mapping) || (def1.type.unwrap_non_null != def2.type.unwrap_non_null) + def_strings = resolved_subtypes.each_with_object([]) do |st, defs| + field = st.graphql_fields_by_name[field_name] + defs << "on #{st.name}:\n#{field.to_sdl.strip} mapping: #{field.resolve_mapping.inspect}" if st.graphql_fields_by_name.key?(field_name) + end + + raise Errors::SchemaError, + "Conflicting definitions for field `#{field_name}` on the subtypes of `#{name}`. " \ + "Their definitions must agree. Defs:\n\n#{def_strings.join("\n\n")}" + end + + def1 + end + end + end + + def subtypes_indexed? + indexed_by_subtype_name = resolve_subtypes.to_h do |subtype, acc| + [subtype.name, subtype.indexed?] + end + + uniq_indexed = indexed_by_subtype_name.values.uniq + + if uniq_indexed.size > 1 + descriptions = indexed_by_subtype_name.map do |name_value| + name, value = name_value + "#{name}: indexed? = #{value}" + end + + raise Errors::SchemaError, + "The #{self.class.name} #{name} has some indexed subtypes, and some non-indexed subtypes. " \ + "All subtypes must be indexed or all must NOT be indexed. Subtypes:\n" \ + "#{descriptions.join("\n")}" + end + + !!uniq_indexed.first + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_type_info.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_type_info.rb new file mode 100644 index 00000000..aafe4a9d --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_type_info.rb @@ -0,0 +1,181 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/json_schema/meta_schema_validator" + +module ElasticGraph + module SchemaDefinition + module Mixins + # Mixin used to specify non-GraphQL type info (datastore index and JSON schema type info). + # Exists as a mixin so we can apply the same consistent API to every place we need to use this. + # Currently it's used in 3 places: + # + # - {SchemaElements::ScalarType}: allows specification of how scalars are represented in JSON schema and the index. + # - {SchemaElements::TypeWithSubfields}: allows customization of how an object type is represented in JSON schema and the index. + # - {SchemaElements::Field}: allows customization of a specific field over the field type's standard JSON schema and the index mapping. + module HasTypeInfo + # @return [Hash] datastore mapping options + def mapping_options + @mapping_options ||= {} + end + + # @return [Hash] JSON schema options + def json_schema_options + @json_schema_options ||= {} + end + + # Set of mapping parameters that it makes sense to allow customization of, based on + # [the Elasticsearch docs](https://www.elastic.co/guide/en/elasticsearch/reference/8.15/mapping-params.html). + CUSTOMIZABLE_DATASTORE_PARAMS = Set[ + :analyzer, + :eager_global_ordinals, + :enabled, + :fields, + :format, + :index, + :meta, # not actually in the doc above. Added to support some `index_configurator` tests on 7.9+. + :norms, + :null_value, + :search_analyzer, + :type, + ] + + # Defines the Elasticsearch/OpenSearch [field mapping type](https://www.elastic.co/guide/en/elasticsearch/reference/7.10/mapping-types.html) + # and [mapping parameters](https://www.elastic.co/guide/en/elasticsearch/reference/7.10/mapping-params.html) for a field or type. + # The options passed here will be included in the generated `datastore_config.yaml` artifact that ElasticGraph uses to configure + # Elasticsearch/OpenSearch. + # + # Can be called multiple times; each time, the options will be merged into the existing options. + # + # This is required on a {SchemaElements::ScalarType}; without it, ElasticGraph would have no way to know how the datatype should be + # indexed in the datastore. + # + # On a {SchemaElements::Field}, this can be used to customize how a field is indexed. For example, `String` fields are normally + # indexed as [keywords](https://www.elastic.co/guide/en/elasticsearch/reference/7.10/keyword.html); to instead index a `String` + # field for full text search, you’d need to configure `mapping type: "text"`. + # + # On a {SchemaElements::ObjectType}, this can be used to use a specific Elasticsearch/OpenSearch data type for something that is + # modeled as an object in GraphQL. For example, we use it for the `GeoLocation` type so they get indexed in Elasticsearch using the + # [geo_point type](https://www.elastic.co/guide/en/elasticsearch/reference/7.10/geo-point.html). + # + # @param options [Hash] mapping options--must be limited to {CUSTOMIZABLE_DATASTORE_PARAMS} + # @return [void] + # + # @example Define the mapping of a custom scalar type + # ElasticGraph.define_schema do |schema| + # schema.scalar_type "URL" do |t| + # t.mapping type: "keyword" + # t.json_schema type: "string", format: "uri" + # end + # end + # + # @example Customize the mapping of a field + # ElasticGraph.define_schema do |schema| + # schema.object_type "Card" do |t| + # t.field "id", "ID!" + # + # t.field "cardholderName", "String" do |f| + # # index this field for full text search + # f.mapping type: "text" + # end + # + # t.field "expYear", "Int" do |f| + # # Use a smaller numeric type to save space in the datastore + # f.mapping type: "short" + # f.json_schema minimum: 2000, maximum: 2099 + # end + # + # t.field "expMonth", "Int" do |f| + # # Use a smaller numeric type to save space in the datastore + # f.mapping type: "byte" + # f.json_schema minimum: 1, maximum: 12 + # end + # + # t.index "cards" + # end + # end + def mapping(**options) + param_diff = (options.keys.to_set - CUSTOMIZABLE_DATASTORE_PARAMS).to_a + + unless param_diff.empty? + raise Errors::SchemaError, "Some configured mapping overrides are unsupported: #{param_diff.inspect}" + end + + mapping_options.update(options) + end + + # Defines the [JSON schema](https://json-schema.org/understanding-json-schema/) validations for this field or type. Validations + # defined here will be included in the generated `json_schemas.yaml` artifact, which is used by the ElasticGraph indexer to + # validate events before indexing their data in the datastore. In addition, the publisher may use `json_schemas.yaml` for code + # generation and to apply validation before publishing an event to ElasticGraph. + # + # Can be called multiple times; each time, the options will be merged into the existing options. + # + # This is _required_ on a {SchemaElements::ScalarType} (since we don’t know how a custom scalar type should be represented in + # JSON!). On a {SchemaElements::Field}, this is optional, but can be used to make the JSON schema validation stricter then it + # would otherwise be. For example, you could use `json_schema maxLength: 30` on a `String` field to limit the length. + # + # You can use any of the JSON schema validation keywords here. In addition, `nullable: false` is supported to configure the + # generated JSON schema to disallow `null` values for the field. Note that if you define a field with a non-nullable GraphQL type + # (e.g. `Int!`), the JSON schema will automatically disallow nulls. However, as explained in the + # {SchemaElements::TypeWithSubfields#field} documentation, we generally recommend against defining non-nullable GraphQL fields. + # `json_schema nullable: false` will disallow `null` values from being indexed, while still keeping the field nullable in the + # GraphQL schema. If you think you might want to make a field non-nullable in the GraphQL schema some day, it’s a good idea to use + # `json_schema nullable: false` now to ensure every indexed record has a non-null value for the field. + # + # @note We recommend using JSON schema validations in a limited fashion. Validations that are appropriate to apply when data is + # entering the system-of-record are often not appropriate on a secondary index like ElasticGraph. Events that violate a JSON + # schema validation will fail to index (typically they will be sent to the dead letter queue and page an oncall engineer). If an + # ElasticGraph instance is meant to contain all the data of some source system, you probably don’t want it applying stricter + # validations than the source system itself has. We recommend limiting your JSON schema validations to situations where + # violations would prevent ElasticGraph from operating correctly. + # + # @param options [Hash] JSON schema options + # @return [void] + # + # @example Define the JSON schema validations of a custom scalar type + # ElasticGraph.define_schema do |schema| + # schema.scalar_type "URL" do |t| + # t.mapping type: "keyword" + # + # # JSON schema has a built-in URI format validator: + # # https://json-schema.org/understanding-json-schema/reference/string.html#resource-identifiers + # t.json_schema type: "string", format: "uri" + # end + # end + # + # @example Define additional validations on a field + # ElasticGraph.define_schema do |schema| + # schema.object_type "Card" do |t| + # t.field "id", "ID!" + # + # t.field "expYear", "Int" do |f| + # # Use JSON schema to ensure the publisher is sending us 4 digit years, not 2 digit years. + # f.json_schema minimum: 2000, maximum: 2099 + # end + # + # t.field "expMonth", "Int" do |f| + # f.json_schema minimum: 1, maximum: 12 + # end + # + # t.index "cards" + # end + # end + def json_schema(**options) + validatable_json_schema = Support::HashUtil.stringify_keys(options) + + if (error_msg = JSONSchema.strict_meta_schema_validator.validate_with_error_message(validatable_json_schema)) + raise Errors::SchemaError, "Invalid JSON schema options set on #{self}:\n\n#{error_msg}" + end + + json_schema_options.update(options) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/implements_interfaces.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/implements_interfaces.rb new file mode 100644 index 00000000..0efc538b --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/implements_interfaces.rb @@ -0,0 +1,122 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" + +module ElasticGraph + module SchemaDefinition + module Mixins + # Mixin for types that can implement interfaces ({SchemaElements::ObjectType} and {SchemaElements::InterfaceType}). + module ImplementsInterfaces + # Declares that the current type implements the specified interface, making the current type a subtype of the interface. The + # current type must define all of the fields of the named interface, with the exact same field types. + # + # @param interface_names [Array] names of interface types implemented by this type + # @return [void] + # + # @example Implement an interface + # ElasticGraph.define_schema do |schema| + # schema.interface_type "Athlete" do |t| + # t.field "name", "String" + # t.field "team", "String" + # end + # + # schema.object_type "BaseballPlayer" do |t| + # t.implements "Athlete" + # t.field "name", "String" + # t.field "team", "String" + # t.field "battingAvg", "Float" + # end + # + # schema.object_type "BasketballPlayer" do |t| + # t.implements "Athlete" + # t.field "name", "String" + # t.field "team", "String" + # t.field "pointsPerGame", "Float" + # end + # end + def implements(*interface_names) + interface_refs = interface_names.map do |interface_name| + schema_def_state.type_ref(interface_name).to_final_form.tap do |interface_ref| + schema_def_state.implementations_by_interface_ref[interface_ref] << self + end + end + + implemented_interfaces.concat(interface_refs) + end + + # @return [Array] list of type references for the interface types implemented by this type + def implemented_interfaces + @implemented_interfaces ||= [] + end + + # Called after the schema definition is complete, before dumping artifacts. Here we validate + # the correctness of interface implementations. We defer it until this time to not require the + # interface and fields to be defined before the `implements` call. + # + # Note that the GraphQL gem on its own supports a form of "interface inheritance": if declaring + # that an object type implements an interface, and the object type is missing one or more of the + # interface fields, the GraphQL gem dynamically adds the missing interface fields to the object + # type (at least, that's the result I noted when dumping the GraphQL SDL after trying that!). + # However, we cannot allow that, because our schema definition is used to generate non-GrapQL + # artifacts (e.g. the JSON schema and the index mapping), and all the artifacts must agree + # on the fields. Therefore, we use this method to verify that the object type fully implements + # the specified interfaces. + # + # @return [void] + # @private + def verify_graphql_correctness! + schema_error_messages = implemented_interfaces.filter_map do |interface_ref| + interface = interface_ref.resolved + + case interface + when SchemaElements::InterfaceType + differences = (_ = interface).interface_fields_by_name.values.filter_map do |interface_field| + my_field_sdl = graphql_fields_by_name[interface_field.name]&.to_sdl(type_structure_only: true) + interface_field_sdl = interface_field.to_sdl(type_structure_only: true) + + if my_field_sdl.nil? + "missing `#{interface_field.name}`" + elsif my_field_sdl != interface_field_sdl + "`#{interface_field_sdl.strip}` vs `#{my_field_sdl.strip}`" + end + end + + unless differences.empty? + "Type `#{name}` does not correctly implement interface `#{interface_ref}` " \ + "due to field differences: #{differences.join("; ")}." + end + when nil + "Type `#{name}` cannot implement `#{interface_ref}` because `#{interface_ref}` is not defined." + else + "Type `#{name}` cannot implement `#{interface_ref}` because `#{interface_ref}` is not an interface." + end + end + + unless schema_error_messages.empty? + raise Errors::SchemaError, schema_error_messages.join("\n\n") + end + end + + # @yield [SchemaElements::Argument] an argument + # @yieldreturn [Boolean] whether or not to include the argument in the generated GraphQL SDL + # @return [String] SDL string of the type + def to_sdl(&field_arg_selector) + name_section = + if implemented_interfaces.empty? + name + else + "#{name} implements #{implemented_interfaces.join(" & ")}" + end + + generate_sdl(name_section: name_section, &field_arg_selector) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/supports_default_value.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/supports_default_value.rb new file mode 100644 index 00000000..4cc7da08 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/supports_default_value.rb @@ -0,0 +1,47 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/graphql_formatter" + +module ElasticGraph + module SchemaDefinition + module Mixins + # A mixin designed to be included in a schema element class that supports default values. + # Designed to be `prepended` so that it can hook into `initialize`. + module SupportsDefaultValue + # @private + def initialize(...) + __skip__ = super # steep can't type this. + @default_value = NO_DEFAULT_PROVIDED + end + + # Used to specify the default value for this field or argument. + # + # @param default_value [Object] default value for this field or argument + # @return [void] + def default(default_value) + @default_value = default_value + end + + # Generates SDL for the default value. Suitable for inclusion in the schema elememnts `#to_sdl`. + # + # @return [String] + def default_value_sdl + return nil if @default_value == NO_DEFAULT_PROVIDED + " = #{Support::GraphQLFormatter.serialize(@default_value)}" + end + + private + + # A sentinel value that we can use to detect when a default has been provided. + # We can't use `nil` to detect if a default has been provided because `nil` is a valid default value! + NO_DEFAULT_PROVIDED = Module.new + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/supports_filtering_and_aggregation.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/supports_filtering_and_aggregation.rb new file mode 100644 index 00000000..6cff035f --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/supports_filtering_and_aggregation.rb @@ -0,0 +1,267 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module Mixins + # Responsible for building object types for filtering and aggregation, from an existing object type. + # + # This is specifically designed to support {SchemaElements::TypeWithSubfields} (where we have the fields directly available) and + # {SchemaElements::UnionType} (where we will need to compute the list of fields by resolving the subtypes and merging their fields). + # + # @private + module SupportsFilteringAndAggregation + # Indicates if this type supports a given feature (e.g. `filterable?`). + def supports?(&feature_predicate) + # If the type uses a custom mapping type we don't know if it can support a feature, so we assume it can't. + # TODO: clean this up using an interface instead of checking mapping options. + return false if has_custom_mapping_type? + + graphql_fields_by_name.values.any?(&feature_predicate) + end + + # Inverse of `supports?`. + def does_not_support?(&feature_predicate) + !supports?(&feature_predicate) + end + + def derived_graphql_types + return [] if graphql_only? + + indexed_agg_type = to_indexed_aggregation_type + indexed_aggregation_pagination_types = + if indexed_agg_type + schema_def_state.factory.build_relay_pagination_types(indexed_agg_type.name) + else + [] # : ::Array[SchemaElements::ObjectType] + end + + sub_aggregation_types = sub_aggregation_types_for_nested_field_references.flat_map do |type| + [type] + schema_def_state.factory.build_relay_pagination_types(type.name, support_pagination: false) do |t| + # Record metadata that is necessary for elasticgraph-graphql to correctly recognize and handle + # this sub-aggregation correctly. + t.runtime_metadata_overrides = {elasticgraph_category: :nested_sub_aggregation_connection} + end + end + + document_pagination_types = + if indexed? + schema_def_state.factory.build_relay_pagination_types(name, include_total_edge_count: true, derived_indexed_types: (_ = self).derived_indexed_types) + elsif schema_def_state.paginated_collection_element_types.include?(name) + schema_def_state.factory.build_relay_pagination_types(name, include_total_edge_count: true) + else + [] # : ::Array[SchemaElements::ObjectType] + end + + sort_order_enum_type = schema_def_state.enums_for_indexed_types.sort_order_enum_for(self) + derived_sort_order_enum_types = [sort_order_enum_type].compact + (sort_order_enum_type&.derived_graphql_types || []) + + to_input_filters + + document_pagination_types + + indexed_aggregation_pagination_types + + sub_aggregation_types + + derived_sort_order_enum_types + + build_aggregation_sub_aggregations_types + [ + indexed_agg_type, + to_grouped_by_type, + to_aggregated_values_type + ].compact + end + + def has_custom_mapping_type? + mapping_type = mapping_options[:type] + mapping_type && mapping_type != "object" + end + + private + + # Converts the type to the corresponding input filter type. + def to_input_filters + return [] if does_not_support?(&:filterable?) + + schema_def_state.factory.build_standard_filter_input_types_for_index_object_type(name) do |t| + graphql_fields_by_name.values.each do |field| + if field.filterable? + t.graphql_fields_by_name[field.name] = field.to_filter_field(parent_type: t) + end + end + end + end + + # Generates the `*SubAggregation` types for all of the `mapping type: "nested"` fields that reference this type. + # A different `*SubAggregation` type needs to be generated for each nested field reference, and for each parent nesting + # context of that nested field reference. This is necessary because we will support different available `sub_aggregations` + # based on the parents of a particular nested field. + # + # For example, given a `Player` object type definition and a `Team` type definition like this: + # + # schema.object_type "Team" do |t| + # t.field "id", "ID!" + # t.field "name", "String" + # t.field "players", "[Player!]!" do |f| + # f.mapping type: "nested" + # end + # t.index "teams" + # end + # + # ...we will generate a `TeamPlayerSubAggregation` type which will have a `sub_aggregations` field which can have + # `parent_team` and `seasons` fields (assuming `Player` has a `seasons` nested field...). + def sub_aggregation_types_for_nested_field_references + schema_def_state.user_defined_field_references_by_type_name.fetch(name) { [] }.select(&:nested?).flat_map do |nested_field_ref| + schema_def_state.sub_aggregation_paths_for(nested_field_ref.parent_type).map do |path| + schema_def_state.factory.new_object_type type_ref.as_sub_aggregation(parent_doc_types: path.parent_doc_types).name do |t| + t.documentation "Return type representing a bucket of `#{name}` objects for a sub-aggregation within each `#{type_ref.as_parent_aggregation(parent_doc_types: path.parent_doc_types).name}`." + + t.field schema_def_state.schema_elements.count_detail, "AggregationCountDetail", graphql_only: true do |f| + f.documentation "Details of the count of `#{name}` documents in a sub-aggregation bucket." + end + + if supports?(&:groupable?) + t.field schema_def_state.schema_elements.grouped_by, type_ref.as_grouped_by.name, graphql_only: true do |f| + f.documentation "Used to specify the `#{name}` fields to group by. The returned values identify each sub-aggregation bucket." + end + end + + if supports?(&:aggregatable?) + t.field schema_def_state.schema_elements.aggregated_values, type_ref.as_aggregated_values.name, graphql_only: true do |f| + f.documentation "Provides computed aggregated values over all `#{name}` documents in a sub-aggregation bucket." + end + end + + if graphql_fields_by_name.values.any?(&:sub_aggregatable?) + sub_aggs_name = type_ref.as_aggregation_sub_aggregations(parent_doc_types: path.parent_doc_types + [name]).name + t.field schema_def_state.schema_elements.sub_aggregations, sub_aggs_name, graphql_only: true do |f| + f.documentation "Used to perform sub-aggregations of `#{t.name}` data." + end + end + end + end + end + end + + # Builds the `*AggregationSubAggregations` types. For example, for an indexed type named `Team` which has nested fields, + # this would generate a `TeamAggregationSubAggregations` type. This type provides access to the various sub-aggregation + # fields. + def build_aggregation_sub_aggregations_types + # The sub-aggregation types do not generate correctly for abstract types, so for now we omit sub-aggregations for abstract types. + return [] if abstract? + + sub_aggregatable_fields = graphql_fields_by_name.values.select(&:sub_aggregatable?) + return [] if sub_aggregatable_fields.empty? + + schema_def_state.sub_aggregation_paths_for(self).map do |path| + agg_sub_aggs_type_ref = type_ref.as_aggregation_sub_aggregations( + parent_doc_types: path.parent_doc_types, + field_path: path.field_path + ) + + schema_def_state.factory.new_object_type agg_sub_aggs_type_ref.name do |t| + under_field_description = "under `#{path.field_path_string}` " unless path.field_path.empty? + t.documentation "Provides access to the `#{schema_def_state.schema_elements.sub_aggregations}` #{under_field_description}within each `#{type_ref.as_parent_aggregation(parent_doc_types: path.parent_doc_types).name}`." + + sub_aggregatable_fields.each do |field| + if field.nested? + unwrapped_type = field.type_for_derived_types.fully_unwrapped + field_type_name = unwrapped_type + .as_sub_aggregation(parent_doc_types: path.parent_doc_types) + .as_connection + .name + + field.define_sub_aggregations_field(parent_type: t, type: field_type_name) do |f| + f.argument schema_def_state.schema_elements.filter, unwrapped_type.as_filter_input.name do |a| + a.documentation "Used to filter the `#{unwrapped_type.name}` documents included in this sub-aggregation based on the provided criteria." + end + + f.argument schema_def_state.schema_elements.first, "Int" do |a| + a.documentation "Determines how many sub-aggregation buckets should be returned." + end + end + else + field_type_name = type_ref.as_aggregation_sub_aggregations( + parent_doc_types: path.parent_doc_types, + field_path: path.field_path + [field] + ).name + + field.define_sub_aggregations_field(parent_type: t, type: field_type_name) + end + end + end + end + end + + def to_indexed_aggregation_type + return nil unless indexed? + + schema_def_state.factory.new_object_type type_ref.as_aggregation.name do |t| + t.documentation "Return type representing a bucket of `#{name}` documents for an aggregations query." + + if supports?(&:groupable?) + t.field schema_def_state.schema_elements.grouped_by, type_ref.as_grouped_by.name, graphql_only: true do |f| + f.documentation "Used to specify the `#{name}` fields to group by. The returned values identify each aggregation bucket." + end + end + + t.field schema_def_state.schema_elements.count, "JsonSafeLong!", graphql_only: true do |f| + f.documentation "The count of `#{name}` documents in an aggregation bucket." + end + + if supports?(&:aggregatable?) + t.field schema_def_state.schema_elements.aggregated_values, type_ref.as_aggregated_values.name, graphql_only: true do |f| + f.documentation "Provides computed aggregated values over all `#{name}` documents in an aggregation bucket." + end + end + + # The sub-aggregation types do not generate correctly for abstract types, so for now we omit sub-aggregations for abstract types. + if !abstract? && supports?(&:sub_aggregatable?) + t.field schema_def_state.schema_elements.sub_aggregations, type_ref.as_aggregation_sub_aggregations.name, graphql_only: true do |f| + f.documentation "Used to perform sub-aggregations of `#{t.name}` data." + end + end + + # Record metadata that is necessary for elasticgraph-graphql to correctly recognize and handle + # this indexed aggregation type correctly. + t.runtime_metadata_overrides = {source_type: name, elasticgraph_category: :indexed_aggregation} + end + end + + def to_grouped_by_type + # If the type uses a custom mapping type we don't know how it can be aggregated, so we assume it needs no aggregation type. + # TODO: clean this up using an interface instead of checking mapping options. + return nil if has_custom_mapping_type? + + new_non_empty_object_type type_ref.as_grouped_by.name do |t| + t.documentation "Type used to specify the `#{name}` fields to group by for aggregations." + + graphql_fields_by_name.values.each do |field| + field.define_grouped_by_field(t) + end + end + end + + def to_aggregated_values_type + # If the type uses a custom mapping type we don't know how it can be aggregated, so we assume it needs no aggregation type. + # TODO: clean this up using an interface instead of checking mapping options. + return nil if has_custom_mapping_type? + + new_non_empty_object_type type_ref.as_aggregated_values.name do |t| + t.documentation "Type used to perform aggregation computations on `#{name}` fields." + + graphql_fields_by_name.values.each do |field| + field.define_aggregated_values_field(t) + end + end + end + + def new_non_empty_object_type(name, &block) + type = schema_def_state.factory.new_object_type(name, &block) + type unless type.graphql_fields_by_name.empty? + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/verifies_graphql_name.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/verifies_graphql_name.rb new file mode 100644 index 00000000..37ed1b0f --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/verifies_graphql_name.rb @@ -0,0 +1,38 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" + +module ElasticGraph + module SchemaDefinition + module Mixins + # Used to verify the validity of the name of GraphQL schema elements. + # + # @note This mixin is designed to be used via `prepend`, so it can add a constructor override that enforces + # the GraphQL name pattern as the object is built. + module VerifiesGraphQLName + # @private + def initialize(...) + __skip__ = super(...) # __skip__ tells Steep to ignore this + + VerifiesGraphQLName.verify_name!(name) + end + + # Raises if the provided name is invalid. + # + # @param name [String] name of GraphQL schema element + # @return [void] + # @raise [Errors::InvalidGraphQLNameError] if the name is invalid + def self.verify_name!(name) + return if GRAPHQL_NAME_PATTERN.match?(name) + raise Errors::InvalidGraphQLNameError, "Not a valid GraphQL name: `#{name}`. #{GRAPHQL_NAME_VALIDITY_DESCRIPTION}" + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/rake_tasks.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/rake_tasks.rb new file mode 100644 index 00000000..c046ac03 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/rake_tasks.rb @@ -0,0 +1,190 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "rake/tasklib" +require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" + +module ElasticGraph + module SchemaDefinition + # Defines rake tasks for managing artifacts generated from a schema definition. + # + # @note {ElasticGraph::Local::RakeTasks} wraps this and provides additional functionality. Most users will not need to interact with + # this class directly. + class RakeTasks < ::Rake::TaskLib + # @private + attr_reader :output + + # @param index_document_sizes [Boolean] When enabled, ElasticGraph will configure the index mappings so that the datastore indexes a + # `_size` field in each document. ElasticGraph itself does not do anything with this field, but it will be available for your use + # in any direct queries (e.g. via Kibana). Important note: this requires the [mapper-size + # plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/8.15/mapper-size.html) to be installed on your datastore cluster. + # You are responsible for ensuring that is installed if you enable this feature. If you enable this and the plugin is not + # installed, you will get errors! + # @param path_to_schema [String, Pathname] path to the main (or only) schema definition file + # @param schema_artifacts_directory [String, Pathname] Directory to dump the schema artifacts in + # @param schema_element_name_form [:camelCase, :snake_case] the form of names for schema elements (fields, arguments, directives) + # generated by ElasticGraph. + # @param schema_element_name_overrides [Hash] overrides for specific names of schema elements (fields, arguments, + # directives) generated by ElasticGraph. For example, to rename the `gt` filter field to `greaterThan`, pass `{gt: "greaterThan"}`. + # @param derived_type_name_formats [Hash] overrides for the naming formats used by ElasticGraph for derived GraphQL + # type names. For example, to use `Metrics` instead of `AggregatedValues` as the suffix for the generated types supporting + # getting aggregated metrid values, pass `{AggregatedValues: "%{base}Metrics"}`. See {SchemaElements::TypeNamer::DEFAULT_FORMATS} + # for the available formats. + # @param type_name_overrides [Hash] overrides for the names of specific GraphQL types. For example, to rename the + # `DateTime` scalar to `Timestamp`, pass `{DateTime: "Timestamp}`. + # @param enum_value_overrides_by_type [Hash>] overrides for the names of specific enum values for + # specific enum types. For example, to rename the `DayOfWeek.MONDAY` enum to `DayOfWeek.MON`, pass `{DayOfWeek: {MONDAY: "MON"}}`. + # @param extension_modules [Array] List of Ruby modules to extend onto the `SchemaDefinition::API` instance. Designed to + # support ElasticGraph extension gems (such as `elasticgraph-apollo`). + # @param enforce_json_schema_version [Boolean] Whether or not to enforce the requirement that the JSON schema version is incremented + # every time dumping the JSON schemas results in a changed artifact. Generally speaking, you will want this to be `true` for any + # ElasticGraph application that is in production as the versioning of JSON schemas is what supports safe schema evolution as it + # allows ElasticGraph to identify which version of the JSON schema the publishing system was operating on when it published an + # event. It can be useful to set it to `false` before your application is in production, as you do not want to be forced to bump + # the version after every single schema change while you are building an initial prototype. + # @param output [IO] used for printing task output + # + # @example Minimal setup with defaults + # ElasticGraph::SchemaDefinition::RakeTasks.new( + # index_document_sizes: false, + # path_to_schema: "config/schema.rb", + # schema_artifacts_directory: "config/schema/artifacts", + # schema_element_name_form: :camelCase + # ) + # + # @example Spell out the full names of the `gt`/`gte`/`lt`/`lte` filter operators + # ElasticGraph::SchemaDefinition::RakeTasks.new( + # index_document_sizes: false, + # path_to_schema: "config/schema.rb", + # schema_artifacts_directory: "config/schema/artifacts", + # schema_element_name_form: :camelCase, + # schema_element_name_overrides: { + # gt: "greaterThan", + # gte: "greaterThanOrEqualTo", + # lt: "lessThan", + # lte: "lessThanOrEqualTo" + # } + # ) + # + # @example Change the `AggregatedValues` type suffix to `Metrics` + # ElasticGraph::SchemaDefinition::RakeTasks.new( + # index_document_sizes: false, + # path_to_schema: "config/schema.rb", + # schema_artifacts_directory: "config/schema/artifacts", + # schema_element_name_form: :camelCase, + # derived_type_name_formats: {AggregatedValues: "Metrics"} + # ) + # + # @example Rename `JsonSafeLong` to `BigInt` + # ElasticGraph::SchemaDefinition::RakeTasks.new( + # index_document_sizes: false, + # path_to_schema: "config/schema.rb", + # schema_artifacts_directory: "config/schema/artifacts", + # schema_element_name_form: :camelCase, + # type_name_overrides: {JsonSafeLong: "BigInt"} + # ) + # + # @example Shorten the names of the `DayOfWeek` enum values + # ElasticGraph::SchemaDefinition::RakeTasks.new( + # index_document_sizes: false, + # path_to_schema: "config/schema.rb", + # schema_artifacts_directory: "config/schema/artifacts", + # schema_element_name_form: :camelCase, + # enum_value_overrides_by_type: { + # DayOfWeek: { + # MONDAY: "MON", + # TUESDAY: "TUE", + # WEDNESDAY: "WED", + # THURSDAY: "THU", + # FRIDAY: "FRI", + # SATURDAY: "SAT", + # SUNDAY: "SUN" + # } + # } + # ) + def initialize( + index_document_sizes:, + path_to_schema:, + schema_artifacts_directory:, + schema_element_name_form:, + schema_element_name_overrides: {}, + derived_type_name_formats: {}, + type_name_overrides: {}, + enum_value_overrides_by_type: {}, + extension_modules: [], + enforce_json_schema_version: true, + output: $stdout + ) + @schema_element_names = SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new( + form: schema_element_name_form, + overrides: schema_element_name_overrides + ) + + @derived_type_name_formats = derived_type_name_formats + @type_name_overrides = type_name_overrides + @enum_value_overrides_by_type = enum_value_overrides_by_type + @index_document_sizes = index_document_sizes + @path_to_schema = path_to_schema + @schema_artifacts_directory = schema_artifacts_directory + @enforce_json_schema_version = enforce_json_schema_version + @extension_modules = extension_modules + @output = output + + define_tasks + end + + private + + def define_tasks + namespace :schema_artifacts do + desc "Dumps all schema artifacts based on the current ElasticGraph schema definition" + task :dump do + schema_artifact_manager.dump_artifacts + end + + desc "Checks the artifacts to make sure they are up-to-date, raising an exception if not" + task :check do + schema_artifact_manager.check_artifacts + end + end + end + + def schema_artifact_manager + require "elastic_graph/schema_definition/schema_artifact_manager" + + # :nocov: -- tests don't cover the `VERBOSE` side + max_diff_lines = ENV["VERBOSE"] ? 999999999 : 50 + # :nocov: + + SchemaArtifactManager.new( + schema_definition_results: schema_definition_results, + schema_artifacts_directory: @schema_artifacts_directory.to_s, + enforce_json_schema_version: @enforce_json_schema_version, + output: @output, + max_diff_lines: max_diff_lines + ) + end + + def schema_definition_results + require "elastic_graph/schema_definition/api" + + API.new( + @schema_element_names, + @index_document_sizes, + extension_modules: @extension_modules, + derived_type_name_formats: @derived_type_name_formats, + type_name_overrides: @type_name_overrides, + enum_value_overrides_by_type: @enum_value_overrides_by_type, + output: @output + ).tap do |api| + api.as_active_instance { load @path_to_schema.to_s } + end.results + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb new file mode 100644 index 00000000..f18256dc --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb @@ -0,0 +1,404 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/errors" +require "elastic_graph/schema_artifacts/runtime_metadata/schema" +require "elastic_graph/schema_artifacts/artifacts_helper_methods" +require "elastic_graph/schema_definition/indexing/event_envelope" +require "elastic_graph/schema_definition/indexing/json_schema_with_metadata" +require "elastic_graph/schema_definition/indexing/relationship_resolver" +require "elastic_graph/schema_definition/indexing/update_target_resolver" +require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" +require "elastic_graph/schema_definition/schema_elements/field_path" +require "elastic_graph/schema_definition/scripting/file_system_repository" +require "elastic_graph/support/memoizable_data" + +module ElasticGraph + module SchemaDefinition + # Provides the results of defining a schema. + # + # @note This class is designed to implement the same interface as `ElasticGraph::SchemaArtifacts::FromDisk`, so that it can be used + # interchangeably with schema artifacts loaded from disk. This allows the artifacts to be used in tests without having to dump them or + # reload them. + class Results < Support::MemoizableData.define(:state) + include Mixins::HasReadableToSAndInspect.new + include SchemaArtifacts::ArtifactsHelperMethods + + # @return [String] the generated GraphQL SDL schema string dumped as `schema.graphql` + def graphql_schema_string + @graphql_schema_string ||= generate_sdl + end + + # @return [Hash] the Elasticsearch/OpenSearch configuration dumped as `datastore_config.yaml` + def datastore_config + @datastore_config ||= generate_datastore_config + end + + # @return [Hash] runtime metadata used by other parts of ElasticGraph and dumped as `runtime_metadata.yaml` + def runtime_metadata + @runtime_metadata ||= build_runtime_metadata + end + + # @param version [Integer] desired JSON schema version + # @return [Hash] the JSON schema for the requested version, if available + # @raise [Errors::NotFoundError] if the requested JSON schema version is not available + def json_schemas_for(version) + unless available_json_schema_versions.include?(version) + raise Errors::NotFoundError, "The requested json schema version (#{version}) is not available. Available versions: #{available_json_schema_versions.to_a.join(", ")}." + end + + @latest_versioned_json_schema ||= merge_field_metadata_into_json_schema(current_public_json_schema).json_schema + end + + # @return [Set] set of available JSON schema versions + def available_json_schema_versions + @available_json_schema_versions ||= Set[latest_json_schema_version] + end + + # @return [Hash] the newly generated JSON schema + def latest_json_schema_version + current_public_json_schema[JSON_SCHEMA_VERSION_KEY] + end + + # @private + def json_schema_version_setter_location + state.json_schema_version_setter_location + end + + # @private + def json_schema_field_metadata_by_type_and_field_name + @json_schema_field_metadata_by_type_and_field_name ||= json_schema_indexing_field_types_by_name + .transform_values(&:json_schema_field_metadata_by_field_name) + end + + # @private + def current_public_json_schema + @current_public_json_schema ||= build_public_json_schema + end + + # @private + def merge_field_metadata_into_json_schema(json_schema) + json_schema_with_metadata_merger.merge_metadata_into(json_schema) + end + + # @private + def unused_deprecated_elements + json_schema_with_metadata_merger.unused_deprecated_elements + end + + # @private + STATIC_SCRIPT_REPO = Scripting::FileSystemRepository.new(::File.join(__dir__.to_s, "scripting", "scripts")) + + # @private + def derived_indexing_type_names + @derived_indexing_type_names ||= state + .object_types_by_name + .values + .flat_map { |type| type.derived_indexed_types.map { |dit| dit.destination_type_ref.name } } + .to_set + end + + private + + def after_initialize + # Record that we are now generating results so that caching can kick in. + state.user_definition_complete = true + end + + def json_schema_with_metadata_merger + @json_schema_with_metadata_merger ||= Indexing::JSONSchemaWithMetadata::Merger.new(self) + end + + def generate_datastore_config + # We need to check this before generating our datastore configuration. + # We can't generate a mapping from a recursively defined schema type. + check_for_circular_dependencies! + + index_templates, indices = state.object_types_by_name.values + .flat_map(&:indices) + .sort_by(&:name) + .partition(&:rollover_config) + + datastore_scripts = (build_dynamic_scripts + STATIC_SCRIPT_REPO.scripts) + + { + "index_templates" => index_templates.to_h { |i| [i.name, i.to_index_template_config] }, + "indices" => indices.to_h { |i| [i.name, i.to_index_config] }, + "scripts" => datastore_scripts.to_h { |s| [s.id, s.to_artifact_payload] } + } + end + + def build_dynamic_scripts + state.object_types_by_name.values + .flat_map(&:derived_indexed_types) + .map(&:painless_script) + end + + def build_runtime_metadata + extra_update_targets_by_object_type_name = identify_extra_update_targets_by_object_type_name + + object_types_by_name = all_types_except_root_query_type + .select { |t| t.respond_to?(:graphql_fields_by_name) } + .to_h { |type| [type.name, (_ = type).runtime_metadata(extra_update_targets_by_object_type_name.fetch(type.name) { [] })] } + + scalar_types_by_name = state.scalar_types_by_name.transform_values(&:runtime_metadata) + + enum_generator = state.factory.new_enums_for_indexed_types + + indexed_enum_types_by_name = state.object_types_by_name.values + .select(&:indexed?) + .filter_map { |type| enum_generator.sort_order_enum_for(_ = type) } + .to_h { |enum_type| [(_ = enum_type).name, (_ = enum_type).runtime_metadata] } + + enum_types_by_name = all_types_except_root_query_type + .grep(SchemaElements::EnumType) # : ::Array[SchemaElements::EnumType] + .to_h { |t| [t.name, t.runtime_metadata] } + .merge(indexed_enum_types_by_name) + + index_definitions_by_name = state.object_types_by_name.values.flat_map(&:indices).to_h do |index| + [index.name, index.runtime_metadata] + end + + SchemaArtifacts::RuntimeMetadata::Schema.new( + object_types_by_name: object_types_by_name, + scalar_types_by_name: scalar_types_by_name, + enum_types_by_name: enum_types_by_name, + index_definitions_by_name: index_definitions_by_name, + schema_element_names: state.schema_elements, + graphql_extension_modules: state.graphql_extension_modules, + static_script_ids_by_scoped_name: STATIC_SCRIPT_REPO.script_ids_by_scoped_name + ) + end + + # Builds a map, keyed by object type name, of extra `update_targets` that have been generated + # from any fields that use `sourced_from` on other types. + def identify_extra_update_targets_by_object_type_name + # The field_path_resolver memoizes some calculations, and we want the same instance to be + # used by all UpdateTargetBuilders to maximize its effectiveness. + field_path_resolver = SchemaElements::FieldPath::Resolver.new + sourced_field_errors = [] # : ::Array[::String] + relationship_errors = [] # : ::Array[::String] + + state.object_types_by_name.values.each_with_object(::Hash.new { |h, k| h[k] = [] }) do |object_type, accum| + fields_with_sources_by_relationship_name = + if object_type.indices.empty? + # only indexed types can have `sourced_from` fields, and resolving `fields_with_sources` on an unindexed union type + # such as `_Entity` when we are using apollo can lead to exceptions when multiple entity types have the same field name + # that use different mapping types. + {} # : ::Hash[::String, ::Array[SchemaElements::Field]] + else + object_type + .fields_with_sources + .group_by { |f| (_ = f.source).relationship_name } + end + + defined_relationships = object_type + .graphql_fields_by_name.values + .select(&:relationship) + .map(&:name) + + (defined_relationships | fields_with_sources_by_relationship_name.keys).each do |relationship_name| + sourced_fields = fields_with_sources_by_relationship_name.fetch(relationship_name) { [] } + relationship_resolver = Indexing::RelationshipResolver.new( + schema_def_state: state, + object_type: object_type, + relationship_name: relationship_name, + sourced_fields: sourced_fields, + field_path_resolver: field_path_resolver + ) + + resolved_relationship, relationship_error = relationship_resolver.resolve + relationship_errors << relationship_error if relationship_error + + if object_type.indices.any? && resolved_relationship && sourced_fields.any? + update_target_resolver = Indexing::UpdateTargetResolver.new( + object_type: object_type, + resolved_relationship: resolved_relationship, + sourced_fields: sourced_fields, + field_path_resolver: field_path_resolver + ) + + update_target, errors = update_target_resolver.resolve + accum[resolved_relationship.related_type.name] << update_target if update_target + sourced_field_errors.concat(errors) + end + end + end.tap do + full_errors = [] # : ::Array[::String] + + if sourced_field_errors.any? + full_errors << "Schema had #{sourced_field_errors.size} error(s) related to `sourced_from` fields:\n\n#{sourced_field_errors.map.with_index(1) { |e, i| "#{i}. #{e}" }.join("\n\n")}" + end + + if relationship_errors.any? + full_errors << "Schema had #{relationship_errors.size} error(s) related to relationship fields:\n\n#{relationship_errors.map.with_index(1) { |e, i| "#{i}. #{e}" }.join("\n\n")}" + end + + unless full_errors.empty? + raise Errors::SchemaError, full_errors.join("\n\n") + end + end + end + + # Generates the SDL defined by your schema. Intended to be called only once + # at the very end (after evaluating the "main" template). `Evaluator` calls this + # automatically at the end. + def generate_sdl + check_for_circular_dependencies! + state.object_types_by_name.values.each(&:verify_graphql_correctness!) + + type_defs = state.factory + .new_graphql_sdl_enumerator(all_types_except_root_query_type) + .map { |sdl| strip_trailing_whitespace(sdl) } + + [type_defs + state.sdl_parts].join("\n\n") + end + + def build_public_json_schema + json_schema_version = state.json_schema_version + if json_schema_version.nil? + raise Errors::SchemaError, "`json_schema_version` must be specified in the schema. To resolve, add `schema.json_schema_version 1` in a schema definition block." + end + + indexed_type_names = state.object_types_by_name.values + .select { |type| type.indexed? && !type.abstract? } + .reject { |type| derived_indexing_type_names.include?(type.name) } + .map(&:name) + + definitions_by_name = json_schema_indexing_field_types_by_name + .transform_values(&:to_json_schema) + .compact + + { + "$schema" => JSON_META_SCHEMA, + JSON_SCHEMA_VERSION_KEY => json_schema_version, + "$defs" => { + "ElasticGraphEventEnvelope" => Indexing::EventEnvelope.json_schema(indexed_type_names, json_schema_version) + }.merge(definitions_by_name) + } + end + + def json_schema_indexing_field_types_by_name + @json_schema_indexing_field_types_by_name ||= state + .types_by_name.values + .reject do |t| + derived_indexing_type_names.include?(t.name) || + # Skip graphql framework types + t.graphql_only? + end + .sort_by(&:name) + .to_h { |type| [type.name, type.to_indexing_field_type] } + end + + def strip_trailing_whitespace(string) + string.gsub(/ +$/, "") + end + + def check_for_circular_dependencies! + return if @no_circular_dependencies + + referenced_types_by_source_type = state.types_by_name + .reject { |_, type| type.graphql_only? } + .each_with_object(::Hash.new { |h, k| h[k] = ::Set.new }) do |(type_name, _), cache| + recursively_add_referenced_types_to(state.type_ref(type_name), cache) + end + + circular_reference_sets = referenced_types_by_source_type + .select { |source_type, referenced_types| referenced_types.include?(source_type) } + .values + .uniq + + if circular_reference_sets.any? + descriptions = circular_reference_sets.map do |set| + "- The set of #{set.to_a} forms a circular reference chain." + end + + raise Errors::SchemaError, "Your schema has self-referential types, which are not allowed, since " \ + "it prevents the datastore mapping and GraphQL schema generation from terminating:\n" \ + "#{descriptions.join("\n")}" + end + + @no_circular_dependencies = true + end + + def recursively_add_referenced_types_to(source_type_ref, references_cache) + return unless (source_type = source_type_ref.as_object_type) + references_set = references_cache[source_type_ref.name] + + # Recursive references are allowed only when its a relation, so skip that case. + source_type.graphql_fields_by_name.values.reject { |f| f.relationship }.each do |field| + field_type = field.type.fully_unwrapped + + if field_type.object? && references_set.add?(field_type.name) + recursively_add_referenced_types_to(field_type, references_cache) + end + + references_set.merge(references_cache[field_type.name]) + end + end + + def all_types_except_root_query_type + @all_types_except_root_query_type ||= state.types_by_name.values.flat_map do |registered_type| + related_types = [registered_type] + registered_type.derived_graphql_types + apply_customizations_to(related_types, registered_type) + related_types + end + end + + def apply_customizations_to(types, registered_type) + built_in_customizers = state.built_in_types_customization_blocks + if built_in_customizers.any? && state.initially_registered_built_in_types.include?(registered_type.name) + types.each do |type| + built_in_customizers.each do |customization_block| + customization_block.call(type) + end + end + end + + unless (unknown_type_names = registered_type.derived_type_customizations_by_name.keys - types.map(&:name)).empty? + raise Errors::SchemaError, + "`customize_derived_types` was called on `#{registered_type.name}` with some unrecognized type names " \ + "(#{unknown_type_names.join(", ")}). Maybe some of the derived GraphQL types are misspelled?" + end + + unless (unknown_type_names = registered_type.derived_field_customizations_by_type_and_field_name.keys - types.map(&:name)).empty? + raise Errors::SchemaError, + "`customize_derived_type_fields` was called on `#{registered_type.name}` with some unrecognized type names " \ + "(#{unknown_type_names.join(", ")}). Maybe some of the derived GraphQL types are misspelled?" + end + + unknown_field_names = (types - [registered_type]).flat_map do |type| + registered_type.derived_type_customizations_for_type(type).each { |b| b.call(type) } + field_customizations_by_name = registered_type.derived_field_customizations_by_name_for_type(type) + + if field_customizations_by_name.any? && !type.respond_to?(:graphql_fields_by_name) + raise Errors::SchemaError, + "`customize_derived_type_fields` was called on `#{registered_type.name}` with a type that can " \ + "never have fields: `#{type.name}`." + end + + field_customizations_by_name.filter_map do |field_name, customization_blocks| + if (field = (_ = type).graphql_fields_by_name[field_name]) + customization_blocks.each { |b| b.call(field) } + nil + else + "#{type.name}.#{field_name}" + end + end + end + + unless unknown_field_names.empty? + raise Errors::SchemaError, + "`customize_derived_type_fields` was called on `#{registered_type.name}` with some unrecognized field names " \ + "(#{unknown_field_names.join(", ")}). Maybe one of the field names was misspelled?" + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb new file mode 100644 index 00000000..9814f111 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb @@ -0,0 +1,482 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "did_you_mean" +require "elastic_graph/constants" +require "elastic_graph/schema_definition/json_schema_pruner" +require "elastic_graph/support/memoizable_data" +require "fileutils" +require "graphql" +require "tempfile" +require "yaml" + +module ElasticGraph + module SchemaDefinition + # Manages schema artifacts. Note: not tested directly. Instead, the `RakeTasks` tests drive this class. + # + # Note that we use `abort` instead of `raise` here for exceptions that require the user to perform an action + # to resolve. The output from `abort` is cleaner (no stack trace, etc) which improves the signal-to-noise + # ratio for the user to (hopefully) make it easier to understand what to do, without needing to wade through + # extra output. + # + # @private + class SchemaArtifactManager + # @dynamic schema_definition_results + attr_reader :schema_definition_results + + def initialize(schema_definition_results:, schema_artifacts_directory:, enforce_json_schema_version:, output:, max_diff_lines: 50) + @schema_definition_results = schema_definition_results + @schema_artifacts_directory = schema_artifacts_directory + @enforce_json_schema_version = enforce_json_schema_version + @output = output + @max_diff_lines = max_diff_lines + + @json_schemas_artifact = new_yaml_artifact( + JSON_SCHEMAS_FILE, + JSONSchemaPruner.prune(schema_definition_results.current_public_json_schema), + extra_comment_lines: [ + "This is the \"public\" JSON schema file and is intended to be provided to publishers so that", + "they can perform code generation and event validation." + ] + ) + + # Here we round-trip the SDL string through the GraphQL gem's formatting logic. This provides + # nice, consistent formatting (alphabetical order, consistent spacing, etc) and also prunes out + # any "orphaned" schema types (that is, types that are defined but never referenced). + # We also prepend a line break so there's a blank line between the comment block and the + # schema elements. + graphql_schema = ::GraphQL::Schema.from_definition(schema_definition_results.graphql_schema_string).to_definition.chomp + + unversioned_artifacts = [ + new_yaml_artifact(DATASTORE_CONFIG_FILE, schema_definition_results.datastore_config), + new_yaml_artifact(RUNTIME_METADATA_FILE, pruned_runtime_metadata(graphql_schema).to_dumpable_hash), + @json_schemas_artifact, + new_raw_artifact(GRAPHQL_SCHEMA_FILE, "\n" + graphql_schema) + ] + + versioned_artifacts = build_desired_versioned_json_schemas(@json_schemas_artifact.desired_contents).values.map do |versioned_schema| + new_versioned_json_schema_artifact(versioned_schema) + end + + @artifacts = (unversioned_artifacts + versioned_artifacts).sort_by(&:file_name) + notify_about_unused_type_name_overrides + notify_about_unused_enum_value_overrides + end + + # Dumps all the schema artifacts to disk. + def dump_artifacts + check_if_needs_json_schema_version_bump do |recommended_json_schema_version| + if @enforce_json_schema_version + # @type var setter_location: ::Thread::Backtrace::Location + # We use `_ =` because while `json_schema_version_setter_location` can be nil, + # it'll never be nil if we get here and we want the type to be non-nilable. + setter_location = _ = schema_definition_results.json_schema_version_setter_location + setter_location_path = ::Pathname.new(setter_location.absolute_path.to_s).relative_path_from(::Dir.pwd) + + abort "A change has been attempted to `json_schemas.yaml`, but the `json_schema_version` has not been correspondingly incremented. Please " \ + "increase the schema's version, and then run the `schema_artifacts:dump` command again.\n\n" \ + "To update the schema version to the expected version, change line #{setter_location.lineno} at `#{setter_location_path}` to:\n" \ + " `schema.json_schema_version #{recommended_json_schema_version}`\n\n" \ + "Alternately, pass `enforce_json_schema_version: false` to `ElasticGraph::SchemaDefinition::RakeTasks.new` to allow the JSON schemas " \ + "file to change without requiring a version bump, but that is only recommended for non-production applications during initial schema prototyping." + else + @output.puts <<~EOS + WARNING: the `json_schemas.yaml` artifact is being updated without the `json_schema_version` being correspondingly incremented. + This is not recommended for production applications, but is currently allowed because you have set `enforce_json_schema_version: false`. + EOS + end + end + + ::FileUtils.mkdir_p(@schema_artifacts_directory) + @artifacts.each { |artifact| artifact.dump(@output) } + end + + # Checks that all schema artifacts are up-to-date, raising an exception if not. + def check_artifacts + out_of_date_artifacts = @artifacts.select(&:out_of_date?) + + if out_of_date_artifacts.empty? + descriptions = @artifacts.map.with_index(1) { |art, i| "#{i}. #{art.file_name}" } + @output.puts <<~EOS + Your schema artifacts are all up to date: + #{descriptions.join("\n")} + + EOS + else + abort artifacts_out_of_date_error(out_of_date_artifacts) + end + end + + private + + def notify_about_unused_type_name_overrides + type_namer = @schema_definition_results.state.type_namer + return if (unused_overrides = type_namer.unused_name_overrides).empty? + + suggester = ::DidYouMean::SpellChecker.new(dictionary: type_namer.used_names.to_a) + warnings = unused_overrides.map.with_index(1) do |(unused_name, _), index| + alternatives = suggester.correct(unused_name).map { |alt| "`#{alt}`" } + "#{index}. The type name override `#{unused_name}` does not match any type in your GraphQL schema and has been ignored." \ + "#{" Possible alternatives: #{alternatives.join(", ")}." unless alternatives.empty?}" + end + + @output.puts <<~EOS + WARNING: #{unused_overrides.size} of the `type_name_overrides` do not match any type(s) in your GraphQL schema: + + #{warnings.join("\n")} + EOS + end + + def notify_about_unused_enum_value_overrides + enum_value_namer = @schema_definition_results.state.enum_value_namer + return if (unused_overrides = enum_value_namer.unused_overrides).empty? + + used_value_names_by_type_name = enum_value_namer.used_value_names_by_type_name + type_suggester = ::DidYouMean::SpellChecker.new(dictionary: used_value_names_by_type_name.keys) + index = 0 + warnings = unused_overrides.flat_map do |type_name, overrides| + if used_value_names_by_type_name.key?(type_name) + value_suggester = ::DidYouMean::SpellChecker.new(dictionary: used_value_names_by_type_name.fetch(type_name)) + overrides.map do |(value_name), _| + alternatives = value_suggester.correct(value_name).map { |alt| "`#{alt}`" } + "#{index += 1}. The enum value override `#{type_name}.#{value_name}` does not match any enum value in your GraphQL schema and has been ignored." \ + "#{" Possible alternatives: #{alternatives.join(", ")}." unless alternatives.empty?}" + end + else + alternatives = type_suggester.correct(type_name).map { |alt| "`#{alt}`" } + ["#{index += 1}. `enum_value_overrides_by_type` has a `#{type_name}` key, which does not match any enum type in your GraphQL schema and has been ignored." \ + "#{" Possible alternatives: #{alternatives.join(", ")}." unless alternatives.empty?}"] + end + end + + @output.puts <<~EOS + WARNING: some of the `enum_value_overrides_by_type` do not match any type(s)/value(s) in your GraphQL schema: + + #{warnings.join("\n")} + EOS + end + + def build_desired_versioned_json_schemas(current_public_json_schema) + versioned_parsed_yamls = ::Dir.glob(::File.join(@schema_artifacts_directory, JSON_SCHEMAS_BY_VERSION_DIRECTORY, "v*.yaml")).map do |file| + ::YAML.safe_load_file(file) + end + [current_public_json_schema] + + results_by_json_schema_version = versioned_parsed_yamls.to_h do |parsed_yaml| + merged_schema = @schema_definition_results.merge_field_metadata_into_json_schema(parsed_yaml) + [merged_schema.json_schema_version, merged_schema] + end + + report_json_schema_merge_errors(results_by_json_schema_version.values) + report_json_schema_merge_warnings + + results_by_json_schema_version.transform_values(&:json_schema) + end + + def report_json_schema_merge_errors(merged_results) + json_schema_versions_by_missing_field = ::Hash.new { |h, k| h[k] = [] } + json_schema_versions_by_missing_type = ::Hash.new { |h, k| h[k] = [] } + json_schema_versions_by_missing_necessary_field = ::Hash.new { |h, k| h[k] = [] } + + merged_results.each do |result| + result.missing_fields.each do |field| + json_schema_versions_by_missing_field[field] << result.json_schema_version + end + + result.missing_types.each do |type| + json_schema_versions_by_missing_type[type] << result.json_schema_version + end + + result.missing_necessary_fields.each do |missing_necessary_field| + json_schema_versions_by_missing_necessary_field[missing_necessary_field] << result.json_schema_version + end + end + + missing_field_errors = json_schema_versions_by_missing_field.map do |field, json_schema_versions| + missing_field_error_for(field, json_schema_versions) + end + + missing_type_errors = json_schema_versions_by_missing_type.map do |type, json_schema_versions| + missing_type_error_for(type, json_schema_versions) + end + + missing_necessary_field_errors = json_schema_versions_by_missing_necessary_field.map do |field, json_schema_versions| + missing_necessary_field_error_for(field, json_schema_versions) + end + + definition_conflict_errors = merged_results + .flat_map { |result| result.definition_conflicts.to_a } + .group_by(&:name) + .map do |name, deprecated_elements| + <<~EOS + The schema definition of `#{name}` has conflicts. To resolve the conflict, remove the unneeded definitions from the following: + + #{format_deprecated_elements(deprecated_elements)} + EOS + end + + errors = missing_field_errors + missing_type_errors + missing_necessary_field_errors + definition_conflict_errors + return if errors.empty? + + abort errors.join("\n\n") + end + + def report_json_schema_merge_warnings + unused_elements = @schema_definition_results.unused_deprecated_elements + return if unused_elements.empty? + + @output.puts <<~EOS + The schema definition has #{unused_elements.size} unneeded reference(s) to deprecated schema elements. These can all be safely deleted: + + #{format_deprecated_elements(unused_elements)} + + EOS + end + + def format_deprecated_elements(deprecated_elements) + descriptions = deprecated_elements + .sort_by { |e| [e.defined_at.path, e.defined_at.lineno] } + .map(&:description) + .uniq + + descriptions.each.with_index(1).map { |desc, idx| "#{idx}. #{desc}" }.join("\n") + end + + def missing_field_error_for(qualified_field, json_schema_versions) + type, field = qualified_field.split(".") + + <<~EOS + The `#{qualified_field}` field (which existed in #{describe_json_schema_versions(json_schema_versions, "and")}) no longer exists in the current schema definition. + ElasticGraph cannot guess what it should do with this field's data when ingesting events at #{old_versions(json_schema_versions)}. + To continue, do one of the following: + + 1. If the `#{qualified_field}` field has been renamed, indicate this by calling `field.renamed_from "#{field}"` on the renamed field. + 2. If the `#{qualified_field}` field has been dropped, indicate this by calling `type.deleted_field "#{field}"` on the `#{type}` type. + 3. Alternately, if no publishers or in-flight events use #{describe_json_schema_versions(json_schema_versions, "or")}, delete #{files_noun_phrase(json_schema_versions)} from `#{JSON_SCHEMAS_BY_VERSION_DIRECTORY}`, and no further changes are required. + EOS + end + + def missing_type_error_for(type, json_schema_versions) + <<~EOS + The `#{type}` type (which existed in #{describe_json_schema_versions(json_schema_versions, "and")}) no longer exists in the current schema definition. + ElasticGraph cannot guess what it should do with this type's data when ingesting events at #{old_versions(json_schema_versions)}. + To continue, do one of the following: + + 1. If the `#{type}` type has been renamed, indicate this by calling `type.renamed_from "#{type}"` on the renamed type. + 2. If the `#{type}` field has been dropped, indicate this by calling `schema.deleted_type "#{type}"` on the schema. + 3. Alternately, if no publishers or in-flight events use #{describe_json_schema_versions(json_schema_versions, "or")}, delete #{files_noun_phrase(json_schema_versions)} from `#{JSON_SCHEMAS_BY_VERSION_DIRECTORY}`, and no further changes are required. + EOS + end + + def missing_necessary_field_error_for(field, json_schema_versions) + path = field.fully_qualified_path.split(".").last + # :nocov: -- we only cover one side of this ternary. + has_or_have = (json_schema_versions.size == 1) ? "has" : "have" + # :nocov: + + <<~EOS + #{describe_json_schema_versions(json_schema_versions, "and")} #{has_or_have} no field that maps to the #{field.field_type} field path of `#{field.fully_qualified_path}`. + Since the field path is required for #{field.field_type}, ElasticGraph cannot ingest events that lack it. To continue, do one of the following: + + 1. If the `#{field.fully_qualified_path}` field has been renamed, indicate this by calling `field.renamed_from "#{path}"` on the renamed field rather than using `deleted_field`. + 2. Alternately, if no publishers or in-flight events use #{describe_json_schema_versions(json_schema_versions, "or")}, delete #{files_noun_phrase(json_schema_versions)} from `#{JSON_SCHEMAS_BY_VERSION_DIRECTORY}`, and no further changes are required. + EOS + end + + def describe_json_schema_versions(json_schema_versions, conjunction) + json_schema_versions = json_schema_versions.sort + + # Steep doesn't support pattern matching yet, so have to skip type checking here. + __skip__ = case json_schema_versions + in [single_version] + "JSON schema version #{single_version}" + in [version1, version2] + "JSON schema versions #{version1} #{conjunction} #{version2}" + else + *versions, last_version = json_schema_versions + "JSON schema versions #{versions.join(", ")}, #{conjunction} #{last_version}" + end + end + + def old_versions(json_schema_versions) + return "this old version" if json_schema_versions.size == 1 + "these old versions" + end + + def files_noun_phrase(json_schema_versions) + return "its file" if json_schema_versions.size == 1 + "their files" + end + + def artifacts_out_of_date_error(out_of_date_artifacts) + # @type var diffs: ::Array[[SchemaArtifact[untyped], ::String]] + diffs = [] + + descriptions = out_of_date_artifacts.map.with_index(1) do |artifact, index| + reason = + if (diff = artifact.diff(color: @output.tty?)) + description, diff = truncate_diff(diff, @max_diff_lines) + diffs << [artifact, diff] + "see [#{diffs.size}] below for the #{description}" + else + "file does not exist" + end + + "#{index}. #{artifact.file_name} (#{reason})" + end + + diffs = diffs.map.with_index(1) do |(artifact, diff), index| + <<~EOS + [#{index}] #{artifact.file_name} diff: + #{diff} + EOS + end + + <<~EOS.strip + #{out_of_date_artifacts.size} schema artifact(s) are out of date. Run `rake schema_artifacts:dump` to update the following artifact(s): + + #{descriptions.join("\n")} + + #{diffs.join("\n")} + EOS + end + + def truncate_diff(diff, lines) + diff_lines = diff.lines + + if diff_lines.size <= lines + ["diff", diff] + else + truncated = diff_lines.first(lines).join + ["first #{lines} lines of the diff", truncated] + end + end + + def new_yaml_artifact(file_name, desired_contents, extra_comment_lines: []) + SchemaArtifact.new( + ::File.join(@schema_artifacts_directory, file_name), + desired_contents, + ->(hash) { ::YAML.dump(hash) }, + ->(string) { ::YAML.safe_load(string) }, + extra_comment_lines + ) + end + + def new_versioned_json_schema_artifact(desired_contents) + # File name depends on the schema_version field in the json schema. + schema_version = desired_contents[JSON_SCHEMA_VERSION_KEY] + + new_yaml_artifact( + ::File.join(JSON_SCHEMAS_BY_VERSION_DIRECTORY, "v#{schema_version}.yaml"), + desired_contents, + extra_comment_lines: [ + "This JSON schema file contains internal ElasticGraph metadata and should be considered private.", + "The unversioned JSON schema file is public and intended to be provided to publishers." + ] + ) + end + + def new_raw_artifact(file_name, desired_contents) + SchemaArtifact.new( + ::File.join(@schema_artifacts_directory, file_name), + desired_contents, + _ = :itself.to_proc, + _ = :itself.to_proc, + [] + ) + end + + def check_if_needs_json_schema_version_bump(&block) + if @json_schemas_artifact.out_of_date? + existing_schema_version = @json_schemas_artifact.existing_dumped_contents&.dig(JSON_SCHEMA_VERSION_KEY) || -1 + desired_schema_version = @json_schemas_artifact.desired_contents[JSON_SCHEMA_VERSION_KEY] + + if existing_schema_version >= desired_schema_version + yield existing_schema_version + 1 + end + end + end + + def pruned_runtime_metadata(graphql_schema_string) + schema = ::GraphQL::Schema.from_definition(graphql_schema_string) + runtime_meta = schema_definition_results.runtime_metadata + + schema_type_names = schema.types.keys + pruned_enum_types = runtime_meta.enum_types_by_name.slice(*schema_type_names) + pruned_scalar_types = runtime_meta.scalar_types_by_name.slice(*schema_type_names) + pruned_object_types = runtime_meta.object_types_by_name.slice(*schema_type_names) + + runtime_meta.with( + enum_types_by_name: pruned_enum_types, + scalar_types_by_name: pruned_scalar_types, + object_types_by_name: pruned_object_types + ) + end + end + + # @private + class SchemaArtifact < Support::MemoizableData.define(:file_name, :desired_contents, :dumper, :loader, :extra_comment_lines) + def dump(output) + if out_of_date? + dirname = File.dirname(file_name) + FileUtils.mkdir_p(dirname) # Create directory if needed. + + ::File.write(file_name, dumped_contents) + output.puts "Dumped schema artifact to `#{file_name}`." + else + output.puts "`#{file_name}` is already up to date." + end + end + + def out_of_date? + (_ = existing_dumped_contents) != desired_contents + end + + def existing_dumped_contents + return nil unless exists? + + # We drop the first 2 lines because it is the comment block containing dynamic elements. + file_contents = ::File.read(file_name).split("\n").drop(2).join("\n") + loader.call(file_contents) + end + + def diff(color:) + return nil unless exists? + + ::Tempfile.create do |f| + f.write(dumped_contents.chomp) + f.fsync + + `git diff --no-index #{file_name} #{f.path}#{" --color" if color}` + .gsub(file_name, "existing_contents") + .gsub(f.path, "/updated_contents") + end + end + + private + + def exists? + return !!@exists if defined?(@exists) + @exists = ::File.exist?(file_name) + end + + def dumped_contents + @dumped_contents ||= "#{comment_preamble}\n#{dumper.call(desired_contents)}" + end + + def comment_preamble + lines = [ + "Generated by `rake schema_artifacts:dump`.", + "DO NOT EDIT BY HAND. Any edits will be lost the next time the rake task is run." + ] + + lines = extra_comment_lines + [""] + lines unless extra_comment_lines.empty? + lines.map { |line| "# #{line}".strip }.join("\n") + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/argument.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/argument.rb new file mode 100644 index 00000000..c3eeb41d --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/argument.rb @@ -0,0 +1,56 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_definition/mixins/has_directives" +require "elastic_graph/schema_definition/mixins/has_documentation" +require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" +require "elastic_graph/schema_definition/mixins/supports_default_value" +require "elastic_graph/schema_definition/mixins/verifies_graphql_name" + +module ElasticGraph + module SchemaDefinition + # Namespace for classes which represent GraphQL schema elements. + module SchemaElements + # Represents a [GraphQL argument](https://spec.graphql.org/October2021/#sec-Language.Arguments). + # + # @!attribute [r] schema_def_state + # @return [State] state of the schema + # @!attribute [r] parent_field + # @return [Field] field which has this argument + # @!attribute [r] name + # @return [String] name of the argument + # @!attribute [r] original_value_type + # @return [TypeReference] type of the argument, as originally provided + # @see #value_type + class Argument < Struct.new(:schema_def_state, :parent_field, :name, :original_value_type) + prepend Mixins::VerifiesGraphQLName + prepend Mixins::SupportsDefaultValue + include Mixins::HasDocumentation + include Mixins::HasDirectives + include Mixins::HasReadableToSAndInspect.new { |a| "#{a.parent_field.parent_type.name}.#{a.parent_field.name}(#{a.name}: #{a.value_type})" } + + # @return [String] GraphQL SDL form of the argument + def to_sdl + "#{formatted_documentation}#{name}: #{value_type}#{default_value_sdl}#{directives_sdl(prefix_with: " ")}" + end + + # When the argument type is an enum, and we're configured with different naming for input vs output enums, + # we need to convert the value type to its input form. Note that this intentionally happens lazily (rather than + # doing this when `Argument` is instantiated), because the referenced type need not exist when the argument + # is defined, and we may not be able to figure out if it's an enum until the type has been defined. So, we + # apply this lazily. + # + # @return [TypeReference] the type of the argument + # @see #original_value_type + def value_type + original_value_type.to_final_form(as_input: true) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb new file mode 100644 index 00000000..29aaf721 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb @@ -0,0 +1,1524 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/graphql/scalar_coercion_adapters/valid_time_zones" +require "elastic_graph/schema_artifacts/runtime_metadata/enum" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Defines all built-in GraphQL types provided by ElasticGraph. + # + # ## Scalar Types + # + # ### Standard GraphQL Scalars + # + # These are defined by the [GraphQL spec](https://spec.graphql.org/October2021/#sec-Scalars.Built-in-Scalars). + # + # Boolean + # : Represents `true` or `false` values. + # + # Float + # : Represents signed double-precision fractional values as specified by + # [IEEE 754](https://en.wikipedia.org/wiki/IEEE_floating_point). + # + # ID + # : Represents a unique identifier that is Base64 obfuscated. It is often used to + # refetch an object or as key for a cache. The ID type appears in a JSON response as a + # String; however, it is not intended to be human-readable. When expected as an input + # type, any string (such as `"VXNlci0xMA=="`) or integer (such as `4`) input value will + # be accepted as an ID. + # + # Int + # : Represents non-fractional signed whole numeric values. Int can represent values between + # -(2^31) and 2^31 - 1. + # + # String + # : Represents textual data as UTF-8 character sequences. This type is most often used by + # GraphQL to represent free-form human-readable text. + # + # ### Additional ElasticGraph Scalars + # + # ElasticGraph defines these additional scalar types. + # + # Cursor + # : An opaque string value representing a specific location in a paginated connection type. + # Returned cursors can be passed back in the next query via the `before` or `after` + # arguments to continue paginating from that point. + # + # Date + # : A date, represented as an [ISO 8601 date string](https://en.wikipedia.org/wiki/ISO_8601). + # + # DateTime + # : A timestamp, represented as an [ISO 8601 time string](https://en.wikipedia.org/wiki/ISO_8601). + # + # JsonSafeLong + # : A numeric type for large integer values that can serialize safely as JSON. While JSON + # itself has no hard limit on the size of integers, the RFC-7159 spec mentions that + # values outside of the range -9,007,199,254,740,991 (-(2^53) + 1) to 9,007,199,254,740,991 + # (2^53 - 1) may not be interopable with all JSON implementations. As it turns out, the + # number implementation used by JavaScript has this issue. When you parse a JSON string that + # contains a numeric value like `4693522397653681111`, the parsed result will contain a + # rounded value like `4693522397653681000`. While this is entirely a client-side problem, + # we want to preserve maximum compatibility with common client languages. Given the ubiquity + # of GraphiQL as a GraphQL client, we want to avoid this problem. Our solution is to support + # two separate types: + # + # - This type (`JsonSafeLong`) is serialized as a number, but limits values to the safely + # serializable range. + # - The `LongString` type supports long values that use all 64 bits, but serializes as a + # string rather than a number, avoiding the JavaScript compatibility problems. For more + # background, see the [JavaScript `Number.MAX_SAFE_INTEGER` + # docs](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER). + # + # LocalTime + # : A local time such as `"23:59:33"` or `"07:20:47.454"` without a time zone or offset, + # formatted based on the [partial-time portion of + # RFC3339](https://datatracker.ietf.org/doc/html/rfc3339#section-5.6). + # + # LongString + # : A numeric type for large integer values in the inclusive range -2^63 (-9,223,372,036,854,775,808) + # to (2^63 - 1) (9,223,372,036,854,775,807). Note that `LongString` values are serialized as strings + # within JSON, to avoid interopability problems with JavaScript. If you want a large integer type + # that serializes within JSON as a number, use `JsonSafeLong`. + # + # TimeZone + # : An [IANA time zone identifier](https://www.iana.org/time-zones), such as `America/Los_Angeles` + # or `UTC`. For a full list of valid identifiers, see the + # [wikipedia article](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List). + # + # Untyped + # : A custom scalar type that allows any type of data, including: + # + # - strings + # - numbers + # - objects and arrays (nested as deeply as you like) + # - booleans + # + # Note: fields of this type are effectively untyped. We recommend it only be used for parts + # of your schema that can't be statically typed. + # + # ## Enum Types + # + # ElasticGraph defines these enum types. Most of these are intended for usage as an _input_ + # argument, but they could be used as a return type in your schema if they meet your needs. + # + # DateGroupingGranularity + # : Enumerates the supported granularities of a `Date`. + # + # DateGroupingTruncationUnit + # : Enumerates the supported truncation units of a `Date`. + # + # DateTimeGroupingGranularity + # : Enumerates the supported granularities of a `DateTime`. + # + # DateTimeGroupingTruncationUnit + # : Enumerates the supported truncation units of a `DateTime`. + # + # DateTimeUnit + # : Enumeration of `DateTime` units. + # + # DateUnit + # : Enumeration of `Date` units. + # + # DayOfWeek + # : Indicates the specific day of the week. + # + # DistanceUnit + # : Enumerates the supported distance units. + # + # LocalTimeGroupingTruncationUnit + # : Enumerates the supported truncation units of a `LocalTime`. + # + # LocalTimeUnit + # : Enumeration of `LocalTime` units. + # + # MatchesQueryAllowedEditsPerTerm + # : Enumeration of allowed values for the `matchesQuery: {allowedEditsPerTerm: ...}` filter option. + # + # ## Object Types + # + # ElasticGraph defines these object types. + # + # AggregationCountDetail + # : Provides detail about an aggregation `count`. + # + # GeoLocation + # : Geographic coordinates representing a location on the Earth's surface. + # + # PageInfo + # : Provides information about the specific fetched page. This implements the + # `PageInfo` specification from the [Relay GraphQL Cursor Connections + # Specification](https://relay.dev/graphql/connections.htm#sec-undefined.PageInfo). + # + # @!attribute [rw] schema_def_api + # @private + # @!attribute [rw] schema_def_state + # @private + # @!attribute [rw] names + # @private + class BuiltInTypes + attr_reader :schema_def_api, :schema_def_state, :names + + # @private + def initialize(schema_def_api, schema_def_state) + @schema_def_api = schema_def_api + @schema_def_state = schema_def_state + @names = schema_def_state.schema_elements + end + + # @private + def register_built_in_types + register_directives + register_standard_graphql_scalars + register_custom_elastic_graph_scalars + register_enum_types + register_date_and_time_grouped_by_types + register_standard_elastic_graph_types + end + + private + + def register_directives + # Note: The `eg` prefix is being used based on a GraphQL Spec recommendation: + # http://spec.graphql.org/October2021/#sec-Type-System.Directives.Custom-Directives + schema_def_api.raw_sdl <<~EOS + """ + Indicates an upper bound on how quickly a query must respond to meet the service-level objective. + ElasticGraph will log a "good event" message if the query latency is less than or equal to this value, + and a "bad event" message if the query latency is greater than this value. These messages can be used + to drive an SLO dashboard. + + Note that the latency compared against this only contains processing time within ElasticGraph itself. + Any time spent on sending the request or response over the network is not included in the comparison. + """ + directive @#{names.eg_latency_slo}(#{names.ms}: Int!) on QUERY + EOS + end + + def register_standard_elastic_graph_types + # This is a special filter on a `String` type, so we don't have a `Text` scalar to generate it from. + schema_def_state.factory.build_standard_filter_input_types_for_index_leaf_type("String", name_prefix: "Text") do |t| + # We can't support filtering on `null` within a list, so make the field non-nullable when it's the + # `ListElementFilterInput` type. See scalar_type.rb for a larger comment explaining the rationale behind this. + equal_to_any_of_type = t.type_ref.list_element_filter_input? ? "[String!]" : "[String]" + t.field names.equal_to_any_of, equal_to_any_of_type do |f| + f.documentation ScalarType::EQUAL_TO_ANY_OF_DOC + end + + t.field names.matches, "String" do |f| + f.documentation <<~EOS + Matches records where the field value matches the provided value using full text search. + + Will be ignored when `null` is passed. + EOS + + f.directive "deprecated", reason: "Use `#{names.matches_query}` instead." + end + + t.field names.matches_query, schema_def_state.type_ref("MatchesQuery").as_filter_input.name do |f| + f.documentation <<~EOS + Matches records where the field value matches the provided query using full text search. + This is more lenient than `#{names.matches_phrase}`: the order of terms is ignored, and, + by default, only one search term is required to be in the field value. + + Will be ignored when `null` is passed. + EOS + end + + t.field names.matches_phrase, schema_def_state.type_ref("MatchesPhrase").as_filter_input.name do |f| + f.documentation <<~EOS + Matches records where the field value has a phrase matching the provided phrase using + full text search. This is stricter than `#{names.matches_query}`: all terms must match + and be in the same order as the provided phrase. + + Will be ignored when `null` is passed. + EOS + end + end.each do |input_type| + field_type = input_type.type_ref.list_filter_input? ? "[String]" : "String" + input_type.documentation <<~EOS + Input type used to specify filters on `#{field_type}` fields that have been indexed for full text search. + + Will be ignored if passed as an empty object (or as `null`). + EOS + + register_input_type(input_type) + end + + register_filter "MatchesQuery" do |t| + t.documentation <<~EOS + Input type used to specify parameters for the `#{names.matches_query}` filtering operator. + + Will be ignored if passed as `null`. + EOS + + t.field names.query, "String!" do |f| + f.documentation "The input query to search for." + end + + t.field names.allowed_edits_per_term, "MatchesQueryAllowedEditsPerTerm!" do |f| + f.documentation <<~EOS + Number of allowed modifications per term to arrive at a match. For example, if set to 'ONE', the input + term 'glue' would match 'blue' but not 'clued', since the latter requires two modifications. + EOS + + f.default "DYNAMIC" + end + + t.field names.require_all_terms, "Boolean!" do |f| + f.documentation <<~EOS + Set to `true` to match only if all terms in `#{names.query}` are found, or + `false` to only require one term to be found. + EOS + + f.default false + end + + # any_of/not don't really make sense on this filter because it doesn't make sense to + # apply an OR operator or negation to the fields of this type since they are all an + # indivisible part of a single filter operation on a specific field. So we remove them + # here. + remove_any_of_and_not_filter_operators_on(t) + end + + register_filter "MatchesPhrase" do |t| + t.documentation <<~EOS + Input type used to specify parameters for the `#{names.matches_phrase}` filtering operator. + + Will be ignored if passed as `null`. + EOS + + t.field names.phrase, "String!" do |f| + f.documentation "The input phrase to search for." + end + + # any_of/not don't really make sense on this filter because it doesn't make sense to + # apply an OR operator or negation to the fields of this type since they are all an + # indivisible part of a single filter operation on a specific field. So we remove them + # here. + remove_any_of_and_not_filter_operators_on(t) + end + + # This is defined as a built-in ElasticGraph type so that we can leverage Elasticsearch/OpenSearch GeoLocation features + # based on the geo-point type: + # https://www.elastic.co/guide/en/elasticsearch/reference/7.10/geo-point.html + schema_def_api.object_type "GeoLocation" do |t| + t.documentation "Geographic coordinates representing a location on the Earth's surface." + + # As per the Elasticsearch docs, the field MUST come in named `lat` in Elastisearch (but we want the full name in GraphQL). + t.field names.latitude, "Float", name_in_index: "lat" do |f| + f.documentation "Angular distance north or south of the Earth's equator, measured in degrees from -90 to +90." + + # Note: we use `nullable: false` because we index it as a single `geo_point` field, and therefore can't + # support a `latitude` without a `longitude` or vice-versa. + f.json_schema minimum: -90, maximum: 90, nullable: false + end + + # As per the Elasticsearch docs, the field MUST come in named `lon` in Elastisearch (but we want the full name in GraphQL). + t.field names.longitude, "Float", name_in_index: "lon" do |f| + f.documentation "Angular distance east or west of the Prime Meridian at Greenwich, UK, measured in degrees from -180 to +180." + + # Note: we use `nullable: false` because we index it as a single `geo_point` field, and therefore can't + # support a `latitude` without a `longitude` or vice-versa. + f.json_schema minimum: -180, maximum: 180, nullable: false + end + + t.mapping type: "geo_point" + end + + # Note: `GeoLocation` is an index leaf type even though it is a GraphQL object type. In the datastore, + # it is indexed as an indivisible `geo_point` field. + schema_def_state.factory.build_standard_filter_input_types_for_index_leaf_type("GeoLocation") do |t| + t.field names.near, schema_def_state.type_ref("GeoLocationDistance").as_filter_input.name do |f| + f.documentation <<~EOS + Matches records where the field's geographic location is within a specified distance from the + location identified by `#{names.latitude}` and `#{names.longitude}`. + + Will be ignored when `null` or an empty object is passed. + EOS + end + end.each { |input_filter| register_input_type(input_filter) } + + register_filter "GeoLocationDistance" do |t| + t.documentation "Input type used to specify distance filtering parameters on `GeoLocation` fields." + + # Note: all 4 of these fields (latitude, longitude, max_distance, unit) are required for this + # filter to operator properly, so they are all non-null fields. + + t.field names.latitude, "Float!" do |f| + f.documentation "Angular distance north or south of the Earth's equator, measured in degrees from -90 to +90." + end + + t.field names.longitude, "Float!" do |f| + f.documentation "Angular distance east or west of the Prime Meridian at Greenwich, UK, measured in degrees from -180 to +180." + end + + t.field names.max_distance, "Float!" do |f| + f.documentation <<~EOS + Maximum distance (of the provided `#{names.unit}`) to consider "near" the location identified + by `#{names.latitude}` and `#{names.longitude}`. + EOS + end + + t.field names.unit, "DistanceUnit!" do |f| + f.documentation "Determines the unit of the specified `#{names.max_distance}`." + end + + # any_of/not don't really make sense on this filter because it doesn't make sense to + # apply an OR operator or negation to the fields of this type since they are all an + # indivisible part of a single filter operation on a specific field. So we remove them + # here. + remove_any_of_and_not_filter_operators_on(t) + end + + # Note: `has_next_page`/`has_previous_page` are required to be non-null by the relay + # spec: https://relay.dev/graphql/connections.htm#sec-undefined.PageInfo. + # The cursors are required to be non-null by the relay spec, but it is nonsensical + # when dealing with an empty collection, and relay itself implements it to be null: + # + # https://github.com/facebook/relay/commit/a17b462b3ff7355df4858a42ddda75f58c161302 + # + # For more context, see: + # https://github.com/rmosolgo/graphql-ruby/pull/2886#issuecomment-618414736 + # https://github.com/facebook/relay/pull/2655 + # + # For now we will make the cursor fields nullable. It would be a breaking change + # to go from non-null to null, but is not a breaking change to make it non-null + # in the future. + register_framework_object_type "PageInfo" do |t| + t.documentation <<~EOS + Provides information about the specific fetched page. This implements the `PageInfo` + specification from the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-undefined.PageInfo). + EOS + + t.field names.has_next_page, "Boolean!", graphql_only: true do |f| + f.documentation "Indicates if there is another page of results available after the current one." + end + + t.field names.has_previous_page, "Boolean!", graphql_only: true do |f| + f.documentation "Indicates if there is another page of results available before the current one." + end + + t.field names.start_cursor, "Cursor", graphql_only: true do |f| + f.documentation <<~EOS + The `Cursor` of the first edge of the current page. This can be passed in the next query as + a `before` argument to paginate backwards. + EOS + end + + t.field names.end_cursor, "Cursor", graphql_only: true do |f| + f.documentation <<~EOS + The `Cursor` of the last edge of the current page. This can be passed in the next query as + a `after` argument to paginate forwards. + EOS + end + end + + schema_def_api.factory.new_input_type("DateTimeGroupingOffsetInput") do |t| + t.documentation <<~EOS + Input type offered when grouping on `DateTime` fields, representing the amount of offset + (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 1 day to change + what day-of-week weeks are considered to start on. + EOS + + t.field names.amount, "Int!" do |f| + f.documentation "Number (positive or negative) of the given `#{names.unit}` to offset the boundaries of the `DateTime` groupings." + end + + t.field names.unit, "DateTimeUnit!" do |f| + f.documentation "Unit of offsetting to apply to the boundaries of the `DateTime` groupings." + end + + register_input_type(t) + end + + schema_def_api.factory.new_input_type("DateGroupingOffsetInput") do |t| + t.documentation <<~EOS + Input type offered when grouping on `Date` fields, representing the amount of offset + (positive or negative) to shift the `Date` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 1 day to change + what day-of-week weeks are considered to start on. + EOS + + t.field names.amount, "Int!" do |f| + f.documentation "Number (positive or negative) of the given `#{names.unit}` to offset the boundaries of the `Date` groupings." + end + + t.field names.unit, "DateUnit!" do |f| + f.documentation "Unit of offsetting to apply to the boundaries of the `Date` groupings." + end + + register_input_type(t) + end + + schema_def_api.factory.new_input_type("DayOfWeekGroupingOffsetInput") do |t| + t.documentation <<~EOS + Input type offered when grouping on `DayOfWeek` fields, representing the amount of offset + (positive or negative) to shift the `DayOfWeek` boundaries of each grouping bucket. + + For example, you can apply an offset of -2 hours to shift `DateTime` values to the prior `DayOfWeek` + when they fall between midnight and 2 AM. + EOS + + t.field names.amount, "Int!" do |f| + f.documentation "Number (positive or negative) of the given `#{names.unit}` to offset the boundaries of the `DayOfWeek` groupings." + end + + t.field names.unit, "DateTimeUnit!" do |f| + f.documentation "Unit of offsetting to apply to the boundaries of the `DayOfWeek` groupings." + end + + register_input_type(t) + end + + schema_def_api.factory.new_input_type("LocalTimeGroupingOffsetInput") do |t| + t.documentation <<~EOS + Input type offered when grouping on `LocalTime` fields, representing the amount of offset + (positive or negative) to shift the `LocalTime` boundaries of each grouping bucket. + + For example, when grouping by `HOUR`, you can shift by 30 minutes to change + what minute-of-hour hours are considered to start on. + EOS + + t.field names.amount, "Int!" do |f| + f.documentation "Number (positive or negative) of the given `#{names.unit}` to offset the boundaries of the `LocalTime` groupings." + end + + t.field names.unit, "LocalTimeUnit!" do |f| + f.documentation "Unit of offsetting to apply to the boundaries of the `LocalTime` groupings." + end + + register_input_type(t) + end + + schema_def_api.factory.new_aggregated_values_type_for_index_leaf_type "NonNumeric" do |t| + t.documentation "A return type used from aggregations to provided aggregated values over non-numeric fields." + end.tap { |t| schema_def_api.state.register_object_interface_or_union_type(t) } + + register_framework_object_type "AggregationCountDetail" do |t| + t.documentation "Provides detail about an aggregation `#{names.count}`." + + t.field names.approximate_value, "JsonSafeLong!", graphql_only: true do |f| + f.documentation <<~EOS + The (approximate) count of documents in this aggregation bucket. + + When documents in an aggregation bucket are sourced from multiple shards, the count may be only + approximate. The `#{names.upper_bound}` indicates the maximum value of the true count, but usually + the true count is much closer to this approximate value (which also provides a lower bound on the + true count). + + When this approximation is known to be exact, the same value will be available from `#{names.exact_value}` + and `#{names.upper_bound}`. + EOS + end + + t.field names.exact_value, "JsonSafeLong", graphql_only: true do |f| + f.documentation <<~EOS + The exact count of documents in this aggregation bucket, if an exact value can be determined. + + When documents in an aggregation bucket are sourced from multiple shards, it may not be possible to + efficiently determine an exact value. When no exact value can be determined, this field will be `null`. + The `#{names.approximate_value}` field--which will never be `null`--can be used to get an approximation + for the count. + EOS + end + + t.field names.upper_bound, "JsonSafeLong!", graphql_only: true do |f| + f.documentation <<~EOS + An upper bound on how large the true count of documents in this aggregation bucket could be. + + When documents in an aggregation bucket are sourced from multiple shards, it may not be possible to + efficiently determine an exact value. The `#{names.approximate_value}` field provides an approximation, + and this field puts an upper bound on the true count. + EOS + end + end + end + + # Registers the standard GraphQL scalar types. Note that the SDL for the scalar type itself isn't + # included in the dumped SDL, but registering it allows us to derive a filter for each, + # which we need. In addition, this lets us define the mapping and JSON schema for each standard + # scalar type. + def register_standard_graphql_scalars + schema_def_api.scalar_type "Boolean" do |t| + t.mapping type: "boolean" + t.json_schema type: "boolean" + end + + schema_def_api.scalar_type "Float" do |t| + t.mapping type: "double" + t.json_schema type: "number" + + t.customize_aggregated_values_type do |avt| + # not nullable, since sum(empty_set) == 0 + avt.field names.approximate_sum, "Float!", graphql_only: true do |f| + f.runtime_metadata_computation_detail empty_bucket_value: 0, function: :sum + + f.documentation <<~EOS + The sum of the field values within this grouping. + + As with all double-precision `Float` values, operations are subject to floating-point loss + of precision, so the value may be approximate. + EOS + end + + define_exact_min_and_max_on_aggregated_values(avt, "Float") do |adjective:, full_name:| + <<~EOS + The value will be "exact" in that the aggregation computation will return + the exact value of the #{adjective} float that has been indexed, without + introducing any new imprecision. However, floats by their nature are + naturally imprecise since they cannot precisely represent all real numbers. + EOS + end + + avt.field names.approximate_avg, "Float", graphql_only: true do |f| + f.runtime_metadata_computation_detail empty_bucket_value: nil, function: :avg + + f.documentation <<~EOS + The average (mean) of the field values within this grouping. + + The computation of this value may introduce additional imprecision (on top of the + natural imprecision of floats) when it deals with intermediary values that are + outside the `JsonSafeLong` range (#{format_number(JSON_SAFE_LONG_MIN)} to #{format_number(JSON_SAFE_LONG_MAX)}). + EOS + end + end + end + + schema_def_api.scalar_type "ID" do |t| + t.mapping type: "keyword" + t.json_schema type: "string" + end + + schema_def_api.scalar_type "Int" do |t| + t.mapping type: "integer" + t.json_schema type: "integer", minimum: INT_MIN, maximum: INT_MAX + + t.prepare_for_indexing_with "ElasticGraph::Indexer::IndexingPreparers::Integer", + defined_at: "elastic_graph/indexer/indexing_preparers/integer" + + define_integral_aggregated_values_for(t) + end + + schema_def_api.scalar_type "String" do |t| + t.mapping type: "keyword" + t.json_schema type: "string" + end + end + + def register_custom_elastic_graph_scalars + schema_def_api.scalar_type "Cursor" do |t| + # Technically, we don't use the mapping or json_schema on this type since it's a return-only + # type and isn't indexed. However, `scalar_type` requires them to be set (since custom scalars + # defined by users will need those set) so we set them here to what they would be if we actually + # used them. + t.mapping type: "keyword" + t.json_schema type: "string" + t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::Cursor", + defined_at: "elastic_graph/graphql/scalar_coercion_adapters/cursor" + + t.documentation <<~EOS + An opaque string value representing a specific location in a paginated connection type. + Returned cursors can be passed back in the next query via the `before` or `after` + arguments to continue paginating from that point. + EOS + end + + schema_def_api.scalar_type "Date" do |t| + t.mapping type: "date", format: DATASTORE_DATE_FORMAT + t.json_schema type: "string", format: "date" + t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::Date", + defined_at: "elastic_graph/graphql/scalar_coercion_adapters/date" + + t.documentation <<~EOS + A date, represented as an [ISO 8601 date string](https://en.wikipedia.org/wiki/ISO_8601). + EOS + + t.customize_aggregated_values_type do |avt| + define_exact_min_max_and_approx_avg_on_aggregated_values(avt, "Date") do |adjective:, full_name:| + <<~EOS + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + EOS + end + end + end + + schema_def_api.scalar_type "DateTime" do |t| + t.mapping type: "date", format: DATASTORE_DATE_TIME_FORMAT + t.json_schema type: "string", format: "date-time" + t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::DateTime", + defined_at: "elastic_graph/graphql/scalar_coercion_adapters/date_time" + + t.documentation <<~EOS + A timestamp, represented as an [ISO 8601 time string](https://en.wikipedia.org/wiki/ISO_8601). + EOS + + date_time_time_of_day_ref = schema_def_state.type_ref("#{t.type_ref}TimeOfDay") + + t.customize_derived_types( + t.type_ref.as_filter_input.to_final_form(as_input: true).name, + t.type_ref.as_list_element_filter_input.to_final_form(as_input: true).name + ) do |ft| + ft.field names.time_of_day, date_time_time_of_day_ref.as_filter_input.name do |f| + f.documentation <<~EOS + Matches records based on the time-of-day of the `DateTime` values. + + Will be ignored when `null` or an empty list is passed. + EOS + end + end + + t.customize_aggregated_values_type do |avt| + define_exact_min_max_and_approx_avg_on_aggregated_values(avt, "DateTime") do |adjective:, full_name:| + <<~EOS + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + EOS + end + end + + register_filter date_time_time_of_day_ref.name do |t| + t.documentation <<~EOS + Input type used to specify filters on the time-of-day of `DateTime` fields. + + Will be ignored if passed as an empty object (or as `null`). + EOS + + fixup_doc = ->(doc_string) do + doc_string.sub("the field value", "the time of day of the `DateTime` field value") + end + + # Unlike a normal `equal_to_any_of` (which allows nullable elements to allow filtering to null values), we make + # it non-nullable here because it's nonsensical to filter to where a DateTime's time-of-day is null. + t.field names.equal_to_any_of, "[LocalTime!]" do |f| + f.documentation fixup_doc.call(ScalarType::EQUAL_TO_ANY_OF_DOC) + end + + t.field names.gt, "LocalTime" do |f| + f.documentation fixup_doc.call(ScalarType::GT_DOC) + end + + t.field names.gte, "LocalTime" do |f| + f.documentation fixup_doc.call(ScalarType::GTE_DOC) + end + + t.field names.lt, "LocalTime" do |f| + f.documentation fixup_doc.call(ScalarType::LT_DOC) + end + + t.field names.lte, "LocalTime" do |f| + f.documentation fixup_doc.call(ScalarType::LTE_DOC) + end + + t.field names.time_zone, "TimeZone!" do |f| + f.documentation "TimeZone to use when comparing the `DateTime` values against the provided `LocalTime` values." + f.default "UTC" + end + + # With our initial implementation of `time_of_day` filtering, it's tricky to support `any_of`/`not` within + # the `time_of_day: {...}` input object. They are still supported outside of `time_of_day` (on the parent + # input object) so no functionality is losts by omitting these. Also, this aligns with our `GeoLocationDistanceFilterInput` + # which is a similarly complex filter where we didn't include them. + remove_any_of_and_not_filter_operators_on(t) + end + end + + schema_def_api.scalar_type "LocalTime" do |t| + t.documentation <<~EOS + A local time such as `"23:59:33"` or `"07:20:47.454"` without a time zone or offset, formatted based on the + [partial-time portion of RFC3339](https://datatracker.ietf.org/doc/html/rfc3339#section-5.6). + EOS + + t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::LocalTime", + defined_at: "elastic_graph/graphql/scalar_coercion_adapters/local_time" + + t.mapping type: "date", format: "HH:mm:ss||HH:mm:ss.S||HH:mm:ss.SS||HH:mm:ss.SSS" + + t.json_schema type: "string", pattern: VALID_LOCAL_TIME_JSON_SCHEMA_PATTERN + + t.customize_aggregated_values_type do |avt| + define_exact_min_max_and_approx_avg_on_aggregated_values(avt, "LocalTime") do |adjective:, full_name:| + <<~EOS + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + EOS + end + end + end + + schema_def_api.scalar_type "TimeZone" do |t| + t.mapping type: "keyword" + t.json_schema type: "string", enum: GraphQL::ScalarCoercionAdapters::VALID_TIME_ZONES.to_a + t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::TimeZone", + defined_at: "elastic_graph/graphql/scalar_coercion_adapters/time_zone" + + t.documentation <<~EOS + An [IANA time zone identifier](https://www.iana.org/time-zones), such as `America/Los_Angeles` or `UTC`. + + For a full list of valid identifiers, see the [wikipedia article](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List). + EOS + end + + schema_def_api.scalar_type "Untyped" do |t| + # Allow any JSON for this type. The list of supported types is taken from: + # + # https://github.com/json-schema-org/json-schema-spec/blob/draft-07/schema.json#L23-L29 + # + # ...except we are omitting `null` here; it'll be added by the nullability decorator if the field is defined as nullable. + t.json_schema type: ["array", "boolean", "integer", "number", "object", "string"] + + # In the index we store this as a JSON string in a `keyword` field. + t.mapping type: "keyword" + + t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::Untyped", + defined_at: "elastic_graph/graphql/scalar_coercion_adapters/untyped" + + t.prepare_for_indexing_with "ElasticGraph::Indexer::IndexingPreparers::Untyped", + defined_at: "elastic_graph/indexer/indexing_preparers/untyped" + + t.documentation <<~EOS + A custom scalar type that allows any type of data, including: + + - strings + - numbers + - objects and arrays (nested as deeply as you like) + - booleans + + Note: fields of this type are effectively untyped. We recommend it only be used for + parts of your schema that can't be statically typed. + EOS + end + + schema_def_api.scalar_type "JsonSafeLong" do |t| + t.mapping type: "long" + t.json_schema type: "integer", minimum: JSON_SAFE_LONG_MIN, maximum: JSON_SAFE_LONG_MAX + t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::JsonSafeLong", + defined_at: "elastic_graph/graphql/scalar_coercion_adapters/longs" + + t.prepare_for_indexing_with "ElasticGraph::Indexer::IndexingPreparers::Integer", + defined_at: "elastic_graph/indexer/indexing_preparers/integer" + + t.documentation <<~EOS + A numeric type for large integer values that can serialize safely as JSON. + + While JSON itself has no hard limit on the size of integers, the RFC-7159 spec + mentions that values outside of the range #{format_number(JSON_SAFE_LONG_MIN)} (-(2^53) + 1) + to #{format_number(JSON_SAFE_LONG_MAX)} (2^53 - 1) may not be interopable with all JSON + implementations. As it turns out, the number implementation used by JavaScript + has this issue. When you parse a JSON string that contains a numeric value like + `4693522397653681111`, the parsed result will contain a rounded value like + `4693522397653681000`. + + While this is entirely a client-side problem, we want to preserve maximum compatibility + with common client languages. Given the ubiquity of GraphiQL as a GraphQL client, + we want to avoid this problem. + + Our solution is to support two separate types: + + - This type (`JsonSafeLong`) is serialized as a number, but limits values to the safely + serializable range. + - The `LongString` type supports long values that use all 64 bits, but serializes as a + string rather than a number, avoiding the JavaScript compatibility problems. + + For more background, see the [JavaScript `Number.MAX_SAFE_INTEGER` + docs](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER). + EOS + + define_integral_aggregated_values_for(t) + end + + schema_def_api.scalar_type "LongString" do |t| + # Note: while this type is returned from GraphQL queries as a string, we still + # require it to be an integer in the JSON documents we index. We want min/max + # validation on input (to avoid ingesting values that are larger than we can + # handle). This is easy to do if we ingest these values as numbers, but hard + # to do if we ingest them as strings. (The `pattern` regex to validate the range + # would be *extremely* complicated). + t.mapping type: "long" + t.json_schema type: "integer", minimum: LONG_STRING_MIN, maximum: LONG_STRING_MAX + t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::LongString", + defined_at: "elastic_graph/graphql/scalar_coercion_adapters/longs" + t.prepare_for_indexing_with "ElasticGraph::Indexer::IndexingPreparers::Integer", + defined_at: "elastic_graph/indexer/indexing_preparers/integer" + + t.documentation <<~EOS + A numeric type for large integer values in the inclusive range -2^63 + (#{format_number(LONG_STRING_MIN)}) to (2^63 - 1) (#{format_number(LONG_STRING_MAX)}). + + Note that `LongString` values are serialized as strings within JSON, to avoid + interopability problems with JavaScript. If you want a large integer type that + serializes within JSON as a number, use `JsonSafeLong`. + EOS + + t.customize_aggregated_values_type do |avt| + # not nullable, since sum(empty_set) == 0 + avt.field names.approximate_sum, "Float!", graphql_only: true do |f| + f.runtime_metadata_computation_detail empty_bucket_value: 0, function: :sum + + f.documentation <<~EOS + The (approximate) sum of the field values within this grouping. + + Sums of large `LongString` values can result in overflow, where the exact sum cannot + fit in a `LongString` return value. This field, as a double-precision `Float`, can + represent larger sums, but the value may only be approximate. + EOS + end + + avt.field names.exact_sum, "JsonSafeLong", graphql_only: true do |f| + f.runtime_metadata_computation_detail empty_bucket_value: 0, function: :sum + + f.documentation <<~EOS + The exact sum of the field values within this grouping, if it fits in a `JsonSafeLong`. + + Sums of large `LongString` values can result in overflow, where the exact sum cannot + fit in a `JsonSafeLong`. In that case, `null` will be returned, and `#{names.approximate_sum}` + can be used to get an approximate value. + EOS + end + + define_exact_min_and_max_on_aggregated_values(avt, "JsonSafeLong") do |adjective:, full_name:| + approx_name = (full_name == "minimum") ? names.approximate_min : names.approximate_max + + <<~EOS + So long as the grouping contains at least one non-null value, and no values exceed the + `JsonSafeLong` range in the underlying indexed field, this will return an exact non-null value. + + If no non-null values are available, or if the #{full_name} value is outside the `JsonSafeLong` + range, `null` will be returned. `#{approx_name}` can be used to differentiate between these + cases and to get an approximate value. + EOS + end + + { + names.exact_min => [:min, "minimum", names.approximate_min, "smallest"], + names.exact_max => [:max, "maximum", names.approximate_max, "largest"] + }.each do |exact_name, (func, full_name, approx_name, adjective)| + avt.field approx_name, "LongString", graphql_only: true do |f| + f.runtime_metadata_computation_detail empty_bucket_value: nil, function: func + + f.documentation <<~EOS + The #{full_name} of the field values within this grouping. + + The aggregation computation performed to identify the #{adjective} value is not able + to maintain exact precision when dealing with values that are outside the `JsonSafeLong` + range (#{format_number(JSON_SAFE_LONG_MIN)} to #{format_number(JSON_SAFE_LONG_MAX)}). + In that case, the `#{exact_name}` field will return `null`, but this field will provide + a value which may be approximate. + EOS + end + end + + avt.field names.approximate_avg, "Float", graphql_only: true do |f| + f.runtime_metadata_computation_detail empty_bucket_value: nil, function: :avg + + f.documentation <<~EOS + The average (mean) of the field values within this grouping. + + Note that the returned value is approximate. Imprecision can be introduced by the computation if + any intermediary values fall outside the `JsonSafeLong` range (#{format_number(JSON_SAFE_LONG_MIN)} + to #{format_number(JSON_SAFE_LONG_MAX)}). + EOS + end + end + end + end + + def register_enum_types + # Elasticsearch and OpenSearch treat weeks as beginning on Monday for date histogram aggregations. + # Note that I can't find clear documentation on this. + # + # https://www.elastic.co/guide/en/elasticsearch/reference/7.10/search-aggregations-bucket-datehistogram-aggregation.html#calendar_intervals + # + # > One week is the interval between the start day_of_week:hour:minute:second and + # > the same day of the week and time of the following week in the specified time zone. + # + # However, we have observed that this is how it behaves. We verify it in this test: + # elasticgraph-graphql/spec/acceptance/elasticgraph_graphql_spec.rb + es_first_day_of_week = "Monday" + + # TODO: Drop support for legacy grouping schema + schema_def_api.enum_type "DateGroupingGranularity" do |t| + t.documentation <<~EOS + Enumerates the supported granularities of a `Date`. + EOS + + t.value "YEAR" do |v| + v.documentation "The year a `Date` falls in." + v.update_runtime_metadata datastore_value: "year" + end + + t.value "QUARTER" do |v| + v.documentation "The quarter a `Date` falls in." + v.update_runtime_metadata datastore_value: "quarter" + end + + t.value "MONTH" do |v| + v.documentation "The month a `Date` falls in." + v.update_runtime_metadata datastore_value: "month" + end + + t.value "WEEK" do |v| + v.documentation "The week, beginning on #{es_first_day_of_week}, a `Date` falls in." + v.update_runtime_metadata datastore_value: "week" + end + + t.value "DAY" do |v| + v.documentation "The exact day of a `Date`." + v.update_runtime_metadata datastore_value: "day" + end + end + + schema_def_api.enum_type "DateGroupingTruncationUnit" do |t| + t.documentation <<~EOS + Enumerates the supported truncation units of a `Date`. + EOS + + t.value "YEAR" do |v| + v.documentation "The year a `Date` falls in." + v.update_runtime_metadata datastore_value: "year" + end + + t.value "QUARTER" do |v| + v.documentation "The quarter a `Date` falls in." + v.update_runtime_metadata datastore_value: "quarter" + end + + t.value "MONTH" do |v| + v.documentation "The month a `Date` falls in." + v.update_runtime_metadata datastore_value: "month" + end + + t.value "WEEK" do |v| + v.documentation "The week, beginning on #{es_first_day_of_week}, a `Date` falls in." + v.update_runtime_metadata datastore_value: "week" + end + + t.value "DAY" do |v| + v.documentation "The exact day of a `Date`." + v.update_runtime_metadata datastore_value: "day" + end + end + + # TODO: Drop support for legacy grouping schema + schema_def_api.enum_type "DateTimeGroupingGranularity" do |t| + t.documentation <<~EOS + Enumerates the supported granularities of a `DateTime`. + EOS + + t.value "YEAR" do |v| + v.documentation "The year a `DateTime` falls in." + v.update_runtime_metadata datastore_value: "year" + end + + t.value "QUARTER" do |v| + v.documentation "The quarter a `DateTime` falls in." + v.update_runtime_metadata datastore_value: "quarter" + end + + t.value "MONTH" do |v| + v.documentation "The month a `DateTime` falls in." + v.update_runtime_metadata datastore_value: "month" + end + + t.value "WEEK" do |v| + v.documentation "The week, beginning on #{es_first_day_of_week}, a `DateTime` falls in." + v.update_runtime_metadata datastore_value: "week" + end + + t.value "DAY" do |v| + v.documentation "The day a `DateTime` falls in." + v.update_runtime_metadata datastore_value: "day" + end + + t.value "HOUR" do |v| + v.documentation "The hour a `DateTime` falls in." + v.update_runtime_metadata datastore_value: "hour" + end + + t.value "MINUTE" do |v| + v.documentation "The minute a `DateTime` falls in." + v.update_runtime_metadata datastore_value: "minute" + end + + t.value "SECOND" do |v| + v.documentation "The second a `DateTime` falls in." + v.update_runtime_metadata datastore_value: "second" + end + end + + schema_def_api.enum_type "DateTimeGroupingTruncationUnit" do |t| + t.documentation <<~EOS + Enumerates the supported truncation units of a `DateTime`. + EOS + + t.value "YEAR" do |v| + v.documentation "The year a `DateTime` falls in." + v.update_runtime_metadata datastore_value: "year" + end + + t.value "QUARTER" do |v| + v.documentation "The quarter a `DateTime` falls in." + v.update_runtime_metadata datastore_value: "quarter" + end + + t.value "MONTH" do |v| + v.documentation "The month a `DateTime` falls in." + v.update_runtime_metadata datastore_value: "month" + end + + t.value "WEEK" do |v| + v.documentation "The week, beginning on #{es_first_day_of_week}, a `DateTime` falls in." + v.update_runtime_metadata datastore_value: "week" + end + + t.value "DAY" do |v| + v.documentation "The day a `DateTime` falls in." + v.update_runtime_metadata datastore_value: "day" + end + + t.value "HOUR" do |v| + v.documentation "The hour a `DateTime` falls in." + v.update_runtime_metadata datastore_value: "hour" + end + + t.value "MINUTE" do |v| + v.documentation "The minute a `DateTime` falls in." + v.update_runtime_metadata datastore_value: "minute" + end + + t.value "SECOND" do |v| + v.documentation "The second a `DateTime` falls in." + v.update_runtime_metadata datastore_value: "second" + end + end + + schema_def_api.enum_type "LocalTimeGroupingTruncationUnit" do |t| + t.documentation <<~EOS + Enumerates the supported truncation units of a `LocalTime`. + EOS + + t.value "HOUR" do |v| + v.documentation "The hour a `LocalTime` falls in." + v.update_runtime_metadata datastore_value: "hour" + end + + t.value "MINUTE" do |v| + v.documentation "The minute a `LocalTime` falls in." + v.update_runtime_metadata datastore_value: "minute" + end + + t.value "SECOND" do |v| + v.documentation "The second a `LocalTime` falls in." + v.update_runtime_metadata datastore_value: "second" + end + end + + schema_def_api.enum_type "DistanceUnit" do |t| + t.documentation "Enumerates the supported distance units." + + # Values here are taken from: https://www.elastic.co/guide/en/elasticsearch/reference/7.10/common-options.html#distance-units + t.value "MILE" do |v| + v.documentation "A United States customary unit of 5,280 feet." + v.update_runtime_metadata datastore_abbreviation: :mi + end + + t.value "YARD" do |v| + v.documentation "A United States customary unit of 3 feet." + v.update_runtime_metadata datastore_abbreviation: :yd + end + + t.value "FOOT" do |v| + v.documentation "A United States customary unit of 12 inches." + v.update_runtime_metadata datastore_abbreviation: :ft + end + + t.value "INCH" do |v| + v.documentation "A United States customary unit equal to 1/12th of a foot." + v.update_runtime_metadata datastore_abbreviation: :in + end + + t.value "KILOMETER" do |v| + v.documentation "A metric system unit equal to 1,000 meters." + v.update_runtime_metadata datastore_abbreviation: :km + end + + t.value "METER" do |v| + v.documentation "The base unit of length in the metric system." + v.update_runtime_metadata datastore_abbreviation: :m + end + + t.value "CENTIMETER" do |v| + v.documentation "A metric system unit equal to 1/100th of a meter." + v.update_runtime_metadata datastore_abbreviation: :cm + end + + t.value "MILLIMETER" do |v| + v.documentation "A metric system unit equal to 1/1,000th of a meter." + v.update_runtime_metadata datastore_abbreviation: :mm + end + + t.value "NAUTICAL_MILE" do |v| + v.documentation "An international unit of length used for air, marine, and space navigation. Equivalent to 1,852 meters." + v.update_runtime_metadata datastore_abbreviation: :nmi + end + end + + schema_def_api.enum_type "DateTimeUnit" do |t| + t.documentation "Enumeration of `DateTime` units." + + # Values here are taken from: https://www.elastic.co/guide/en/elasticsearch/reference/7.10/common-options.html#time-units + t.value "DAY" do |v| + v.documentation "The time period of a full rotation of the Earth with respect to the Sun." + v.update_runtime_metadata datastore_abbreviation: :d, datastore_value: 86_400_000 + end + + t.value "HOUR" do |v| + v.documentation "1/24th of a day." + v.update_runtime_metadata datastore_abbreviation: :h, datastore_value: 3_600_000 + end + + t.value "MINUTE" do |v| + v.documentation "1/60th of an hour." + v.update_runtime_metadata datastore_abbreviation: :m, datastore_value: 60_000 + end + + t.value "SECOND" do |v| + v.documentation "1/60th of a minute." + v.update_runtime_metadata datastore_abbreviation: :s, datastore_value: 1_000 + end + + t.value "MILLISECOND" do |v| + v.documentation "1/1000th of a second." + v.update_runtime_metadata datastore_abbreviation: :ms, datastore_value: 1 + end + + # These units, which Elasticsearch and OpenSearch support, only make sense to use when using the + # Date nanoseconds type: + # + # https://www.elastic.co/guide/en/elasticsearch/reference/7.10/date_nanos.html + # + # However, we currently only use the standard `Date` type, which has millisecond granularity, + # For now these sub-millisecond granularities aren't useful to support, so we're not including + # them at this time. + # + # t.value "MICROSECOND" do |v| + # v.documentation "1/1000th of a millisecond." + # v.update_runtime_metadata datastore_abbreviation: :micros + # end + # + # t.value "NANOSECOND" do |v| + # v.documentation "1/1000th of a microsecond." + # v.update_runtime_metadata datastore_abbreviation: :nanos + # end + end + + schema_def_api.enum_type "DateUnit" do |t| + t.documentation "Enumeration of `Date` units." + + # Values here are taken from: https://www.elastic.co/guide/en/elasticsearch/reference/7.10/common-options.html#time-units + t.value "DAY" do |v| + v.documentation "The time period of a full rotation of the Earth with respect to the Sun." + v.update_runtime_metadata datastore_abbreviation: :d, datastore_value: 86_400_000 + end + end + + schema_def_api.enum_type "LocalTimeUnit" do |t| + t.documentation "Enumeration of `LocalTime` units." + + # Values here are taken from: https://www.elastic.co/guide/en/elasticsearch/reference/7.10/common-options.html#time-units + t.value "HOUR" do |v| + v.documentation "1/24th of a day." + v.update_runtime_metadata datastore_abbreviation: :h, datastore_value: 3_600_000 + end + + t.value "MINUTE" do |v| + v.documentation "1/60th of an hour." + v.update_runtime_metadata datastore_abbreviation: :m, datastore_value: 60_000 + end + + t.value "SECOND" do |v| + v.documentation "1/60th of a minute." + v.update_runtime_metadata datastore_abbreviation: :s, datastore_value: 1_000 + end + + t.value "MILLISECOND" do |v| + v.documentation "1/1000th of a second." + v.update_runtime_metadata datastore_abbreviation: :ms, datastore_value: 1 + end + end + + schema_def_api.enum_type "MatchesQueryAllowedEditsPerTerm" do |t| + t.documentation "Enumeration of allowed values for the `#{names.matches_query}: {#{names.allowed_edits_per_term}: ...}` filter option." + + t.value "NONE" do |v| + v.documentation "No allowed edits per term." + v.update_runtime_metadata datastore_abbreviation: :"0" + end + + t.value "ONE" do |v| + v.documentation "One allowed edit per term." + v.update_runtime_metadata datastore_abbreviation: :"1" + end + + t.value "TWO" do |v| + v.documentation "Two allowed edits per term." + v.update_runtime_metadata datastore_abbreviation: :"2" + end + + t.value "DYNAMIC" do |v| + v.documentation "Allowed edits per term is dynamically chosen based on the length of the term." + v.update_runtime_metadata datastore_abbreviation: :AUTO + end + end + end + + def register_date_and_time_grouped_by_types + # DateGroupedBy + date = schema_def_state.type_ref("Date") + register_framework_object_type date.as_grouped_by.name do |t| + t.documentation "Allows for grouping `Date` values based on the desired return type." + t.runtime_metadata_overrides = {elasticgraph_category: :date_grouped_by_object} + + t.field names.as_date, "Date", graphql_only: true do |f| + f.documentation "Used when grouping on the full `Date` value." + define_date_grouping_arguments(f, omit_timezone: true) + end + + t.field names.as_day_of_week, "DayOfWeek", graphql_only: true do |f| + f.documentation "An alternative to `#{names.as_date}` for when grouping on the day-of-week is desired." + define_day_of_week_grouping_arguments(f, omit_timezone: true) + end + end + + # DateTimeGroupedBy + date_time = schema_def_state.type_ref("DateTime") + register_framework_object_type date_time.as_grouped_by.name do |t| + t.documentation "Allows for grouping `DateTime` values based on the desired return type." + t.runtime_metadata_overrides = {elasticgraph_category: :date_grouped_by_object} + + t.field names.as_date_time, "DateTime", graphql_only: true do |f| + f.documentation "Used when grouping on the full `DateTime` value." + define_date_time_grouping_arguments(f) + end + + t.field names.as_date, "Date", graphql_only: true do |f| + f.documentation "An alternative to `#{names.as_date_time}` for when grouping on just the date is desired." + define_date_grouping_arguments(f) + end + + t.field names.as_time_of_day, "LocalTime", graphql_only: true do |f| + f.documentation "An alternative to `#{names.as_date_time}` for when grouping on just the time-of-day is desired." + define_local_time_grouping_arguments(f) + end + + t.field names.as_day_of_week, "DayOfWeek", graphql_only: true do |f| + f.documentation "An alternative to `#{names.as_date_time}` for when grouping on the day-of-week is desired." + define_day_of_week_grouping_arguments(f) + end + end + + schema_def_api.enum_type "DayOfWeek" do |t| + t.documentation "Indicates the specific day of the week." + + t.value "MONDAY" do |v| + v.documentation "Monday." + end + + t.value "TUESDAY" do |v| + v.documentation "Tuesday." + end + + t.value "WEDNESDAY" do |v| + v.documentation "Wednesday." + end + + t.value "THURSDAY" do |v| + v.documentation "Thursday." + end + + t.value "FRIDAY" do |v| + v.documentation "Friday." + end + + t.value "SATURDAY" do |v| + v.documentation "Saturday." + end + + t.value "SUNDAY" do |v| + v.documentation "Sunday." + end + end + end + + def define_date_grouping_arguments(grouping_field, omit_timezone: false) + define_calendar_type_grouping_arguments(grouping_field, schema_def_state.type_ref("Date"), <<~EOS, omit_timezone: omit_timezone) + For example, when grouping by `WEEK`, you can shift by 1 day to change what day-of-week weeks are considered to start on. + EOS + end + + def define_date_time_grouping_arguments(grouping_field) + define_calendar_type_grouping_arguments(grouping_field, schema_def_state.type_ref("DateTime"), <<~EOS) + For example, when grouping by `WEEK`, you can shift by 1 day to change what day-of-week weeks are considered to start on. + EOS + end + + def define_local_time_grouping_arguments(grouping_field) + define_calendar_type_grouping_arguments(grouping_field, schema_def_state.type_ref("LocalTime"), <<~EOS) + For example, when grouping by `HOUR`, you can apply an offset of -5 minutes to shift `LocalTime` + values to the prior hour when they fall between the the top of an hour and 5 after. + EOS + end + + def define_day_of_week_grouping_arguments(grouping_field, omit_timezone: false) + define_calendar_type_grouping_arguments(grouping_field, schema_def_state.type_ref("DayOfWeek"), <<~EOS, omit_timezone: omit_timezone, omit_truncation_unit: true) + For example, you can apply an offset of -2 hours to shift `DateTime` values to the prior `DayOfWeek` + when they fall between midnight and 2 AM. + EOS + end + + def define_calendar_type_grouping_arguments(grouping_field, calendar_type, offset_example_description, omit_timezone: false, omit_truncation_unit: false) + define_grouping_argument_offset(grouping_field, calendar_type, offset_example_description) + define_grouping_argument_time_zone(grouping_field, calendar_type) unless omit_timezone + define_grouping_argument_truncation_unit(grouping_field, calendar_type) unless omit_truncation_unit + end + + def define_grouping_argument_offset(grouping_field, calendar_type, example_description) + grouping_field.argument schema_def_state.schema_elements.offset, "#{calendar_type.name}GroupingOffsetInput" do |a| + a.documentation <<~EOS + Amount of offset (positive or negative) to shift the `#{calendar_type.name}` boundaries of each grouping bucket. + + #{example_description.strip} + EOS + end + end + + def define_grouping_argument_time_zone(grouping_field, calendar_type) + grouping_field.argument schema_def_state.schema_elements.time_zone, "TimeZone!" do |a| + a.documentation "The time zone to use when determining which grouping a `#{calendar_type.name}` value falls in." + a.default "UTC" + end + end + + def define_grouping_argument_truncation_unit(grouping_field, calendar_type) + grouping_field.argument schema_def_state.schema_elements.truncation_unit, "#{calendar_type.name}GroupingTruncationUnit!" do |a| + a.documentation "Determines the grouping truncation unit for this field." + end + end + + def define_integral_aggregated_values_for(scalar_type, long_type: "JsonSafeLong") + scalar_type_name = scalar_type.name + scalar_type.customize_aggregated_values_type do |t| + # not nullable, since sum(empty_set) == 0 + t.field names.approximate_sum, "Float!", graphql_only: true do |f| + f.runtime_metadata_computation_detail empty_bucket_value: 0, function: :sum + + f.documentation <<~EOS + The (approximate) sum of the field values within this grouping. + + Sums of large `#{scalar_type_name}` values can result in overflow, where the exact sum cannot + fit in a `#{long_type}` return value. This field, as a double-precision `Float`, can + represent larger sums, but the value may only be approximate. + EOS + end + + t.field names.exact_sum, long_type, graphql_only: true do |f| + f.runtime_metadata_computation_detail empty_bucket_value: 0, function: :sum + + f.documentation <<~EOS + The exact sum of the field values within this grouping, if it fits in a `#{long_type}`. + + Sums of large `#{scalar_type_name}` values can result in overflow, where the exact sum cannot + fit in a `#{long_type}`. In that case, `null` will be returned, and `#{names.approximate_sum}` + can be used to get an approximate value. + EOS + end + + define_exact_min_and_max_on_aggregated_values(t, scalar_type_name) do |adjective:, full_name:| + <<~EOS + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + EOS + end + + t.field names.approximate_avg, "Float", graphql_only: true do |f| + f.runtime_metadata_computation_detail empty_bucket_value: nil, function: :avg + + f.documentation <<~EOS + The average (mean) of the field values within this grouping. + + Note that the returned value is approximate. Imprecision can be introduced by the computation if + any intermediary values fall outside the `JsonSafeLong` range (#{format_number(JSON_SAFE_LONG_MIN)} + to #{format_number(JSON_SAFE_LONG_MAX)}). + EOS + end + end + end + + def define_exact_min_max_and_approx_avg_on_aggregated_values(aggregated_values_type, scalar_type, &block) + define_exact_min_and_max_on_aggregated_values(aggregated_values_type, scalar_type, &block) + + aggregated_values_type.field names.approximate_avg, scalar_type, graphql_only: true do |f| + f.runtime_metadata_computation_detail empty_bucket_value: nil, function: :avg + + f.documentation <<~EOS + The average (mean) of the field values within this grouping. + The returned value will be rounded to the nearest `#{scalar_type}` value. + EOS + end + end + + def define_exact_min_and_max_on_aggregated_values(aggregated_values_type, scalar_type) + { + names.exact_min => [:min, "minimum", "smallest"], + names.exact_max => [:max, "maximum", "largest"] + }.each do |name, (func, full_name, adjective)| + discussion = yield(adjective: adjective, full_name: full_name) + + aggregated_values_type.field name, scalar_type, graphql_only: true do |f| + f.runtime_metadata_computation_detail empty_bucket_value: nil, function: func + + f.documentation ["The #{full_name} of the field values within this grouping.", discussion].compact.join("\n\n") + end + end + end + + def register_framework_object_type(name) + schema_def_api.object_type(name) do |t| + t.graphql_only true + yield t + end + end + + def format_number(num) + abs_value_formatted = num.to_s.reverse.scan(/\d{1,3}/).join(",").reverse + (num < 0) ? "-#{abs_value_formatted}" : abs_value_formatted + end + + def register_filter(type, &block) + register_input_type(schema_def_state.factory.new_filter_input_type(type, &block)) + end + + def register_input_type(input_type) + schema_def_state.register_input_type(input_type) + end + + def remove_any_of_and_not_filter_operators_on(type) + type.graphql_fields_by_name.delete(names.any_of) + type.graphql_fields_by_name.delete(names.not) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/deprecated_element.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/deprecated_element.rb new file mode 100644 index 00000000..04201847 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/deprecated_element.rb @@ -0,0 +1,21 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # @private + DeprecatedElement = ::Data.define(:schema_def_state, :name, :defined_at, :defined_via) do + # @implements DeprecatedElement + def description + "`#{defined_via}` at #{defined_at.path}:#{defined_at.lineno}" + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/directive.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/directive.rb new file mode 100644 index 00000000..ee199a04 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/directive.rb @@ -0,0 +1,40 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_definition/mixins/verifies_graphql_name" +require "elastic_graph/support/graphql_formatter" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Represents a [GraphQL directive](https://spec.graphql.org/October2021/#sec-Language.Directives). + # + # @!attribute [r] name + # @return [String] name of the directive + # @!attribute [r] arguments + # @return [Hash] directive arguments + # @!parse class Directive < ::Data; end + class Directive < ::Data.define(:name, :arguments) + prepend Mixins::VerifiesGraphQLName + + # @return [String] GraphQL SDL form of the directive + def to_sdl + %(@#{name}#{Support::GraphQLFormatter.format_args(**arguments)}) + end + + # Duplicates this directive on another GraphQL schema element. + # + # @param element [Argument, EnumType, EnumValue, Field, ScalarType, TypeWithSubfields, UnionType] schema element + # @return [void] + def duplicate_on(element) + element.directive name, arguments + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_type.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_type.rb new file mode 100644 index 00000000..b8350288 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_type.rb @@ -0,0 +1,189 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/enum" +require "elastic_graph/schema_definition/indexing/field_type/enum" +require "elastic_graph/schema_definition/mixins/can_be_graphql_only" +require "elastic_graph/schema_definition/mixins/has_derived_graphql_type_customizations" +require "elastic_graph/schema_definition/mixins/has_directives" +require "elastic_graph/schema_definition/mixins/has_documentation" +require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" +require "elastic_graph/schema_definition/mixins/verifies_graphql_name" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # {include:API#enum_type} + # + # @example Define an enum type + # ElasticGraph.define_schema do |schema| + # schema.enum_type "Currency" do |t| + # # in the block, `t` is an EnumType + # t.value "USD" + # end + # end + # + # @!attribute [r] schema_def_state + # @return [State] state of the schema + # @!attribute [rw] type_ref + # @private + # @!attribute [rw] for_output + # @return [Boolean] `true` if this enum is used for both input and output; `false` if it is for input only + # @!attribute [r] values_by_name + # @return [Hash] map of enum values, keyed by name + class EnumType < Struct.new(:schema_def_state, :type_ref, :for_output, :values_by_name) + # @dynamic type_ref, graphql_only? + prepend Mixins::VerifiesGraphQLName + include Mixins::CanBeGraphQLOnly + include Mixins::HasDocumentation + include Mixins::HasDirectives + include Mixins::HasDerivedGraphQLTypeCustomizations + include Mixins::HasReadableToSAndInspect.new { |e| e.name } + + # @private + def initialize(schema_def_state, name) + # @type var values_by_name: ::Hash[::String, EnumValue] + values_by_name = {} + super(schema_def_state, schema_def_state.type_ref(name).to_final_form, true, values_by_name) + + # :nocov: -- currently all invocations have a block + yield self if block_given? + # :nocov: + end + + # @return [String] name of the enum type + def name + type_ref.name + end + + # @return [TypeReference] reference to `AggregatedValues` type to use for this enum. + def aggregated_values_type + schema_def_state.type_ref("NonNumeric").as_aggregated_values + end + + # Defines an enum value for the current enum type. + # + # @param value_name [String] name of the enum value + # @yield [EnumValue] enum value so it can be further customized + # @return [void] + # + # @example Define an enum type with multiple enum values + # ElasticGraph.define_schema do |schema| + # schema.enum_type "Currency" do |t| + # t.value "USD" do |v| + # v.documentation "US Dollars." + # end + # + # t.value "JPY" do |v| + # v.documentation "Japanese Yen." + # end + # end + # end + def value(value_name, &block) + alternate_original_name = value_name + value_name = schema_def_state.enum_value_namer.name_for(name, value_name.to_s) + + if values_by_name.key?(value_name) + raise Errors::SchemaError, "Duplicate value on Enum::Type #{name}: #{value_name}" + end + + if value_name.length > DEFAULT_MAX_KEYWORD_LENGTH + raise Errors::SchemaError, "Enum value `#{name}.#{value_name}` is too long: it is #{value_name.length} characters but cannot exceed #{DEFAULT_MAX_KEYWORD_LENGTH} characters." + end + + values_by_name[value_name] = schema_def_state.factory.new_enum_value(value_name, alternate_original_name, &block) + end + + # Defines multiple enum values. In contrast to {#value}, the enum values cannot be customized + # further via a block. + # + # @param value_names [Array] names of the enum values + # @return [void] + # + # @example Define an enum type with multiple enum values + # ElasticGraph.define_schema do |schema| + # schema.enum_type "Currency" do |t| + # t.values "USD", "JPY", "CAD", "GBP" + # end + # end + def values(*value_names) + value_names.flatten.each { |name| value(name) } + end + + # @return [SchemaArtifacts::RuntimeMetadata::Enum::Type] runtime metadata for this enum type + def runtime_metadata + runtime_metadata_values_by_name = values_by_name + .transform_values(&:runtime_metadata) + .compact + + SchemaArtifacts::RuntimeMetadata::Enum::Type.new(values_by_name: runtime_metadata_values_by_name) + end + + # @return [String] GraphQL SDL form of the enum type + def to_sdl + if values_by_name.empty? + raise Errors::SchemaError, "Enum type #{name} has no values, but enums must have at least one value." + end + + <<~EOS + #{formatted_documentation}enum #{name} #{directives_sdl(suffix_with: " ")}{ + #{values_by_name.values.map(&:to_sdl).flat_map { |s| s.split("\n") }.join("\n ")} + } + EOS + end + + # @private + def derived_graphql_types + # Derived GraphQL types must be generated for an output enum. For an enum type that is only + # used as an input, we do not need derived types. + return [] unless for_output + + derived_scalar_types = schema_def_state.factory.new_scalar_type(name) do |t| + t.mapping type: "keyword" + t.json_schema type: "string" + t.graphql_only graphql_only? + end.derived_graphql_types + + if (input_enum = as_input).equal?(self) + derived_scalar_types + else + [input_enum] + derived_scalar_types + end + end + + # @return [Indexing::FieldType::Enum] indexing representation of this enum type + def to_indexing_field_type + Indexing::FieldType::Enum.new(values_by_name.keys) + end + + # @return [false] enum types are never directly indexed + def indexed? + false + end + + # @return [EnumType] converts the enum type to its input form for when different naming is used for input vs output enums. + def as_input + input_name = type_ref + .as_input_enum # To apply the configured format for input enums. + .to_final_form # To handle a type name override of the input enum. + .name + + return self if input_name == name + + schema_def_state.factory.new_enum_type(input_name) do |t| + t.for_output = false # flag that it's not used as an output enum, and therefore `derived_graphql_types` will be empty on it. + t.graphql_only true # input enums are always GraphQL-only. + t.documentation doc_comment + directives.each { |dir| dir.duplicate_on(t) } + values_by_name.each { |_, val| val.duplicate_on(t) } + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_value.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_value.rb new file mode 100644 index 00000000..6a4f8531 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_value.rb @@ -0,0 +1,73 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_definition/mixins/has_directives" +require "elastic_graph/schema_definition/mixins/has_documentation" +require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" +require "elastic_graph/schema_definition/mixins/verifies_graphql_name" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Represents a value of a [GraphQL enum type](https://spec.graphql.org/October2021/#sec-Enums). + # + # @!attribute [r] schema_def_state + # @return [State] state of the schema + # @!attribute [r] name + # @return [String] name of the value + # @!attribute [r] runtime_metadata + # @return [SchemaElements::RuntimeMetadata::Enum::Value] runtime metadata + class EnumValue < Struct.new(:schema_def_state, :name, :runtime_metadata) + prepend Mixins::VerifiesGraphQLName + include Mixins::HasDocumentation + include Mixins::HasDirectives + include Mixins::HasReadableToSAndInspect.new { |v| v.name } + + # @private + def initialize(schema_def_state, name, original_name) + runtime_metadata = SchemaArtifacts::RuntimeMetadata::Enum::Value.new( + sort_field: nil, + datastore_value: nil, + datastore_abbreviation: nil, + alternate_original_name: (original_name if original_name != name) + ) + + super(schema_def_state, name, runtime_metadata) + yield self + end + + # @return [String] GraphQL SDL form of the enum value + def to_sdl + "#{formatted_documentation}#{name}#{directives_sdl(prefix_with: " ")}" + end + + # Duplicates this enum value on another {EnumType}. + # + # @param other_enum_type [EnumType] enum type to duplicate this value onto + # @return [void] + def duplicate_on(other_enum_type) + other_enum_type.value name do |v| + v.documentation doc_comment + directives.each { |dir| dir.duplicate_on(v) } + v.update_runtime_metadata(**runtime_metadata.to_h) + end + end + + # Updates the runtime metadata. + # + # @param [Hash] updates to apply to the runtime metadata + # @return [void] + def update_runtime_metadata(**updates) + self.runtime_metadata = runtime_metadata.with(**updates) + end + + private :runtime_metadata= + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_value_namer.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_value_namer.rb new file mode 100644 index 00000000..7a1d1ec6 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_value_namer.rb @@ -0,0 +1,89 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/errors" +require "elastic_graph/support/hash_util" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Abstraction for generating names for GraphQL enum values. This allows users to customize the + # naming of our built-in enum values. + # + # @private + class EnumValueNamer < ::Struct.new(:overrides_by_type_name) + def initialize(overrides_by_type_name = {}) + overrides_by_type_name = Support::HashUtil + .stringify_keys(overrides_by_type_name) # : ::Hash[::String, ::Hash[::String, ::String]] + + @used_value_names_by_type_name = ::Hash.new { |h, k| h[k] = [] } + validate_overrides(overrides_by_type_name) + super(overrides_by_type_name: overrides_by_type_name) + end + + # Returns the name that should be used for the given `type_name` and `value_name`. + def name_for(type_name, value_name) + @used_value_names_by_type_name[type_name] << value_name + overrides_by_type_name.dig(type_name, value_name) || value_name + end + + # Returns the overrides that did not wind up being used. Unused overrides usually happen + # because of a typo, and can be safely removed. + def unused_overrides + overrides_by_type_name.filter_map do |type_name, overrides| + if @used_value_names_by_type_name.key?(type_name) + unused_overrides = overrides.except(*@used_value_names_by_type_name.fetch(type_name)) + [type_name, unused_overrides] unless unused_overrides.empty? + else + [type_name, overrides] + end + end.to_h + end + + # Full set of enum type and value names that were used. Can be used to provide suggestions + # for when there are `unused_overrides`. + def used_value_names_by_type_name + @used_value_names_by_type_name.dup + end + + private + + def validate_overrides(overrides_by_type_name) + duplicate_problems = overrides_by_type_name.flat_map do |type_name, overrides| + overrides + .group_by { |k, v| v } + .transform_values { |kv_pairs| kv_pairs.map(&:first) } + .select { |_, v| v.size > 1 } + .map do |override, source_names| + "Multiple `#{type_name}` enum value overrides (#{source_names.sort.join(", ")}) map to the same name (#{override}), which is not supported." + end + end + + invalid_name_problems = overrides_by_type_name.flat_map do |type_name, overrides| + overrides.filter_map do |source_name, override| + unless GRAPHQL_NAME_PATTERN.match(override) + "`#{override}` (the override for `#{type_name}.#{source_name}`) is not a valid GraphQL type name. " + + GRAPHQL_NAME_VALIDITY_DESCRIPTION + end + end + end + + notify_problems(duplicate_problems + invalid_name_problems) + end + + def notify_problems(problems) + return if problems.empty? + + raise Errors::ConfigError, "Provided `enum_value_overrides_by_type_name` have #{problems.size} problem(s):\n\n" \ + "#{problems.map.with_index(1) { |problem, i| "#{i}. #{problem}" }.join("\n\n")}" + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enums_for_indexed_types.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enums_for_indexed_types.rb new file mode 100644 index 00000000..3f6ef96b --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enums_for_indexed_types.rb @@ -0,0 +1,82 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/enum" +require "elastic_graph/schema_artifacts/runtime_metadata/sort_field" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Responsible for generating enum types based on specific indexed types. + # + # @private + class EnumsForIndexedTypes + def initialize(schema_def_state) + @schema_def_state = schema_def_state + end + + # Generates a `SortOrder` enum type for the given indexed type. + def sort_order_enum_for(indexed_type) + return nil unless indexed_type.indexed? + + build_enum(indexed_type, :sort_order, :sortable?, "sorted") do |enum_type, field_path| + value_name_parts = field_path.map(&:name) + index_field = field_path.map(&:name_in_index).join(".") + + {asc: "ascending", desc: "descending"}.each do |dir, dir_description| + enum_type.value((value_name_parts + [dir.to_s.upcase]).join("_")) do |v| + v.update_runtime_metadata sort_field: SchemaArtifacts::RuntimeMetadata::SortField.new(index_field, dir) + v.documentation "Sorts #{dir_description} by the `#{graphql_field_path_description(field_path)}` field." + + wrapped_enum_value = @schema_def_state.factory.new_sort_order_enum_value(v, field_path) + + field_path.each do |field| + field.sort_order_enum_value_customizations.each { |block| block.call(wrapped_enum_value) } + end + end + end + end + end + + private + + def build_enum(indexed_type, category, field_predicate, past_tense_verb, &block) + derived_type_ref = indexed_type.type_ref.as_static_derived_type(category) + + enum = @schema_def_state.factory.new_enum_type(derived_type_ref.name) do |enum_type| + enum_type.documentation "Enumerates the ways `#{indexed_type.name}`s can be #{past_tense_verb}." + define_enum_values_for_type(enum_type, indexed_type, field_predicate, &block) + end.as_input + + enum unless enum.values_by_name.empty? + end + + def define_enum_values_for_type(enum_type, object_type, field_predicate, parents: [], &block) + object_type + .graphql_fields_by_name.values + .select(&field_predicate) + .each { |f| define_enum_values_for_field(enum_type, f, field_predicate, parents: parents, &block) } + end + + def define_enum_values_for_field(enum_type, field, field_predicate, parents:, &block) + path = parents + [field] + + if (object_type = field.type.fully_unwrapped.as_object_type) + define_enum_values_for_type(enum_type, object_type, field_predicate, parents: path, &block) + else + block.call(enum_type, path) + end + end + + def graphql_field_path_description(field_path) + field_path.map(&:name).join(".") + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field.rb new file mode 100644 index 00000000..8ebea5be --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field.rb @@ -0,0 +1,1104 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" +require "elastic_graph/constants" +require "elastic_graph/schema_definition/indexing/field" +require "elastic_graph/schema_definition/indexing/field_reference" +require "elastic_graph/schema_definition/mixins/has_directives" +require "elastic_graph/schema_definition/mixins/has_documentation" +require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" +require "elastic_graph/schema_definition/mixins/has_type_info" +require "elastic_graph/schema_definition/mixins/verifies_graphql_name" +require "elastic_graph/support/graphql_formatter" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Represents a [GraphQL field](https://spec.graphql.org/October2021/#sec-Language.Fields). + # + # @example Define a GraphQL field + # ElasticGraph.define_schema do |schema| + # schema.object_type "Widget" do |t| + # t.field "id", "ID" do |f| + # # `f` in this block is a Field object + # end + # end + # end + # + # @!attribute [r] name + # @return [String] name of the field + # @!attribute [r] schema_def_state + # @return [State] schema definition state + # @!attribute [r] graphql_only + # @return [Boolean] true if this field exists only in the GraphQL schema and is not indexed + # @!attribute [r] name_in_index + # @return [String] the name of this field in the datastore index + # + # @!attribute [rw] original_type + # @private + # @!attribute [rw] parent_type + # @private + # @!attribute [rw] original_type_for_derived_types + # @private + # @!attribute [rw] accuracy_confidence + # @private + # @!attribute [rw] filter_customizations + # @private + # @!attribute [rw] grouped_by_customizations + # @private + # @!attribute [rw] sub_aggregations_customizations + # @private + # @!attribute [rw] aggregated_values_customizations + # @private + # @!attribute [rw] sort_order_enum_value_customizations + # @private + # @!attribute [rw] args + # @private + # @!attribute [rw] sortable + # @private + # @!attribute [rw] filterable + # @private + # @!attribute [rw] aggregatable + # @private + # @!attribute [rw] groupable + # @private + # @!attribute [rw] source + # @private + # @!attribute [rw] runtime_field_script + # @private + # @!attribute [rw] relationship + # @private + # @!attribute [rw] singular_name + # @private + # @!attribute [rw] computation_detail + # @private + # @!attribute [rw] non_nullable_in_json_schema + # @private + # @!attribute [rw] backing_indexing_field + # @private + # @!attribute [rw] as_input + # @private + # @!attribute [rw] legacy_grouping_schema + # @private + class Field < Struct.new( + :name, :original_type, :parent_type, :original_type_for_derived_types, :schema_def_state, :accuracy_confidence, + :filter_customizations, :grouped_by_customizations, :sub_aggregations_customizations, + :aggregated_values_customizations, :sort_order_enum_value_customizations, + :args, :sortable, :filterable, :aggregatable, :groupable, :graphql_only, :source, :runtime_field_script, :relationship, :singular_name, + :computation_detail, :non_nullable_in_json_schema, :backing_indexing_field, :as_input, + :legacy_grouping_schema, :name_in_index + ) + include Mixins::HasDocumentation + include Mixins::HasDirectives + include Mixins::HasTypeInfo + include Mixins::HasReadableToSAndInspect.new { |f| "#{f.parent_type.name}.#{f.name}: #{f.type}" } + + # @private + def initialize( + name:, type:, parent_type:, schema_def_state:, + accuracy_confidence: :high, name_in_index: name, + runtime_metadata_graphql_field: SchemaArtifacts::RuntimeMetadata::GraphQLField::EMPTY, + type_for_derived_types: nil, graphql_only: nil, singular: nil, + sortable: nil, filterable: nil, aggregatable: nil, groupable: nil, + backing_indexing_field: nil, as_input: false, legacy_grouping_schema: false + ) + type_ref = schema_def_state.type_ref(type) + super( + name: name, + original_type: type_ref, + parent_type: parent_type, + original_type_for_derived_types: type_for_derived_types ? schema_def_state.type_ref(type_for_derived_types) : type_ref, + schema_def_state: schema_def_state, + accuracy_confidence: accuracy_confidence, + filter_customizations: [], + grouped_by_customizations: [], + sub_aggregations_customizations: [], + aggregated_values_customizations: [], + sort_order_enum_value_customizations: [], + args: {}, + sortable: sortable, + filterable: filterable, + aggregatable: aggregatable, + groupable: groupable, + graphql_only: graphql_only, + source: nil, + runtime_field_script: nil, + # Note: we named the keyword argument `singular` (with no `_name` suffix) for consistency with + # other schema definition APIs, which also use `singular:` instead of `singular_name:`. We include + # the `_name` suffix on the attribute for clarity. + singular_name: singular, + name_in_index: name_in_index, + non_nullable_in_json_schema: false, + backing_indexing_field: backing_indexing_field, + as_input: as_input, + legacy_grouping_schema: legacy_grouping_schema + ) + + if name != name_in_index && name_in_index&.include?(".") && !graphql_only + raise Errors::SchemaError, "#{self} has an invalid `name_in_index`: #{name_in_index.inspect}. Only `graphql_only: true` fields can have a `name_in_index` that references a child field." + end + + schema_def_state.register_user_defined_field(self) + yield self if block_given? + end + + # @private + @@initialize_param_names = instance_method(:initialize).parameters.map(&:last).to_set + + # must come after we capture the initialize params. + prepend Mixins::VerifiesGraphQLName + + # @return [TypeReference] the type of this field + def type + # Here we lazily convert the `original_type` to an input type as needed. This must be lazy because + # the logic of `as_input` depends on detecting whether the type is an enum type, which it may not + # be able to do right away--we assume not if we can't tell, and retry every time this method is called. + original_type.to_final_form(as_input: as_input) + end + + # @return [TypeReference] the type of the corresponding field on derived types (usually this is the same as {#type}). + # + # @private + def type_for_derived_types + original_type_for_derived_types.to_final_form(as_input: as_input) + end + + # @note For each field defined in your schema that is filterable, a corresponding filtering field will be created on the + # `*FilterInput` type derived from the parent object type. + # + # Registers a customization callback that will be applied to the corresponding filtering field that will be generated for this + # field. + # + # @yield [Field] derived filtering field + # @return [void] + # @see #customize_aggregated_values_field + # @see #customize_grouped_by_field + # @see #customize_sort_order_enum_values + # @see #customize_sub_aggregations_field + # @see #on_each_generated_schema_element + # + # @example Mark `CampaignFilterInput.organizationId` with `@deprecated` + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "id", "ID" + # + # t.field "organizationId", "ID" do |f| + # f.customize_filter_field do |ff| + # ff.directive "deprecated" + # end + # end + # + # t.index "campaigns" + # end + # end + def customize_filter_field(&customization_block) + filter_customizations << customization_block + end + + # @note For each field defined in your schema that is aggregatable, a corresponding `aggregatedValues` field will be created on the + # `*AggregatedValues` type derived from the parent object type. + # + # Registers a customization callback that will be applied to the corresponding `aggregatedValues` field that will be generated for + # this field. + # + # @yield [Field] derived aggregated values field + # @return [void] + # @see #customize_filter_field + # @see #customize_grouped_by_field + # @see #customize_sort_order_enum_values + # @see #customize_sub_aggregations_field + # @see #on_each_generated_schema_element + # + # @example Mark `CampaignAggregatedValues.adImpressions` with `@deprecated` + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "id", "ID" + # + # t.field "adImpressions", "Int" do |f| + # f.customize_aggregated_values_field do |avf| + # avf.directive "deprecated" + # end + # end + # + # t.index "campaigns" + # end + # end + def customize_aggregated_values_field(&customization_block) + aggregated_values_customizations << customization_block + end + + # @note For each field defined in your schema that is groupable, a corresponding `groupedBy` field will be created on the + # `*AggregationGroupedBy` type derived from the parent object type. + # + # Registers a customization callback that will be applied to the corresponding `groupedBy` field that will be generated for this + # field. + # + # @yield [Field] derived grouped by field + # @return [void] + # @see #customize_aggregated_values_field + # @see #customize_filter_field + # @see #customize_sort_order_enum_values + # @see #customize_sub_aggregations_field + # @see #on_each_generated_schema_element + # + # @example Mark `CampaignGroupedBy.organizationId` with `@deprecated` + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "id", "ID" + # + # t.field "organizationId", "ID" do |f| + # f.customize_grouped_by_field do |gbf| + # gbf.directive "deprecated" + # end + # end + # + # t.index "campaigns" + # end + # end + def customize_grouped_by_field(&customization_block) + grouped_by_customizations << customization_block + end + + # @note For each field defined in your schema that is sub-aggregatable (e.g. list fields indexed using the `nested` mapping type), + # a corresponding field will be created on the `*AggregationSubAggregations` type derived from the parent object type. + # + # Registers a customization callback that will be applied to the corresponding `subAggregations` field that will be generated for + # this field. + # + # @yield [Field] derived sub-aggregations field + # @return [void] + # @see #customize_aggregated_values_field + # @see #customize_filter_field + # @see #customize_grouped_by_field + # @see #customize_sort_order_enum_values + # @see #on_each_generated_schema_element + # + # @example Mark `TransactionAggregationSubAggregations.fees` with `@deprecated` + # ElasticGraph.define_schema do |schema| + # schema.object_type "Transaction" do |t| + # t.field "id", "ID" + # + # t.field "fees", "[Money!]!" do |f| + # f.mapping type: "nested" + # + # f.customize_sub_aggregations_field do |saf| + # # Adds a `@deprecated` directive to the `PaymentAggregationSubAggregations.fees` + # # field without also adding it to the `Payment.fees` field. + # saf.directive "deprecated" + # end + # end + # + # t.index "transactions" + # end + # + # schema.object_type "Money" do |t| + # t.field "amount", "Int" + # t.field "currency", "String" + # end + # end + def customize_sub_aggregations_field(&customization_block) + sub_aggregations_customizations << customization_block + end + + # @note for each sortable field, enum values will be generated on the derived sort order enum type allowing you to + # sort by the field `ASC` or `DESC`. + # + # Registers a customization callback that will be applied to the corresponding enum values that will be generated for this field + # on the derived `SortOrder` enum type. + # + # @yield [SortOrderEnumValue] derived sort order enum value + # @return [void] + # @see #customize_aggregated_values_field + # @see #customize_filter_field + # @see #customize_grouped_by_field + # @see #customize_sub_aggregations_field + # @see #on_each_generated_schema_element + # + # @example Mark `CampaignSortOrder.organizationId_(ASC|DESC)` with `@deprecated` + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "id", "ID" + # + # t.field "organizationId", "ID" do |f| + # f.customize_sort_order_enum_values do |soev| + # soev.directive "deprecated" + # end + # end + # + # t.index "campaigns" + # end + # end + def customize_sort_order_enum_values(&customization_block) + sort_order_enum_value_customizations << customization_block + end + + # When you define a {Field} on an {ObjectType} or {InterfaceType}, ElasticGraph generates up to 6 different GraphQL schema elements + # for it: + # + # * A {Field} is generated on the parent {ObjectType} or {InterfaceType} (that is, this field itself). This is used by clients to + # ask for values for the field in a response. + # * A {Field} may be generated on the `*FilterInput` {InputType} derived from the parent {ObjectType} or {InterfaceType}. This is + # used by clients to specify how the query should filter. + # * A {Field} may be generated on the `*AggregationGroupedBy` {ObjectType} derived from the parent {ObjectType} or {InterfaceType}. + # This is used by clients to specify how aggregations should be grouped. + # * A {Field} may be generated on the `*AggregatedValues` {ObjectType} derived from the parent {ObjectType} or {InterfaceType}. + # This is used by clients to apply aggregation functions (e.g. `sum`, `max`, `min`, etc) to a set of field values for a group. + # * A {Field} may be generated on the `*AggregationSubAggregations` {ObjectType} derived from the parent {ObjectType} or + # {InterfaceType}. This is used by clients to perform sub-aggregations on list fields indexed using the `nested` mapping type. + # * Multiple {EnumValue}s (both `*_ASC` and `*_DESC`) are generated on the `*SortOrder` {EnumType} derived from the parent indexed + # {ObjectType}. This is used by clients to sort by a field. + # + # This method registers a customization callback which is applied to every element that is generated for this field. + # + # @yield [Field, EnumValue] the schema element + # @return [void] + # @see #customize_aggregated_values_field + # @see #customize_filter_field + # @see #customize_grouped_by_field + # @see #customize_sort_order_enum_values + # @see #customize_sub_aggregations_field + # + # @example + # ElasticGraph.define_schema do |schema| + # schema.object_type "Transaction" do |t| + # t.field "id", "ID" + # + # t.field "amount", "Int" do |f| + # f.on_each_generated_schema_element do |element| + # # Adds a `@deprecated` directive to every GraphQL schema element generated for `amount`: + # # + # # - The `Transaction.amount` field. + # # - The `TransactionFilterInput.amount` field. + # # - The `TransactionAggregationGroupedBy.amount` field. + # # - The `TransactionAggregatedValues.amount` field. + # # - The `TransactionSortOrder.amount_ASC` and`TransactionSortOrder.amount_DESC` enum values. + # element.directive "deprecated" + # end + # end + # + # t.index "transactions" + # end + # end + def on_each_generated_schema_element(&customization_block) + customization_block.call(self) + customize_filter_field(&customization_block) + customize_aggregated_values_field(&customization_block) + customize_grouped_by_field(&customization_block) + customize_sub_aggregations_field(&customization_block) + customize_sort_order_enum_values(&customization_block) + end + + # (see Mixins::HasTypeInfo#json_schema) + def json_schema(nullable: nil, **options) + if options.key?(:type) + raise Errors::SchemaError, "Cannot override JSON schema type of field `#{name}` with `#{options.fetch(:type)}`" + end + + case nullable + when true + raise Errors::SchemaError, "`nullable: true` is not allowed on a field--just declare the GraphQL field as being nullable (no `!` suffix) instead." + when false + self.non_nullable_in_json_schema = true + end + + super(**options) + end + + # Configures ElasticGraph to source a field’s value from a related object. This can be used to denormalize data at ingestion time to + # support filtering, grouping, sorting, or aggregating data on a field from a related object. + # + # @param relationship [String] name of a relationship defined with {TypeWithSubfields#relates_to_one} using an inbound foreign key + # which contains the the field you wish to source values from + # @param field_path [String] dot-separated path to the field on the related type containing values that should be copied to this + # field + # @return [void] + # + # @example Source `City.currency` from `Country.currency` + # ElasticGraph.define_schema do |schema| + # schema.object_type "Country" do |t| + # t.field "id", "ID" + # t.field "name", "String" + # t.field "currency", "String" + # t.relates_to_one "capitalCity", "City", via: "capitalCityId", dir: :out + # t.index "countries" + # end + # + # schema.object_type "City" do |t| + # t.field "id", "ID" + # t.field "name", "String" + # t.relates_to_one "capitalOf", "Country", via: "capitalCityId", dir: :in + # + # t.field "currency", "String" do |f| + # f.sourced_from "capitalOf", "currency" + # end + # + # t.index "cities" + # end + # end + def sourced_from(relationship, field_path) + self.source = schema_def_state.factory.new_field_source( + relationship_name: relationship, + field_path: field_path + ) + end + + # @private + def runtime_script(script) + self.runtime_field_script = script + end + + # Registers an old name that this field used to have in a prior version of the schema. + # + # @note In situations where this API applies, ElasticGraph will give you an error message indicating that you need to use this API + # or {TypeWithSubfields#deleted_field}. Likewise, when ElasticGraph no longer needs to know about this, it'll give you a warning + # indicating the call to this method can be removed. + # + # @param old_name [String] old name this field used to have in a prior version of the schema + # @return [void] + # + # @example Indicate that `Widget.description` used to be called `Widget.notes`. + # ElasticGraph.define_schema do |schema| + # schema.object_type "Widget" do |t| + # t.field "description", "String" do |f| + # f.renamed_from "notes" + # end + # end + # end + def renamed_from(old_name) + schema_def_state.register_renamed_field( + parent_type.name, + from: old_name, + to: name, + defined_at: caller_locations(1, 1).first, # : ::Thread::Backtrace::Location + defined_via: %(field.renamed_from "#{old_name}") + ) + end + + # @private + def to_sdl(type_structure_only: false, default_value_sdl: nil, &arg_selector) + if type_structure_only + "#{name}#{args_sdl(joiner: ", ", &arg_selector)}: #{type.name}" + else + args_sdl = args_sdl(joiner: "\n ", after_opening_paren: "\n ", &arg_selector) + "#{formatted_documentation}#{name}#{args_sdl}: #{type.name}#{default_value_sdl} #{directives_sdl}".strip + end + end + + # Indicates if this field is sortable. Sortable fields will have corresponding `_ASC` and `_DESC` values generated in the + # sort order {EnumType} of the parent indexed type. + # + # By default, the sortability is inferred by the field type and mapping. For example, list fields are not sortable, + # and fields mapped as `text` are not sortable either. Fields are sortable in most other cases. + # + # The `sortable: true` option can be used to force a field to be sortable. + # + # @return [Boolean] true if this field is sortable + def sortable? + return sortable unless sortable.nil? + + # List fields are not sortable by default. We'd need to provide the datastore a sort mode option: + # https://www.elastic.co/guide/en/elasticsearch/reference/current/sort-search-results.html#_sort_mode_option + return false if type.list? + + # Boolean fields are not sortable by default. + # - Boolean: sorting all falses before all trues (or whatever) is not generally interesting. + return false if type.unwrap_non_null.boolean? + + # Elasticsearch/OpenSearch do not support sorting text fields: + # > Text fields are not used for sorting... + # (from https://www.elastic.co/guide/en/elasticsearch/reference/current/the datastore.html#text) + return false if text? + + # If the type uses custom mapping type we don't know how if the datastore can sort by it, so we assume it's not sortable. + return false if type.as_object_type&.has_custom_mapping_type? + + # Default every other field to being sortable. + true + end + + # Indicates if this field is filterable. Filterable fields will be available in the GraphQL schema under the `filter` argument. + # + # Most fields are filterable, except when: + # + # - It's a relation. Relation fields require us to load the related data from another index and can't be filtered on. + # - The field is an object type that isn't itself filterable (e.g. due to having no filterable fields or whatever). + # - Explicitly disabled with `filterable: false`. + # + # @return [Boolean] + def filterable? + # Object types that use custom index mappings (as `GeoLocation` does) aren't filterable + # by default since we can't guess what datastore filtering capabilities they have. We've implemented + # filtering support for `GeoLocation` fields, though, so we need to explicitly make it fliterable here. + # TODO: clean this up using an interface instead of checking for `GeoLocation`. + return true if type.fully_unwrapped.name == "GeoLocation" + + return false if relationship || type.fully_unwrapped.as_object_type&.does_not_support?(&:filterable?) + return true if filterable.nil? + filterable + end + + # Indicates if this field is groupable. Groupable fields will be available under `groupedBy` for an aggregations query. + # + # Groupability is inferred based on the field type and mapping type, or you can use the `groupable: true` option to force it. + # + # @return [Boolean] + def groupable? + # If the groupability of the field was specified explicitly when the field was defined, use the specified value. + return groupable unless groupable.nil? + + # We don't want the `id` field of an indexed type to be available to group by, because it's the unique primary key + # and the groupings would each contain one document. It's simpler and more efficient to just query the raw documents + # instead. + return false if parent_type.indexed? && name == "id" + + return false if relationship || type.fully_unwrapped.as_object_type&.does_not_support?(&:groupable?) + + # We don't support grouping an entire list of values, but we do support grouping on individual values in a list. + # However, we only do so when a `singular_name` has been provided (so that we know what to call the grouped_by field). + # The semantics are a little odd (since one document can be duplicated in multiple grouping buckets) so we're ok + # with not offering it by default--the user has to opt-in by telling us what to call the field in its singular form. + return list_field_groupable_by_single_values? if type.list? && type.fully_unwrapped.leaf? + + # Nested fields will be supported through specific nested aggregation support, and do not + # work as expected when grouping on the root document type. + return false if nested? + + # Text fields cannot be efficiently grouped on, so make them non-groupable by default. + return false if text? + + # In all other cases, default to being groupable. + true + end + + # Indicates if this field is aggregatable. Aggregatable fields will be available under `aggregatedValues` for an aggregations query. + # + # Aggregatability is inferred based on the field type and mapping type, or you can use the `aggregatable: true` option to force it. + # + # @return [Boolean] + def aggregatable? + return aggregatable unless aggregatable.nil? + return false if relationship + + # We don't yet support aggregating over subfields of a `nested` field. + # TODO: add support for aggregating over subfields of `nested` fields. + return false if nested? + + # Text fields are not efficiently aggregatable (and you'll often get errors from the datastore if you attempt to aggregate them). + return false if text? + + type_for_derived_types.fully_unwrapped.as_object_type&.supports?(&:aggregatable?) || index_leaf? + end + + # Indicates if this field can be used as the basis for a sub-aggregation. Sub-aggregatable fields will be available under + # `subAggregations` for an aggregations query. + # + # Only nested fields, and object fields which have nested fields, can be sub-aggregated. + # + # @return [Boolean] + def sub_aggregatable? + return false if relationship + + nested? || type_for_derived_types.fully_unwrapped.as_object_type&.supports?(&:sub_aggregatable?) + end + + # Defines an argument on the field. + # + # @note ElasticGraph takes care of defining arguments for all the query features it supports, so there is generally no need to use + # this API, and it has no way to interpret arbitrary arguments defined on a field. However, it can be useful for extensions that + # extend the {ElasticGraph::GraphQL} query engine. For example, {ElasticGraph::Apollo} uses this API to satisfy the [Apollo + # federation subgraph spec](https://www.apollographql.com/docs/federation/federation-spec/). + # + # @param name [String] name of the argument + # @param value_type [String] type of the argument in GraphQL SDL syntax + # @yield [Argument] for further customization + # + # @example Define an argument on a field + # ElasticGraph.define_schema do |schema| + # schema.object_type "Product" do |t| + # t.field "name", "String" do |f| + # f.argument "language", "String" + # end + # end + # end + def argument(name, value_type, &block) + args[name] = schema_def_state.factory.new_argument( + self, + name, + schema_def_state.type_ref(value_type), + &block + ) + end + + # The index mapping type in effect for this field. This could come from either the field definition or from the type definition. + # + # @return [String] + def mapping_type + backing_indexing_field&.mapping_type || (resolve_mapping || {})["type"] + end + + # @private + def list_field_groupable_by_single_values? + (type.list? || backing_indexing_field&.type&.list?) && !singular_name.nil? + end + + # @private + def define_aggregated_values_field(parent_type) + return unless aggregatable? + + unwrapped_type_for_derived_types = type_for_derived_types.fully_unwrapped + aggregated_values_type = + if index_leaf? + unwrapped_type_for_derived_types.resolved.aggregated_values_type + else + unwrapped_type_for_derived_types.as_aggregated_values + end + + parent_type.field name, aggregated_values_type.name, name_in_index: name_in_index, graphql_only: true do |f| + f.documentation derived_documentation("Computed aggregate values for the `#{name}` field") + aggregated_values_customizations.each { |block| block.call(f) } + end + end + + # @private + def define_grouped_by_field(parent_type) + return unless (field_name = grouped_by_field_name) + + parent_type.field field_name, grouped_by_field_type_name, name_in_index: name_in_index, graphql_only: true do |f| + add_grouped_by_field_documentation(f) + + define_legacy_timestamp_grouping_arguments_if_needed(f) if legacy_grouping_schema + + grouped_by_customizations.each { |block| block.call(f) } + end + end + + # @private + def grouped_by_field_type_name + unwrapped_type = type_for_derived_types.fully_unwrapped + if unwrapped_type.scalar_type_needing_grouped_by_object? && !legacy_grouping_schema + unwrapped_type.with_reverted_override.as_grouped_by.name + elsif unwrapped_type.leaf? + unwrapped_type.name + else + unwrapped_type.as_grouped_by.name + end + end + + # @private + def add_grouped_by_field_documentation(field) + text = if list_field_groupable_by_single_values? + derived_documentation( + "The individual value from `#{name}` for this group", + list_field_grouped_by_doc_note("`#{name}`") + ) + elsif type.list? && type.fully_unwrapped.object? + derived_documentation( + "The `#{name}` field value for this group", + list_field_grouped_by_doc_note("the selected subfields of `#{name}`") + ) + elsif type_for_derived_types.fully_unwrapped.scalar_type_needing_grouped_by_object? && !legacy_grouping_schema + derived_documentation("Offers the different grouping options for the `#{name}` value within this group") + else + derived_documentation("The `#{name}` field value for this group") + end + + field.documentation text + end + + # @private + def grouped_by_field_name + return nil unless groupable? + list_field_groupable_by_single_values? ? singular_name : name + end + + # @private + def define_sub_aggregations_field(parent_type:, type:) + parent_type.field name, type, name_in_index: name_in_index, graphql_only: true do |f| + f.documentation derived_documentation("Used to perform a sub-aggregation of `#{name}`") + sub_aggregations_customizations.each { |c| c.call(f) } + + yield f if block_given? + end + end + + # @private + def to_filter_field(parent_type:, for_single_value: !type_for_derived_types.list?) + type_prefix = text? ? "Text" : type_for_derived_types.fully_unwrapped.name + filter_type = schema_def_state + .type_ref(type_prefix) + .as_static_derived_type(filter_field_category(for_single_value)) + .name + + params = to_h + .slice(*@@initialize_param_names) + .merge(type: filter_type, parent_type: parent_type, name_in_index: name_in_index, type_for_derived_types: nil) + + schema_def_state.factory.new_field(**params).tap do |f| + f.documentation derived_documentation( + "Used to filter on the `#{name}` field", + "Will be ignored if `null` or an empty object is passed" + ) + + filter_customizations.each { |c| c.call(f) } + end + end + + # @private + def define_relay_pagination_arguments! + argument schema_def_state.schema_elements.first.to_sym, "Int" do |a| + a.documentation <<~EOS + Used in conjunction with the `after` argument to forward-paginate through the `#{name}`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `#{name}`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + EOS + end + + argument schema_def_state.schema_elements.after.to_sym, "Cursor" do |a| + a.documentation <<~EOS + Used to forward-paginate through the `#{name}`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + EOS + end + + argument schema_def_state.schema_elements.last.to_sym, "Int" do |a| + a.documentation <<~EOS + Used in conjunction with the `before` argument to backward-paginate through the `#{name}`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `#{name}`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + EOS + end + + argument schema_def_state.schema_elements.before.to_sym, "Cursor" do |a| + a.documentation <<~EOS + Used to backward-paginate through the `#{name}`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + EOS + end + end + + # Converts this field to an `Indexing::FieldReference`, which contains all the attributes involved + # in building an `Indexing::Field`. Notably, we cannot actually convert to an `Indexing::Field` at + # the point this method is called, because the referenced field type may not have been defined + # yet. We don't need an actual `Indexing::Field` until the very end of the schema definition process, + # when we are dumping the artifacts. However, we need this at field definition time so that we + # can correctly detect duplicate indexing field issues when a field is defined. (This is used + # in `TypeWithSubfields#field`). + # + # @private + def to_indexing_field_reference + return nil if graphql_only + + Indexing::FieldReference.new( + name: name, + name_in_index: name_in_index, + type: non_nullable_in_json_schema ? type.wrap_non_null : type, + mapping_options: mapping_options, + json_schema_options: json_schema_options, + accuracy_confidence: accuracy_confidence, + source: source, + runtime_field_script: runtime_field_script + ) + end + + # Converts this field to its `IndexingField` form. + # + # @private + def to_indexing_field + to_indexing_field_reference&.resolve + end + + # @private + def resolve_mapping + to_indexing_field&.mapping + end + + # Returns the string paths to the list fields that we need to index counts for. + # We do this to support the ability to filter on the size of a list. + # + # @private + def paths_to_lists_for_count_indexing(has_list_ancestor: false) + self_path = (has_list_ancestor || type.list?) ? [name_in_index] : [] + + nested_paths = + # Nested fields get indexed as separate hidden documents: + # https://www.elastic.co/guide/en/elasticsearch/reference/8.8/nested.html + # + # Given that, the counts of any `nested` list subfields will go in a `__counts` field on the + # separate hidden document. + if !nested? && (object_type = type.fully_unwrapped.as_object_type) + object_type.indexing_fields_by_name_in_index.values.flat_map do |sub_field| + sub_field.paths_to_lists_for_count_indexing(has_list_ancestor: has_list_ancestor || type.list?).map do |sub_path| + "#{name_in_index}#{LIST_COUNTS_FIELD_PATH_KEY_SEPARATOR}#{sub_path}" + end + end + else + [] + end + + self_path + nested_paths + end + + # Indicates if this field is a leaf value in the index. Note that GraphQL leaf values + # are always leaf values in the index but the inverse is not always true. For example, + # a `GeoLocation` field is not a leaf in GraphQL (because `GeoLocation` is an object + # type with subfields) but in the index we use a single `geo_point` mapping type, which + # is a single unit, so we consider it an index leaf. + # + # @private + def index_leaf? + type_for_derived_types.fully_unwrapped.leaf? || DATASTORE_PROPERTYLESS_OBJECT_TYPES.include?(mapping_type) + end + + # @private + ACCURACY_SCORES = { + # :high is assigned to `Field`s that are generated directly from GraphQL fields or :extra_fields. + # For these, we know everything available to us in the schema about them. + high: 3, + + # :medium is assigned to `Field`s that are inferred from the id fields required by a relation. + # We make logical guesses about the `indexing_field_type` but if the field is also manually defined, + # it could be slightly different (e.g. additional json schema validations), so we have medium + # confidence of these. + medium: 2, + + # :low is assigned to the ElastcField inferred for the foreign key of an inbound relation. The + # nullability/cardinality of the foreign key field cannot be known from the relation metadata, + # so we just guess what seems safest (`[:nullable]`). If the field is defined another way + # we should prefer it, so we give these fields :low confidence. + low: 1 + } + + # Given two fields, picks the one that is most accurate. If they have the same accuracy + # confidence, yields to a block to force it to deal with the discrepancy, unless the fields + # are exactly equal (in which case we can return either). + # + # @private + def self.pick_most_accurate_from(field1, field2, to_comparable: ->(it) { it }) + return field1 if to_comparable.call(field1) == to_comparable.call(field2) + yield if field1.accuracy_confidence == field2.accuracy_confidence + # Array#max_by can return nil (when called on an empty array), but our steep type is non-nil. + # Since it's not smart enough to realize the non-empty-array-usage of `max_by` won't return nil, + # we have to cast it to untyped here. + _ = [field1, field2].max_by { |f| ACCURACY_SCORES.fetch(f.accuracy_confidence) } + end + + # Indicates if the field uses the `nested` mapping type. + # + # @private + def nested? + mapping_type == "nested" + end + + # Records the `ComputationDetail` that should be on the `runtime_metadata_graphql_field`. + # + # @private + def runtime_metadata_computation_detail(empty_bucket_value:, function:) + self.computation_detail = SchemaArtifacts::RuntimeMetadata::ComputationDetail.new( + empty_bucket_value: empty_bucket_value, + function: function + ) + end + + # Lazily creates and returns a GraphQLField using the field's {#name_in_index}, {#computation_detail}, + # and {#relationship}. + # + # @private + def runtime_metadata_graphql_field + SchemaArtifacts::RuntimeMetadata::GraphQLField.new( + name_in_index: name_in_index, + computation_detail: computation_detail, + relation: relationship&.runtime_metadata + ) + end + + private + + def args_sdl(joiner:, after_opening_paren: "", &arg_selector) + selected_args = args.values.select(&arg_selector) + args_sdl = selected_args.map(&:to_sdl).flat_map { |s| s.split("\n") }.join(joiner) + return nil if args_sdl.empty? + "(#{after_opening_paren}#{args_sdl})" + end + + # Indicates if the field uses the `text` mapping type. + def text? + mapping_type == "text" + end + + def define_legacy_timestamp_grouping_arguments_if_needed(grouping_field) + case type.fully_unwrapped.name + when "Date" + grouping_field.argument schema_def_state.schema_elements.granularity, "DateGroupingGranularity!" do |a| + a.documentation "Determines the grouping granularity for this field." + end + + grouping_field.argument schema_def_state.schema_elements.offset_days, "Int" do |a| + a.documentation <<~EOS + Number of days (positive or negative) to shift the `Date` boundaries of each date grouping bucket. + + For example, when grouping by `YEAR`, this can be used to align the buckets with fiscal or school years instead of calendar years. + EOS + end + when "DateTime" + grouping_field.argument schema_def_state.schema_elements.granularity, "DateTimeGroupingGranularity!" do |a| + a.documentation "Determines the grouping granularity for this field." + end + + grouping_field.argument schema_def_state.schema_elements.time_zone, "TimeZone" do |a| + a.documentation "The time zone to use when determining which grouping a `DateTime` value falls in." + a.default "UTC" + end + + grouping_field.argument schema_def_state.schema_elements.offset, "DateTimeGroupingOffsetInput" do |a| + a.documentation <<~EOS + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change what day-of-week weeks are considered to start on. + EOS + end + end + end + + def list_field_grouped_by_doc_note(individual_value_selection_description) + <<~EOS.strip + Note: `#{name}` is a collection field, but selecting this field will group on individual values of #{individual_value_selection_description}. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `#{name}` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `#{name}` multiple times for a single document, that document will only be included in the group + once + EOS + end + + # Determines the suffix of the filter field derived for this field. The suffix used determines + # the filtering capabilities (e.g. filtering on a single value vs a list of values with `any_satisfy`). + def filter_field_category(for_single_value) + return :filter_input if for_single_value + + # For an index leaf field, there are no further nesting paths to traverse. We want to directly + # use a `ListFilterInput` type (e.g. `IntListFilterInput`) to offer `any_satisfy` filtering at this level. + return :list_filter_input if index_leaf? + + # If it's a list-of-objects field we require the user to tell us what mapping type they want to + # use, which determines the suffix (and is handled below). Otherwise, we want to use `FieldsListFilterInput`. + # We are within a list filtering context (as indicated by `for_single_value` being false) without + # being at an index leaf field, so we must use `FieldsListFilterInput` as there are further nesting paths + # on the document and we want to provide `any_satisfy` at the leaf fields. + return :fields_list_filter_input unless type_for_derived_types.list? + + case mapping_type + when "nested" then :list_filter_input + when "object" then :fields_list_filter_input + else + raise Errors::SchemaError, <<~EOS + `#{parent_type.name}.#{name}` is a list-of-objects field, but the mapping type has not been explicitly specified. Elasticsearch and OpenSearch + offer two ways to index list-of-objects fields. It cannot be changed on an existing field without dropping the index and recreating it (losing + any existing indexed data!), and there are nuanced tradeoffs involved here, so ElasticGraph provides no default mapping in this situation. + + If you're currently prototyping and don't want to spend time weighing this tradeoff, we recommend you do this: + + ``` + t.field "#{name}", "#{type.name}" do |f| + # Here we are opting for flexibility (nested) over pure performance (object). + # TODO: evaluate if we want to stick with `nested` before going to production. + f.mapping type: "nested" + end + ``` + + Read on for details of the tradeoff involved here. + + ----------------------------------------------------------------------------------------------------------------------------- + + Here are the options: + + 1) `f.mapping type: "object"` will cause each field path to be indexed as a separate "flattened" list. + + For example, given a `Film` document like this: + + ``` + { + "name": "The Empire Strikes Back", + "characters": [ + {"first": "Luke", "last": "Skywalker"}, + {"first": "Han", "last": "Solo"} + ] + } + ``` + + ...the data will look like this in the inverted Lucene index: + + ``` + { + "name": "The Empire Strikes Back", + "characters.first": ["Luke", "Han"], + "characters.last": ["Skywalker", "Solo"] + } + ``` + + This is highly efficient, but there is no way to search on multiple fields of a character and be sure that the matching values came from the same character. + ElasticGraph models this in the filtering API it offers for this case: + + ``` + query { + films(filter: { + characters: { + first: {#{schema_def_state.schema_elements.any_satisfy}: {#{schema_def_state.schema_elements.equal_to_any_of}: ["Luke"]}} + last: {#{schema_def_state.schema_elements.any_satisfy}: {#{schema_def_state.schema_elements.equal_to_any_of}: ["Skywalker"]}} + } + }) { + # ... + } + } + ``` + + As suggested by this filtering API, this will match any film that has a character with a first name of "Luke" and a character + with the last name of "Skywalker", but this could be satisfied by two separate characters. + + 2) `f.mapping type: "nested"` will cause each _object_ in the list to be indexed as a separate hidden document, preserving the independence of each. + + Given a `Film` document like "The Empire Strikes Back" from above, the `nested` type will index separate hidden documents for each character. This + allows ElasticGraph to offer this filtering API instead: + + ``` + query { + films(filter: { + characters: {#{schema_def_state.schema_elements.any_satisfy}: { + first: {#{schema_def_state.schema_elements.equal_to_any_of}: ["Luke"]} + last: {#{schema_def_state.schema_elements.equal_to_any_of}: ["Skywalker"]} + }} + }) { + # ... + } + } + ``` + + As suggested by this filtering API, this will only match films that have a character named "Luke Skywalker". However, the Elasticsearch docs[^1][^2] warn + that the `nested` mapping type can lead to performance problems, and index sorting cannot be configured[^3] when the `nested` type is used. + + [^1]: https://www.elastic.co/guide/en/elasticsearch/reference/8.10/nested.html + [^2]: https://www.elastic.co/guide/en/elasticsearch/reference/8.10/joining-queries.html + [^3]: https://www.elastic.co/guide/en/elasticsearch/reference/8.10/index-modules-index-sorting.html + EOS + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field_path.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field_path.rb new file mode 100644 index 00000000..8a079790 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field_path.rb @@ -0,0 +1,112 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Represents a potentially nested path to a field. + # + # @private + class FieldPath < Data.define(:first_part, :last_part, :path_parts) + # The type of the field (based purely on the last part; the parent parts aren't interesting here). + def type + last_part.type + end + + def path + path_parts.map(&:name).join(".") + end + + # The full path to the field in the index. + def path_in_index + path_parts.map(&:name_in_index).join(".") + end + + # The full name of the field path, including the parent type name, such as "Widget.nested.some_field". + def fully_qualified_path + "#{first_part.parent_type.name}.#{path}" + end + + # The full name of the field path in the index, including the parent type name, such as "Widget.nested.some_field". + def fully_qualified_path_in_index + "#{first_part.parent_type.name}.#{path_in_index}" + end + + # The full description of the field path, including the parent type name, and field type, + # such as "Widget.nested.some_field: ID". + def full_description + "#{fully_qualified_path}: #{type.name}" + end + + # We hide `new` because `FieldPath` is only intended to be instantiated from a `Resolver` instance. + # Importantly, `Resolver` provides an invariant that we want: a `FieldPath` is never instantiated + # with an empty list of path parts. (This is important for the steep type checking, so it can count + # on `last_part` being non-nil). + private_class_method :new + + # Responsible for resolving a particular field path (given as a string) into a `FieldPath` object. + # + # Important: this class optimizes performance by memoizing some things based on the current state + # of the ElasticGraph schema. It's intended to be used AFTER the schema is fully defined (e.g. + # as part of dumping schema artifacts). Using it before the schema has fully been defined requires + # that you discard the instance after using it, as it won't be aware of additions to the schema + # and may yield inaccurate results. + class Resolver + def initialize + @indexing_fields_by_public_name_by_type = ::Hash.new do |hash, type| + hash[type] = type + .indexing_fields_by_name_in_index + .values + .to_h { |f| [f.name, f] } + end + end + + # Resolves the given `path_string` relative to the given `type`. + # Returns `nil` if no field at that path can be found. + # + # Requires a block which will be called to determine if a parent field is valid to resolve through. + # For example, the caller may want to disallow all parent list fields, or disallow `nested` parent + # list fields while allowing `object` parent list fields. + def resolve_public_path(type, path_string) + field = nil # : Field? + + path_parts = path_string.split(".").map do |field_name| + return nil unless type + return nil if field && !yield(field) + return nil unless (field = @indexing_fields_by_public_name_by_type.dig(type, field_name)) + type = field.type.unwrap_list.as_object_type + field + end + + return nil if path_parts.empty? + + FieldPath.send(:new, path_parts.first, path_parts.last, path_parts) + end + + # Determines the nested paths in the given `path_string` + # Returns `nil` if no field at that path can be found, and + # returns `[]` if no nested paths are found. + # + # Nested paths are represented as the full path to the nested fields + # For example: a `path_string` of "foo.bar.baz" might have + # nested paths ["foo", "foo.bar.baz"] + def determine_nested_paths(type, path_string) + field_path = resolve_public_path(type, path_string) { true } + return nil unless field_path + + parts_so_far = [] # : ::Array[::String] + field_path.path_parts.filter_map do |field| + parts_so_far << field.name + parts_so_far.join(".") if field.nested? + end + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field_source.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field_source.rb new file mode 100644 index 00000000..b55990f6 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field_source.rb @@ -0,0 +1,16 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # @private + FieldSource = ::Data.define(:relationship_name, :field_path) + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/graphql_sdl_enumerator.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/graphql_sdl_enumerator.rb new file mode 100644 index 00000000..bafb84cd --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/graphql_sdl_enumerator.rb @@ -0,0 +1,113 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Responsible for enumerating the SDL strings for all GraphQL types, both explicitly defined and derived. + # + # @private + class GraphQLSDLEnumerator + include ::Enumerable + # @dynamic schema_def_state + attr_reader :schema_def_state + + def initialize(schema_def_state, all_types_except_root_query_type) + @schema_def_state = schema_def_state + @all_types_except_root_query_type = all_types_except_root_query_type + end + + # Yields the SDL for each GraphQL type, including both explicitly defined + # GraphQL types and derived GraphqL types. + def each(&block) + all_types = enumerate_all_types.sort_by(&:name) + all_type_names = all_types.map(&:name).to_set + + all_types.each do |type| + next if STOCK_GRAPHQL_SCALARS.include?(type.name) + yield type.to_sdl { |arg| all_type_names.include?(arg.value_type.fully_unwrapped.name) } + end + end + + private + + def enumerate_all_types + [root_query_type].compact + @all_types_except_root_query_type + end + + def aggregation_efficiency_hints_for(derived_indexed_types) + return nil if derived_indexed_types.empty? + + hints = derived_indexed_types.map do |type| + derived_indexing_type = @schema_def_state.types_by_name.fetch(type.destination_type_ref.name) + alternate_field_name = (_ = derived_indexing_type).plural_root_query_field_name + grouping_field = type.id_source + + " - The root `#{alternate_field_name}` field groups by `#{grouping_field}`" + end + + <<~EOS + Note: aggregation queries are relatively expensive, and some fields have been pre-aggregated to allow + more efficient queries for some common aggregation cases: + + #{hints.join("\n")} + EOS + end + + def root_query_type + # Some of our tests need to define their own root `Query` type, so here we avoid + # generating `Query` if an sdl part exists that already defines it. + return nil if @schema_def_state.sdl_parts.flat_map { |sdl| sdl.lines }.any? { |line| line.start_with?("type Query") } + + new_built_in_object_type "Query" do |t| + t.documentation "The query entry point for the entire schema." + + @schema_def_state.types_by_name.values.select(&:indexed?).sort_by(&:name).each do |type| + # @type var indexed_type: Mixins::HasIndices & _Type + indexed_type = _ = type + + t.relates_to_many( + indexed_type.plural_root_query_field_name, + indexed_type.name, + via: "ignore", + dir: :in, + singular: indexed_type.singular_root_query_field_name + ) do |f| + f.documentation "Fetches `#{indexed_type.name}`s based on the provided arguments." + indexed_type.root_query_fields_customizations&.call(f) + end + + # Add additional efficiency hints to the aggregation field documentation if we have any such hints. + # This needs to be outside the `relates_to_many` block because `relates_to_many` adds its own "suffix" to + # the field documentation, and here we add another one. + if (agg_efficiency_hint = aggregation_efficiency_hints_for(indexed_type.derived_indexed_types)) + agg_name = @schema_def_state.schema_elements.normalize_case("#{indexed_type.singular_root_query_field_name}_aggregations") + agg_field = t.graphql_fields_by_name.fetch(agg_name) + agg_field.documentation "#{agg_field.doc_comment}\n\n#{agg_efficiency_hint}" + end + end + end + end + + def new_built_in_object_type(name, &block) + new_object_type name do |type| + @schema_def_state.built_in_types_customization_blocks.each do |customization_block| + customization_block.call(type) + end + + block.call(type) + end + end + + def new_object_type(name, &block) + @schema_def_state.factory.new_object_type(name, &block) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/input_field.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/input_field.rb new file mode 100644 index 00000000..7030244b --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/input_field.rb @@ -0,0 +1,31 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" +require "elastic_graph/schema_definition/mixins/supports_default_value" +require "elastic_graph/schema_definition/schema_elements/field" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # A decorator that wraps a `Field` in order to provide additional functionality that + # we need to support on fields on input types (but not on fields on return types). + # + # For example, fields on input types support default values, but return type fields do not. + # + # @private + class InputField < DelegateClass(Field) + prepend Mixins::SupportsDefaultValue + + def to_sdl(type_structure_only: false, default_value_sdl: self.default_value_sdl, &arg_selector) + super(type_structure_only: type_structure_only, default_value_sdl: default_value_sdl, &arg_selector) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/input_type.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/input_type.rb new file mode 100644 index 00000000..1d91cde6 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/input_type.rb @@ -0,0 +1,60 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" +require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" +require "elastic_graph/schema_definition/schema_elements/type_with_subfields" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Represents a GraphQL `input` (used primarily for filtering). + # + # @private + class InputType < DelegateClass(TypeWithSubfields) + include Mixins::HasReadableToSAndInspect.new { |t| t.name } + + def initialize(schema_def_state, name) + schema_def_state.factory.new_type_with_subfields( + :input, name, + wrapping_type: self, + field_factory: schema_def_state.factory.method(:new_input_field) + ) do |type| + # Here we clear `reserved_field_names` because those field names are reserved precisely for our usage + # here on input filters. If we don't set this to an empty set we'll get exceptions in `new_filter` above + # when we generate our standard filter operators. + # + # Note: we opt-out of the reserved field names here rather then opting in on the other `TypeWithSubfields` + # subtypes because this is the only case where we don't want the reserved field name check applied (but we + # have multiple subtypes where we do want it applied). + type.reserved_field_names = Set.new + + super(type) + graphql_only true + yield self + end + end + + def runtime_metadata(extra_update_targets) + SchemaArtifacts::RuntimeMetadata::ObjectType.new( + update_targets: extra_update_targets, + index_definition_names: [], + graphql_fields_by_name: graphql_fields_by_name.transform_values(&:runtime_metadata_graphql_field), + elasticgraph_category: nil, + source_type: nil, + graphql_only_return_type: false + ) + end + + def derived_graphql_types + [] + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/interface_type.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/interface_type.rb new file mode 100644 index 00000000..50f897f6 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/interface_type.rb @@ -0,0 +1,72 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" +require "elastic_graph/schema_definition/mixins/has_indices" +require "elastic_graph/schema_definition/mixins/has_subtypes" +require "elastic_graph/schema_definition/mixins/implements_interfaces" +require "elastic_graph/schema_definition/mixins/supports_filtering_and_aggregation" +require "elastic_graph/schema_definition/schema_elements/type_with_subfields" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # {include:API#interface_type} + # + # @example Define an interface + # ElasticGraph.define_schema do |schema| + # schema.interface_type "Athlete" do |t| + # # in the block, `t` is an InterfaceType + # end + # end + class InterfaceType < DelegateClass(TypeWithSubfields) + # As of the October 2021 GraphQL spec, interfaces can now implement other interfaces: + # http://spec.graphql.org/October2021/#sec-Interfaces.Interfaces-Implementing-Interfaces + # Originated from: graphql/graphql-spec#373 + include Mixins::ImplementsInterfaces + include Mixins::SupportsFilteringAndAggregation + include Mixins::HasIndices + include Mixins::HasSubtypes + include Mixins::HasReadableToSAndInspect.new { |t| t.name } + + # @private + def initialize(schema_def_state, name) + field_factory = schema_def_state.factory.method(:new_field) + schema_def_state.factory.new_type_with_subfields(:interface, name, wrapping_type: self, field_factory: field_factory) do |type| + __skip__ = super(type) do + yield self + end + end + end + + # This contains more than just the proper interface fields; it also contains the fields from the + # subtypes, which winds up being used to generate an input filter and aggregation type. + # + # For just the interface fields, use `interface_fields_by_name`. + # + # @private + def graphql_fields_by_name + merged_fields_from_subtypes_by_name = super # delegates to the `HasSubtypes` definition. + # The interface field definitions should take precedence over the merged fields from the subtypes. + merged_fields_from_subtypes_by_name.merge(interface_fields_by_name) + end + + # @private + def interface_fields_by_name + __getobj__.graphql_fields_by_name + end + + private + + def resolve_subtypes + schema_def_state.implementations_by_interface_ref[type_ref] + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/list_counts_state.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/list_counts_state.rb new file mode 100644 index 00000000..9ce93976 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/list_counts_state.rb @@ -0,0 +1,40 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # @private + class ListCountsState < ::Data.define( + # the path from the root to the current list counts field + :path_to_list_counts, + # the path within the list counts field + :path_from_list_counts + ) + # @dynamic path_to_list_counts, path_from_list_counts, with + + def self.new_list_counts_field(at:) + new(path_to_list_counts: at, path_from_list_counts: "") + end + + INITIAL = new_list_counts_field(at: LIST_COUNTS_FIELD) + + def [](subpath) + with(path_from_list_counts: "#{path_from_list_counts}#{subpath}.") + end + + def path_to_count_subfield(subpath) + count_subfield = (path_from_list_counts + subpath).gsub(".", LIST_COUNTS_FIELD_PATH_KEY_SEPARATOR) + "#{path_to_list_counts}.#{count_subfield}" + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/object_type.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/object_type.rb new file mode 100644 index 00000000..db98936c --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/object_type.rb @@ -0,0 +1,53 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" +require "elastic_graph/errors" +require "elastic_graph/schema_definition/mixins/has_indices" +require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" +require "elastic_graph/schema_definition/mixins/implements_interfaces" +require "elastic_graph/schema_definition/schema_elements/type_with_subfields" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # {include:API#object_type} + # + # @example Define an object type + # ElasticGraph.define_schema do |schema| + # schema.object_type "Money" do |t| + # # in the block, `t` is an ObjectType + # end + # end + class ObjectType < DelegateClass(TypeWithSubfields) + # DelegateClass(TypeWithSubfields) provides the following methods: + # @dynamic name, type_ref, to_sdl, derived_graphql_types, to_indexing_field_type, current_sources, index_field_runtime_metadata_tuples, graphql_only?, relay_pagination_type + include Mixins::SupportsFilteringAndAggregation + + # `include HasIndices` provides the following methods: + # @dynamic runtime_metadata, derived_indexed_types, indices, indexed?, abstract? + include Mixins::HasIndices + + # `include ImplementsInterfaces` provides the following methods: + # @dynamic verify_graphql_correctness! + include Mixins::ImplementsInterfaces + include Mixins::HasReadableToSAndInspect.new { |t| t.name } + + # @private + def initialize(schema_def_state, name) + field_factory = schema_def_state.factory.method(:new_field) + schema_def_state.factory.new_type_with_subfields(:type, name, wrapping_type: self, field_factory: field_factory) do |type| + __skip__ = super(type) do + yield self + end + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/relationship.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/relationship.rb new file mode 100644 index 00000000..0be6c34b --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/relationship.rb @@ -0,0 +1,218 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" +require "elastic_graph/errors" +require "elastic_graph/schema_definition/schema_elements/field" +require "elastic_graph/support/hash_util" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Wraps a {Field} to provide additional relationship-specific functionality when defining a field via + # {TypeWithSubfields#relates_to_one} or {TypeWithSubfields#relates_to_many}. + # + # @example Define relationships between two types + # ElasticGraph.define_schema do |schema| + # schema.object_type "Orchestra" do |t| + # t.field "id", "ID" + # t.relates_to_many "musicians", "Musician", via: "orchestraId", dir: :in, singular: "musician" do |r| + # # In this block, `r` is a `Relationship`. + # end + # t.index "orchestras" + # end + # + # schema.object_type "Musician" do |t| + # t.field "id", "ID" + # t.field "instrument", "String" + # t.relates_to_one "orchestra", "Orchestra", via: "orchestraId", dir: :out do |r| + # # In this block, `r` is a `Relationship`. + # end + # t.index "musicians" + # end + # end + class Relationship < DelegateClass(Field) + # @dynamic related_type + + # @return [ObjectType, InterfaceType, UnionType] the type this relationship relates to + attr_reader :related_type + + # @private + def initialize(field, cardinality:, related_type:, foreign_key:, direction:) + super(field) + @cardinality = cardinality + @related_type = related_type + @foreign_key = foreign_key + @direction = direction + @equivalent_field_paths_by_local_path = {} + @additional_filter = {} + end + + # Adds additional filter conditions to a relationship beyond the foreign key. + # + # @param filter [Hash, Hash] additional filter conditions for this relationship + # @return [void] + # + # @example Define additional filter conditions on a `relates_to_one` relationship + # ElasticGraph.define_schema do |schema| + # schema.object_type "Orchestra" do |t| + # t.field "id", "ID" + # t.relates_to_many "musicians", "Musician", via: "orchestraId", dir: :in, singular: "musician" + # t.relates_to_one "firstViolin", "Musician", via: "orchestraId", dir: :in do |r| + # r.additional_filter isFirstViolon: true + # end + # + # t.index "orchestras" + # end + # + # schema.object_type "Musician" do |t| + # t.field "id", "ID" + # t.field "instrument", "String" + # t.field "isFirstViolon", "Boolean" + # t.relates_to_one "orchestra", "Orchestra", via: "orchestraId", dir: :out + # t.index "musicians" + # end + # end + def additional_filter(filter) + stringified_filter = Support::HashUtil.stringify_keys(filter) + @additional_filter = Support::HashUtil.deep_merge(@additional_filter, stringified_filter) + end + + # Indicates that `path` (a field on the related type) is the equivalent of `locally_named` on this type. + # + # Use this API to specify a local field's equivalent path on the related type. This must be used on relationships used by + # {Field#sourced_from} when the local type uses {Indexing::Index#route_with} or {Indexing::Index#rollover} so that + # ElasticGraph can determine what field from the related type to use to route the update requests to the correct index and shard. + # + # @param path [String] path to a routing or rollover field on the related type + # @param locally_named [String] path on the local type to the equivalent field + # @return [void] + # + # @example + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "id", "ID!" + # t.field "name", "String" + # t.field "createdAt", "DateTime" + # + # t.relates_to_one "launchPlan", "CampaignLaunchPlan", via: "campaignId", dir: :in do |r| + # r.equivalent_field "campaignCreatedAt", locally_named: "createdAt" + # end + # + # t.field "launchDate", "Date" do |f| + # f.sourced_from "launchPlan", "launchDate" + # end + # + # t.index "campaigns"do |i| + # i.rollover :yearly, "createdAt" + # end + # end + # + # schema.object_type "CampaignLaunchPlan" do |t| + # t.field "id", "ID" + # t.field "campaignId", "ID" + # t.field "campaignCreatedAt", "DateTime" + # t.field "launchDate", "Date" + # + # t.index "campaign_launch_plans" + # end + # end + def equivalent_field(path, locally_named: path) + if @equivalent_field_paths_by_local_path.key?(locally_named) + raise Errors::SchemaError, "`equivalent_field` has been called multiple times on `#{parent_type.name}.#{name}` with the same " \ + "`locally_named` value (#{locally_named.inspect}), but each local field can have only one `equivalent_field`." + else + @equivalent_field_paths_by_local_path[locally_named] = path + end + end + + # Gets the `routing_value_source` from this relationship for the given `index`, based on the configured + # routing used by `index` and the configured equivalent fields. + # + # Returns the GraphQL field name (not the `name_in_index`). + # + # @private + def routing_value_source_for_index(index) + return nil unless index.uses_custom_routing? + + @equivalent_field_paths_by_local_path.fetch(index.routing_field_path.path) do |local_need| + yield local_need + end + end + + # Gets the `rollover_timestamp_value_source` from this relationship for the given `index`, based on the + # configured equivalent fields and the rollover configuration used by `index`. + # + # Returns the GraphQL field name (not the `name_in_index`). + # + # @private + def rollover_timestamp_value_source_for_index(index) + return nil unless (rollover_config = index.rollover_config) + + @equivalent_field_paths_by_local_path.fetch(rollover_config.timestamp_field_path.path) do |local_need| + yield local_need + end + end + + # @private + def validate_equivalent_fields(field_path_resolver) + resolved_related_type = (_ = related_type.as_object_type) # : indexableType + + @equivalent_field_paths_by_local_path.flat_map do |local_path_string, related_type_path_string| + errors = [] # : ::Array[::String] + + local_path = resolve_and_validate_field_path(parent_type, local_path_string, field_path_resolver) do |error| + errors << error + end + + related_type_path = resolve_and_validate_field_path(resolved_related_type, related_type_path_string, field_path_resolver) do |error| + errors << error + end + + if local_path && related_type_path && local_path.type.unwrap_non_null != related_type_path.type.unwrap_non_null + errors << "Field `#{related_type_path.full_description}` is defined as an equivalent of " \ + "`#{local_path.full_description}` via an `equivalent_field` definition on `#{parent_type.name}.#{name}`, " \ + "but their types do not agree. To continue, change one or the other so that they agree." + end + + errors + end + end + + # @private + def many? + @cardinality == :many + end + + # @private + def runtime_metadata + field_path_resolver = SchemaElements::FieldPath::Resolver.new + resolved_related_type = (_ = related_type.unwrap_list.as_object_type) # : indexableType + foreign_key_nested_paths = field_path_resolver.determine_nested_paths(resolved_related_type, @foreign_key) + foreign_key_nested_paths ||= [] # : ::Array[::String] + SchemaArtifacts::RuntimeMetadata::Relation.new(foreign_key: @foreign_key, direction: @direction, additional_filter: @additional_filter, foreign_key_nested_paths: foreign_key_nested_paths) + end + + private + + def resolve_and_validate_field_path(type, field_path_string, field_path_resolver) + field_path = field_path_resolver.resolve_public_path(type, field_path_string) do |parent_field| + !parent_field.type.list? + end + + if field_path.nil? + yield "Field `#{type.name}.#{field_path_string}` (referenced from an `equivalent_field` defined on " \ + "`#{parent_type.name}.#{name}`) does not exist. Either define it or correct the `equivalent_field` definition." + end + + field_path + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/scalar_type.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/scalar_type.rb new file mode 100644 index 00000000..31e0c6cd --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/scalar_type.rb @@ -0,0 +1,310 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/scalar_type" +require "elastic_graph/schema_definition/indexing/field_type/scalar" +require "elastic_graph/schema_definition/mixins/can_be_graphql_only" +require "elastic_graph/schema_definition/mixins/has_derived_graphql_type_customizations" +require "elastic_graph/schema_definition/mixins/has_directives" +require "elastic_graph/schema_definition/mixins/has_documentation" +require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" +require "elastic_graph/schema_definition/mixins/has_type_info" +require "elastic_graph/schema_definition/mixins/verifies_graphql_name" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # {include:API#scalar_type} + # + # @example Define a scalar type + # ElasticGraph.define_schema do |schema| + # schema.scalar_type "URL" do |t| + # t.mapping type: "keyword" + # t.json_schema type: "string", format: "uri" + # end + # end + # + # @!attribute [r] schema_def_state + # @return [State] schema definition state + # @!attribute [rw] type_ref + # @private + # @!attribute [rw] mapping_type + # @private + # @!attribute [rw] runtime_metadata + # @private + # @!attribute [rw] aggregated_values_customizations + # @private + class ScalarType < Struct.new(:schema_def_state, :type_ref, :mapping_type, :runtime_metadata, :aggregated_values_customizations) + # `Struct.new` provides the following methods: + # @dynamic type_ref, runtime_metadata + prepend Mixins::VerifiesGraphQLName + include Mixins::CanBeGraphQLOnly + include Mixins::HasDocumentation + include Mixins::HasDirectives + include Mixins::HasDerivedGraphQLTypeCustomizations + include Mixins::HasReadableToSAndInspect.new { |t| t.name } + + # `HasTypeInfo` provides the following methods: + # @dynamic mapping_options, json_schema_options + include Mixins::HasTypeInfo + + # @dynamic graphql_only? + + # @private + def initialize(schema_def_state, name) + super(schema_def_state, schema_def_state.type_ref(name).to_final_form) + + # Default the runtime metadata before yielding, so it can be overridden as needed. + self.runtime_metadata = SchemaArtifacts::RuntimeMetadata::ScalarType.new( + coercion_adapter_ref: SchemaArtifacts::RuntimeMetadata::ScalarType::DEFAULT_COERCION_ADAPTER_REF, + indexing_preparer_ref: SchemaArtifacts::RuntimeMetadata::ScalarType::DEFAULT_INDEXING_PREPARER_REF + ) + + yield self + + missing = [ + ("`mapping`" if mapping_options.empty?), + ("`json_schema`" if json_schema_options.empty?) + ].compact + + if missing.any? + raise Errors::SchemaError, "Scalar types require `mapping` and `json_schema` to be configured, but `#{name}` lacks #{missing.join(" and ")}." + end + end + + # @return [String] name of the scalar type + def name + type_ref.name + end + + # (see Mixins::HasTypeInfo#mapping) + def mapping(**options) + self.mapping_type = options.fetch(:type) do + raise Errors::SchemaError, "Must specify a mapping `type:` on custom scalars but was missing on the `#{name}` type." + end + + super + end + + # Specifies the scalar coercion adapter that should be used for this scalar type. The scalar coercion adapter is responsible + # for validating and coercing scalar input values, and converting scalar return values to a form suitable for JSON serialization. + # + # @note For examples of scalar coercion adapters, see `ElasticGraph::GraphQL::ScalarCoercionAdapters`. + # @note If the `defined_at` require path requires any directories be put on the Ruby `$LOAD_PATH`, you are responsible for doing + # that before booting {ElasticGraph::GraphQL}. + # + # @param adapter_name [String] fully qualified Ruby class name of the adapter + # @param defined_at [String] the `require` path of the adapter + # @return [void] + # + # @example Register a coercion adapter + # ElasticGraph.define_schema do |schema| + # schema.scalar_type "PhoneNumber" do |t| + # t.mapping type: "keyword" + # t.json_schema type: "string", pattern: "^\\+[1-9][0-9]{1,14}$" + # t.coerce_with "CoercionAdapters::PhoneNumber", defined_at: "./coercion_adapters/phone_number" + # end + # end + def coerce_with(adapter_name, defined_at:) + self.runtime_metadata = runtime_metadata.with(coercion_adapter_ref: { + "extension_name" => adapter_name, + "require_path" => defined_at + }).tap(&:load_coercion_adapter) # verify the adapter is valid. + end + + # Specifies an indexing preparer that should be used for this scalar type. The indexing preparer is responsible for preparing + # scalar values before indexing them, performing any desired formatting or normalization. + # + # @note For examples of scalar coercion adapters, see `ElasticGraph::Indexer::IndexingPreparers`. + # @note If the `defined_at` require path requires any directories be put on the Ruby `$LOAD_PATH`, you are responsible for doing + # that before booting {ElasticGraph::GraphQL}. + # + # @param preparer_name [String] fully qualified Ruby class name of the indexing preparer + # @param defined_at [String] the `require` path of the preparer + # @return [void] + # + # @example Register an indexing preparer + # ElasticGraph.define_schema do |schema| + # schema.scalar_type "PhoneNumber" do |t| + # t.mapping type: "keyword" + # t.json_schema type: "string", pattern: "^\\+[1-9][0-9]{1,14}$" + # + # t.prepare_for_indexing_with "IndexingPreparers::PhoneNumber", + # defined_at: "./indexing_preparers/phone_number" + # end + # end + def prepare_for_indexing_with(preparer_name, defined_at:) + self.runtime_metadata = runtime_metadata.with(indexing_preparer_ref: { + "extension_name" => preparer_name, + "require_path" => defined_at + }).tap(&:load_indexing_preparer) # verify the preparer is valid. + end + + # @return [String] the GraphQL SDL form of this scalar + def to_sdl + "#{formatted_documentation}scalar #{name} #{directives_sdl}" + end + + # Registers a block which will be used to customize the derived `*AggregatedValues` object type. + # + # @private + def customize_aggregated_values_type(&block) + self.aggregated_values_customizations = block + end + + # @private + def aggregated_values_type + if aggregated_values_customizations + type_ref.as_aggregated_values + else + schema_def_state.type_ref("NonNumeric").as_aggregated_values + end + end + + # @private + def to_indexing_field_type + Indexing::FieldType::Scalar.new(scalar_type: self) + end + + # @private + def derived_graphql_types + return [] if graphql_only? + + pagination_types = + if schema_def_state.paginated_collection_element_types.include?(name) + schema_def_state.factory.build_relay_pagination_types(name, include_total_edge_count: true) + else + [] # : ::Array[ObjectType] + end + + (to_input_filters + pagination_types).tap do |derived_types| + if (aggregated_values_type = to_aggregated_values_type) + derived_types << aggregated_values_type + end + end + end + + # @private + def indexed? + false + end + + private + + EQUAL_TO_ANY_OF_DOC = <<~EOS + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + EOS + + GT_DOC = <<~EOS + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + EOS + + GTE_DOC = <<~EOS + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + EOS + + LT_DOC = <<~EOS + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + EOS + + LTE_DOC = <<~EOS + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + EOS + + def to_input_filters + # Note: all fields on inputs should be nullable, to support parameterized queries where + # the parameters are allowed to be set to `null`. We also now support nulls within lists. + + # For floats, we may want to remove the `equal_to_any_of` operator at some point. + # In many languages. checking exact equality with floats is problematic. + # For example, in IRB: + # + # 2.7.1 :003 > 0.3 == (0.1 + 0.2) + # => false + # + # However, it's not yet clear if that issue will come up with GraphQL, because + # float values are serialized on the wire as JSON, using an exact decimal + # string representation. So for now we are keeping `equal_to_any_of`. + schema_def_state.factory.build_standard_filter_input_types_for_index_leaf_type(name) do |t| + # Normally, we use a nullable type for `equal_to_any_of`, to allow a filter expression like this: + # + # filter: {optional_field: {equal_to_any_of: [null]}} + # + # That filter expression matches documents where `optional_field == null`. However, + # we cannot support this: + # + # filter: {tags: {any_satisfy: {equal_to_any_of: [null]}}} + # + # We can't support that because we implement filtering on `null` using an `exists` query: + # https://www.elastic.co/guide/en/elasticsearch/reference/8.10/query-dsl-exists-query.html + # + # ...but that works based on the field existing (or not), and does not let us filter on the + # presence or absence of `null` within a list. + # + # So, here we make the field non-null if we're in an `any_satisfy` context (as indicated by + # the type ending with `ListElementFilterInput`). + equal_to_any_of_type = t.type_ref.list_element_filter_input? ? "[#{name}!]" : "[#{name}]" + + t.field schema_def_state.schema_elements.equal_to_any_of, equal_to_any_of_type do |f| + f.documentation EQUAL_TO_ANY_OF_DOC + end + + if mapping_type_efficiently_comparable? + t.field schema_def_state.schema_elements.gt, name do |f| + f.documentation GT_DOC + end + + t.field schema_def_state.schema_elements.gte, name do |f| + f.documentation GTE_DOC + end + + t.field schema_def_state.schema_elements.lt, name do |f| + f.documentation LT_DOC + end + + t.field schema_def_state.schema_elements.lte, name do |f| + f.documentation LTE_DOC + end + end + end + end + + def to_aggregated_values_type + return nil unless (customization_block = aggregated_values_customizations) + schema_def_state.factory.new_aggregated_values_type_for_index_leaf_type(name, &customization_block) + end + + # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html + # https://www.elastic.co/guide/en/elasticsearch/reference/7.13/number.html#number + NUMERIC_TYPES = %w[long integer short byte double float half_float scaled_float unsigned_long].to_set + DATE_TYPES = %w[date date_nanos].to_set + # The Elasticsearch/OpenSearch docs do not exhaustively give a list of types on which range queries are efficient, + # but the docs are clear that it is efficient on numeric and date types, and is inefficient on string + # types: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-range-query.html + COMPARABLE_TYPES = NUMERIC_TYPES | DATE_TYPES + + def mapping_type_efficiently_comparable? + COMPARABLE_TYPES.include?(mapping_type) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/sort_order_enum_value.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/sort_order_enum_value.rb new file mode 100644 index 00000000..1a9cb3b8 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/sort_order_enum_value.rb @@ -0,0 +1,36 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" +require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" +require "elastic_graph/schema_definition/schema_elements/enum_value" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Simple wrapper around an {EnumValue} so that we can expose the `sort_order_field_path` to {Field} customization callbacks. + class SortOrderEnumValue < DelegateClass(EnumValue) + include Mixins::HasReadableToSAndInspect.new { |v| v.name } + + # @dynamic sort_order_field_path + + # @return [Array] path to the field from the root of the indexed {ObjectType} + attr_reader :sort_order_field_path + + # @private + def initialize(enum_value, sort_order_field_path) + # We've told steep that SortOrderEnumValue is subclass of EnumValue + # but here are supering to the `DelegateClass`'s initialize, not `EnumValue`'s, + # so we have to use `__skip__` + __skip__ = super(enum_value) + @sort_order_field_path = sort_order_field_path + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/sub_aggregation_path.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/sub_aggregation_path.rb new file mode 100644 index 00000000..008c11f2 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/sub_aggregation_path.rb @@ -0,0 +1,66 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Abstraction responsible for identifying paths to sub-aggregations, and, on that basis, determining + # what the type names should be. + # + # @private + SubAggregationPath = ::Data.define( + # List of index document types within which the target type exists. This contains the set of parent + # index document types--that is, types which are indexed or are themselves used as a `nested` field + # on a parent of it. Parent objects which are not "index documents" (e.g. directly at an index level + # or a nested field level) are omitted; we omit them because we don't offer sub-aggregations for such + # a field, and the set of sub-aggregations we are going to offer is the basis for generating separate + # `*SubAggregation` types. + :parent_doc_types, + # List of fields forming a path from the last parent doc type. + :field_path + ) do + # @implements SubAggregationPath + + # Determines the set of sub aggregation paths for the given type. + def self.paths_for(type, schema_def_state:) + root_paths = type.indexed? ? [SubAggregationPath.new([type.name], [])] : [] # : ::Array[SubAggregationPath] + + non_relation_field_refs = schema_def_state + .user_defined_field_references_by_type_name.fetch(type.name) { [] } + # Relationship fields are the only case where types can reference each other in circular fashion. + # If we don't reject that case here, we can get stuck in infinite recursion. + .reject(&:relationship) + + root_paths + non_relation_field_refs.flat_map do |field_ref| + # Here we call `schema_def_state.sub_aggregation_paths_for` rather than directly + # recursing to give schema_def_state a chance to cache the results. + parent_paths = schema_def_state.sub_aggregation_paths_for(field_ref.parent_type) + + if field_ref.nested? + parent_paths.map { |path| path.plus_parent(field_ref.type_for_derived_types.fully_unwrapped.name) } + else + parent_paths.map { |path| path.plus_field(field_ref) } + end + end + end + + def plus_parent(parent) + with(parent_doc_types: parent_doc_types + [parent], field_path: []) + end + + def plus_field(field) + with(field_path: field_path + [field]) + end + + def field_path_string + field_path.map(&:name).join(".") + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_namer.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_namer.rb new file mode 100644 index 00000000..dbee741c --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_namer.rb @@ -0,0 +1,237 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "did_you_mean" +require "elastic_graph/constants" +require "elastic_graph/errors" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Abstraction for generating derived GraphQL type names based on a collection of formats. A default set of formats is included, and + # overrides can be provided to customize the format we use for naming derived types. + class TypeNamer < ::Struct.new(:formats, :regexes, :name_overrides, :reverse_overrides) + # Initializes a new `TypeNamer` with the provided format overrides. + # The keys in `overrides` must match the keys in `DEFAULT_FORMATS` and the values must have + # the same placeholders as are present in the default formats. + # + # @private + def initialize(format_overrides: {}, name_overrides: {}) + @used_names = [] + name_overrides = name_overrides.transform_keys(&:to_s) + + validate_format_overrides(format_overrides) + validate_name_overrides(name_overrides) + + formats = DEFAULT_FORMATS.merge(format_overrides) + regexes = formats.transform_values { |format| /\A#{format.gsub(PLACEHOLDER_REGEX, "(\\w+)")}\z/ } + reverse_overrides = name_overrides.to_h { |k, v| [v, k] } + + super(formats: formats, regexes: regexes, name_overrides: name_overrides, reverse_overrides: reverse_overrides) + end + + # Returns the configured name for the given `standard_name`. + # + # By default, the returned name will just be the string form of the given `standard_name`, but if + # the `TypeNamer` was instantiated with an override for the given `standard_name`, that will be + # returned instead. + # + # @private + def name_for(standard_name) + string_name = standard_name.to_s + @used_names << string_name + name_overrides.fetch(string_name, string_name) + end + + # If the given `potentially_overriden_name` is an overridden name, returns the name from before the + # override was applied. Note: this may not be the true "original" name that ElasticGraph would have + # have used (e.g. it could still be customized by `formats`) but it will be the name that would + # be used without any `name_overrides`. + # + # @private + def revert_override_for(potentially_overriden_name) + reverse_overrides.fetch(potentially_overriden_name, potentially_overriden_name) + end + + # Generates a derived type name based on the provided format name and arguments. The given arguments must match + # the placeholders in the format. If the format name is unknown or the arguments are invalid, a `Errors::ConfigError` is raised. + # + # Note: this does not apply any configured `name_overrides`. It's up to the caller to apply that when desired. + # + # @private + def generate_name_for(format_name, **args) + format = formats.fetch(format_name) do + suggestions = FORMAT_SUGGESTER.correct(format_name).map(&:inspect) + raise Errors::ConfigError, "Unknown format name: #{format_name.inspect}. Possible alternatives: #{suggestions.join(", ")}." + end + + expected_placeholders = REQUIRED_PLACEHOLDERS.fetch(format_name) + if (missing_placeholders = expected_placeholders - args.keys).any? + raise Errors::ConfigError, "The arguments (#{args.inspect}) provided for `#{format_name}` format (#{format.inspect}) omits required key(s): #{missing_placeholders.join(", ")}." + end + + if (extra_placeholders = args.keys - expected_placeholders).any? + raise Errors::ConfigError, "The arguments (#{args.inspect}) provided for `#{format_name}` format (#{format.inspect}) contains extra key(s): #{extra_placeholders.join(", ")}." + end + + format % args + end + + # Given a `name` that has been generated for the given `format`, extracts the `base` parameter value that was used + # to generate `name`. + # + # Raises an error if the given `format` does not support `base` extraction. (To extract `base`, it's required that + # `base` is the only placeholder in the format.) + # + # Returns `nil` if the given `format` does support `base` extraction but `name` does not match the `format`. + # + # @private + def extract_base_from(name, format:) + unless REQUIRED_PLACEHOLDERS.fetch(format) == [:base] + raise Errors::InvalidArgumentValueError, "The `#{format}` format does not support base extraction." + end + + regexes.fetch(format).match(name)&.captures&.first + end + + # Indicates if the given `name` matches the format for the provided `format_name`. + # + # Note: our formats are not "mutually exclusive"--some names can match more than one format, so the + # fact that a name matches a format does not guarantee it was generated by that format. + # + # @private + def matches_format?(name, format_name) + regexes.fetch(format_name).match?(name) + end + + # Returns a hash containing the entries of `name_overrides` which have not been used. + # These are likely to be typos, and they can be used to warn the user. + # + # @private + def unused_name_overrides + name_overrides.except(*@used_names.uniq) + end + + # Returns a set containing all names that got passed to `name_for`: essentially, these are the + # candidates for valid name overrides. + # + # Can be used (in conjunction with `unused_name_overrides`) to provide suggested + # alternatives to the user. + # + # @private + def used_names + @used_names.to_set + end + + # Extracts the names of the placeholders from the provided format. + # + # @private + def self.placeholders_in(format) + format.scan(PLACEHOLDER_REGEX).flatten.map(&:to_sym) + end + + # The default formats used for derived GraphQL type names. These formats can be customized by providing `derived_type_name_formats` + # to {RakeTasks} or {Local::RakeTasks}. + # + # @return [Hash] + DEFAULT_FORMATS = { + AggregatedValues: "%{base}AggregatedValues", + Aggregation: "%{base}Aggregation", + Connection: "%{base}Connection", + Edge: "%{base}Edge", + FieldsListFilterInput: "%{base}FieldsListFilterInput", + FilterInput: "%{base}FilterInput", + GroupedBy: "%{base}GroupedBy", + InputEnum: "%{base}Input", + ListElementFilterInput: "%{base}ListElementFilterInput", + ListFilterInput: "%{base}ListFilterInput", + SortOrder: "%{base}SortOrder", + SubAggregation: "%{parent_types}%{base}SubAggregation", + SubAggregations: "%{parent_agg_type}%{field_path}SubAggregations" + }.freeze + + private + + # https://rubular.com/r/EJMY0zHZiC5HQm + PLACEHOLDER_REGEX = /%\{(\w+)\}/ + + REQUIRED_PLACEHOLDERS = DEFAULT_FORMATS.transform_values { |format| placeholders_in(format) } + FORMAT_SUGGESTER = ::DidYouMean::SpellChecker.new(dictionary: DEFAULT_FORMATS.keys) + DEFINITE_ENUM_FORMATS = [:SortOrder].to_set + DEFINITE_OBJECT_FORMATS = DEFAULT_FORMATS.keys.to_set - DEFINITE_ENUM_FORMATS - [:InputEnum].to_set + TYPES_THAT_CANNOT_BE_OVERRIDDEN = STOCK_GRAPHQL_SCALARS.union(["Query"]).freeze + + def validate_format_overrides(format_overrides) + format_problems = format_overrides.flat_map do |format_name, format| + validate_format(format_name, format) + end + + notify_problems(format_problems, "Provided derived type name formats") + end + + def validate_format(format_name, format) + if (required_placeholders = REQUIRED_PLACEHOLDERS[format_name]) + placeholders = self.class.placeholders_in(format) + placeholder_problems = [] # : ::Array[String] + + if (missing_placeholders = required_placeholders - placeholders).any? + placeholder_problems << "The #{format_name} format #{format.inspect} is missing required placeholders: #{missing_placeholders.join(", ")}. " \ + "Example valid format: #{DEFAULT_FORMATS.fetch(format_name).inspect}." + end + + if (extra_placeholders = placeholders - required_placeholders).any? + placeholder_problems << "The #{format_name} format #{format.inspect} has excess placeholders: #{extra_placeholders.join(", ")}. " \ + "Example valid format: #{DEFAULT_FORMATS.fetch(format_name).inspect}." + end + + example_name = format % placeholders.to_h { |placeholder| [placeholder.to_sym, placeholder.capitalize] } + unless GRAPHQL_NAME_PATTERN.match(example_name) + placeholder_problems << "The #{format_name} format #{format.inspect} does not produce a valid GraphQL type name. " + + GRAPHQL_NAME_VALIDITY_DESCRIPTION + end + + placeholder_problems + else + suggestions = FORMAT_SUGGESTER.correct(format_name).map(&:inspect) + ["Unknown format name: #{format_name.inspect}. Possible alternatives: #{suggestions.join(", ")}."] + end + end + + def validate_name_overrides(name_overrides) + duplicate_problems = name_overrides + .group_by { |k, v| v } + .transform_values { |kv_pairs| kv_pairs.map(&:first) } + .select { |_, v| v.size > 1 } + .map do |override, source_names| + "Multiple names (#{source_names.sort.join(", ")}) map to the same override: #{override}, which is not supported." + end + + invalid_name_problems = name_overrides.filter_map do |source_name, override| + unless GRAPHQL_NAME_PATTERN.match(override) + "`#{override}` (the override for `#{source_name}`) is not a valid GraphQL type name. " + + GRAPHQL_NAME_VALIDITY_DESCRIPTION + end + end + + cant_override_problems = TYPES_THAT_CANNOT_BE_OVERRIDDEN.intersection(name_overrides.keys).map do |type_name| + "`#{type_name}` cannot be overridden because it is part of the GraphQL spec." + end + + notify_problems(duplicate_problems + invalid_name_problems + cant_override_problems, "Provided type name overrides") + end + + def notify_problems(problems, source_description) + return if problems.empty? + + raise Errors::ConfigError, "#{source_description} have #{problems.size} problem(s):\n\n" \ + "#{problems.map.with_index(1) { |problem, i| "#{i}. #{problem}" }.join("\n\n")}" + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_reference.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_reference.rb new file mode 100644 index 00000000..e000e92c --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_reference.rb @@ -0,0 +1,353 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" +require "elastic_graph/schema_definition/mixins/verifies_graphql_name" +require "elastic_graph/schema_definition/schema_elements/type_namer" +require "elastic_graph/support/memoizable_data" +require "forwardable" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Represents a reference to a type. This is basically just a name of a type, + # with the ability to resolve it to an actual type object on demand. In addition, + # we provide some useful logic that is based entirely on the type name. + # + # This is necessary because GraphQL does not require that types are defined + # before they are referenced. (And also you can have circular type dependencies). + # Therefore, we need to use a reference to a type initially, and can later resolve + # it to a concrete type object as needed. + # + # @private + class TypeReference < Support::MemoizableData.define(:name, :schema_def_state) + extend Forwardable + # @dynamic type_namer + def_delegator :schema_def_state, :type_namer + + # Extracts the type without any non-null or list wrappings it has. + def fully_unwrapped + schema_def_state.type_ref(unwrapped_name) + end + + # Removes any non-null wrappings the type has. + def unwrap_non_null + schema_def_state.type_ref(name.delete_suffix("!")) + end + + def wrap_non_null + return self if non_null? + schema_def_state.type_ref("#{name}!") + end + + # Removes the list wrapping if this is a list. + # + # If the outer wrapping is non-null, unwraps that as well. + def unwrap_list + schema_def_state.type_ref(unwrap_non_null.name.delete_prefix("[").delete_suffix("]")) + end + + # Returns the `ObjectType`, `UnionType` or `InterfaceType` object to which this + # type name refers, if it is the name of one of those kinds of types. + # + # Ignores any non-null wrapping on the type, if there is one. + def as_object_type + type = _ = unwrap_non_null.resolved + type if type.respond_to?(:graphql_fields_by_name) + end + + # Returns `true` if this is known to be an object type of some sort (including interface types, + # union types, and proper object types). + # + # Returns `false` if this is known to be a leaf type of some sort (either a scalar or enum). + # Returns `false` if this is a list type (either a list of objects or leafs). + # + # Raises an error if it cannot be determined either from the name or by resolving the type. + # + # Ignores any non-null wrapping on the type, if there is one. + def object? + return unwrap_non_null.object? if non_null? + + if (resolved_type = resolved) + return resolved_type.respond_to?(:graphql_fields_by_name) + end + + # For derived GraphQL types, the name usually implies what kind of type it is. + # The derived types get generated last, so this prediate may be called before the + # type has been defined. + case schema_kind_implied_by_name + when :object + true + when :enum + false + else + # If we can't determine the type from the name, just raise an error. + raise Errors::SchemaError, "Type `#{name}` cannot be resolved. Is it misspelled?" + end + end + + def enum? + return unwrap_non_null.enum? if non_null? + + if (resolved_type = resolved) + return resolved_type.is_a?(EnumType) + end + + # For derived GraphQL types, the name usually implies what kind of type it is. + # The derived types get generated last, so this prediate may be called before the + # type has been defined. + case schema_kind_implied_by_name + when :object + false + when :enum + true + else + # If we can't determine the type from the name, just raise an error. + raise Errors::SchemaError, "Type `#{name}` cannot be resolved. Is it misspelled?" + end + end + + # Returns `true` if this is known to be a scalar type or enum type. + # Returns `false` if this is known to be an object type or list type of any sort. + # + # Raises an error if it cannot be determined either from the name or by resolving the type. + # + # Ignores any non-null wrapping on the type, if there is one. + def leaf? + !list? && !object? + end + + # Returns `true` if this is a list type. + # + # Ignores any non-null wrapping on the type, if there is one. + def list? + name.start_with?("[") + end + + # Returns `true` if this is a non-null type. + def non_null? + name.end_with?("!") + end + + def boolean? + name == "Boolean" + end + + def to_s + name + end + + def resolved + schema_def_state.types_by_name[name] + end + + def unwrapped_name + name + .sub(/\A\[+/, "") # strip `[` characters from the start: https://rubular.com/r/tHVBBQkQUMMVVz + .sub(/[\]!]+\z/, "") # strip `]` and `!` characters from the end: https://rubular.com/r/pC8C0i7EpvHDbf + end + + # Generally speaking, scalar types have `grouped_by` fields which are scalars of the same types, + # and object types have `grouped_by` fields which are special `[object_type]GroupedBy` types. + # + # ...except for some special cases (Date and DateTime), which this predicate detects. + def scalar_type_needing_grouped_by_object? + %w[Date DateTime].include?(type_namer.revert_override_for(name)) + end + + # Returns a new `TypeReference` with any type name overrides reverted (to provide the "original" type name). + def with_reverted_override + schema_def_state.type_ref(type_namer.revert_override_for(name)) + end + + # Returns all the JSON schema array/nullable layers of a type, from outermost to innermost. + # For example, [[Int]] will return [:nullable, :array, :nullable, :array, :nullable] + def json_schema_layers + @json_schema_layers ||= begin + layers, inner_type = peel_json_schema_layers_once + + if layers.empty? || inner_type == self + layers + else + layers + inner_type.json_schema_layers + end + end + end + + # Most of ElasticGraph's derived GraphQL types have a static suffix (e.g. the full type name + # is source_type + suffix). This is a map of all of these. + STATIC_FORMAT_NAME_BY_CATEGORY = TypeNamer::REQUIRED_PLACEHOLDERS.filter_map do |format_name, placeholders| + if placeholders == [:base] + as_snake_case = SchemaArtifacts::RuntimeMetadata::SchemaElementNamesDefinition::SnakeCaseConverter + .normalize_case(format_name.to_s) + .delete_prefix("_") + + [as_snake_case.to_sym, format_name] + end + end.to_h + + # Converts the TypeReference to its final form (i.e. the from that will be used in rendered schema artifacts). + # This handles multiple bits of type name customization based on the configured `type_name_overrides` and + # `derived_type_name_formats` settings (via the `TypeNamer`): + # + # - If the `as_input` is `true` and this is a reference to an enum type, converts to the `InputEnum` format. + # - If there is a configured name override that applies to this type, uses it. + def to_final_form(as_input: false) + unwrapped = fully_unwrapped + inner_name = type_namer.name_for(unwrapped.name) + + if as_input && schema_def_state.type_ref(inner_name).enum? + inner_name = type_namer.name_for( + type_namer.generate_name_for(:InputEnum, base: inner_name) + ) + end + + renamed_with_same_wrappings(inner_name) + end + + # Builds a `TypeReference` for a statically named derived type for the given `category. + # + # In addition, a dynamic method `as_[category]` is also provided (defined further below). + def as_static_derived_type(category) + renamed_with_same_wrappings(type_namer.generate_name_for( + STATIC_FORMAT_NAME_BY_CATEGORY.fetch(category), + base: fully_unwrapped.name + )) + end + + # Generates the type name used for a sub-aggregation. This type has `grouped_by`, `aggregated_values`, + # `count` and `sub_aggregations` sub-fields to expose the different bits of aggregation functionality. + # + # The type name is based both on the type reference name and on the set of `parent_doc_types` + # that exist above it. The `parent_doc_types` are used in the name because we plan to offer different sub-aggregations + # under it based on where it is in the document structure. A type which is `nested` at multiple levels in different + # document contexts needs separate types generated for each case so that we can offer the correct contextual + # sub-aggregations that can be offered for each case. + def as_sub_aggregation(parent_doc_types:) + renamed_with_same_wrappings(type_namer.generate_name_for( + :SubAggregation, + base: fully_unwrapped.name, + parent_types: parent_doc_types.join + )) + end + + # Generates the type name used for a `sub_aggregations` field. A `sub_aggregations` field is + # available alongside `grouped_by`, `count`, and `aggregated_values` on an aggregation or + # sub-aggregation node. This type is used in two situations: + # + # 1. It is used directly under `nodes`/`edges { node }` on an Aggregation or SubAggregation. + # It provides access to each of the sub-aggregations that are available in that context. + # 2. It is used underneath that `SubAggregations` object for single object fields which have + # fields under them that are sub-aggregatable. + # + # The fields (and types of those fields) used for one of these types is contextual based on + # what the parent doc types are (so that we can offer sub-aggregations of the parent doc types!) + # and the field path (for the 2nd case). + def as_aggregation_sub_aggregations(parent_doc_types: [fully_unwrapped.name], field_path: []) + field_part = field_path.map { |f| to_title_case(f.name) }.join + + renamed_with_same_wrappings(type_namer.generate_name_for( + :SubAggregations, + parent_agg_type: parent_aggregation_type(parent_doc_types), + field_path: field_part + )) + end + + def as_parent_aggregation(parent_doc_types:) + schema_def_state.type_ref(parent_aggregation_type(parent_doc_types)) + end + + # Here we iterate over our mapping and generate dynamic methods for each category. + STATIC_FORMAT_NAME_BY_CATEGORY.keys.each do |category| + define_method(:"as_#{category}") do + # @type self: TypeReference + as_static_derived_type(category) + end + end + + def list_filter_input? + matches_format_of?(:list_filter_input) + end + + def list_element_filter_input? + matches_format_of?(:list_element_filter_input) + end + + # These methods are defined dynamically above: + # @dynamic as_aggregated_values + # @dynamic as_grouped_by + # @dynamic as_aggregation + # @dynamic as_connection + # @dynamic as_edge + # @dynamic as_fields_list_filter_input + # @dynamic as_filter_input + # @dynamic as_input_enum + # @dynamic as_list_element_filter_input, list_element_filter_input? + # @dynamic as_list_filter_input, list_filter_input? + # @dynamic as_sort_order + + private + + def after_initialize + Mixins::VerifiesGraphQLName.verify_name!(unwrapped_name) + end + + def peel_json_schema_layers_once + if list? + return [[:array], unwrap_list] if non_null? + return [[:nullable, :array], unwrap_list] + end + + return [[], unwrap_non_null] if non_null? + [[:nullable], self] + end + + def matches_format_of?(category) + format_name = STATIC_FORMAT_NAME_BY_CATEGORY.fetch(category) + type_namer.matches_format?(name, format_name) + end + + def parent_aggregation_type(parent_doc_types) + __skip__ = case parent_doc_types + in [single_parent_type] + type_namer.generate_name_for(:Aggregation, base: single_parent_type) + in [*parent_types, last_parent_type] + type_namer.generate_name_for(:SubAggregation, parent_types: parent_types.join, base: last_parent_type) + else + raise Errors::SchemaError, "Unexpected `parent_doc_types`: #{parent_doc_types.inspect}. `parent_doc_types` must not be empty." + end + end + + def renamed_with_same_wrappings(new_name) + pre_wrappings, post_wrappings = name.split(GRAPHQL_NAME_WITHIN_LARGER_STRING_PATTERN) + schema_def_state.type_ref("#{pre_wrappings}#{new_name}#{post_wrappings}") + end + + ENUM_FORMATS = TypeNamer::DEFINITE_ENUM_FORMATS + OBJECT_FORMATS = TypeNamer::DEFINITE_OBJECT_FORMATS + + def schema_kind_implied_by_name + name = type_namer.revert_override_for(self.name) + return :enum if ENUM_FORMATS.any? { |f| type_namer.matches_format?(name, f) } + return :object if OBJECT_FORMATS.any? { |f| type_namer.matches_format?(name, f) } + + if (as_output_enum_name = type_namer.extract_base_from(name, format: :InputEnum)) + :enum if ENUM_FORMATS.any? { |f| type_namer.matches_format?(as_output_enum_name, f) } + end + end + + def to_title_case(name) + CamelCaseConverter.normalize_case(name).sub(/\A(\w)/, &:upcase) + end + + CamelCaseConverter = SchemaArtifacts::RuntimeMetadata::SchemaElementNamesDefinition::CamelCaseConverter + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_with_subfields.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_with_subfields.rb new file mode 100644 index 00000000..bb0d3d5f --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_with_subfields.rb @@ -0,0 +1,578 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/schema_artifacts/runtime_metadata/relation" +require "elastic_graph/schema_definition/indexing/field" +require "elastic_graph/schema_definition/indexing/field_type/object" +require "elastic_graph/schema_definition/mixins/can_be_graphql_only" +require "elastic_graph/schema_definition/mixins/has_derived_graphql_type_customizations" +require "elastic_graph/schema_definition/mixins/has_directives" +require "elastic_graph/schema_definition/mixins/has_documentation" +require "elastic_graph/schema_definition/mixins/has_type_info" +require "elastic_graph/schema_definition/mixins/supports_filtering_and_aggregation" +require "elastic_graph/schema_definition/mixins/verifies_graphql_name" +require "elastic_graph/schema_definition/schema_elements/list_counts_state" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Defines common functionality for all GraphQL types that have subfields: + # + # - {InputType} + # - {InterfaceType} + # - {ObjectType} + # + # @abstract + # + # @!attribute [rw] schema_kind + # @private + # @!attribute [rw] schema_def_state + # @private + # @!attribute [rw] type_ref + # @private + # @!attribute [rw] reserved_field_names + # @private + # @!attribute [rw] graphql_fields_by_name + # @private + # @!attribute [rw] indexing_fields_by_name_in_index + # @private + # @!attribute [rw] field_factory + # @private + # @!attribute [rw] wrapping_type + # @private + # @!attribute [rw] relay_pagination_type + # @private + class TypeWithSubfields < Struct.new( + :schema_kind, :schema_def_state, :type_ref, :reserved_field_names, + :graphql_fields_by_name, :indexing_fields_by_name_in_index, :field_factory, + :wrapping_type, :relay_pagination_type + ) + prepend Mixins::VerifiesGraphQLName + include Mixins::CanBeGraphQLOnly + include Mixins::HasDocumentation + include Mixins::HasDirectives + include Mixins::HasDerivedGraphQLTypeCustomizations + include Mixins::HasTypeInfo + + # The following methods are provided by `Struct.new`: + # @dynamic type_ref + + # The following methods are provided by `SupportsFilteringAndAggregation`: + # @dynamic derived_graphql_types + + # The following methods are provided by `CanBeGraphQLOnly`: + # @dynamic graphql_only? + + # @private + def initialize(schema_kind, schema_def_state, name, wrapping_type:, field_factory:) + # `any_satisfy`, `any_of`/`all_of`, and `not` are "reserved" field names. They are reserved for usage by + # ElasticGraph itself in the `*FilterInput` types it generates. If we allow them to be used as field + # names, we'll run into conflicts when we later generate the `*FilterInput` type. + # + # Note that we don't have the same kind of conflict for the other filtering operators (e.g. + # `equal_to_any_of`, `gt`, etc) because on the generated filter structure, those are leaf + # nodes. They never exist alongside document field names on a filter type, but these do, + # so we have to guard against them here. + reserved_field_names = [ + schema_def_state.schema_elements.all_of, + schema_def_state.schema_elements.any_of, + schema_def_state.schema_elements.any_satisfy, + schema_def_state.schema_elements.not + ].to_set + + # @type var graphql_fields_by_name: ::Hash[::String, Field] + graphql_fields_by_name = {} + # @type var indexing_fields_by_name_in_index: ::Hash[::String, Field] + indexing_fields_by_name_in_index = {} + + super( + schema_kind, + schema_def_state, + schema_def_state.type_ref(name).to_final_form, + reserved_field_names, + graphql_fields_by_name, + indexing_fields_by_name_in_index, + field_factory, + wrapping_type, + false + ) + + yield self + end + + # @return [String] the name of this GraphQL type + def name + type_ref.name + end + + # Defines a [GraphQL field](https://spec.graphql.org/October2021/#sec-Language.Fields) on this type. + # + # @param name [String] name of the field + # @param type [String] type of the field as a [type reference](https://spec.graphql.org/October2021/#sec-Type-References). The named type must be + # one of {BuiltInTypes ElasticGraph's built-in types} or a type that has been defined in your schema. + # @param graphql_only [Boolean] if `true`, ElasticGraph will define the field as a GraphQL field but omit it from the indexing + # artifacts (`json_schemas.yaml` and `datastore_config.yaml`). This can be used along with `name_in_index` to support careful + # schema evolution. + # @param indexing_only [Boolean] if `true`, ElasticGraph will define the field for indexing (in the `json_schemas.yaml` and + # `datastore_config.yaml` schema artifact) but will omit it from the GraphQL schema. This can be useful to begin indexing a field + # before you expose it in GraphQL so that you can fully backfill it first. + # @option options [String] name_in_index the name of the field in the datastore index. Can be used to back a GraphQL field with a + # differently named field in the index. + # @option options [String] singular can be used on a list field (e.g. `t.field "tags", "[String!]!", singular: "tag"`) to tell + # ElasticGraph what the singular form of a field's name is. When provided, ElasticGraph will define a `groupedBy` field (using the + # singular form) allowing clients to group by individual values from the field. + # @option options [Boolean] aggregatable force-enables or disables the ability for aggregation queries to aggregate over this field. + # When not provided, ElasticGraph will infer field aggregatability based on the field's GraphQL type and mapping type. + # @option options [Boolean] filterable force-enables or disables the ability for queries to filter by this field. When not provided, + # ElasticGraph will infer field filterability based on the field's GraphQL type and mapping type. + # @option options [Boolean] groupable force-enables or disables the ability for aggregation queries to group by this field. When + # not provided, ElasticGraph will infer field groupability based on the field's GraphQL type and mapping type. + # @option options [Boolean] sortable force-enables or disables the ability for queries to sort by this field. When not provided, + # ElasticGraph will infer field sortability based on the field's GraphQL type and mapping type. + # @yield [Field] the field for further customization + # @return [void] + # + # @see #paginated_collection_field + # @see #relates_to_many + # @see #relates_to_one + # + # @note Be careful about defining non-nullable fields. Changing a field’s type from non-nullable (e.g. `Int!`) to nullable (e.g. + # `Int`) is a breaking change for clients. Making a field non-nullable may also prevent you from applying permissioning to a field + # via an AuthZ layer (as such a layer would have no way to force a field value to `null` when for a client denied field access). + # Therefore, we recommend limiting your use of `!` to only a few situations such as defining a type’s primary key (e.g. + # `t.field "id", "ID!"`) or defining a list field (e.g. `t.field "authors", "[String!]!"`) since empty lists already provide a + # "no data" representation. You can still configure the ElasticGraph indexer to require a non-null value for a field using + # `f.json_schema nullable: false`. + # + # @note ElasticGraph’s understanding of datastore capabilities may override your configured + # `aggregatable`/`filterable`/`groupable`/`sortable` options. For example, a field indexed as `text` for full text search will + # not be sortable or groupable even if you pass `sortable: true, groupable: true` when defining the field, because [text fields + # cannot be efficiently sorted by or grouped on](https://www.elastic.co/guide/en/elasticsearch/reference/8.15/text.html#text). + # + # @example Define a field with documentation + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "id", "ID" do |f| + # f.documentation "The Campaign's identifier." + # end + # end + # end + # + # @example Omit a new field from the GraphQL schema until its data has been backfilled + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "id", "ID" + # + # # TODO: remove `indexing_only: true` once the data for this field has been fully backfilled + # t.field "endDate", "Date", indexing_only: true + # end + # end + # + # @example Use `graphql_only` to introduce a new name for an existing field + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "id", "ID" + # + # t.field "endOn", "Date" do |f| + # f.directive "deprecated", reason: "Use `endDate` instead." + # end + # + # # We've decided we want to call the field `endDate` instead of `endOn`, but the data + # # for this field is currently indexed in `endOn`, so we can use `graphql_only` and + # # `name_in_index` to expose the existing data under a new field name. + # t.field "endDate", "Date", name_in_index: "endOn", graphql_only: true + # end + # end + def field(name, type, graphql_only: false, indexing_only: false, **options) + if reserved_field_names.include?(name) + raise Errors::SchemaError, "Invalid field name: `#{self.name}.#{name}`. `#{name}` is reserved for use by " \ + "ElasticGraph as a filtering operator. To use it for a field name, add " \ + "the `schema_element_name_overrides` option (on `ElasticGraph::SchemaDefinition::RakeTasks.new`) to " \ + "configure an alternate name for the `#{name}` operator." + end + + options = {name_in_index: nil}.merge(options) if graphql_only + + field_factory.call( + name: name, + type: type, + graphql_only: graphql_only, + parent_type: wrapping_type, + **options + ) do |field| + yield field if block_given? + + unless indexing_only + register_field(field.name, field, graphql_fields_by_name, "GraphQL", :indexing_only) + end + + unless graphql_only + register_field(field.name_in_index, field, indexing_fields_by_name_in_index, "indexing", :graphql_only) do |f| + f.to_indexing_field_reference + end + end + end + end + + # Registers the name of a field that existed in a prior version of the schema but has been deleted. + # + # @note In situations where this API applies, ElasticGraph will give you an error message indicating that you need to use this API + # or {Field#renamed_from}. Likewise, when ElasticGraph no longer needs to know about this, it'll give you a warning indicating + # the call to this method can be removed. + # + # @param field_name [String] name of field that used to exist but has been deleted + # @return [void] + # + # @example Indicate that `Widget.description` has been deleted + # ElasticGraph.define_schema do |schema| + # schema.object_type "Widget" do |t| + # t.deleted_field "description" + # end + # end + def deleted_field(field_name) + schema_def_state.register_deleted_field( + name, + field_name, + defined_at: caller_locations(2, 1).first, # : ::Thread::Backtrace::Location + defined_via: %(type.deleted_field "#{field_name}") + ) + end + + # Registers an old name that this type used to have in a prior version of the schema. + # + # @note In situations where this API applies, ElasticGraph will give you an error message indicating that you need to use this API + # or {API#deleted_type}. Likewise, when ElasticGraph no longer needs to know about this, it'll give you a warning indicating + # the call to this method can be removed. + # + # @param old_name [String] old name this field used to have in a prior version of the schema + # @return [void] + # + # @example Indicate that `Widget` used to be called `Component`. + # ElasticGraph.define_schema do |schema| + # schema.object_type "Widget" do |t| + # t.renamed_from "Component" + # end + # end + def renamed_from(old_name) + schema_def_state.register_renamed_type( + name, + from: old_name, + defined_at: caller_locations(2, 1).first, # : ::Thread::Backtrace::Location + defined_via: %(type.renamed_from "#{old_name}") + ) + end + + # An alternative to {#field} for when you have a list field that you want exposed as a [paginated Relay + # connection](https://relay.dev/graphql/connections.htm) rather than as a simple list. + # + # @note Bear in mind that pagination does not have much efficiency benefit in this case: all elements of the collection will be + # retrieved when fetching this field from the datastore. The pagination implementation will just trim down the collection before + # returning it. + # + # @param name [String] name of the field + # @param element_type [String] name of the type of element in the collection + # @param name_in_index [String] the name of the field in the datastore index. Can be used to back a GraphQL field with a + # differently named field in the index. + # @param singular [String] indicates what the singular form of a field's name is. When provided, ElasticGraph will define a + # `groupedBy` field (using the singular form) allowing clients to group by individual values from the field. + # @yield [Field] the field for further customization + # @return [void] + # + # @see #field + # @see #relates_to_many + # @see #relates_to_one + # + # @example Define `Author.books` as a paginated collection field + # ElasticGraph.define_schema do |schema| + # schema.object_type "Author" do |t| + # t.field "id", "ID" + # t.field "name", "String" + # t.paginated_collection_field "books", "String" + # t.index "authors" + # end + # end + def paginated_collection_field(name, element_type, name_in_index: name, singular: nil, &block) + element_type_ref = schema_def_state.type_ref(element_type).to_final_form + element_type = element_type_ref.name + + schema_def_state.paginated_collection_element_types << element_type + + backing_indexing_field = field(name, "[#{element_type}!]!", indexing_only: true, name_in_index: name_in_index, &block) + + field( + name, + element_type_ref.as_connection.name, + name_in_index: name_in_index, + type_for_derived_types: "[#{element_type}]", + groupable: !!singular, + sortable: false, + graphql_only: true, + singular: singular, + backing_indexing_field: backing_indexing_field + ) do |f| + f.define_relay_pagination_arguments! + block&.call(f) + end + end + + # Defines a "has one" relationship between the current indexed type and another indexed type by defining a field clients + # can use to navigate across indexed types in a single GraphQL query. + # + # @param field_name [String] name of the relationship field + # @param type [String] name of the related type + # @param via [String] name of the foreign key field + # @param dir [:in, :out] direction of the foreign key. Use `:in` for an inbound foreign key that resides on the related type and + # references the `id` of this type. Use `:out` for an outbound foreign key that resides on this type and references the `id` of + # the related type. + # @yield [Relationship] the generated relationship fields, for further customization + # @return [void] + # + # @see #field + # @see #relates_to_many + # + # @example Use `relates_to_one` to define `Player.team` + # ElasticGraph.define_schema do |schema| + # schema.object_type "Team" do |t| + # t.field "id", "ID" + # t.field "name", "String" + # t.field "homeCity", "String" + # t.index "teams" + # end + # + # schema.object_type "Player" do |t| + # t.field "id", "ID" + # t.field "name", "String" + # t.relates_to_one "team", "Team", via: "teamId", dir: :out + # t.index "players" + # end + # end + def relates_to_one(field_name, type, via:, dir:, &block) + foreign_key_type = schema_def_state.type_ref(type).non_null? ? "ID!" : "ID" + relates_to(field_name, type, via: via, dir: dir, foreign_key_type: foreign_key_type, cardinality: :one, related_type: type, &block) + end + + # Defines a "has many" relationship between the current indexed type and another indexed type by defining a pair of fields clients + # can use to navigate across indexed types in a single GraphQL query. The pair of generated fields will be [Relay Connection + # types](https://relay.dev/graphql/connections.htm#sec-Connection-Types) allowing you to filter, sort, paginate, and aggregated the + # related data. + # + # @param field_name [String] name of the relationship field + # @param type [String] name of the related type + # @param via [String] name of the foreign key field + # @param dir [:in, :out] direction of the foreign key. Use `:in` for an inbound foreign key that resides on the related type and + # references the `id` of this type. Use `:out` for an outbound foreign key that resides on this type and references the `id` of + # the related type. + # @param singular [String] singular form of the `field_name`; will be used (along with an `Aggregations` suffix) for the name of + # the generated aggregations field + # @yield [Relationship] the generated relationship fields, for further customization + # @return [void] + # + # @see #field + # @see #paginated_collection_field + # @see #relates_to_one + # + # @example Use `relates_to_many` to define `Team.players` and `Team.playerAggregations` + # ElasticGraph.define_schema do |schema| + # schema.object_type "Team" do |t| + # t.field "id", "ID" + # t.field "name", "String" + # t.field "homeCity", "String" + # t.relates_to_many "players", "Player", via: "teamId", dir: :in, singular: "player" + # t.index "teams" + # end + # + # schema.object_type "Player" do |t| + # t.field "id", "ID" + # t.field "name", "String" + # t.field "teamId", "ID" + # t.index "players" + # end + # end + def relates_to_many(field_name, type, via:, dir:, singular:) + foreign_key_type = (dir == :out) ? "[ID!]!" : "ID" + type_ref = schema_def_state.type_ref(type).to_final_form + + relates_to(field_name, type_ref.as_connection.name, via: via, dir: dir, foreign_key_type: foreign_key_type, cardinality: :many, related_type: type) do |f| + f.argument schema_def_state.schema_elements.filter, type_ref.as_filter_input.name do |a| + a.documentation "Used to filter the returned `#{field_name}` based on the provided criteria." + end + + f.argument schema_def_state.schema_elements.order_by, "[#{type_ref.as_sort_order.name}!]" do |a| + a.documentation "Used to specify how the returned `#{field_name}` should be sorted." + end + + f.define_relay_pagination_arguments! + + yield f if block_given? + end + + aggregations_name = schema_def_state.schema_elements.normalize_case("#{singular}_aggregations") + relates_to(aggregations_name, type_ref.as_aggregation.as_connection.name, via: via, dir: dir, foreign_key_type: foreign_key_type, cardinality: :many, related_type: type) do |f| + f.argument schema_def_state.schema_elements.filter, type_ref.as_filter_input.name do |a| + a.documentation "Used to filter the `#{type}` documents that get aggregated over based on the provided criteria." + end + + f.define_relay_pagination_arguments! + + yield f if block_given? + + f.documentation f.derived_documentation("Aggregations over the `#{field_name}` data") + end + end + + # Converts the type to GraphQL SDL syntax. + # + # @private + def to_sdl(&field_arg_selector) + generate_sdl(name_section: name, &field_arg_selector) + end + + # @private + def generate_sdl(name_section:, &field_arg_selector) + <<~SDL + #{formatted_documentation}#{schema_kind} #{name_section} #{directives_sdl(suffix_with: " ")}{ + #{fields_sdl(&field_arg_selector)} + } + SDL + end + + # @private + def aggregated_values_type + schema_def_state.type_ref("NonNumeric").as_aggregated_values + end + + # @private + def indexed? + false + end + + # @private + def to_indexing_field_type + Indexing::FieldType::Object.new( + type_name: name, + subfields: indexing_fields_by_name_in_index.values.map(&:to_indexing_field).compact, + mapping_options: mapping_options, + json_schema_options: json_schema_options + ) + end + + # @private + def current_sources + indexing_fields_by_name_in_index.values.flat_map do |field| + child_field_sources = field.type.fully_unwrapped.as_object_type&.current_sources || [] + [field.source&.relationship_name || SELF_RELATIONSHIP_NAME] + child_field_sources + end + end + + # @private + def index_field_runtime_metadata_tuples( + # path from the overall document root + path_prefix: "", + # the source of the parent field + parent_source: SELF_RELATIONSHIP_NAME, + # tracks the state of the list counts field + list_counts_state: ListCountsState::INITIAL + ) + indexing_fields_by_name_in_index.flat_map do |name, field| + path = path_prefix + name + source = field.source&.relationship_name || parent_source + index_field = SchemaArtifacts::RuntimeMetadata::IndexField.new(source: source) + + list_count_field_tuples = field.paths_to_lists_for_count_indexing.map do |subpath| + [list_counts_state.path_to_count_subfield(subpath), index_field] # : [::String, SchemaArtifacts::RuntimeMetadata::IndexField] + end + + if (object_type = field.type.fully_unwrapped.as_object_type) + new_list_counts_state = + if field.type.list? && field.nested? + ListCountsState.new_list_counts_field(at: "#{path}.#{LIST_COUNTS_FIELD}") + else + list_counts_state[name] + end + + object_type.index_field_runtime_metadata_tuples( + path_prefix: "#{path}.", + parent_source: source, + list_counts_state: new_list_counts_state + ) + else + [[path, index_field]] # : ::Array[[::String, SchemaArtifacts::RuntimeMetadata::IndexField]] + end + list_count_field_tuples + end + end + + private + + def fields_sdl(&arg_selector) + graphql_fields_by_name.values + .map { |f| f.to_sdl(&arg_selector) } + .flat_map { |sdl| sdl.split("\n") } + .join("\n ") + end + + def register_field(name, field, registry, registry_type, only_option_to_fix, &to_comparable) + if (existing_field = registry[name]) + field = Field.pick_most_accurate_from(field, existing_field, to_comparable: to_comparable || ->(f) { f }) do + raise Errors::SchemaError, "Duplicate #{registry_type} field on Type #{self.name}: #{name}. " \ + "To resolve this, set `#{only_option_to_fix}: true` on one of the fields." + end + end + + registry[name] = field + end + + def relates_to(field_name, type, via:, dir:, foreign_key_type:, cardinality:, related_type:) + field(field_name, type, sortable: false, filterable: false, groupable: false, graphql_only: true) do |field| + relationship = schema_def_state.factory.new_relationship( + field, + cardinality: cardinality, + related_type: schema_def_state.type_ref(related_type).to_final_form, + foreign_key: via, + direction: dir + ) + + yield relationship if block_given? + + field.relationship = relationship + + if dir == :out + register_inferred_foreign_key_fields(from_type: [via, foreign_key_type], to_other: ["id", "ID!"], related_type: relationship.related_type) + else + register_inferred_foreign_key_fields(from_type: ["id", "ID!"], to_other: [via, foreign_key_type], related_type: relationship.related_type) + end + end + end + + def register_inferred_foreign_key_fields(from_type:, to_other:, related_type:) + # The root `Query` object shouldn't have inferred foreign key fields (it's not indexed). + return if name.to_s == "Query" + + from_field_name, from_type_name = from_type + field(from_field_name, from_type_name, indexing_only: true, accuracy_confidence: :medium) + + # If it's a self-referential, we also should add a foreign key field for the other end of the relation. + if name == related_type.unwrap_non_null.name + # This must be `:low` confidence for cases where we have a self-referential type that goes both + # directions, such as: + # + # s.object_type "MyTypeBothDirections" do |t| + # t.relates_to_one "parent", "MyTypeBothDirections!", via: "children_ids", dir: :in + # t.relates_to_many "children", "MyTypeBothDirections", via: "children_ids", dir: :out + # end + # + # In such a circumstance, the `from_type` side may be more accurate (and will be defined on the `field` + # call above) and we want it preferred over this definition here. + to_field_name, to_type_name = to_other + field(to_field_name, to_type_name, indexing_only: true, accuracy_confidence: :low) + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/union_type.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/union_type.rb new file mode 100644 index 00000000..a883ed90 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/union_type.rb @@ -0,0 +1,157 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_definition/indexing/index" +require "elastic_graph/schema_definition/mixins/can_be_graphql_only" +require "elastic_graph/schema_definition/mixins/has_derived_graphql_type_customizations" +require "elastic_graph/schema_definition/mixins/has_directives" +require "elastic_graph/schema_definition/mixins/has_documentation" +require "elastic_graph/schema_definition/mixins/has_indices" +require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" +require "elastic_graph/schema_definition/mixins/has_subtypes" +require "elastic_graph/schema_definition/mixins/supports_filtering_and_aggregation" +require "elastic_graph/schema_definition/mixins/verifies_graphql_name" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + # {include:API#union_type} + # + # @example Define a union type + # ElasticGraph.define_schema do |schema| + # schema.object_type "Card" do |t| + # # ... + # end + # + # schema.object_type "BankAccount" do |t| + # # ... + # end + # + # schema.object_type "BitcoinWallet" do |t| + # # ... + # end + # + # schema.union_type "FundingSource" do |t| + # t.subtype "Card" + # t.subtypes "BankAccount", "BitcoinWallet" + # end + # end + # + # @!attribute [r] schema_def_state + # @return [State] state of the schema + # @!attribute [rw] type_ref + # @private + # @!attribute [rw] subtype_refs + # @private + class UnionType < Struct.new(:schema_def_state, :type_ref, :subtype_refs) + prepend Mixins::VerifiesGraphQLName + include Mixins::CanBeGraphQLOnly + include Mixins::HasDocumentation + include Mixins::HasDirectives + include Mixins::SupportsFilteringAndAggregation + include Mixins::HasIndices + include Mixins::HasSubtypes + include Mixins::HasDerivedGraphQLTypeCustomizations + include Mixins::HasReadableToSAndInspect.new { |t| t.name } + + # @private + def initialize(schema_def_state, name) + super(schema_def_state, schema_def_state.type_ref(name).to_final_form, Set.new) do + yield self + end + end + + # @return [String] the name of the union type + def name + type_ref.name + end + + # Defines a subtype of this union type. + # + # @param name [String] the name of an object type which is a member of this union type + # @return [void] + # + # @example + # ElasticGraph.define_schema do |schema| + # schema.object_type "Card" do |t| + # # ... + # end + # + # schema.union_type "FundingSource" do |t| + # t.subtype "Card" + # end + # end + def subtype(name) + type_ref = schema_def_state.type_ref(name.to_s).to_final_form + + if subtype_refs.include?(type_ref) + raise Errors::SchemaError, "Duplicate subtype on UnionType #{self.name}: #{name}" + end + + subtype_refs << type_ref + end + + # Defines multiple subtypes of this union type. + # + # @param names [Array] names of object types which are members of this union type + # @return [void] + # + # @example Define a union type + # ElasticGraph.define_schema do |schema| + # schema.object_type "BankAccount" do |t| + # # ... + # end + # + # schema.object_type "BitcoinWallet" do |t| + # # ... + # end + # + # schema.union_type "FundingSource" do |t| + # t.subtypes "BankAccount", "BitcoinWallet" + # end + # end + def subtypes(*names) + names.flatten.each { |n| subtype(n) } + end + + # @return [String] the formatted GraphQL SDL of the union type + def to_sdl + if subtype_refs.empty? + raise Errors::SchemaError, "UnionType type #{name} has no subtypes, but must have at least one." + end + + "#{formatted_documentation}union #{name} #{directives_sdl(suffix_with: " ")}= #{subtype_refs.map(&:name).to_a.join(" | ")}" + end + + # @private + def verify_graphql_correctness! + # Nothing to verify. `verify_graphql_correctness!` will be called on each subtype automatically. + end + + # Various things check `mapping_options` on indexed types (usually object types, but can also happen on union types). + # We need to implement `mapping_options` here to satisfy those method calls, but we will never use custom mapping on + # a union type so we hardcode it to return nil. + # + # @private + def mapping_options + {} + end + + private + + def resolve_subtypes + subtype_refs.map do |ref| + ref.as_object_type || raise( + Errors::SchemaError, "The subtype `#{ref}` of the UnionType `#{name}` is not a defined object type." + ) + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/file_system_repository.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/file_system_repository.rb new file mode 100644 index 00000000..41079042 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/file_system_repository.rb @@ -0,0 +1,77 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/schema_definition/scripting/script" +require "elastic_graph/support/memoizable_data" +require "pathname" + +module ElasticGraph + module SchemaDefinition + # @private + module Scripting + # A simple abstraction that supports loading static scripts off of disk. The given directory + # is expected to have a sub-directory per script context, with individual scripts under the + # context sub-directories. The language is inferred from the script file extensions. + # + # @private + class FileSystemRepository < Support::MemoizableData.define(:dir) + # Based on https://www.elastic.co/guide/en/elasticsearch/reference/8.5/modules-scripting.html + SUPPORTED_LANGUAGES_BY_EXTENSION = { + ".painless" => "painless", + ".expression" => "expression", + ".mustache" => "mustache", + ".java" => "java" + } + + # The `Script` objects available in this file system repository. + def scripts + @scripts ||= ::Pathname.new(dir).children.sort.flat_map do |context_dir| + unless context_dir.directory? + raise Errors::InvalidScriptDirectoryError, "`#{dir}` has a file (#{context_dir}) that is not a context directory as expected." + end + + context_dir.children.sort.map do |script_file| + unless script_file.file? + raise Errors::InvalidScriptDirectoryError, "`#{dir}` has extra directory nesting (#{script_file}) that is unexpected." + end + + language = SUPPORTED_LANGUAGES_BY_EXTENSION[script_file.extname] || raise( + Errors::InvalidScriptDirectoryError, "`#{dir}` has a file (`#{script_file}`) that has an unrecognized file extension: #{script_file.extname}." + ) + + Script.new( + name: script_file.basename.sub_ext("").to_s, + source: script_file.read.strip, + language: language, + context: context_dir.basename.to_s + ) + end + end.tap { |all_scripts| verify_no_duplicates!(all_scripts) } + end + + # Map of script ids keyed by the `scoped_name` to allow easy lookup of the ids. + def script_ids_by_scoped_name + @script_ids_by_scoped_name ||= scripts.to_h { |s| [s.scoped_name, s.id] } + end + + private + + def verify_no_duplicates!(scripts) + duplicate_scoped_names = scripts.group_by(&:scoped_name).select do |scoped_name, scripts_with_scoped_name| + scripts_with_scoped_name.size > 1 + end.keys + + if duplicate_scoped_names.any? + raise Errors::InvalidScriptDirectoryError, "`#{dir}` has multiple scripts with the same scoped name, which is not allowed: #{duplicate_scoped_names.join(", ")}." + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/script.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/script.rb new file mode 100644 index 00000000..8ef82362 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/script.rb @@ -0,0 +1,48 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "digest/md5" +require "elastic_graph/support/memoizable_data" + +module ElasticGraph + module SchemaDefinition + module Scripting + # @private + class Script < Support::MemoizableData.define(:name, :source, :language, :context) + # The id we use when storing the script in the datastore. The id is based partially on a hash of + # the source code to make script safely evolveable: when the source code of a script changes, its + # id changes, and the old and new versions continue to be accessible in the datastore, allowing + # old and new versions of the deployed ElasticGraph application to be running at the same time + # (as happens during a zero-downtime rolled-out deploy). Scripts are invoked by their id, so we + # can trust that when the code tries to use a specific version of a script, it'll definitely use + # that version. + def id + @id ||= "#{context}_#{name}_#{::Digest::MD5.hexdigest(source)}" + end + + # The `name` scoped with the `context`. Due to how we structure static scripts on + # the file system (nested under a directory that names the `context`), a given `name` + # is only guaranteed to be unique within the scope of a given `context`. The `scoped_name` + # is how we will refer to a script from elsewhere in the code when we want to use it. + def scoped_name + @scoped_name ||= "#{context}/#{name}" + end + + def to_artifact_payload + { + "context" => context, + "script" => { + "lang" => language, + "source" => source + } + } + end + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/scripts/field/as_day_of_week.painless b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/scripts/field/as_day_of_week.painless new file mode 100644 index 00000000..9905e58c --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/scripts/field/as_day_of_week.painless @@ -0,0 +1,24 @@ +// Check if required params are missing +if (params.offset_ms == null) { + throw new IllegalArgumentException("Missing required parameter: offset_ms"); +} +if (params.time_zone == null) { + throw new IllegalArgumentException("Missing required parameter: time_zone"); +} + +// Set variables used in the loop +ZoneId zoneId = ZoneId.of(params.time_zone); +List results = new ArrayList(); + +for (ZonedDateTime timestamp : doc[params.field]) { + // Convert the timestamp to the specified time zone + ZonedDateTime zonedTimestamp = timestamp.withZoneSameInstant(zoneId); + + // Adjust the timestamp based on the offset_ms parameter + ZonedDateTime adjustedTimestamp = zonedTimestamp.plus(params.offset_ms, ChronoUnit.MILLIS); + + // Format and add the result to the list + results.add(adjustedTimestamp.getDayOfWeek().name()); +} + +return results; diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/scripts/field/as_time_of_day.painless b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/scripts/field/as_time_of_day.painless new file mode 100644 index 00000000..df6b643d --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/scripts/field/as_time_of_day.painless @@ -0,0 +1,41 @@ +// Check if required params are missing +if (params.offset_ms == null) { + throw new IllegalArgumentException("Missing required parameter: offset_ms"); +} +if (params.time_zone == null) { + throw new IllegalArgumentException("Missing required parameter: time_zone"); +} +if (params.interval == null) { + throw new IllegalArgumentException("Missing required parameter: interval"); +} + +// Set variables used in the loop +ZoneId zoneId = ZoneId.of(params.time_zone); +ChronoUnit intervalUnit; +if (params.interval == "hour") { + intervalUnit = ChronoUnit.HOURS; +} else if (params.interval == "minute") { + intervalUnit = ChronoUnit.MINUTES; +} else if (params.interval == "second") { + intervalUnit = ChronoUnit.SECONDS; +} else { + throw new IllegalArgumentException("Invalid interval value: " + params.interval); +} +DateTimeFormatter formatter = DateTimeFormatter.ISO_LOCAL_TIME; +List results = new ArrayList(); + +for (ZonedDateTime timestamp : doc[params.field]) { + // Convert the timestamp to the specified time zone + ZonedDateTime zonedTimestamp = timestamp.withZoneSameInstant(zoneId); + + // Adjust the timestamp based on the offset_ms parameter + ZonedDateTime adjustedTimestamp = zonedTimestamp.plus(params.offset_ms, ChronoUnit.MILLIS); + + // Truncate the timestamp to the specified interval + adjustedTimestamp = adjustedTimestamp.truncatedTo(intervalUnit); + + // Format and add the result to the list + results.add(adjustedTimestamp.format(formatter)); +} + +return results; diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/scripts/filter/by_time_of_day.painless b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/scripts/filter/by_time_of_day.painless new file mode 100644 index 00000000..23a84902 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/scripts/filter/by_time_of_day.painless @@ -0,0 +1,22 @@ +ZoneId zoneId = ZoneId.of(params.time_zone); + +for (ZonedDateTime timestamp : doc[params.field]) { + long docValue = timestamp + .withZoneSameInstant(zoneId) + .toLocalTime() + .toNanoOfDay(); + + // Perform comparisons based on whichever params are set. + // ElasticGraph takes care of passing us param values as nano-of-day so that we + // can directly and efficiently compare against `docValue`. + if ((params.gte == null || docValue >= params.gte) && + (params.gt == null || docValue > params.gt) && + (params.lte == null || docValue <= params.lte) && + (params.lt == null || docValue < params.lt) && + (params.equal_to_any_of == null || params.equal_to_any_of.contains(docValue))) { + return true; + } +} + +// No timestamp values matched the params, so return `false`. +return false; diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/scripts/update/index_data.painless b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/scripts/update/index_data.painless new file mode 100644 index 00000000..87ef03d7 --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/scripting/scripts/update/index_data.painless @@ -0,0 +1,93 @@ +Map source = ctx._source; +String sourceId = params.sourceId; +String relationship = params.relationship; + +// Numbers in JSON appear to be parsed as doubles, but we want the version stored as a long, so we need to cast it here. +long eventVersion = (long) params.version; + +if (source.__sources == null) { + source.__sources = []; +} + +if (source.__versions == null) { + source.__versions = [:]; +} + +if (source.__versions[relationship] == null) { + source.__versions[relationship] = [:]; +} + +Map relationshipVersionsMap = source.__versions.get(relationship); +List previousSourceIdsForRelationship = relationshipVersionsMap.keySet().stream().filter(id -> id != sourceId).collect(Collectors.toList()); + +if (previousSourceIdsForRelationship.size() > 0) { + String previousIdDescription = previousSourceIdsForRelationship.size() == 1 ? previousSourceIdsForRelationship.get(0) : previousSourceIdsForRelationship.toString(); + throw new IllegalArgumentException( + "Cannot update document " + params.id + " " + + "with data from related " + relationship + " " + sourceId + " " + + "because the related " + relationship + " has apparently changed (was: " + previousSourceIdsForRelationship + "), " + + "but mutations of relationships used with `sourced_from` are not supported because " + + "allowing it could break ElasticGraph's out-of-order processing guarantees." + ); +} + +// While the version in `__versions` is going to be used for the doc version in the future, for now +// we need to continue getting it from `__sourceVersions`. Both our old version and this versions of this +// script keep the value in `__sourceVersions` up-to-date, whereas the old script only writes it to +// `__sourceVersions`. Until we have completely migrated off of the old script for all ElasticGraph +// clusters, we need to keep using it. +// +// Later, after the old script is no longer used by any clusters, we'll stop using `__sourceVersions`. +// TODO: switch to `__versions` when we no longer need to maintain compatibility with the old version of the script. +Number _versionForSourceType = source.get("__sourceVersions")?.get(params.sourceType)?.get(sourceId); +Number _versionForRelationship = relationshipVersionsMap.get(sourceId); + +// Our JSON schema requires event versions to be non-negative, so we can safely use Long.MIN_VALUE as a stand-in when the value is null. +long versionForSourceType = _versionForSourceType == null ? Long.MIN_VALUE : _versionForSourceType.longValue(); +long versionForRelationship = _versionForRelationship == null ? Long.MIN_VALUE : _versionForRelationship.longValue(); + +// Pick the larger of the two versions as our doc version. Note that `Math.max` didn't work for me here for +// reasons I don't understand, but a simple ternary works fine. +// +// In theory, we could just use `versionForSourceType` as the `docVersion` (and not even check `__versions` at all) +// since both the old version and this version maintain the doc version in `__sourceVersions`. However, that would +// prevent this version of the script from being forward-compatible with the planned next version of this script. +// In the next version, we plan to stop writing to `__sourceVersions`, and as we can't deploy that change atomically, +// this version of the script will continue to run after that has begun to be used. So this version of the script +// must consider which version is greater here, and not simply trust either version value. +long docVersion = versionForSourceType > versionForRelationship ? versionForSourceType : versionForRelationship; + +if (docVersion >= eventVersion) { + throw new IllegalArgumentException("ElasticGraph update was a no-op: [" + + params.id + "]: version conflict, current version [" + + docVersion + "] is higher or equal to the one provided [" + + eventVersion + "]"); +} else { + source.putAll(params.data); + Map __counts = params.__counts; + + if (__counts != null) { + if (source.__counts == null) { + source.__counts = [:]; + } + + source.__counts.putAll(__counts); + } + + source.id = params.id; + source.__versions[relationship][sourceId] = eventVersion; + + // Record the relationship in `__sources` if it's not already there. We maintain it as an append-only set using a sorted list. + // This ensures deterministic ordering of its elements regardless of event ingestion order, and lets us check membership in O(log N) time. + // + // As per https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/Collections.html#binarySearch(java.util.List,java.lang.Object): + // + // > Returns the index of the search key, if it is contained in the list; otherwise, (-(insertion point) - 1). + // > The insertion point is defined as the point at which the key would be inserted into the list: the index + // > of the first element greater than the key, or list.size() if all elements in the list are less than the + // > specified key. Note that this guarantees that the return value will be >= 0 if and only if the key is found. + int sourceBinarySearchResult = Collections.binarySearch(source.__sources, relationship); + if (sourceBinarySearchResult < 0) { + source.__sources.add(-sourceBinarySearchResult - 1, relationship); + } +} diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/state.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/state.rb new file mode 100644 index 00000000..a7868e4c --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/state.rb @@ -0,0 +1,212 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/errors" +require "elastic_graph/schema_definition/factory" +require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" +require "elastic_graph/schema_definition/schema_elements/enum_value_namer" +require "elastic_graph/schema_definition/schema_elements/type_namer" +require "elastic_graph/schema_definition/schema_elements/sub_aggregation_path" + +module ElasticGraph + module SchemaDefinition + # Encapsulates all state that needs to be managed while a schema is defined. + # This is separated from `API` to make it easy to expose some state management + # helper methods to our internal code without needing to expose it as part of + # the public API. + # + # @private + class State < Struct.new( + :api, + :schema_elements, + :index_document_sizes, + :types_by_name, + :object_types_by_name, + :scalar_types_by_name, + :enum_types_by_name, + :implementations_by_interface_ref, + :sdl_parts, + :paginated_collection_element_types, + :user_defined_fields, + :renamed_types_by_old_name, + :deleted_types_by_old_name, + :renamed_fields_by_type_name_and_old_field_name, + :deleted_fields_by_type_name_and_old_field_name, + :json_schema_version, + :json_schema_version_setter_location, + :graphql_extension_modules, + :initially_registered_built_in_types, + :built_in_types_customization_blocks, + :user_definition_complete, + :sub_aggregation_paths_by_type, + :type_refs_by_name, + :output, + :type_namer, + :enum_value_namer + ) + include Mixins::HasReadableToSAndInspect.new + + def self.with( + api:, + schema_elements:, + index_document_sizes:, + derived_type_name_formats:, + type_name_overrides:, + enum_value_overrides_by_type:, + output: $stdout + ) + # @type var types_by_name: SchemaElements::typesByNameHash + types_by_name = {} + + new( + api: api, + schema_elements: schema_elements, + index_document_sizes: index_document_sizes, + types_by_name: types_by_name, + object_types_by_name: {}, + scalar_types_by_name: {}, + enum_types_by_name: {}, + implementations_by_interface_ref: ::Hash.new { |h, k| h[k] = ::Set.new }, + sdl_parts: [], + paginated_collection_element_types: ::Set.new, + user_defined_fields: ::Set.new, + renamed_types_by_old_name: {}, + deleted_types_by_old_name: {}, + renamed_fields_by_type_name_and_old_field_name: ::Hash.new { |h, k| h[k] = {} }, + deleted_fields_by_type_name_and_old_field_name: ::Hash.new { |h, k| h[k] = {} }, + json_schema_version_setter_location: nil, + json_schema_version: nil, + graphql_extension_modules: [], + initially_registered_built_in_types: ::Set.new, + built_in_types_customization_blocks: [], + user_definition_complete: false, + sub_aggregation_paths_by_type: {}, + type_refs_by_name: {}, + type_namer: SchemaElements::TypeNamer.new( + format_overrides: derived_type_name_formats, + name_overrides: type_name_overrides + ), + enum_value_namer: SchemaElements::EnumValueNamer.new(enum_value_overrides_by_type), + output: output + ) + end + + # @dynamic index_document_sizes? + alias_method :index_document_sizes?, :index_document_sizes + + def type_ref(name) + # Type references are immutable and can be safely cached. Here we cache them because we've observed + # it having a noticeable impact on our test suite runtime. + type_refs_by_name[name] ||= factory.new_type_reference(name) + end + + def register_object_interface_or_union_type(type) + register_type(type, object_types_by_name) + end + + def register_enum_type(type) + register_type(type, enum_types_by_name) + end + + def register_scalar_type(type) + register_type(type, scalar_types_by_name) + end + + def register_input_type(type) + register_type(type) + end + + def register_renamed_type(type_name, from:, defined_at:, defined_via:) + renamed_types_by_old_name[from] = factory.new_deprecated_element( + type_name, + defined_at: defined_at, + defined_via: defined_via + ) + end + + def register_deleted_type(type_name, defined_at:, defined_via:) + deleted_types_by_old_name[type_name] = factory.new_deprecated_element( + type_name, + defined_at: defined_at, + defined_via: defined_via + ) + end + + def register_renamed_field(type_name, from:, to:, defined_at:, defined_via:) + renamed_fields_by_type_name_and_old_field_name[type_name][from] = factory.new_deprecated_element( + to, + defined_at: defined_at, + defined_via: defined_via + ) + end + + def register_deleted_field(type_name, field_name, defined_at:, defined_via:) + deleted_fields_by_type_name_and_old_field_name[type_name][field_name] = factory.new_deprecated_element( + field_name, + defined_at: defined_at, + defined_via: defined_via + ) + end + + # Registers the given `field` as a user-defined field, unless the user definitions are complete. + def register_user_defined_field(field) + user_defined_fields << field + end + + def user_defined_field_references_by_type_name + @user_defined_field_references_by_type_name ||= begin + unless user_definition_complete + raise Errors::SchemaError, "Cannot access `user_defined_field_references_by_type_name` until the schema definition is complete." + end + + @user_defined_field_references_by_type_name ||= user_defined_fields + .group_by { |f| f.type.fully_unwrapped.name } + end + end + + def factory + @factory ||= Factory.new(self) + end + + def enums_for_indexed_types + @enums_for_indexed_types ||= factory.new_enums_for_indexed_types + end + + def sub_aggregation_paths_for(type) + sub_aggregation_paths_by_type.fetch(type) do + SchemaElements::SubAggregationPath.paths_for(type, schema_def_state: self).uniq.tap do |paths| + # Cache our results if the user has finished their schema definition. Otherwise, it's not safe to cache. + # :nocov: -- we never execute this with `user_definition_complete == false` + sub_aggregation_paths_by_type[type] = paths if user_definition_complete + # :nocov: + end + end + end + + private + + RESERVED_TYPE_NAMES = [EVENT_ENVELOPE_JSON_SCHEMA_NAME].to_set + + def register_type(type, additional_type_index = nil) + name = (_ = type).name + + if RESERVED_TYPE_NAMES.include?(name) + raise Errors::SchemaError, "`#{name}` cannot be used as a schema type because it is a reserved name." + end + + if types_by_name.key?(name) + raise Errors::SchemaError, "Duplicate definition for type #{name} detected. Each type can only be defined once." + end + + additional_type_index[name] = type if additional_type_index + types_by_name[name] = type + end + end + end +end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/test_support.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/test_support.rb new file mode 100644 index 00000000..1ab64dad --- /dev/null +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/test_support.rb @@ -0,0 +1,113 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/schema_definition/api" +require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" + +module ElasticGraph + module SchemaDefinition + # Mixin designed to facilitate writing tests that define schemas. + # + # @private + module TestSupport + extend self + + def define_schema( + schema_element_name_form:, + schema_element_name_overrides: {}, + index_document_sizes: true, + json_schema_version: 1, + extension_modules: [], + derived_type_name_formats: {}, + type_name_overrides: {}, + enum_value_overrides_by_type: {}, + output: nil, + &block + ) + schema_elements = SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new( + form: schema_element_name_form, + overrides: schema_element_name_overrides + ) + + define_schema_with_schema_elements( + schema_elements, + index_document_sizes: index_document_sizes, + json_schema_version: json_schema_version, + extension_modules: extension_modules, + derived_type_name_formats: derived_type_name_formats, + type_name_overrides: type_name_overrides, + enum_value_overrides_by_type: enum_value_overrides_by_type, + output: output, + &block + ) + end + + def define_schema_with_schema_elements( + schema_elements, + index_document_sizes: true, + json_schema_version: 1, + extension_modules: [], + derived_type_name_formats: {}, + type_name_overrides: {}, + enum_value_overrides_by_type: {}, + output: nil + ) + api = API.new( + schema_elements, + index_document_sizes, + extension_modules: extension_modules, + derived_type_name_formats: derived_type_name_formats, + type_name_overrides: type_name_overrides, + enum_value_overrides_by_type: enum_value_overrides_by_type, + output: output || $stdout + ) + + yield api if block_given? + + # Set the json_schema_version to the provided value, if needed. + if !json_schema_version.nil? && api.state.json_schema_version.nil? + api.json_schema_version json_schema_version + end + + api.results + end + + DOC_COMMENTS = ( + '(?:^ *"""\\n' + # opening sequence of `"""` on its own line. + '(?:[^"]|(?:"(?!")))*' + # any sequence characters with no `""` sequence. (either no `"` or `"` is not followed by another) + '\\n *"""\\n)' # closing sequence of `"""` on its own line. + ) + + def type_def_from(sdl, type, include_docs: false) + type_def_keyword = '^(type|input|enum|union|interface|scalar|directive)\b' + # capture from the start of the type definition for `type` until the next type definition (or `\z` for end of string) + type_extraction_regex = /(#{DOC_COMMENTS}?#{type_def_keyword} #{type}\b.*?)(?:(?:#{DOC_COMMENTS}?#{type_def_keyword})|\z)/m + # type_defs = sdl.scan(type_extraction_regex).map(&:first) + type_defs = sdl.scan(type_extraction_regex).map { |match| [match].flatten.first } + + if type_defs.size >= 2 + # :nocov: -- only executed when a mistake has been made; causes a failing test. + raise Errors::SchemaError, + "Expected to find 0 or 1 type definition for #{type}, but found #{type_defs.size}. Type defs:\n\n#{type_defs.join("\n\n")}" + # :nocov: + end + + result = type_defs.first&.strip + result &&= strip_docs(result) unless include_docs + result + end + + def strip_docs(string) + string + .gsub(/#{DOC_COMMENTS}/o, "") # strip doc comments + .gsub("\n\n", "\n") # cleanup formatting so we don't have extra blank lines. + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/api.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/api.rbs new file mode 100644 index 00000000..20d1680a --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/api.rbs @@ -0,0 +1,79 @@ +module ElasticGraph + def self.define_schema: () { (SchemaDefinition::API) -> void } -> void + + module SchemaDefinition + type jsonSchemaLayer = :nullable | :array + type jsonSchemaLayersArray = ::Array[jsonSchemaLayer] + + interface _NamedElement + def name: () -> ::String + end + + interface _HasSchemaDefState + def schema_def_state: () -> State + end + + interface _Type + include _NamedElement + def type_ref: () -> SchemaElements::TypeReference + def graphql_only?: () -> bool + def indexed?: () -> bool + def to_sdl: () ?{ (SchemaElements::Field::argument) -> boolish } -> ::String + def derived_graphql_types: () -> ::Array[SchemaElements::graphQLType] + def to_indexing_field_type: () -> Indexing::_FieldType + end + + interface _IndexableType + include _Type + def abstract?: () -> bool + def indices: () -> ::Array[Indexing::Index] + def runtime_metadata: (::Array[SchemaArtifacts::RuntimeMetadata::UpdateTarget]) -> SchemaArtifacts::RuntimeMetadata::ObjectType + def current_sources: () -> ::Array[::String] + def index_field_runtime_metadata_tuples: ( + ?path_prefix: ::String, + ?parent_source: ::String, + ?list_counts_state: SchemaElements::ListCountsState + ) -> ::Array[[::String, SchemaArtifacts::RuntimeMetadata::IndexField]] + def derived_indexed_types: () -> ::Array[Indexing::DerivedIndexedType] + def verify_graphql_correctness!: () -> void + def relay_pagination_type: () -> bool + end + + # Here we make a type alias that should be used in place of `_IndexableType`. All types + # wind up including `Mixins::HasDirectives` but steep doesn't know that `_IndexableType` + # instances therefore have the `HasDirectives` methods. This works around that: `indexableType` + # is a type that satisfies the `_IndexableType` interface _and_ mixes in `HasDirectives`. + # + # We could list other mixins here but will do that as there is need over time. + type indexableType = _IndexableType & Mixins::HasDirectives & Mixins::SupportsFilteringAndAggregation & Mixins::HasIndices + + class API + attr_reader state: State + attr_reader factory: Factory + + def initialize: ( + SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + bool, + ?extension_modules: ::Array[::Module], + ?derived_type_name_formats: ::Hash[::Symbol, ::String], + ?type_name_overrides: ::Hash[::Symbol, ::String], + ?enum_value_overrides_by_type: ::Hash[::Symbol, ::Hash[::Symbol, ::String]], + ?output: io + ) -> void + + def raw_sdl: (::String) -> void + def object_type: (::String) { (SchemaElements::ObjectType) -> void } -> void + def interface_type: (::String) { (SchemaElements::InterfaceType) -> void } -> void + def enum_type: (::String) { (SchemaElements::EnumType) -> void } -> void + def union_type: (::String) { (SchemaElements::UnionType) -> void } -> void + def scalar_type: (::String) { (SchemaElements::ScalarType) -> void } -> void + def deleted_type: (::String) -> void + def as_active_instance: { () -> void } -> void + @results: Results? + def results: () -> Results + def json_schema_version: (::Integer) -> void + def register_graphql_extension: (::Module, defined_at: ::String, **untyped) -> void + def on_built_in_types: () { (SchemaElements::graphQLType) -> void } -> void + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/factory.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/factory.rbs new file mode 100644 index 00000000..14c98b7f --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/factory.rbs @@ -0,0 +1,149 @@ +module ElasticGraph + module SchemaDefinition + class Factory + @state: State + def initialize: (State) -> void + + def self.prevent_non_factory_instantiation_of: (::Class) -> ::Method + + def new_deprecated_element: ( + ::String, + defined_at: ::Thread::Backtrace::Location?, + defined_via: ::String + ) -> SchemaElements::DeprecatedElement + @@deprecated_element_new: ::Method + + def new_argument: (SchemaElements::Field, ::String, SchemaElements::TypeReference) ?{ (SchemaElements::Argument) -> void } -> SchemaElements::Argument + @@argument_new: ::Method + + def new_built_in_types: (API) -> SchemaElements::BuiltInTypes + @@built_in_types_new: ::Method + + def new_directive: (::String, SchemaElements::directiveArgHash) -> SchemaElements::Directive + @@directive_new: ::Method + + def new_enum_type: (::String) ?{ (SchemaElements::EnumType) -> void } -> SchemaElements::EnumType + @@enum_type_new: ::Method + + def new_enum_value: (::String, ::String) ?{ (SchemaElements::EnumValue) -> void } -> SchemaElements::EnumValue + @@enum_value_new: ::Method + + def new_enums_for_indexed_types: () -> SchemaElements::EnumsForIndexedTypes + @@enums_for_indexed_types_new: ::Method + + def new_field: ( + name: ::String, + type: ::String, + parent_type: SchemaElements::anyObjectType, + ?filter_type: ::String?, + ?name_in_index: ::String, + ?accuracy_confidence: SchemaElements::Field::accuracyConfidence, + ?sortable: bool?, + ?filterable: bool?, + ?aggregatable: bool?, + ?groupable: bool?, + ?graphql_only: bool?, + ?runtime_metadata_graphql_field: SchemaArtifacts::RuntimeMetadata::GraphQLField + ) ?{ (SchemaElements::Field) -> void } -> SchemaElements::Field + @@field_new: ::Method + + def new_graphql_sdl_enumerator: (::Array[SchemaElements::graphQLType]) -> SchemaElements::GraphQLSDLEnumerator + @@graphql_sdl_enumerator_new: ::Method + + def new_input_field: (**untyped) { (SchemaElements::InputField) -> void } -> SchemaElements::InputField + @@input_field_new: ::Method + + def new_input_type: (::String) { (SchemaElements::InputType) -> void } -> SchemaElements::InputType + @@input_type_new: ::Method + + def new_filter_input_type: (::String, ?name_prefix: ::String, ?category: ::Symbol) { (SchemaElements::InputType) -> void } -> SchemaElements::InputType + + def build_standard_filter_input_types_for_index_leaf_type: ( + ::String, + ?name_prefix: ::String + ) { (SchemaElements::InputType) -> void } -> ::Array[SchemaElements::InputType] + + def build_standard_filter_input_types_for_index_object_type: ( + ::String, + ?name_prefix: ::String + ) { (SchemaElements::InputType) -> void } -> ::Array[SchemaElements::InputType] + + def build_relay_pagination_types: ( + ::String, + ?include_total_edge_count: bool, + ?derived_indexed_types: ::Array[Indexing::DerivedIndexedType], + ?support_pagination: bool + ) ?{ (SchemaElements::ObjectType) -> void } -> ::Array[SchemaElements::ObjectType] + + def new_interface_type: (::String) { (SchemaElements::InterfaceType) -> void } -> SchemaElements::InterfaceType + @@interface_type_new: ::Method + + def new_object_type: (::String) ?{ (SchemaElements::ObjectType) -> void } -> SchemaElements::ObjectType + @@object_type_new: ::Method + + def new_scalar_type: (::String) { (SchemaElements::ScalarType) -> void } -> SchemaElements::ScalarType + @@scalar_type_new: ::Method + + def new_sort_order_enum_value: (SchemaElements::EnumValue, ::Array[SchemaElements::Field]) -> SchemaElements::SortOrderEnumValue + @@sort_order_enum_value_new: ::Method + + def new_type_reference: (::String) -> SchemaElements::TypeReference + @@type_reference_new: ::Method + + def new_type_with_subfields: ( + SchemaElements::schemaKind, + ::String, + wrapping_type: SchemaElements::anyObjectType, + field_factory: ::Method + ) ?{ (SchemaElements::TypeWithSubfields) -> void } -> SchemaElements::TypeWithSubfields + @@type_with_subfields_new: ::Method + + def new_union_type: (::String) { (SchemaElements::UnionType) -> void } -> SchemaElements::UnionType + @@union_type_new: ::Method + + def new_field_source: (relationship_name: ::String, field_path: ::String) -> SchemaElements::FieldSource + @@field_source_new: ::Method + + def new_relationship: ( + SchemaElements::Field, + cardinality: SchemaElements::Relationship::cardinality, + related_type: SchemaElements::TypeReference, + foreign_key: ::String, + direction: SchemaElements::foreignKeyDirection + ) -> SchemaElements::Relationship + @@relationship_new: ::Method + + def new_aggregated_values_type_for_index_leaf_type: ( + ::String + ) { (SchemaElements::ObjectType) -> void } -> SchemaElements::ObjectType + + private + + def new_list_filter_input_type: ( + ::String, + name_prefix: ::String, + any_satisfy_type_category: ::Symbol + ) -> SchemaElements::InputType + + def new_list_element_filter_input_type: ( + ::String, + name_prefix: ::String + ) { (SchemaElements::InputType) -> void } -> SchemaElements::InputType + + def new_fields_list_filter_input_type: ( + ::String, + name_prefix: ::String + ) -> SchemaElements::InputType + + def define_list_counts_filter_field_on: (SchemaElements::InputType) -> void + + def edge_type_for: (::String) -> SchemaElements::ObjectType + def connection_type_for: ( + ::String, + bool, + ::Array[Indexing::DerivedIndexedType], + bool + ) ?{ (SchemaElements::ObjectType) -> void } -> SchemaElements::ObjectType + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_fields/append_only_set.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_fields/append_only_set.rbs new file mode 100644 index 00000000..f9cfd76c --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_fields/append_only_set.rbs @@ -0,0 +1,17 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + module DerivedFields + class AppendOnlySet + include _DerivedField + def self.new: (::String, ::String) -> instance + + private + + IDEMPOTENTLY_INSERT_VALUES: ::String + IDEMPOTENTLY_INSERT_VALUE: ::String + end + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_fields/field_initializer_support.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_fields/field_initializer_support.rbs new file mode 100644 index 00000000..9a7a3040 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_fields/field_initializer_support.rbs @@ -0,0 +1,15 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + module DerivedFields + module FieldInitializerSupport + EMPTY_PAINLESS_LIST: ::String + EMPTY_PAINLESS_MAP: ::String + + def self.build_empty_value_initializers: (::String, leaf_value: ::String | :leave_unset) -> ::Array[::String] + def self.default_source_field_to_empty: (::String, ::String) -> ::String + end + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_fields/immutable_value.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_fields/immutable_value.rbs new file mode 100644 index 00000000..2d587f7d --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_fields/immutable_value.rbs @@ -0,0 +1,33 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + module DerivedFields + class ImmutableValueSuperclass + attr_reader destination_field: ::String + attr_reader source_field: ::String + attr_reader nullable: bool + attr_reader can_change_from_null: bool + + def initialize: ( + destination_field: ::String, + source_field: ::String, + nullable: bool, + can_change_from_null: bool + ) -> void + + def with: ( + ?destination_field: ::String, + ?source_field: ::String, + ?nullable: bool, + ?can_change_from_null: bool + ) -> ImmutableValue + end + + class ImmutableValue < ImmutableValueSuperclass + include _DerivedField + IDEMPOTENTLY_SET_VALUE: ::String + end + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value.rbs new file mode 100644 index 00000000..c2a9a7ee --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value.rbs @@ -0,0 +1,15 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + module DerivedFields + class MinOrMaxValue + include _DerivedField + attr_reader min_or_max: :min | :max + def self.new: (::String, ::String, :min | :max) -> instance + + def self.function_def: (:min | :max) -> ::String + end + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_indexed_type.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_indexed_type.rbs new file mode 100644 index 00000000..b82264ab --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_indexed_type.rbs @@ -0,0 +1,57 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + interface _DerivedField + def source_field: () -> ::String + def destination_field: () -> ::String + def apply_operation_returning_update_status: () -> ::String + def function_definitions: () -> ::Array[::String] + def setup_statements: () -> ::Array[::String] + end + + class DerivedIndexedTypeStruct + attr_reader source_type: SchemaElements::ObjectType + attr_reader destination_type_ref: SchemaElements::TypeReference + attr_reader id_source: ::String + attr_reader routing_value_source: ::String? + attr_reader rollover_timestamp_value_source: ::String? + attr_reader fields: ::Array[_DerivedField] + + def initialize: ( + source_type: SchemaElements::ObjectType, + destination_type_ref: SchemaElements::TypeReference, + id_source: ::String, + routing_value_source: ::String?, + rollover_timestamp_value_source: ::String?, + fields: ::Array[_DerivedField] + ) -> void + end + + class DerivedIndexedType < DerivedIndexedTypeStruct + def initialize: ( + source_type: SchemaElements::ObjectType, + destination_type_ref: SchemaElements::TypeReference, + id_source: ::String, + routing_value_source: ::String?, + rollover_timestamp_value_source: ::String?, + ) { (self) -> void } -> void + + def append_only_set: (::String, from: ::String) -> void + def min_value: (::String, from: ::String) -> void + def max_value: (::String, from: ::String) -> void + def immutable_value: (::String, from: ::String, ?nullable: bool, ?can_change_from_null: bool) -> void + def painless_script: () -> Scripting::Script + def runtime_metadata_for_source_type: () -> SchemaArtifacts::RuntimeMetadata::UpdateTarget + + private + + def generate_script: () -> ::String + def apply_update_statement: (_DerivedField) -> ::String + def was_noop_variable: (_DerivedField) -> ::String + + SCRIPT_ERRORS_VAR: ::String + STATIC_SETUP_STATEMENTS: ::String + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/event_envelope.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/event_envelope.rbs new file mode 100644 index 00000000..aa609f3d --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/event_envelope.rbs @@ -0,0 +1,9 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + module EventEnvelope + def self.json_schema: (::Array[::String], ::Integer) -> ::Hash[::String, untyped] + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs new file mode 100644 index 00000000..8257090f --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs @@ -0,0 +1,64 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + class FieldSupertype + attr_reader name: ::String + attr_reader name_in_index: ::String + attr_reader type: SchemaElements::TypeReference + attr_reader json_schema_layers: jsonSchemaLayersArray + attr_reader indexing_field_type: _FieldType + attr_reader accuracy_confidence: Field::accuracyConfidence + attr_reader json_schema_customizations: ::Hash[::Symbol, untyped] + attr_reader mapping_customizations: ::Hash[::Symbol, untyped] + attr_reader source: SchemaElements::FieldSource? + attr_accessor runtime_field_script: ::String? + + def initialize: ( + name: ::String, + name_in_index: ::String, + type: SchemaElements::TypeReference, + json_schema_layers: jsonSchemaLayersArray, + indexing_field_type: _FieldType, + accuracy_confidence: Field::accuracyConfidence, + json_schema_customizations: ::Hash[::Symbol, untyped], + mapping_customizations: ::Hash[::Symbol, untyped], + source: SchemaElements::FieldSource?, + runtime_field_script: ::String? + ) -> Field + + def with: ( + ?name: ::String, + ?name_in_index: ::String, + ?type: SchemaElements::TypeReference, + ?json_schema_layers: jsonSchemaLayersArray, + ?indexing_field_type: _FieldType, + ?accuracy_confidence: Field::accuracyConfidence, + ?json_schema_customizations: ::Hash[::Symbol, untyped], + ?mapping_customizations: ::Hash[::Symbol, untyped], + ?source: SchemaElements::FieldSource?, + ?runtime_field_script: ::String? + ) -> Field + end + + class Field < FieldSupertype + JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE: ::Hash[::String, untyped] + + type accuracyConfidence = SchemaElements::Field::accuracyConfidence + @mapping: ::Hash[::String, untyped]? + def mapping: () -> ::Hash[::String, untyped] + def json_schema: () -> ::Hash[::String, untyped] + def json_schema_metadata: () -> JSONSchemaFieldMetadata + + def self.normalized_mapping_hash_for: (::Array[Field]) -> ::Hash[::String, untyped] + + def inner_json_schema: () -> ::Hash[::String, untyped] + def outer_json_schema_customizations: () -> ::Hash[::String, untyped] + + def user_specified_json_schema_customizations_go_on_outside?: () -> bool + def process_layer: (::Symbol, ::Hash[::String, untyped]) -> ::Hash[::String, untyped] + def make_nullable: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + def make_array: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_reference.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_reference.rbs new file mode 100644 index 00000000..92344d3e --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_reference.rbs @@ -0,0 +1,40 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + class FieldReference + attr_reader name: ::String + attr_reader name_in_index: ::String + attr_reader type: SchemaElements::TypeReference + attr_reader mapping_options: ::Hash[::Symbol, untyped] + attr_reader json_schema_options: ::Hash[::Symbol, untyped] + attr_reader accuracy_confidence: Field::accuracyConfidence + attr_reader source: SchemaElements::FieldSource? + attr_reader runtime_field_script: ::String? + + def initialize: ( + name: ::String, + name_in_index: ::String, + type: SchemaElements::TypeReference, + mapping_options: ::Hash[::Symbol, untyped], + json_schema_options: ::Hash[::Symbol, untyped], + accuracy_confidence: Field::accuracyConfidence, + source: SchemaElements::FieldSource?, + runtime_field_script: ::String? + ) -> void + + def with: ( + ?name: ::String, + ?name_in_index: ::String, + ?type: SchemaElements::TypeReference, + ?mapping_options: ::Hash[::Symbol, untyped], + ?json_schema_options: ::Hash[::Symbol, untyped], + ?accuracy_confidence: Field::accuracyConfidence, + ?source: SchemaElements::FieldSource?, + ?runtime_field_script: ::String? + ) -> FieldReference + + def resolve: () -> Field? + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs new file mode 100644 index 00000000..5808a070 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs @@ -0,0 +1,12 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + interface _FieldType + def to_mapping: () -> ::Hash[::String, untyped] + def to_json_schema: () -> ::Hash[::String, untyped] + def json_schema_field_metadata_by_field_name: () -> ::Hash[::String, JSONSchemaFieldMetadata] + def format_field_json_schema_customizations: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/enum.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/enum.rbs new file mode 100644 index 00000000..7f8f9afd --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/enum.rbs @@ -0,0 +1,13 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + module FieldType + class Enum + include _FieldType + attr_reader enum_value_names: ::Array[::String] + def initialize: (::Array[::String]) -> void + end + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/object.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/object.rbs new file mode 100644 index 00000000..f977d239 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/object.rbs @@ -0,0 +1,40 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + module FieldType + class ObjectSuperType + attr_reader type_name: ::String + attr_reader subfields: ::Array[Field] + attr_reader mapping_options: Mixins::HasTypeInfo::optionsHash + attr_reader json_schema_options: Mixins::HasTypeInfo::optionsHash + + def initialize: ( + type_name: ::String, + subfields: ::Array[Field], + mapping_options: Mixins::HasTypeInfo::optionsHash, + json_schema_options: Mixins::HasTypeInfo::optionsHash, + ) -> void + + def with: ( + ?type_name: ::String, + ?subfields: ::Array[Field], + ?mapping_options: Mixins::HasTypeInfo::optionsHash, + ?json_schema_options: Mixins::HasTypeInfo::optionsHash, + ) -> Object + end + + class Object < ObjectSuperType + include _FieldType + + @to_mapping: ::Hash[::String, untyped]? + @to_json_schema: ::Hash[::String, untyped]? + + private + + def json_schema_typename_field: () -> ::Hash[::String, untyped] + def validate_sourced_fields_have_no_json_schema_overrides: (::Array[Field]) -> void + end + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/scalar.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/scalar.rbs new file mode 100644 index 00000000..32cff29a --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/scalar.rbs @@ -0,0 +1,14 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + module FieldType + class Scalar + include _FieldType + attr_reader scalar_type: SchemaElements::ScalarType + + def initialize: (scalar_type: SchemaElements::ScalarType) -> Scalar + end + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/union.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/union.rbs new file mode 100644 index 00000000..d100ef66 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/union.rbs @@ -0,0 +1,16 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + module FieldType + class UnionSuperType + attr_reader subtypes_by_name: ::Hash[::String, Object] + def self.new: (::Hash[::String, Object]) -> instance + end + + class Union < UnionSuperType + include _FieldType + end + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/index.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/index.rbs new file mode 100644 index 00000000..a7208011 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/index.rbs @@ -0,0 +1,20 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + # Note: this is a partial signature definition (`index.rb` is ignored in `Steepfile`) + class Index + attr_reader name: ::String + attr_reader routing_field_path: SchemaElements::FieldPath + attr_reader rollover_config: RolloverConfig? + def uses_custom_routing?: () -> bool + def to_index_config: () -> ::Hash[::String, untyped] + def to_index_template_config: () -> ::Hash[::String, untyped] + def runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::IndexDefinition + + private + + def date_and_datetime_types: () -> ::Array[::String] + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/json_schema_field_metadata.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/json_schema_field_metadata.rbs new file mode 100644 index 00000000..b025eda1 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/json_schema_field_metadata.rbs @@ -0,0 +1,17 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + class JSONSchemaFieldMetadata + attr_reader type: ::String + attr_reader name_in_index: ::String + + def initialize: ( + type: ::String, + name_in_index: ::String + ) -> void + + def to_dumpable_hash: () -> {"type" => ::String, "nameInIndex" => ::String} + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/json_schema_with_metadata.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/json_schema_with_metadata.rbs new file mode 100644 index 00000000..dcc37b60 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/json_schema_with_metadata.rbs @@ -0,0 +1,98 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + class JSONSchemaWithMetadataSupertype + attr_reader json_schema: ::Hash[::String, untyped] + attr_reader missing_fields: ::Set[::String] + attr_reader missing_types: ::Set[::String] + attr_reader definition_conflicts: ::Set[SchemaElements::DeprecatedElement] + attr_reader missing_necessary_fields: ::Array[JSONSchemaWithMetadata::MissingNecessaryField] + + def initialize: ( + json_schema: ::Hash[::String, untyped], + missing_fields: ::Set[::String], + missing_types: ::Set[::String], + definition_conflicts: ::Set[SchemaElements::DeprecatedElement], + missing_necessary_fields: ::Array[JSONSchemaWithMetadata::MissingNecessaryField] + ) -> void + + def with: ( + ?json_schema: ::Hash[::String, untyped], + ?missing_fields: ::Set[::String], + ?missing_types: ::Set[::String], + ?definition_conflicts: ::Set[SchemaElements::DeprecatedElement], + ?missing_necessary_fields: ::Array[JSONSchemaWithMetadata::MissingNecessaryField] + ) -> instance + end + + class JSONSchemaWithMetadata < JSONSchemaWithMetadataSupertype + def json_schema_version: () -> ::Integer + + class Merger + @field_metadata_by_type_and_field_name: ::Hash[::String, ::Hash[::String, JSONSchemaFieldMetadata]] + @renamed_types_by_old_name: ::Hash[::String, SchemaElements::DeprecatedElement] + @deleted_types_by_old_name: ::Hash[::String, SchemaElements::DeprecatedElement] + @renamed_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]] + @deleted_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]] + @state: State + @derived_indexing_type_names: ::Set[::String] + + attr_reader unused_deprecated_elements: ::Set[SchemaElements::DeprecatedElement] + + def initialize: (Results) -> void + def merge_metadata_into: (::Hash[::String, untyped]) -> JSONSchemaWithMetadata + + private + + def determine_current_type_name: ( + ::String, + missing_types: ::Set[::String], + definition_conflicts: ::Set[SchemaElements::DeprecatedElement] + ) -> ::String? + + def field_metadata_for: ( + ::String, + ::String, + missing_fields: ::Set[::String], + definition_conflicts: ::Set[SchemaElements::DeprecatedElement] + ) -> JSONSchemaFieldMetadata? + + def identify_missing_necessary_fields: ( + ::Hash[::String, untyped], + ::Hash[::String, ::String] + ) -> ::Array[MissingNecessaryField] + + def identify_missing_necessary_fields_for_index_def: ( + indexableType, + Index, + JSONSchemaResolver, + ::Integer + ) -> ::Array[MissingNecessaryField] + + class JSONSchemaResolver + @state: State + @old_type_name_by_current_name: ::Hash[::String, ::String] + @meta_by_old_type_and_name_in_index: ::Hash[::String, ::Hash[::String, ::Hash[::String, untyped]]] + + def initialize: (State, ::Hash[::String, untyped], ::Hash[::String, ::String]) -> void + def necessary_path_missing?: (SchemaElements::FieldPath) -> bool + + private + + def necessary_path_part_missing?: (::String, ::String) { (::Hash[::String, untyped]) -> void } -> bool + end + end + + class MissingNecessaryField + attr_reader field_type: ::String + attr_reader fully_qualified_path: ::String + + def initialize: ( + field_type: ::String, + fully_qualified_path: ::String + ) -> void + end + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/list_counts_mapping.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/list_counts_mapping.rbs new file mode 100644 index 00000000..843649d3 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/list_counts_mapping.rbs @@ -0,0 +1,12 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + module ListCountsMapping + def self.merged_into: ( + ::Hash[::String, untyped], + for_type: SchemaElements::anyObjectType + ) -> ::Hash[::String, untyped] + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/relationship_resolver.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/relationship_resolver.rbs new file mode 100644 index 00000000..cc6563d0 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/relationship_resolver.rbs @@ -0,0 +1,48 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + class RelationshipResolver + def initialize: ( + schema_def_state: State, + object_type: indexableType, + relationship_name: ::String, + sourced_fields: ::Array[SchemaElements::Field], + field_path_resolver: SchemaElements::FieldPath::Resolver + ) -> void + + def resolve: () -> untyped + + private + + attr_reader schema_def_state: State + attr_reader object_type: indexableType + attr_reader relationship_name: ::String + attr_reader sourced_fields: ::Array[SchemaElements::Field] + attr_reader field_path_resolver: SchemaElements::FieldPath::Resolver + + def relationship_error_prefix: () -> ::String + def validate_foreign_key: ( + indexableType, + SchemaArtifacts::RuntimeMetadata::Relation + ) -> ::String? + def relationship_description: () -> ::String + end + + class ResolvedRelationship + attr_reader relationship_name: ::String + attr_reader relationship_field: SchemaElements::Field + attr_reader relationship: SchemaElements::Relationship + attr_reader related_type: indexableType + attr_reader relation_metadata: SchemaArtifacts::RuntimeMetadata::Relation + + def initialize: ( + ::String, + SchemaElements::Field, + SchemaElements::Relationship, + indexableType, + SchemaArtifacts::RuntimeMetadata::Relation + ) -> void + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/rollover_config.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/rollover_config.rbs new file mode 100644 index 00000000..7743acfa --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/rollover_config.rbs @@ -0,0 +1,29 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + class RolloverConfigSupertype + attr_reader frequency: SchemaArtifacts::RuntimeMetadata::IndexDefinition::Rollover::frequency + attr_reader timestamp_field_path: SchemaElements::FieldPath + + def initialize: ( + SchemaArtifacts::RuntimeMetadata::IndexDefinition::Rollover::frequency, + SchemaElements::FieldPath + ) -> void + + def self.with: ( + frequency: SchemaArtifacts::RuntimeMetadata::IndexDefinition::Rollover::frequency, + timestamp_field_path: SchemaElements::FieldPath + ) -> RolloverConfig + + def with: ( + ?frequency: SchemaArtifacts::RuntimeMetadata::IndexDefinition::Rollover::frequency, + ?timestamp_field_path: SchemaElements::FieldPath + ) -> RolloverConfig + end + + class RolloverConfig < RolloverConfigSupertype + def runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::IndexDefinition::Rollover + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_factory.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_factory.rbs new file mode 100644 index 00000000..c0b903cc --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_factory.rbs @@ -0,0 +1,22 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + module UpdateTargetFactory + def self.new_normal_indexing_update_target: ( + type: ::String, + relationship: ::String, + id_source: ::String, + data_params: SchemaArtifacts::RuntimeMetadata::paramsHash, + routing_value_source: ::String?, + rollover_timestamp_value_source: ::String? + ) -> SchemaArtifacts::RuntimeMetadata::UpdateTarget + + private + + self.@standard_metadata_params: SchemaArtifacts::RuntimeMetadata::paramsHash? + def self.standard_metadata_params: () -> SchemaArtifacts::RuntimeMetadata::paramsHash + def self.single_value_param_from: (::String) -> SchemaArtifacts::RuntimeMetadata::_Param + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_resolver.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_resolver.rbs new file mode 100644 index 00000000..e3ab355f --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_resolver.rbs @@ -0,0 +1,49 @@ +module ElasticGraph + module SchemaDefinition + module Indexing + class UpdateTargetResolver + def initialize: ( + object_type: indexableType, + resolved_relationship: ResolvedRelationship, + sourced_fields: ::Array[SchemaElements::Field], + field_path_resolver: SchemaElements::FieldPath::Resolver, + ) -> void + + def resolve: () -> [SchemaArtifacts::RuntimeMetadata::UpdateTarget?, ::Array[::String]] + + private + + attr_reader object_type: indexableType + attr_reader resolved_relationship: ResolvedRelationship + attr_reader sourced_fields: ::Array[SchemaElements::Field] + attr_reader field_path_resolver: SchemaElements::FieldPath::Resolver + + def validate_relationship: () -> ::Array[::String] + def relationship_error_prefix: () -> ::String + + def resolve_data_params: () -> [ + ::Hash[::String, SchemaArtifacts::RuntimeMetadata::DynamicParam], + ::Array[::String] + ] + + def resolve_field_source: (_FieldSourceAdapter) -> [::String?, ::String?] + + interface _FieldSourceAdapter + def get_field_source: (SchemaElements::Relationship, Index) { + (::String) -> bot + } -> ::String? + + def cannot_update_reason: (indexableType, ::String) -> ::String + end + + module RoutingSourceAdapter + extend _FieldSourceAdapter + end + + module RolloverTimestampSourceAdapter + extend _FieldSourceAdapter + end + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/json_schema_pruner.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/json_schema_pruner.rbs new file mode 100644 index 00000000..8c5f323a --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/json_schema_pruner.rbs @@ -0,0 +1,9 @@ +module ElasticGraph + module SchemaDefinition + class JSONSchemaPruner + def self.prune: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + def self.referenced_type_names: (::Array[::String], ::Hash[::String, untyped]) -> ::Set[::String] + def self.collect_ref_names: (::Hash[::String, untyped]) -> ::Array[::String] + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/can_be_graphql_only.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/can_be_graphql_only.rbs new file mode 100644 index 00000000..c37ae350 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/can_be_graphql_only.rbs @@ -0,0 +1,11 @@ +module ElasticGraph + module SchemaDefinition + module Mixins + module CanBeGraphQLOnly + @graphql_only: bool? + def graphql_only: (bool) -> void + def graphql_only?: () -> bool + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_derived_graphql_type_customizations.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_derived_graphql_type_customizations.rbs new file mode 100644 index 00000000..d4813d4a --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_derived_graphql_type_customizations.rbs @@ -0,0 +1,25 @@ +module ElasticGraph + module SchemaDefinition + module Mixins + module HasDerivedGraphQLTypeCustomizations: _Type + def customize_derived_types: (*(:all | ::String)) { (_Type) -> void } -> void + def customize_derived_type_fields: ( + ::String, *::String + ) { (SchemaElements::Field) -> void } -> void + + def derived_type_customizations_for_type: (_Type) -> ::Array[^(_Type) -> void] + + def derived_field_customizations_by_name_for_type: ( + _Type + ) -> ::Hash[::String, ::Array[^(SchemaElements::Field) -> void]] + + attr_reader derived_type_customizations_by_name: ::Hash[::String, ::Array[^(_Type) -> void]] + attr_reader derived_field_customizations_by_type_and_field_name: ::Hash[::String, ::Hash[::String, ::Array[^(SchemaElements::Field) -> void]]] + + private + + attr_reader derived_type_customizations_for_all_types: ::Array[^(_Type) -> void] + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_directives.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_directives.rbs new file mode 100644 index 00000000..cb719259 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_directives.rbs @@ -0,0 +1,11 @@ +module ElasticGraph + module SchemaDefinition + module Mixins + module HasDirectives: _HasSchemaDefState + def directive: (::String, ?SchemaElements::directiveArgHash) -> void + def directives_sdl: (?suffix_with: ::String, ?prefix_with: ::String) -> ::String + attr_reader directives: ::Array[SchemaElements::Directive] + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_documentation.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_documentation.rbs new file mode 100644 index 00000000..d6c3dab2 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_documentation.rbs @@ -0,0 +1,13 @@ +module ElasticGraph + module SchemaDefinition + module Mixins + module HasDocumentation + attr_accessor doc_comment: ::String? + def documentation: (::String?) -> void + def append_to_documentation: (::String) -> void + def formatted_documentation: () -> ::String? + def derived_documentation: (::String, ?::String?) -> ::String + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_indices.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_indices.rbs new file mode 100644 index 00000000..e47ef252 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_indices.rbs @@ -0,0 +1,26 @@ +module ElasticGraph + module SchemaDefinition + module Mixins + module HasIndices + attr_reader indices: ::Array[Indexing::Index] + attr_accessor runtime_metadata_overrides: ::Hash[::Symbol, untyped] + def index: (::String, ::Hash[::Symbol, ::String | ::Integer]) ?{ (Indexing::Index) -> void } -> ::Array[Indexing::Index] + def indexed?: () -> bool + def runtime_metadata: (::Array[SchemaArtifacts::RuntimeMetadata::UpdateTarget]) -> SchemaArtifacts::RuntimeMetadata::ObjectType + def derived_indexed_types: () -> ::Array[Indexing::DerivedIndexedType] + def derive_indexed_type_fields: ( + ::String, + from_id: ::String, + ?route_with: ::String, + ?rollover_with: ::String + ) ?{ ( Indexing::DerivedIndexedType) -> void } -> Indexing::DerivedIndexedType + def root_query_fields: (plural: ::String, ?singular: ::String?) ?{ (SchemaElements::Field) -> void } -> void + def plural_root_query_field_name: () -> ::String + def singular_root_query_field_name: () -> ::String + def root_query_fields_customizations: () -> (^(SchemaElements::Field) -> void)? + def fields_with_sources: () -> ::Array[SchemaElements::Field] + def indexing_fields_by_name_in_index: () -> ::Hash[::String, SchemaElements::Field] + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect.rbs new file mode 100644 index 00000000..17108d4f --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect.rbs @@ -0,0 +1,9 @@ +module ElasticGraph + module SchemaDefinition + module Mixins + class HasReadableToSAndInspect < ::Module + def initialize: () ?{ (untyped) -> ::String } -> void + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_subtypes.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_subtypes.rbs new file mode 100644 index 00000000..e8d1b045 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_subtypes.rbs @@ -0,0 +1,32 @@ +module ElasticGraph + module SchemaDefinition + module Mixins + interface _AbstractType + include _Type + def resolve_subtypes: () -> ::Array[SchemaElements::ObjectType] + def schema_def_state: () -> State + end + + module HasSubtypes: _AbstractType + def recursively_resolve_subtypes: () -> ::Array[SchemaElements::ObjectType] + def graphql_fields_by_name: () -> ::Hash[::String, SchemaElements::Field] + def indexing_fields_by_name_in_index: () -> ::Hash[::String, SchemaElements::Field] + def abstract?: () -> true + def current_sources: () -> ::Array[::String] + def index_field_runtime_metadata_tuples: ( + ?path_prefix: ::String, + ?parent_source: ::String, + ?list_counts_state: SchemaElements::ListCountsState + ) -> ::Array[[::String, SchemaArtifacts::RuntimeMetadata::IndexField]] + + private + + def merge_fields_by_name_from_subtypes: () { + (SchemaElements::ObjectType) -> ::Hash[::String, SchemaElements::Field] + } -> ::Hash[::String, SchemaElements::Field] + + def subtypes_indexed?: () -> bool + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_type_info.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_type_info.rbs new file mode 100644 index 00000000..1f3c1391 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_type_info.rbs @@ -0,0 +1,16 @@ +module ElasticGraph + module SchemaDefinition + module Mixins + module HasTypeInfo + CUSTOMIZABLE_DATASTORE_PARAMS: ::Set[::Symbol] + + type optionsHash = ::Hash[::Symbol, untyped] + attr_reader mapping_options: optionsHash + attr_reader json_schema_options: optionsHash + + def mapping: (**untyped) -> void + def json_schema: (**untyped) -> void + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/implements_interfaces.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/implements_interfaces.rbs new file mode 100644 index 00000000..b733b437 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/implements_interfaces.rbs @@ -0,0 +1,11 @@ +module ElasticGraph + module SchemaDefinition + module Mixins + module ImplementsInterfaces: SchemaElements::TypeWithSubfields + def implements: (*::String) -> void + attr_reader implemented_interfaces: ::Array[SchemaElements::TypeReference] + def verify_graphql_correctness!: () -> void + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/supports_default_value.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/supports_default_value.rbs new file mode 100644 index 00000000..aee721c3 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/supports_default_value.rbs @@ -0,0 +1,12 @@ +module ElasticGraph + module SchemaDefinition + module Mixins + module SupportsDefaultValue + NO_DEFAULT_PROVIDED: Module + @default_value: untyped + def default: (untyped) -> void + def default_value_sdl: () -> ::String? + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/supports_filtering_and_aggregation.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/supports_filtering_and_aggregation.rbs new file mode 100644 index 00000000..d03ff33d --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/supports_filtering_and_aggregation.rbs @@ -0,0 +1,34 @@ +module ElasticGraph + module SchemaDefinition + module Mixins + interface _FilterableAndAggregatableType + def schema_def_state: () -> State + def name: () -> ::String + def type_ref: () -> SchemaElements::TypeReference + def mapping_options: () -> HasTypeInfo::optionsHash + def indexed?: () -> bool + def abstract?: () -> bool + def graphql_only?: () -> bool + def runtime_metadata: (::Array[SchemaArtifacts::RuntimeMetadata::UpdateTarget]) -> SchemaArtifacts::RuntimeMetadata::ObjectType + def graphql_fields_by_name: () -> ::Hash[::String, SchemaElements::Field] + end + + module SupportsFilteringAndAggregation: _FilterableAndAggregatableType + def supports?: () { (SchemaElements::Field) -> bool } -> bool + def does_not_support?: () { (SchemaElements::Field) -> bool } -> bool + def derived_graphql_types: () -> ::Array[SchemaElements::graphQLType] + def has_custom_mapping_type?: () -> bool + + private + + def to_input_filters: () -> ::Array[SchemaElements::InputType] + def sub_aggregation_types_for_nested_field_references: () -> ::Array[SchemaElements::ObjectType] + def build_aggregation_sub_aggregations_types: () -> ::Array[SchemaElements::ObjectType] + def to_indexed_aggregation_type: () -> SchemaElements::ObjectType? + def to_grouped_by_type: () -> SchemaElements::ObjectType? + def to_aggregated_values_type: () -> SchemaElements::ObjectType? + def new_non_empty_object_type: (::String) { (SchemaElements::ObjectType) -> void } -> SchemaElements::ObjectType? + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/verifies_graphql_name.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/verifies_graphql_name.rbs new file mode 100644 index 00000000..1ae0c4c2 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/verifies_graphql_name.rbs @@ -0,0 +1,9 @@ +module ElasticGraph + module SchemaDefinition + module Mixins + module VerifiesGraphQLName: _NamedElement + def self.verify_name!: (::String) -> void + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/rake_tasks.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/rake_tasks.rbs new file mode 100644 index 00000000..64dbc2a4 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/rake_tasks.rbs @@ -0,0 +1,36 @@ +module ElasticGraph + module SchemaDefinition + class RakeTasks < ::Rake::TaskLib + def initialize: ( + index_document_sizes: bool, + path_to_schema: ::String | ::Pathname, + schema_artifacts_directory: ::String | ::Pathname, + schema_element_name_form: :camelCase | :snake_case, + ?schema_element_name_overrides: ::Hash[::Symbol, ::String], + ?derived_type_name_formats: ::Hash[::Symbol, ::String], + ?type_name_overrides: ::Hash[::Symbol, ::String], + ?enum_value_overrides_by_type: ::Hash[::Symbol, ::Hash[::Symbol, ::String]], + ?extension_modules: ::Array[::Module], + ?enforce_json_schema_version: bool, + ?output: io + ) -> void + + private + + @derived_type_name_formats: ::Hash[::Symbol, ::String] + @type_name_overrides: ::Hash[::Symbol, ::String] + @enum_value_overrides_by_type: ::Hash[::Symbol, ::Hash[::Symbol, ::String]] + @schema_element_names: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + @index_document_sizes: bool + @path_to_schema: ::String | ::Pathname + @schema_artifacts_directory: ::String | ::Pathname + @enforce_json_schema_version: bool + @extension_modules: ::Array[::Module] + @output: io + + def define_tasks: () -> void + def schema_artifact_manager: () -> SchemaArtifactManager + def schema_definition_results: () -> Results + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs new file mode 100644 index 00000000..6f488421 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs @@ -0,0 +1,54 @@ +module ElasticGraph + module SchemaDefinition + class ResultsSupertype + def initialize: (State) -> void + attr_reader state: State + end + + class Results < ResultsSupertype + include _SchemaArtifacts + + def json_schema_version_setter_location: () -> ::Thread::Backtrace::Location? + def json_schema_field_metadata_by_type_and_field_name: () -> ::Hash[::String, ::Hash[::String, Indexing::JSONSchemaFieldMetadata]] + def current_public_json_schema: () -> ::Hash[::String, untyped] + def merge_field_metadata_into_json_schema: (::Hash[::String, untyped]) -> Indexing::JSONSchemaWithMetadata + def unused_deprecated_elements: () -> ::Set[SchemaElements::DeprecatedElement] + def derived_indexing_type_names: () -> ::Set[::String] + + @graphql_schema_string: ::String? + @datastore_config: ::Hash[::String, untyped] + @runtime_metadata: SchemaArtifacts::RuntimeMetadata::Schema? + @current_json_schemas: ::Hash[::String, untyped]? + @static_script_repo: Scripting::FileSystemRepository? + @available_json_schema_versions: ::Set[::Integer]? + @no_circular_dependencies: bool? + @field_path_resolver: SchemaElements::FieldPath::Resolver? + @json_schema_indexing_field_types_by_name: ::Hash[::String, Indexing::_FieldType]? + @derived_indexing_type_names: ::Set[::String]? + @json_schema_field_metadata_by_type_and_field_name: ::Hash[::String, ::Hash[::String, Indexing::JSONSchemaFieldMetadata]]? + @current_public_json_schema: ::Hash[::String, untyped]? + @latest_versioned_json_schema: ::Hash[::String, untyped]? + @json_schema_with_metadata_merger: Indexing::JSONSchemaWithMetadata::Merger? + + STATIC_SCRIPT_REPO: Scripting::FileSystemRepository + + private + + def json_schema_with_metadata_merger: () -> Indexing::JSONSchemaWithMetadata::Merger + def generate_datastore_config: () -> ::Hash[::String, untyped] + def build_dynamic_scripts: () -> ::Array[Scripting::Script] + def build_runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::Schema + def identify_extra_update_targets_by_object_type_name: () -> ::Hash[::String, ::Array[SchemaArtifacts::RuntimeMetadata::UpdateTarget]] + def generate_sdl: () -> ::String + def build_public_json_schema: () -> ::Hash[::String, untyped] + def json_schema_indexing_field_types_by_name: () -> ::Hash[::String, Indexing::_FieldType] + def strip_trailing_whitespace: (::String) -> ::String + def check_for_circular_dependencies!: () -> void + def recursively_add_referenced_types_to: (SchemaElements::TypeReference, ::Hash[::String, ::Set[::String]]) -> void + + @all_types_except_root_query_type: Array[SchemaElements::graphQLType]? + def all_types_except_root_query_type: () -> ::Array[SchemaElements::graphQLType] + def apply_customizations_to: (::Array[SchemaElements::graphQLType], SchemaElements::graphQLType) -> void + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs new file mode 100644 index 00000000..8c675303 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs @@ -0,0 +1,85 @@ +module ElasticGraph + module SchemaDefinition + class SchemaArtifactManager + attr_reader schema_definition_results: Results + + def initialize: ( + schema_definition_results: Results, + schema_artifacts_directory: ::String, + enforce_json_schema_version: bool, + output: io, + ?max_diff_lines: ::Integer + ) -> void + + def dump_artifacts: () -> void + def check_artifacts: () -> void + + private + + @schema_definition_results: Results + @schema_artifacts_directory: ::String + @enforce_json_schema_version: bool + @output: io + @max_diff_lines: ::Integer + @artifacts: ::Array[SchemaArtifact[untyped]] + @json_schemas_artifact: SchemaArtifact[untyped] + + def notify_about_unused_type_name_overrides: () -> void + def notify_about_unused_enum_value_overrides: () -> void + def build_desired_versioned_json_schemas: (::Hash[::String, untyped]) -> ::Hash[::Integer, ::Hash[::String, untyped]] + def report_json_schema_merge_errors: (::Array[Indexing::JSONSchemaWithMetadata]) -> void + def report_json_schema_merge_warnings: () -> void + def format_deprecated_elements: (::Enumerable[SchemaElements::DeprecatedElement]) -> ::String + def missing_field_error_for: (::String, ::Array[::Integer]) -> ::String + def missing_type_error_for: (::String, ::Array[::Integer]) -> ::String + def missing_necessary_field_error_for: (Indexing::JSONSchemaWithMetadata::MissingNecessaryField, ::Array[::Integer]) -> ::String + def describe_json_schema_versions: (::Array[::Integer], ::String) -> ::String + def old_versions: (::Array[::Integer]) -> ::String + def files_noun_phrase: (::Array[::Integer]) -> ::String + def artifacts_out_of_date_error: (::Array[SchemaArtifact[untyped]]) -> ::String + def truncate_diff: (::String, ::Integer) -> [::String, ::String] + + def new_yaml_artifact: ( + ::String, + ::Hash[::String, untyped], + ?extra_comment_lines: ::Array[::String] + ) -> SchemaArtifact[::Hash[::String, untyped]] + + def new_versioned_json_schema_artifact: (::Hash[::String, untyped]) -> SchemaArtifact[::Hash[::String, untyped]] + def new_raw_artifact: (::String, ::String) -> SchemaArtifact[::String] + def check_if_needs_json_schema_version_bump: () { (::Integer) -> void } -> void + def pruned_runtime_metadata: (::String) -> SchemaArtifacts::RuntimeMetadata::Schema + end + + class SchemaArtifactSupertype[T] + attr_reader file_name: ::String + attr_reader desired_contents: T + attr_reader dumper: ^(T) -> ::String + attr_reader loader: ^(::String) -> T + attr_reader extra_comment_lines: ::Array[::String] + + def initialize: ( + ::String, + T, + ^(T) -> ::String, + ^(::String) -> T, + ::Array[::String]) -> void + end + + class SchemaArtifact[T] < SchemaArtifactSupertype[T] + def dump: (io) -> void + def out_of_date?: () -> bool + def existing_dumped_contents: () -> T? + def diff: (color: bool) -> ::String? + + private + + @exists: bool? + def exists?: () -> bool + + @dumped_contents: ::String? + def dumped_contents: () -> ::String + def comment_preamble: () -> ::String + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/argument.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/argument.rbs new file mode 100644 index 00000000..22652f35 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/argument.rbs @@ -0,0 +1,23 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class ArgumentSuperType + attr_reader schema_def_state: State + attr_reader parent_field: Field + attr_reader name: ::String + attr_reader original_value_type: TypeReference + + def initialize: (State, Field, ::String, TypeReference) -> void + end + + class Argument < ArgumentSuperType + include Mixins::VerifiesGraphQLName + include Mixins::SupportsDefaultValue + include Mixins::HasDocumentation + include Mixins::HasDirectives + + def value_type: () -> TypeReference + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/built_in_types.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/built_in_types.rbs new file mode 100644 index 00000000..182f5d97 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/built_in_types.rbs @@ -0,0 +1,11 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Note: this is a partial signature definition (`built_in_types.rb` is ignored in `Steepfile`) + class BuiltInTypes + def initialize: (API, State) -> void + def register_built_in_types: () -> void + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/deprecated_element.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/deprecated_element.rbs new file mode 100644 index 00000000..b2b3d1f9 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/deprecated_element.rbs @@ -0,0 +1,21 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class DeprecatedElement + attr_reader schema_def_state: State + attr_reader name: ::String + attr_reader defined_at: ::Thread::Backtrace::Location + attr_reader defined_via: ::String + + def initialize: ( + schema_def_state: State, + name: ::String, + defined_at: ::Thread::Backtrace::Location, + defined_via: ::String + ) -> void + + def description: () -> ::String + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/directive.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/directive.rbs new file mode 100644 index 00000000..4269f1c3 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/directive.rbs @@ -0,0 +1,19 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + type directiveArg = ::Integer | ::Symbol | ::String | directiveArgHash | Array[directiveArg] | bool + type directiveArgHash = ::Hash[::Symbol, directiveArg] + + class DirectiveSupertype + def initialize: (::String, directiveArgHash) -> void + attr_reader name: ::String + attr_reader arguments: directiveArgHash + end + + class Directive < DirectiveSupertype + def to_sdl: () -> ::String + def duplicate_on: (Mixins::HasDirectives) -> void + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/enum_type.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/enum_type.rbs new file mode 100644 index 00000000..518792bc --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/enum_type.rbs @@ -0,0 +1,32 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class EnumTypeSuperclass + attr_reader schema_def_state: State + attr_reader type_ref: TypeReference + attr_accessor for_output: bool + attr_reader values_by_name: ::Hash[::String, EnumValue] + + def initialize: ( + State, + TypeReference, + bool, + ::Hash[::String, EnumValue]) -> void + end + + class EnumType < EnumTypeSuperclass + include Mixins::CanBeGraphQLOnly + include Mixins::HasDirectives + include Mixins::HasDocumentation + include Mixins::HasDerivedGraphQLTypeCustomizations + include _Type + def initialize: (State, ::String) ?{ (EnumType) -> void } -> void + def aggregated_values_type: () -> TypeReference + def value: (::String) ?{ (EnumValue) -> void } -> void + def values: (*::String) -> void + def runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::Enum::Type + def as_input: () -> EnumType + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/enum_value.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/enum_value.rbs new file mode 100644 index 00000000..c7d01338 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/enum_value.rbs @@ -0,0 +1,26 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class EnumValueSupertype + attr_reader schema_def_state: State + attr_reader name: ::String + attr_reader runtime_metadata: SchemaArtifacts::RuntimeMetadata::Enum::Value + + def initialize: (State, ::String, SchemaArtifacts::RuntimeMetadata::Enum::Value) -> void + + private + + attr_writer runtime_metadata: SchemaArtifacts::RuntimeMetadata::Enum::Value + end + + class EnumValue < EnumValueSupertype + include Mixins::HasDirectives + include Mixins::HasDocumentation + def initialize: (State, ::String, ::String) ?{ (EnumValue) -> void } -> void + def to_sdl: () -> ::String + def duplicate_on: (EnumType) -> void + def update_runtime_metadata: (**untyped) -> void + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/enum_value_namer.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/enum_value_namer.rbs new file mode 100644 index 00000000..d8d3a44a --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/enum_value_namer.rbs @@ -0,0 +1,23 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class EnumValueNamerSupertype + attr_reader overrides_by_type_name: ::Hash[::String, ::Hash[::String, ::String]] + def initialize: (overrides_by_type_name: ::Hash[::String, ::Hash[::String, ::String]]) -> void + end + + class EnumValueNamer < EnumValueNamerSupertype + @used_value_names_by_type_name: ::Hash[::String, ::Array[::String]] + def initialize: (?::Hash[::String | ::Symbol, ::Hash[::String | ::Symbol, ::String]]) -> void + def name_for: (::String, ::String) -> ::String + def unused_overrides: () -> ::Hash[::String, ::Hash[::String, ::String]] + def used_value_names_by_type_name: () -> ::Hash[::String, ::Array[::String]] + + private + + def validate_overrides: (::Hash[::String, ::Hash[::String, ::String]]) -> void + def notify_problems: (::Array[::String]) -> void + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/enums_for_indexed_types.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/enums_for_indexed_types.rbs new file mode 100644 index 00000000..6c4a3147 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/enums_for_indexed_types.rbs @@ -0,0 +1,10 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class EnumsForIndexedTypes + def initialize: (State) -> void + def sort_order_enum_for: (Mixins::_FilterableAndAggregatableType) -> EnumType? + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field.rbs new file mode 100644 index 00000000..2037b3db --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field.rbs @@ -0,0 +1,101 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Note: this is a partial signature definition (`field.rb` is ignored in `Steepfile`) + class Field + include Mixins::HasDocumentation + include Mixins::HasDirectives + include Mixins::HasTypeInfo + + type argument = untyped + type mapping = untyped + type jsonSchema = untyped + type accuracyConfidence = :high | :medium | :low + + attr_reader name: ::String + attr_reader name_in_index: ::String + attr_reader original_type: TypeReference + attr_reader original_type_for_derived_types: TypeReference + attr_reader parent_type: indexableType + attr_reader schema_def_state: State + attr_reader accuracy_confidence: accuracyConfidence + attr_accessor computation_detail: SchemaArtifacts::RuntimeMetadata::ComputationDetail + attr_reader filter_customizations: ::Array[^(Field) -> void] + attr_reader sort_order_enum_value_customizations: ::Array[^(SortOrderEnumValue) -> void] + attr_reader non_nullable_in_json_schema: bool + attr_reader source: FieldSource? + attr_accessor relationship: Relationship? + attr_reader singular_name: ::String? + attr_reader as_input: bool + + def initialize: ( + name: ::String, + type: ::String, + schema_def_state: State, + ?filter_type: ::String?, + ?name_in_index: ::String, + ?accuracy_confidence: SchemaElements::Field::accuracyConfidence, + ?sortable: bool?, + ?filterable: bool?, + ?aggregatable: bool?, + ?groupable: bool?, + ?graphql_only: bool?, + ?singular: ::String?, + ?as_input: bool + ) ?{ (Field) -> void } -> void + + def runtime_script: (::String?) -> void + def type: () -> TypeReference + def type_for_derived_types: () -> TypeReference + def customize_filter_field: () { (Field) -> void } -> void + def customize_sort_order_enum_values: () { (SortOrderEnumValue) -> void } -> void + def customize_grouped_by_field: () { (Field) -> void } -> void + def customize_sub_aggregations_field: () { (Field) -> void } -> void + def customize_aggregated_values_field: () { (Field) -> void } -> void + def on_each_generated_schema_element: () { (Field | SortOrderEnumValue) -> void } -> void + def renamed_from: (::String) -> void + def argument: (::String, ::String) ?{ (Argument) -> void } -> void + def define_relay_pagination_arguments!: () -> void + def filterable?: () -> bool + def groupable?: () -> bool + def list_field_groupable_by_single_values?: () -> bool + def aggregatable?: () -> bool + def sub_aggregatable?: () -> bool + def define_aggregated_values_field: (ObjectType) -> void + def define_grouped_by_field: (ObjectType) -> void + def define_sub_aggregations_field: (parent_type: ObjectType, type: ::String) ?{ (Field) -> void } -> void + def define_old_aggregation_fields: (ObjectType) -> void + def to_indexing_field_reference: () -> Indexing::FieldReference? + def to_indexing_field: () -> Indexing::Field? + def resolve_mapping: () -> ::Hash[::String, untyped]? + def mapping_type: () -> ::String? + def grouped_by_field_name: () -> ::String + def to_filter_field: (parent_type: anyObjectType, ?for_single_value: bool) -> Field + def to_sdl: (?type_structure_only: bool, ?default_value_sdl: ::String?) ?{ (argument) -> boolish } -> ::String + def sourced_from: (::String, ::String) -> void + def paths_to_lists_for_count_indexing: (?has_list_ancestor: bool) -> ::Array[::String] + def index_leaf?: () -> bool + + ACCURACY_SCORES: ::Hash[accuracyConfidence, ::Integer] + + def self.pick_most_accurate_from: ( + Field, + Field, + ?to_comparable: ^(Field) -> Object? + ) { () -> void } -> Field + + def nested?: () -> bool + + def runtime_metadata_computation_detail: (empty_bucket_value: ::Numeric?, function: ::Symbol) -> void + def runtime_metadata_graphql_field: () -> SchemaArtifacts::RuntimeMetadata::GraphQLField + + private + + def text?: () -> bool + def args_sdl: (joiner: ::String, ?after_opening_paren: ::String) ?{ (argument) -> boolish } -> ::String + def list_field_grouped_by_doc_note: (::String) -> ::String + def filter_field_suffix: (bool) -> ::String + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field_path.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field_path.rbs new file mode 100644 index 00000000..945b41db --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field_path.rbs @@ -0,0 +1,30 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class FieldPathSuperType + attr_reader first_part: Field + attr_reader last_part: Field + attr_reader path_parts: ::Array[Field] + def initialize: (first_part: Field, last_part: Field, path_parts: ::Array[Field]) -> void + end + + class FieldPath < FieldPathSuperType + def type: () -> TypeReference + def path: () -> ::String + def path_in_index: () -> ::String + def fully_qualified_path: () -> ::String + def fully_qualified_path_in_index: () -> ::String + def full_description: () -> ::String + + def self.new: (Field, Field, ::Array[Field]) -> instance + + class Resolver + @indexing_fields_by_public_name_by_type: ::Hash[indexableType, ::Hash[::String, Field]] + def initialize: () -> void + def resolve_public_path: (indexableType, ::String) { (Field) -> bool } -> FieldPath? + def determine_nested_paths: (indexableType, ::String) -> ::Array[::String]? + end + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field_source.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field_source.rbs new file mode 100644 index 00000000..2cf3c755 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field_source.rbs @@ -0,0 +1,25 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class FieldSourceSupertype + attr_reader relationship_name: ::String + attr_reader field_path: ::String + + def initialize: (::String, ::String) -> void + + def self.with: ( + relationship_name: ::String, + field_path: ::String + ) -> FieldSource + + def with: ( + ?relationship_name: ::String, + ?field_path: ::String + ) -> FieldSource + end + + class FieldSource < FieldSourceSupertype + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/graphql_sdl_enumerator.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/graphql_sdl_enumerator.rbs new file mode 100644 index 00000000..851538bb --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/graphql_sdl_enumerator.rbs @@ -0,0 +1,24 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class GraphQLSDLEnumerator + include ::Enumerable[::String] + attr_reader schema_def_state: State + + def initialize: (State, ::Array[graphQLType]) -> void + def each: () { (String) -> void } -> void + + private + + @schema_def_state: State + @all_types_except_root_query_type: ::Array[graphQLType] + + def enumerate_all_types: () -> ::Array[graphQLType] + def aggregation_efficiency_hints_for: (::Array[Indexing::DerivedIndexedType]) -> ::String? + def root_query_type: () -> ObjectType? + def new_built_in_object_type: (::String) { (ObjectType) -> void } -> ObjectType + def new_object_type: (::String) { (ObjectType) -> void } -> ObjectType + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/input_field.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/input_field.rbs new file mode 100644 index 00000000..505037bd --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/input_field.rbs @@ -0,0 +1,13 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class InputFieldSupertype < Field + def initialize: (Field) -> void + end + + class InputField < Field + prepend Mixins::SupportsDefaultValue + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/input_type.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/input_type.rbs new file mode 100644 index 00000000..ec3e2c1c --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/input_type.rbs @@ -0,0 +1,18 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class InputTypeSupertype < TypeWithSubfields + def initialize: (TypeWithSubfields) -> void + end + + # Technically, InputType isn't a subclass of TypeWithSubfields (we use `DelegateClass` instead), + # but that's an implementation detail. In the type system, it is a subtype in terms of it having + # the same interface as `TypeWithSubfields` plus some additional methods. + class InputType < InputTypeSupertype + def initialize: (State, ::String) { (InputType) -> void } -> void + def runtime_metadata: (::Array[SchemaArtifacts::RuntimeMetadata::UpdateTarget]) -> SchemaArtifacts::RuntimeMetadata::ObjectType + def derived_graphql_types: () -> ::Array[graphQLType] + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/interface_type.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/interface_type.rbs new file mode 100644 index 00000000..5dd0f84a --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/interface_type.rbs @@ -0,0 +1,20 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class InterfaceTypeSupertype < TypeWithSubfields + def initialize: (TypeWithSubfields) -> void + attr_reader __getobj__: TypeWithSubfields + end + + # Technically, InterfaceType isn't a subclass of TypeWithSubfields (we use `DelegateClass` instead), + # but that's an implementation detail. In the type system, it is a subtype in terms of it having + # the same interface as `TypeWithSubfields` plus some additional methods. + class InterfaceType < InterfaceTypeSupertype + include Mixins::ImplementsInterfaces + + def initialize: (State, ::String) { (InterfaceType) -> void } -> void + def interface_fields_by_name: () -> ::Hash[::String, Field] + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/list_counts_state.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/list_counts_state.rbs new file mode 100644 index 00000000..7d9755d1 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/list_counts_state.rbs @@ -0,0 +1,19 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class ListCountsState + attr_reader path_to_list_counts: ::String + attr_reader path_from_list_counts: ::String + def self.new: (path_to_list_counts: ::String, path_from_list_counts: ::String) -> ListCountsState + def with: (?path_to_list_counts: ::String, ?path_from_list_counts: ::String) -> ListCountsState + + def self.new_list_counts_field: (at: ::String) -> ListCountsState + + INITIAL: ListCountsState + + def []: (::String) -> ListCountsState + def path_to_count_subfield: (::String) -> ::String + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/object_type.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/object_type.rbs new file mode 100644 index 00000000..3982f7d5 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/object_type.rbs @@ -0,0 +1,21 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class ObjectTypeSupertype < TypeWithSubfields + def initialize: (TypeWithSubfields) -> void + end + + # Technically, ObjectType isn't a subclass of TypeWithSubfields (we use `DelegateClass` instead), + # but that's an implementation detail. In the type system, it is a subtype in terms of it having + # the same interface as `TypeWithSubfields` plus some additional methods. + class ObjectType < ObjectTypeSupertype + include Mixins::HasIndices + include Mixins::ImplementsInterfaces + include Mixins::SupportsFilteringAndAggregation + include _IndexableType + + def initialize: (State, ::String) ?{ (ObjectType) -> void } -> void + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/relationship.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/relationship.rbs new file mode 100644 index 00000000..24fad2ab --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/relationship.rbs @@ -0,0 +1,45 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class RelationshipSupertype < Field + def initialize: (Field) -> void + end + + class Relationship < RelationshipSupertype + type cardinality = :one | :many + attr_reader related_type: TypeReference + + @cardinality: cardinality + @related_type: TypeReference + @foreign_key: ::String + @direction: foreignKeyDirection + @equivalent_field_paths_by_local_path: ::Hash[::String, ::String] + @additional_filter: ::Hash[::String, untyped] + + def initialize: ( + Field, + cardinality: cardinality, + related_type: TypeReference, + foreign_key: ::String, + direction: foreignKeyDirection + ) -> void + + def additional_filter: (::Hash[::String, untyped]) -> void + def equivalent_field: (::String, ?locally_named: ::String) -> void + def routing_value_source_for_index: [T] (Indexing::Index) { (::String) -> bot } -> ::String? + def rollover_timestamp_value_source_for_index: [T] (Indexing::Index) { (::String) -> bot } -> ::String? + def validate_equivalent_fields: (SchemaElements::FieldPath::Resolver) -> ::Array[::String] + def many?: () -> bool + def runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::Relation + + private + + def resolve_and_validate_field_path: ( + indexableType, + ::String, + SchemaElements::FieldPath::Resolver + ) { (::String) -> void } -> SchemaElements::FieldPath? + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/scalar_type.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/scalar_type.rbs new file mode 100644 index 00000000..39d95a31 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/scalar_type.rbs @@ -0,0 +1,51 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class ScalarTypeSuperClass + attr_accessor schema_def_state: State + attr_accessor type_ref: TypeReference + attr_accessor mapping_type: ::String + attr_accessor runtime_metadata: SchemaArtifacts::RuntimeMetadata::ScalarType + attr_accessor aggregated_values_customizations: (^(ObjectType) -> void)? + + def initialize: (State, TypeReference) -> void + end + + class ScalarType < ScalarTypeSuperClass + include _Type + include Mixins::CanBeGraphQLOnly + include Mixins::HasTypeInfo + include Mixins::HasDocumentation + include Mixins::HasDirectives + include Mixins::HasDerivedGraphQLTypeCustomizations + + def initialize: (State, ::String) { (ScalarType) -> void } -> void + def aggregated_values_type: () -> TypeReference + def coerce_with: (::String, defined_at: ::String) -> void + def prepare_for_indexing_with: (::String, defined_at: ::String) -> void + def customize_aggregated_values_type: () { (ObjectType) -> void } -> void + + def runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::ScalarType + def mapping_options: () -> Mixins::HasTypeInfo::optionsHash + def json_schema_options: () -> Mixins::HasTypeInfo::optionsHash + + private + + EQUAL_TO_ANY_OF_DOC: ::String + GT_DOC: ::String + GTE_DOC: ::String + LT_DOC: ::String + LTE_DOC: ::String + + def to_input_filters: () -> ::Array[SchemaElements::InputType] + def to_aggregated_values_type: () -> SchemaElements::ObjectType? + + NUMERIC_TYPES: ::Set[::String] + DATE_TYPES: ::Set[::String] + COMPARABLE_TYPES: ::Set[::String] + + def mapping_type_efficiently_comparable?: () -> bool + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/sort_order_enum_value.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/sort_order_enum_value.rbs new file mode 100644 index 00000000..e05576a9 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/sort_order_enum_value.rbs @@ -0,0 +1,10 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class SortOrderEnumValue < EnumValue + attr_reader sort_order_field_path: ::Array[Field] + def initialize: (EnumValue, ::Array[Field]) -> void + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/sub_aggregation_path.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/sub_aggregation_path.rbs new file mode 100644 index 00000000..40c1c57b --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/sub_aggregation_path.rbs @@ -0,0 +1,18 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class SubAggregationPath + attr_reader parent_doc_types: ::Array[::String] + attr_reader field_path: ::Array[Field] + + def self.new: (::Array[::String], ::Array[Field]) -> SubAggregationPath + def self.paths_for: (Mixins::SupportsFilteringAndAggregation, schema_def_state: State) -> ::Array[SubAggregationPath] + + def plus_parent: (::String) -> SubAggregationPath + def plus_field: (Field) -> SubAggregationPath + def with: (?parent_doc_types: ::Array[::String], ?field_path: ::Array[Field]) -> SubAggregationPath + def field_path_string: () -> ::String + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_namer.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_namer.rbs new file mode 100644 index 00000000..d539d853 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_namer.rbs @@ -0,0 +1,54 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + class TypeNamerSupertype + attr_reader name_overrides: ::Hash[::String, ::String] + attr_reader reverse_overrides: ::Hash[::String, ::String] + attr_reader formats: TypeNamer::formatHash + attr_reader regexes: ::Hash[::Symbol, ::Regexp] + + def initialize: ( + name_overrides: ::Hash[::String, ::String], + reverse_overrides: ::Hash[::String, ::String], + formats: TypeNamer::formatHash, + regexes: ::Hash[::Symbol, ::Regexp] + ) -> void + end + + class TypeNamer < TypeNamerSupertype + type formatHash = ::Hash[::Symbol, ::String] + + def initialize: ( + ?format_overrides: ::Hash[::Symbol, ::String], + ?name_overrides: ::Hash[::String, ::String] | ::Hash[::Symbol, ::String], + ) -> void + + def name_for: (::Symbol | ::String) -> ::String + def revert_override_for: (::String) -> ::String + def generate_name_for: (::Symbol, **::String) -> ::String + def extract_base_from: (::String, format: ::Symbol) -> ::String? + def matches_format?: (::String, ::Symbol) -> bool + def unused_name_overrides: () -> ::Hash[::String, ::String] + def used_names: () -> ::Set[::String] + def self.placeholders_in: (::String) -> ::Array[::String] + + private + + @used_names: ::Array[::String] + + PLACEHOLDER_REGEX: ::Regexp + DEFAULT_FORMATS: formatHash + REQUIRED_PLACEHOLDERS: ::Hash[::Symbol, ::Array[::String]] + FORMAT_SUGGESTER: ::DidYouMean::SpellChecker + DEFINITE_ENUM_FORMATS: ::Set[::Symbol] + DEFINITE_OBJECT_FORMATS: ::Set[::Symbol] + TYPES_THAT_CANNOT_BE_OVERRIDDEN: ::Set[::String] + + def validate_format_overrides: (::Hash[::Symbol, ::String]) -> void + def validate_format: (::Symbol, ::String) -> ::Array[::String] + def validate_name_overrides: (::Hash[::String, ::String]) -> void + def notify_problems: (::Array[::String], ::String) -> void + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_reference.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_reference.rbs new file mode 100644 index 00000000..810258c6 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_reference.rbs @@ -0,0 +1,76 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + type anyObjectType = InputType | InterfaceType | ObjectType | UnionType + type graphQLType = _Type & (EnumType | anyObjectType | ScalarType) & Mixins::HasDirectives + + class TypeReferenceSupertype + attr_reader name: ::String + attr_reader schema_def_state: State + def initialize: (String, State) -> void + end + + class TypeReference < TypeReferenceSupertype + extend ::Forwardable + def type_namer: () -> TypeNamer + def fully_unwrapped: () -> TypeReference + def unwrap_non_null: () -> TypeReference + def wrap_non_null: () -> TypeReference + def unwrap_list: () -> TypeReference + def as_object_type: () -> anyObjectType? + def object?: () -> bool + def enum?: () -> bool + def leaf?: () -> bool + def list?: () -> bool + def non_null?: () -> bool + def boolean?: () -> bool + def resolved: () -> graphQLType? + def unwrapped_name: () -> ::String + def scalar_type_needing_grouped_by_object?: () -> bool + def with_reverted_override: () -> TypeReference + + @json_schema_layers: jsonSchemaLayersArray? + def json_schema_layers: () -> jsonSchemaLayersArray + + def to_final_form: (?as_input: bool) -> TypeReference + + STATIC_FORMAT_NAME_BY_CATEGORY: ::Hash[::Symbol, ::Symbol] + + def as_aggregated_values: () -> TypeReference + def as_aggregation: () -> TypeReference + def as_grouped_by: () -> TypeReference + def as_aggregation_sub_aggregations: (?parent_doc_types: ::Array[::String], ?field_path: ::Array[Field]) -> TypeReference + def as_connection: () -> TypeReference + def as_edge: () -> TypeReference + def as_fields_list_filter_input: () -> TypeReference + def as_filter_input: () -> TypeReference + def as_input_enum: () -> TypeReference + def as_list_element_filter_input: () -> TypeReference + def as_list_filter_input: () -> TypeReference + def as_parent_aggregation: (parent_doc_types: ::Array[::String]) -> TypeReference + def as_sort_order: () -> TypeReference + def as_static_derived_type: (::Symbol) -> TypeReference + def as_sub_aggregation: (parent_doc_types: ::Array[::String]) -> TypeReference + + def list_filter_input?: () -> bool + def list_element_filter_input?: () -> bool + + private + + def peel_json_schema_layers_once: () -> [jsonSchemaLayersArray, TypeReference] + + def matches_format_of?: (::Symbol) -> bool + def parent_aggregation_type: (::Array[::String]) -> ::String + def renamed_with_same_wrappings: (::String) -> TypeReference + + + ENUM_FORMATS: ::Set[::Symbol] + OBJECT_FORMATS: ::Set[::Symbol] + def schema_kind_implied_by_name: () -> (:enum | :object)? + + def to_title_case: (::String) -> ::String + CamelCaseConverter: singleton(SchemaArtifacts::RuntimeMetadata::SchemaElementNamesDefinition::CamelCaseConverter) + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_with_subfields.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_with_subfields.rbs new file mode 100644 index 00000000..4b182741 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_with_subfields.rbs @@ -0,0 +1,113 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + type schemaKind = :type | :input | :interface + type typesByNameHash = ::Hash[::String, graphQLType] + type foreignKeyDirection = :in | :out + + class TypeWithSubfieldsSuperType + def initialize: ( + schemaKind, + State, + TypeReference, + ::Set[::String], + ::Hash[String, Field], + ::Hash[String, Field], + ::Method, + anyObjectType, + bool + ) ?{ (TypeWithSubfields) -> void } -> void + + attr_accessor schema_kind: schemaKind + attr_accessor type_ref: TypeReference + attr_accessor schema_def_state: State + attr_accessor reserved_field_names: ::Set[::String] + attr_accessor graphql_fields_by_name: ::Hash[::String, Field] + attr_accessor indexing_fields_by_name_in_index: ::Hash[::String, Field] + attr_accessor field_factory: ::Method + attr_accessor wrapping_type: anyObjectType + attr_accessor relay_pagination_type: bool + end + + class TypeWithSubfields < TypeWithSubfieldsSuperType + include Mixins::CanBeGraphQLOnly + include Mixins::HasDirectives + include Mixins::HasDocumentation + include _Type + include Mixins::HasDerivedGraphQLTypeCustomizations + include Mixins::HasTypeInfo + + def initialize: ( + schemaKind, + State, + ::String, + wrapping_type: anyObjectType, + field_factory: ::Method + ) ?{ (TypeWithSubfields) -> void } -> void + + def field: ( + ::String, + ::String, + ?graphql_only: bool, + ?indexing_only: bool, + **untyped + # ?name_in_index: ::String, + # ?mapping: SchemaElements::Field::mapping, + # ?json_schema: SchemaElements::Field::jsonSchema, + # ?sortable: bool?, + # ?filterable: bool?, + # ?aggregatable: bool?, + # ?groupable: bool? + ) ?{ (Field) -> void } -> Field + + def aggregated_values_type: () -> TypeReference + def deleted_field: (::String) -> void + def renamed_from: (::String) -> void + + def paginated_collection_field: ( + ::String, + ::String, + ?name_in_index: ::String, + ?singular: ::String? + ) ?{ (Field) -> void } -> Field + + def relates_to_one: (::String, ::String, via: ::String, dir: foreignKeyDirection) ?{ (Field) -> void } -> void + def relates_to_many: (::String, ::String, via: ::String, dir: foreignKeyDirection, singular: ::String) ?{ (Field) -> void } -> void + + def generate_sdl: (name_section: ::String) ?{ (Field::argument) -> boolish } -> String + def current_sources: () -> ::Array[::String] + def index_field_runtime_metadata_tuples: ( + ?path_prefix: ::String, + ?parent_source: ::String, + ?list_counts_state: ListCountsState + ) -> ::Array[[::String, SchemaArtifacts::RuntimeMetadata::IndexField]] + + private + + def fields_sdl: () ?{ (Field::argument) -> boolish } -> String + + def register_field: ( + ::String, + Field, + ::Hash[::String, Field], + ::String, + ::Symbol + ) ?{ (Field) -> Object? } -> void + + def relates_to: ( + ::String, + ::String, + via: ::String, + dir: foreignKeyDirection, + foreign_key_type: ::String, + cardinality: Relationship::cardinality, + related_type: ::String) ?{ (Field) -> void } -> void + + def register_inferred_foreign_key_fields: ( + from_type: [::String, ::String], + to_other: [::String, ::String], + related_type: TypeReference) -> void + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/union_type.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/union_type.rbs new file mode 100644 index 00000000..2cb1b3ca --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/union_type.rbs @@ -0,0 +1,25 @@ +module ElasticGraph + module SchemaDefinition + module SchemaElements + # Note: this is a partial signature definition (`union_type.rb` is ignored in `Steepfile`) + class UnionType + include _IndexableType + include Mixins::CanBeGraphQLOnly + include Mixins::HasDocumentation + include Mixins::HasIndices + include Mixins::HasDirectives + include Mixins::SupportsFilteringAndAggregation + include Mixins::HasDerivedGraphQLTypeCustomizations + + attr_reader schema_def_state: State + attr_reader subtype_refs: ::Set[TypeReference] + + def initialize: (State, ::String) { (UnionType) -> void } -> void + def graphql_fields_by_name: () -> ::Hash[String, Field] + def subtype: (::String) -> void + def subtypes: (*::String) -> void + def mapping_options: () -> ::Hash[::Symbol, untyped] + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/scripting/file_system_repository.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/scripting/file_system_repository.rbs new file mode 100644 index 00000000..bffd17fc --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/scripting/file_system_repository.rbs @@ -0,0 +1,23 @@ +module ElasticGraph + module SchemaDefinition + module Scripting + class FileSystemRepositorySupertype + attr_reader dir: ::String + def initialize: (::String) -> void + def self.with: (dir: ::String) -> FileSystemRepository + def with: (?dir: ::String) -> FileSystemRepository + end + + class FileSystemRepository < FileSystemRepositorySupertype + SUPPORTED_LANGUAGES_BY_EXTENSION: ::Hash[::String, datastoreScriptLanguage] + + attr_reader scripts: ::Array[Script] + attr_reader script_ids_by_scoped_name: ::Hash[::String, ::String] + + private + + def verify_no_duplicates!: (::Array[Script]) -> (void | bot) + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/scripting/script.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/scripting/script.rbs new file mode 100644 index 00000000..1ef1aa65 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/scripting/script.rbs @@ -0,0 +1,40 @@ +module ElasticGraph + module SchemaDefinition + module Scripting + class ScriptSupertype + attr_reader name: ::String + attr_reader source: ::String + attr_reader language: datastoreScriptLanguage + attr_reader context: datastoreScriptContext + + def initialize: ( + name: ::String, + source: ::String, + language: datastoreScriptLanguage, + context: datastoreScriptContext + ) -> void + + def self.with: ( + name: ::String, + source: ::String, + language: datastoreScriptLanguage, + context: datastoreScriptContext + ) -> Script + + def with: ( + ?name: ::String, + ?source: ::String, + ?language: datastoreScriptLanguage, + ?context: datastoreScriptContext + ) -> Script + end + + class Script < ScriptSupertype + attr_reader id: ::String + attr_reader scoped_name: ::String + + def to_artifact_payload: () -> datastoreScriptPayloadHash + end + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/state.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/state.rbs new file mode 100644 index 00000000..13af10d7 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/state.rbs @@ -0,0 +1,101 @@ +module ElasticGraph + module SchemaDefinition + class StateSupertype + attr_reader api: API + attr_reader schema_elements: SchemaArtifacts::RuntimeMetadata::SchemaElementNames + attr_reader index_document_sizes: bool + attr_reader types_by_name: SchemaElements::typesByNameHash + attr_reader object_types_by_name: ::Hash[::String, indexableType] + attr_reader scalar_types_by_name: ::Hash[::String, SchemaElements::ScalarType] + attr_reader enum_types_by_name: ::Hash[::String, SchemaElements::EnumType] + attr_reader implementations_by_interface_ref: ::Hash[SchemaElements::TypeReference, ::Set[SchemaElements::TypeWithSubfields]] + attr_reader sdl_parts: ::Array[::String] + attr_reader paginated_collection_element_types: ::Set[::String] + attr_reader user_defined_fields: ::Set[SchemaElements::Field] + attr_reader renamed_types_by_old_name: ::Hash[::String, SchemaElements::DeprecatedElement] + attr_reader deleted_types_by_old_name: ::Hash[::String, SchemaElements::DeprecatedElement] + attr_reader renamed_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]] + attr_reader deleted_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]] + attr_accessor json_schema_version: ::Integer? + attr_accessor json_schema_version_setter_location: ::Thread::Backtrace::Location? + attr_reader graphql_extension_modules: ::Array[SchemaArtifacts::RuntimeMetadata::Extension] + attr_accessor initially_registered_built_in_types: ::Set[::String] + attr_accessor built_in_types_customization_blocks: ::Array[^(SchemaElements::graphQLType) -> void] + attr_accessor user_definition_complete: bool + attr_accessor sub_aggregation_paths_by_type: ::Hash[Mixins::SupportsFilteringAndAggregation, ::Array[SchemaElements::SubAggregationPath]] + attr_accessor type_refs_by_name: ::Hash[::String, SchemaElements::TypeReference] + attr_reader type_namer: SchemaElements::TypeNamer + attr_reader enum_value_namer: SchemaElements::EnumValueNamer + attr_accessor output: io + + def initialize: ( + api: API, + schema_elements: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + index_document_sizes: bool, + types_by_name: SchemaElements::typesByNameHash, + object_types_by_name: ::Hash[::String, indexableType], + scalar_types_by_name: ::Hash[::String, SchemaElements::ScalarType], + enum_types_by_name: ::Hash[::String, SchemaElements::EnumType], + implementations_by_interface_ref: ::Hash[SchemaElements::TypeReference, ::Set[SchemaElements::TypeWithSubfields]], + sdl_parts: ::Array[::String], + paginated_collection_element_types: ::Set[::String], + user_defined_fields: ::Set[SchemaElements::Field], + renamed_types_by_old_name: ::Hash[::String, SchemaElements::DeprecatedElement], + deleted_types_by_old_name: ::Hash[::String, SchemaElements::DeprecatedElement], + renamed_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]], + deleted_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]], + json_schema_version: Integer?, + json_schema_version_setter_location: ::Thread::Backtrace::Location?, + graphql_extension_modules: ::Array[SchemaArtifacts::RuntimeMetadata::Extension], + initially_registered_built_in_types: ::Set[::String], + built_in_types_customization_blocks: ::Array[^(SchemaElements::graphQLType) -> void], + user_definition_complete: bool, + sub_aggregation_paths_by_type: ::Hash[Mixins::SupportsFilteringAndAggregation, ::Array[SchemaElements::SubAggregationPath]], + type_refs_by_name: ::Hash[::String, SchemaElements::TypeReference], + type_namer: SchemaElements::TypeNamer, + enum_value_namer: SchemaElements::EnumValueNamer, + output: io + ) -> void + end + + class State < StateSupertype + def self.with: ( + api: API, + schema_elements: SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + index_document_sizes: bool, + derived_type_name_formats: ::Hash[::Symbol, ::String], + type_name_overrides: ::Hash[::String, ::String] | ::Hash[::Symbol, ::String], + enum_value_overrides_by_type: ::Hash[::String | ::Symbol, ::Hash[::String | ::Symbol, ::String]], + ?output: io + ) -> State + + def index_document_sizes?: () -> bool + def type_ref: (::String) -> SchemaElements::TypeReference + + def register_object_interface_or_union_type: (SchemaElements::ObjectType | SchemaElements::InterfaceType | SchemaElements::UnionType) -> void + def register_enum_type: (SchemaElements::EnumType) -> void + def register_scalar_type: (SchemaElements::ScalarType) -> void + def register_input_type: (SchemaElements::InputType) -> void + def register_renamed_type: (::String, from: ::String, defined_at: ::Thread::Backtrace::Location?, defined_via: ::String) -> void + def register_deleted_type: (::String, defined_at: ::Thread::Backtrace::Location?, defined_via: ::String) -> void + def register_renamed_field: (::String, from: ::String, to: ::String, defined_at: ::Thread::Backtrace::Location?, defined_via: ::String) -> void + def register_deleted_field: (::String, ::String, defined_at: ::Thread::Backtrace::Location?, defined_via: ::String) -> void + def register_user_defined_field: (SchemaElements::Field) -> void + + @factory: Factory? + def factory: () -> Factory + + @enums_for_indexed_types: SchemaElements::EnumsForIndexedTypes? + def enums_for_indexed_types: () -> SchemaElements::EnumsForIndexedTypes + + def sub_aggregation_paths_for: (Mixins::SupportsFilteringAndAggregation) -> ::Array[SchemaElements::SubAggregationPath] + + @user_defined_field_references_by_type_name: ::Hash[::String, ::Array[SchemaElements::Field]]? + def user_defined_field_references_by_type_name: () -> ::Hash[::String, ::Array[SchemaElements::Field]] + private + + RESERVED_TYPE_NAMES: ::Set[::String] + def register_type: [T] (T & SchemaElements::graphQLType, ?::Hash[::String, T]?) -> T + end + end +end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/test_support.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/test_support.rbs new file mode 100644 index 00000000..57d80754 --- /dev/null +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/test_support.rbs @@ -0,0 +1,34 @@ +module ElasticGraph + module SchemaDefinition + module TestSupport + def define_schema: ( + schema_element_name_form: SchemaArtifacts::RuntimeMetadata::SchemaElementNames::form, + ?schema_element_name_overrides: ::Hash[::Symbol, ::String], + ?index_document_sizes: bool, + ?json_schema_version: ::Integer, + ?extension_modules: ::Array[::Module], + ?derived_type_name_formats: ::Hash[::Symbol, ::String], + ?type_name_overrides: ::Hash[::Symbol, ::String], + ?enum_value_overrides_by_type: ::Hash[::Symbol, ::Hash[::Symbol, ::String]], + ?output: io? + ) ?{ (API) -> void } -> Results + + def define_schema_with_schema_elements: ( + SchemaArtifacts::RuntimeMetadata::SchemaElementNames, + ?index_document_sizes: bool, + ?json_schema_version: ::Integer, + ?extension_modules: ::Array[::Module], + ?derived_type_name_formats: ::Hash[::Symbol, ::String], + ?type_name_overrides: ::Hash[::Symbol, ::String], + ?enum_value_overrides_by_type: ::Hash[::Symbol, ::Hash[::Symbol, ::String]], + ?output: io? + ) ?{ (API) -> void } -> Results + + DOC_COMMENTS: ::String + + def type_def_from: (::String, ::String, ?include_docs: bool) -> ::String + + def strip_docs: (::String) -> ::String + end + end +end diff --git a/elasticgraph-schema_definition/sig/kernel.rbs b/elasticgraph-schema_definition/sig/kernel.rbs new file mode 100644 index 00000000..3c30fd88 --- /dev/null +++ b/elasticgraph-schema_definition/sig/kernel.rbs @@ -0,0 +1,3 @@ +module Kernel + def DelegateClass: (Class) -> Class +end diff --git a/elasticgraph-schema_definition/sig/method.rbs b/elasticgraph-schema_definition/sig/method.rbs new file mode 100644 index 00000000..d19f73a6 --- /dev/null +++ b/elasticgraph-schema_definition/sig/method.rbs @@ -0,0 +1,3 @@ +class Method + def call: (*untyped) ?{ (*untyped) -> untyped } -> untyped | ... +end diff --git a/elasticgraph-schema_definition/sig/rake.rbs b/elasticgraph-schema_definition/sig/rake.rbs new file mode 100644 index 00000000..7c3ceb39 --- /dev/null +++ b/elasticgraph-schema_definition/sig/rake.rbs @@ -0,0 +1,5 @@ +module Rake + module DSL + def task: (*untyped) ?{ (untyped, untyped) -> void } -> void | ... + end +end diff --git a/elasticgraph-schema_definition/spec/fixtures/script_repos/double_nested_script_files/filter/filter/by_age.painless b/elasticgraph-schema_definition/spec/fixtures/script_repos/double_nested_script_files/filter/filter/by_age.painless new file mode 100644 index 00000000..b3c11ac9 --- /dev/null +++ b/elasticgraph-schema_definition/spec/fixtures/script_repos/double_nested_script_files/filter/filter/by_age.painless @@ -0,0 +1 @@ +// Painless code would go here. diff --git a/elasticgraph-schema_definition/spec/fixtures/script_repos/duplicate_name_in_different_contexts/filter/by_age.painless b/elasticgraph-schema_definition/spec/fixtures/script_repos/duplicate_name_in_different_contexts/filter/by_age.painless new file mode 100644 index 00000000..b3c11ac9 --- /dev/null +++ b/elasticgraph-schema_definition/spec/fixtures/script_repos/duplicate_name_in_different_contexts/filter/by_age.painless @@ -0,0 +1 @@ +// Painless code would go here. diff --git a/elasticgraph-schema_definition/spec/fixtures/script_repos/duplicate_name_in_different_contexts/update/by_age.painless b/elasticgraph-schema_definition/spec/fixtures/script_repos/duplicate_name_in_different_contexts/update/by_age.painless new file mode 100644 index 00000000..b3c11ac9 --- /dev/null +++ b/elasticgraph-schema_definition/spec/fixtures/script_repos/duplicate_name_in_different_contexts/update/by_age.painless @@ -0,0 +1 @@ +// Painless code would go here. diff --git a/elasticgraph-schema_definition/spec/fixtures/script_repos/duplicate_name_with_different_lang/filter/by_age.java b/elasticgraph-schema_definition/spec/fixtures/script_repos/duplicate_name_with_different_lang/filter/by_age.java new file mode 100644 index 00000000..a8ea1628 --- /dev/null +++ b/elasticgraph-schema_definition/spec/fixtures/script_repos/duplicate_name_with_different_lang/filter/by_age.java @@ -0,0 +1 @@ +// Java code would go here. diff --git a/elasticgraph-schema_definition/spec/fixtures/script_repos/duplicate_name_with_different_lang/filter/by_age.painless b/elasticgraph-schema_definition/spec/fixtures/script_repos/duplicate_name_with_different_lang/filter/by_age.painless new file mode 100644 index 00000000..b3c11ac9 --- /dev/null +++ b/elasticgraph-schema_definition/spec/fixtures/script_repos/duplicate_name_with_different_lang/filter/by_age.painless @@ -0,0 +1 @@ +// Painless code would go here. diff --git a/elasticgraph-schema_definition/spec/fixtures/script_repos/multiple_contexts_and_languages/filter/UsingMath.java b/elasticgraph-schema_definition/spec/fixtures/script_repos/multiple_contexts_and_languages/filter/UsingMath.java new file mode 100644 index 00000000..a8ea1628 --- /dev/null +++ b/elasticgraph-schema_definition/spec/fixtures/script_repos/multiple_contexts_and_languages/filter/UsingMath.java @@ -0,0 +1 @@ +// Java code would go here. diff --git a/elasticgraph-schema_definition/spec/fixtures/script_repos/multiple_contexts_and_languages/filter/by_age.painless b/elasticgraph-schema_definition/spec/fixtures/script_repos/multiple_contexts_and_languages/filter/by_age.painless new file mode 100644 index 00000000..b3c11ac9 --- /dev/null +++ b/elasticgraph-schema_definition/spec/fixtures/script_repos/multiple_contexts_and_languages/filter/by_age.painless @@ -0,0 +1 @@ +// Painless code would go here. diff --git a/elasticgraph-schema_definition/spec/fixtures/script_repos/multiple_contexts_and_languages/score/by_edit_distance.expression b/elasticgraph-schema_definition/spec/fixtures/script_repos/multiple_contexts_and_languages/score/by_edit_distance.expression new file mode 100644 index 00000000..a8ee9d8f --- /dev/null +++ b/elasticgraph-schema_definition/spec/fixtures/script_repos/multiple_contexts_and_languages/score/by_edit_distance.expression @@ -0,0 +1 @@ +// Lucene expression syntax would go here. diff --git a/elasticgraph-schema_definition/spec/fixtures/script_repos/multiple_contexts_and_languages/update/template1.mustache b/elasticgraph-schema_definition/spec/fixtures/script_repos/multiple_contexts_and_languages/update/template1.mustache new file mode 100644 index 00000000..37389605 --- /dev/null +++ b/elasticgraph-schema_definition/spec/fixtures/script_repos/multiple_contexts_and_languages/update/template1.mustache @@ -0,0 +1 @@ +{{! mustache code would go here}} diff --git a/elasticgraph-schema_definition/spec/fixtures/script_repos/unnested_script_files/by_age.painless b/elasticgraph-schema_definition/spec/fixtures/script_repos/unnested_script_files/by_age.painless new file mode 100644 index 00000000..b3c11ac9 --- /dev/null +++ b/elasticgraph-schema_definition/spec/fixtures/script_repos/unnested_script_files/by_age.painless @@ -0,0 +1 @@ +// Painless code would go here. diff --git a/elasticgraph-schema_definition/spec/fixtures/script_repos/unsupported_language/filter/by_age.rb b/elasticgraph-schema_definition/spec/fixtures/script_repos/unsupported_language/filter/by_age.rb new file mode 100644 index 00000000..26e5869d --- /dev/null +++ b/elasticgraph-schema_definition/spec/fixtures/script_repos/unsupported_language/filter/by_age.rb @@ -0,0 +1,9 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# Script contents would go here. diff --git a/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/rake_tasks_spec.rb b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/rake_tasks_spec.rb new file mode 100644 index 00000000..44d4ae97 --- /dev/null +++ b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/rake_tasks_spec.rb @@ -0,0 +1,1089 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "bundler" +require "elastic_graph/constants" +require "elastic_graph/schema_definition/rake_tasks" +require "elastic_graph/schema_definition/schema_elements/type_namer" +require "graphql" +require "graphql/language/block_string" +require "yaml" + +module ElasticGraph + module SchemaDefinition + RSpec.describe RakeTasks, :rake_task do + describe "schema_artifacts:dump", :in_temp_dir do + it "idempotently dumps all schema artifacts, and is able to check if they are current with `:check`" do + write_elastic_graph_schema_def_code(json_schema_version: 1) + expect_all_artifacts_out_of_date_because_they_havent_been_dumped + + expect { + output = run_rake("schema_artifacts:dump") + expect(output.lines).to include( + a_string_including("Dumped", DATASTORE_CONFIG_FILE), + a_string_including("Dumped", RUNTIME_METADATA_FILE), + a_string_including("Dumped", JSON_SCHEMAS_FILE), + a_string_including("Dumped", versioned_json_schema_file(1)), + a_string_including("Dumped", GRAPHQL_SCHEMA_FILE) + ) + }.to change { read_artifact(DATASTORE_CONFIG_FILE) } + .from(a_falsy_value) + # we expect `number_of_shards: 5` instead of `number_of_shards: 3` because the env-specific + # overrides specified in the config YAML files should not influence the dumped artifacts. + # We don't dump separate artifacts per environment, and thus shouldn't include overrides. + .to(a_string_including("components:", "number_of_shards: 5", "update_ComponentDesigner_from_Component")) + .and change { read_artifact(RUNTIME_METADATA_FILE) } + .from(a_falsy_value) + .to(a_string_including("script_id: update_ComponentDesigner_from_Component_").and(excluding("ruby/object"))) + .and change { read_artifact(JSON_SCHEMAS_FILE) } + .from(a_falsy_value) + .to(a_string_including("\n Component:", "\njson_schema_version: 1")) + .and change { read_artifact(GRAPHQL_SCHEMA_FILE) } + .from(a_falsy_value) + .to(a_string_including("type Component {", "directive @fromExtensionModule")) + + # Verify the data is dumped in Alphabetical order for consistency, and is pruned + # (Except for `EVENT_ENVELOPE_JSON_SCHEMA_NAME` -- it goes first). + definition_names = YAML.safe_load(read_artifact(JSON_SCHEMAS_FILE)).fetch("$defs").keys + expect(definition_names).to eq(%w[ElasticGraphEventEnvelope Component ElectricalPart ID MechanicalPart Size String]) + expect(YAML.safe_load(read_artifact(DATASTORE_CONFIG_FILE)).fetch("indices").keys).to eq %w[ + component_designers components electrical_parts mechanical_parts + ] + + expect_up_to_date_artifacts + + # It should not write anything new, because the core contents have not changed. + expect { + output = run_rake("schema_artifacts:dump") + expect(output.lines).to include(a_string_including("already up to date")) + }.to maintain { read_artifact(DATASTORE_CONFIG_FILE) } + .and maintain { read_artifact(RUNTIME_METADATA_FILE) } + .and maintain { read_artifact(JSON_SCHEMAS_FILE) } + .and maintain { read_artifact(GRAPHQL_SCHEMA_FILE) } + + write_elastic_graph_schema_def_code(component_suffix: "2", component_extras: "schema.deleted_type 'Component'", json_schema_version: 2) + + expect_out_of_date_artifacts_with_details(<<~EOS.strip) + - component_designers: + + component_designers2: + EOS + + expect_out_of_date_artifacts_with_details(<<~EOS.strip, test_color: true) + \e[31m- component_designers:\e[m + \e[32m+\e[m\e[32m component_designers2:\e[m + EOS + + expect { + output = run_rake("schema_artifacts:dump") + expect(output.lines).to include( + a_string_including("Dumped", DATASTORE_CONFIG_FILE), + a_string_including("Dumped", RUNTIME_METADATA_FILE), + a_string_including("Dumped", JSON_SCHEMAS_FILE), + a_string_including("Dumped", versioned_json_schema_file(1)), + a_string_including("Dumped", versioned_json_schema_file(2)), + a_string_including("Dumped", GRAPHQL_SCHEMA_FILE) + ) + }.to change { read_artifact(DATASTORE_CONFIG_FILE) } + .from(a_string_including("components:", "update_ComponentDesigner_from_Component")) + # we expect `number_of_shards: 5` instead of `number_of_shards: 3` because the env-specific + # overrides specified in the config YAML files should not influence the dumped artifacts. + # We don't dump separate artifacts per environment, and thus shouldn't include overrides. + .to(a_string_including("components2:", "number_of_shards: 5", "update_ComponentDesigner2_from_Component2").and(excluding("components:", "update_ComponentDesigner_from_Component"))) + .and change { read_artifact(RUNTIME_METADATA_FILE) } + .from(a_string_including("script_id: update_ComponentDesigner_from_Component_")) + .to(a_string_including("script_id: update_ComponentDesigner2_from_Component2_")) + .and change { read_artifact(JSON_SCHEMAS_FILE) } + .from(a_string_including("\n Component:", "\njson_schema_version: 1")) + .to(a_string_including("\n Component2:", "\njson_schema_version: 2").and(excluding("\n Component:"))) + .and change { read_artifact(GRAPHQL_SCHEMA_FILE) } + .from(a_string_including("type Component {")) + .to(a_string_including("type Component2 {").and(excluding("Component "))) + + expect_up_to_date_artifacts + + delete_artifact versioned_json_schema_file(2) + expect_missing_versioned_json_schema_artifact "v2.yaml" + end + + it "throws an error if the json_schemas artifact is (attempted to be) changed without json_schema_version being bumped" do + write_elastic_graph_schema_def_code(json_schema_version: 1) + expect_all_artifacts_out_of_date_because_they_havent_been_dumped + + # Should succeed, for first artifact. + expect { + output = run_rake("schema_artifacts:dump") + expect(output.lines).to include( + a_string_including("Dumped", JSON_SCHEMAS_FILE), + a_string_including("Dumped", versioned_json_schema_file(1)) + ) + }.to change { read_artifact(JSON_SCHEMAS_FILE) } + .from(a_falsy_value) + .to(a_string_including("\njson_schema_version: 1\n")) + .and change { read_artifact(versioned_json_schema_file(1)) } + .from(a_falsy_value) + .to(a_string_including("\njson_schema_version: 1\n")) + + expect_up_to_date_artifacts + + write_elastic_graph_schema_def_code(json_schema_version: 2) + + # Should succeed, it is ok to update the schema_version without underlying contents changing. + expect { + output = run_rake("schema_artifacts:dump") + expect(output.lines).to include( + a_string_including("Dumped", JSON_SCHEMAS_FILE), + a_string_including("Dumped", versioned_json_schema_file(2)) + ) + }.to change { read_artifact(JSON_SCHEMAS_FILE) } + .from(a_string_including("\njson_schema_version: 1")) + .to(a_string_including("\njson_schema_version: 2")) + .and change { read_artifact(versioned_json_schema_file(2)) } + .from(a_falsy_value) + .to(a_string_including("\njson_schema_version: 2\n")) + + write_elastic_graph_schema_def_code(component_suffix: "2", json_schema_version: 2, component_extras: "t.renamed_from 'Component'") + expect_out_of_date_artifacts + + expect { + run_rake("schema_artifacts:dump") + }.to abort_with a_string_including( + "A change has been attempted to `json_schemas.yaml`", + "`schema.json_schema_version 3`" + ).and matching(json_schema_version_setter_location_regex) + + # Still out of date. + expect_out_of_date_artifacts + + # Decreasing the json_schema_version should also result in a failure. + write_elastic_graph_schema_def_code(component_suffix: "2", json_schema_version: 1, component_extras: "t.renamed_from 'Component'") + expect_out_of_date_artifacts + + expect { + run_rake("schema_artifacts:dump") + }.to abort_with a_string_including( + "A change has been attempted to `json_schemas.yaml`", + "`schema.json_schema_version 3`" + ).and matching(json_schema_version_setter_location_regex) + + write_elastic_graph_schema_def_code(component_suffix: "2", json_schema_version: 3, component_extras: "t.renamed_from 'Component'") + + # Now dump should succeed, as schema_version has been bumped. + expect { + output = run_rake("schema_artifacts:dump") + expect(output.lines).to include( + a_string_including("Dumped", JSON_SCHEMAS_FILE), + a_string_including("Dumped", versioned_json_schema_file(3)) + ) + }.to change { read_artifact(JSON_SCHEMAS_FILE) } + .from(a_string_including("\njson_schema_version: 2")) + .to(a_string_including("\njson_schema_version: 3")) + .and change { read_artifact(versioned_json_schema_file(3)) } + .from(a_falsy_value) + .to(a_string_including("\njson_schema_version: 3\n")) + + # Should be able to run `schema_artifacts:dump` idempotently. + output = run_rake("schema_artifacts:dump") + expect(output.lines).to include( + a_string_including("is already up to date", JSON_SCHEMAS_FILE), + a_string_including("is already up to date", versioned_json_schema_file(3)) + ) + + write_elastic_graph_schema_def_code(component_suffix: "3", json_schema_version: 3, component_extras: "t.renamed_from 'Component'") + expect_out_of_date_artifacts + + expect { + run_rake("schema_artifacts:dump") + }.to abort_with a_string_including( + "A change has been attempted to `json_schemas.yaml`", + "`schema.json_schema_version 4`" + ).and matching(json_schema_version_setter_location_regex) + + expect { + output = run_rake("schema_artifacts:dump", enforce_json_schema_version: false) + expect(output.lines).to include( + a_string_including("Dumped", JSON_SCHEMAS_FILE), + a_string_including("Dumped", versioned_json_schema_file(3)) + ) + }.to change { read_artifact(JSON_SCHEMAS_FILE) } + .and change { read_artifact(versioned_json_schema_file(3)) } + end + + it "allows the derived GraphQL type name formats to be customized" do + # Disable documentation comment wrapping that the GraphQL gem does when formatting an SDL string. + # We need to disable it because the customized derived type formats used below change the length + # of comment lines and cause the documentation to wrap at different points, making it hard to + # compare SDL strings below. + allow(::GraphQL::Language::BlockString).to receive(:break_line) do |line, length, &block| + block.call(line) + end + + write_elastic_graph_schema_def_code(json_schema_version: 1) + run_rake("schema_artifacts:dump") + + # We strip the comment preamble so we can compare it with an SDL string that lacks it below. + uncustomized_graphql_schema = read_artifact(GRAPHQL_SCHEMA_FILE).sub(/^(#[^\n]+\n)+/, "").strip + + derived_type_name_formats = SchemaElements::TypeNamer::DEFAULT_FORMATS.transform_values do |format| + "Prefix#{format}" + end + + run_rake( + "schema_artifacts:dump", + derived_type_name_formats: derived_type_name_formats, + type_name_overrides: { + PrefixComponentGroupedBy: "PrefixComponentGroupedBy457" + } + ) + + customized_graphql_schema = read_artifact(GRAPHQL_SCHEMA_FILE) + + # Our overrides should have added `Prefix` types, where non existed before... + expect(uncustomized_graphql_schema.scan(/\bPrefix\w+\b/)).to be_empty + expect(customized_graphql_schema.scan(/\bPrefix\w+\b/)).not_to be_empty + + # ...and completely renamed the `ComponentGroupedBy` type... + expect(uncustomized_graphql_schema.scan(/\bComponentGroupedBy\b/)).not_to be_empty + expect(customized_graphql_schema.scan(/\bComponentGroupedBy\b/)).to be_empty + + # ...to `PrefixComponentGroupedBy457`. + expect(uncustomized_graphql_schema.scan(/\bPrefixComponentGroupedBy457\b/)).to be_empty + expect(customized_graphql_schema.scan(/\bPrefixComponentGroupedBy457\b/)).not_to be_empty + + unprefixed_schema = ::GraphQL::Schema.from_definition( + customized_graphql_schema + .gsub("PrefixComponentGroupedBy457", "PrefixComponentGroupedBy") + .gsub(/\b(?:Prefix)+(\w+)\b/) { |t| $1 } + ).to_definition.strip + + expect(unprefixed_schema).to eq(uncustomized_graphql_schema) + end + + it "generates separate input vs output enums by default, but allows them to be the same if desired" do + write_elastic_graph_schema_def_code(json_schema_version: 1) + + run_rake("schema_artifacts:dump") + expect(enum_types_in_dumped_graphql_schema).to contain_exactly( + "ComponentDesignerSortOrderInput", + "ComponentSortOrderInput", + "ElectricalPartSortOrderInput", + "MechanicalPartSortOrderInput", + "PartSortOrderInput", + "Size", + "SizeInput" + ) + + run_rake("schema_artifacts:dump", derived_type_name_formats: {InputEnum: "%{base}"}) + expect(enum_types_in_dumped_graphql_schema).to contain_exactly( + "ComponentDesignerSortOrder", + "ComponentSortOrder", + "ElectricalPartSortOrder", + "MechanicalPartSortOrder", + "PartSortOrder", + "Size" + ) + end + + does_not_match_warning_snippet = "does not match any type in your GraphQL schema" + + it "respects type name overrides for all types (both core and derived), except standard GraphQL ones like `Int`" do + original_types = graphql_types_defined_in(CommonSpecHelpers.stock_schema_artifacts(for_context: :graphql).graphql_schema_string) + + # In this test, we evaluate our main test schema because it exercises such a wide variety of cases. + ::File.write("schema.rb", <<~EOS) + load "#{CommonSpecHelpers::REPO_ROOT}/config/schema.rb" + EOS + + exclusions = SchemaElements::TypeNamer::TYPES_THAT_CANNOT_BE_OVERRIDDEN + expect(original_types).to include(*exclusions.to_a) + overrides = (original_types - exclusions.to_a).to_h { |name| [name, "Pre#{name}"] } + + output = run_rake( + "schema_artifacts:dump", + type_name_overrides: overrides.merge({"Widgets" => "Unused"}), + enum_value_overrides_by_type: { + "PreColor" => {"GREAN" => "GREENISH", "MAGENTA" => "RED"}, + "DateGroupingTruncationUnitInput" => {"DAY" => "DAILY"}, + "Nonsense" => {"FOO" => "BAR"} + } + ) + + expect(output).to match( + /WARNING: \d+ of the `type_name_overrides` do not match any type\(s\) in your GraphQL schema/ + ).and include( + "The type name override `Widgets` #{does_not_match_warning_snippet} and has been ignored. Possible alternatives: `Widget`" + ) + + expect(output[/WARNING: some of the `enum_value_overrides_by_type`.*\z/m].lines.first(6).join).to eq(<<~EOS) + WARNING: some of the `enum_value_overrides_by_type` do not match any type(s)/value(s) in your GraphQL schema: + + 1. The enum value override `PreColor.GREAN` does not match any enum value in your GraphQL schema and has been ignored. Possible alternatives: `GREEN`. + 2. The enum value override `PreColor.MAGENTA` does not match any enum value in your GraphQL schema and has been ignored. + 3. `enum_value_overrides_by_type` has a `DateGroupingTruncationUnitInput` key, which does not match any enum type in your GraphQL schema and has been ignored. Possible alternatives: `PreDateGroupingTruncationUnitInput`, `DateGroupingTruncationUnit`. + 4. `enum_value_overrides_by_type` has a `Nonsense` key, which does not match any enum type in your GraphQL schema and has been ignored. + EOS + + overriden_types = graphql_types_defined_in(read_artifact(GRAPHQL_SCHEMA_FILE)) + + # We should have lots of types starting with `Pre`... + expect(overriden_types.grep(/\APre[A-Z]/).size).to be > 50 + # ...and the only types that do not start with `Pre` should be our standard exclusions. + expect(overriden_types.grep_v(/\APre[A-Z]/)).to match_array(exclusions) + end + + it "respects type name overrides for all core types (excluding derived types), except standard GraphQL ones like `Int`" do + derived_type_suffixes = SchemaElements::TypeNamer::DEFAULT_FORMATS.values.map do |format| + format.split("}").last + end + derived_type_regex = /#{derived_type_suffixes.join("|")}\z/ + + exclusions = SchemaElements::TypeNamer::TYPES_THAT_CANNOT_BE_OVERRIDDEN + schema_string = CommonSpecHelpers.stock_schema_artifacts(for_context: :graphql).graphql_schema_string + original_core_types = graphql_types_defined_in(schema_string).reject do |t| + t.start_with?("__") || derived_type_regex.match?(t) || exclusions.include?(t) + end + + # In this test, we evaluate our main test schema because it exercises such a wide variety of cases. + ::File.write("schema.rb", <<~EOS) + load "#{CommonSpecHelpers::REPO_ROOT}/config/schema.rb" + EOS + + overrides = original_core_types.to_h { |name| [name, "Pre#{name}"] } + + output = run_rake("schema_artifacts:dump", type_name_overrides: overrides) + expect(output).to exclude(does_not_match_warning_snippet) + + overriden_types = graphql_types_defined_in(read_artifact(GRAPHQL_SCHEMA_FILE)) + + # We should have lots of types starting with `Pre`... + expect(overriden_types.grep(/\APre[A-Z]/).size).to be > 50 + # ...and almost no types that do not start with `Pre`: just the exclusions, types derived from them, and a few others. + filtered_types = overriden_types.grep_v(/\APre[A-Z]/).grep_v(/\A(#{exclusions.join("|")})/) + allowed_list = %w[ + AggregationCountDetail + DateGroupedBy DateGroupingOffsetInput DateGroupingTruncationUnitInput + DateTimeGroupedBy DateTimeGroupingOffsetInput DateTimeGroupingTruncationUnitInput + DateTimeUnitInput DateUnitInput + DayOfWeekGroupingOffsetInput + LocalTimeGroupingOffsetInput LocalTimeGroupingTruncationUnitInput LocalTimeUnitInput + NonNumericAggregatedValues TextFilterInput + MatchesQueryFilterInput MatchesQueryAllowedEditsPerTermInput MatchesPhraseFilterInput + ] + + expect(filtered_types).to match_array(allowed_list) + end + + it "dumps the ElasticGraph JSON schema metadata only on the internal versioned JSON schema, omitting it from the public copy" do + write_elastic_graph_schema_def_code(json_schema_version: 1) + run_rake("schema_artifacts:dump") + + expect(::YAML.safe_load(read_artifact(JSON_SCHEMAS_FILE)).dig("$defs", "Component", "properties", "id")).to eq( + json_schema_for_keyword_type("ID") + ) + + expect(::YAML.safe_load(read_artifact(versioned_json_schema_file(1))).dig("$defs", "Component", "properties", "id")).to eq( + json_schema_for_keyword_type("ID", { + "ElasticGraph" => { + "type" => "ID!", + "nameInIndex" => "id" + } + }) + ) + end + + it "keeps the ElasticGraph JSON schema metadata up-to-date on all versioned JSON schemas" do + write_elastic_graph_schema_def_code(json_schema_version: 1) + run_rake("schema_artifacts:dump") + + expect(::YAML.safe_load(read_artifact(versioned_json_schema_file(1))).dig("$defs", "Component", "properties", "name")).to eq( + json_schema_for_keyword_type("String", { + "ElasticGraph" => { + "type" => "String!", + "nameInIndex" => "name" + } + }) + ) + + # Here we add a new field `another: String` + write_elastic_graph_schema_def_code(json_schema_version: 2, component_name_extras: "\nt.field 'another', 'String!'") + run_rake("schema_artifacts:dump") + + # It's not added to v1.yaml... + loaded_v1 = ::YAML.safe_load(read_artifact(versioned_json_schema_file(1))) + expect(loaded_v1.dig("$defs", "Component", "properties", "name")).to eq( + json_schema_for_keyword_type("String", { + "ElasticGraph" => { + "type" => "String!", + "nameInIndex" => "name" + } + }) + ) + expect(loaded_v1.dig("$defs", "Component", "properties", "another")).to eq(nil) + + # ..but is added to v2.yaml. + loaded_v2 = ::YAML.safe_load(read_artifact(versioned_json_schema_file(2))) + expect(loaded_v2.dig("$defs", "Component", "properties", "name")).to eq( + json_schema_for_keyword_type("String", { + "ElasticGraph" => { + "type" => "String!", + "nameInIndex" => "name" + } + }) + ) + expect(loaded_v2.dig("$defs", "Component", "properties", "another")).to eq( + json_schema_for_keyword_type("String", { + "ElasticGraph" => { + "type" => "String!", + "nameInIndex" => "another" + } + }) + ) + + # Here we keep the newly added field `another: String` and also change the `name_in_index` of `name`. + write_elastic_graph_schema_def_code(json_schema_version: 2, component_name_extras: ", name_in_index: 'name2'\nt.field 'another', 'String!'") + run_rake("schema_artifacts:dump") + + # The `name_in_index` for `name` should be changed to `name2` in the v1 schema... + loaded_v1 = ::YAML.safe_load(read_artifact(versioned_json_schema_file(1))) + expect(loaded_v1.dig("$defs", "Component", "properties", "name")).to eq( + json_schema_for_keyword_type("String", { + "ElasticGraph" => { + "type" => "String!", + "nameInIndex" => "name2" + } + }) + ) + expect(loaded_v1.dig("$defs", "Component", "properties", "another")).to eq(nil) + + # ...and in the v1 schema. + loaded_v2 = ::YAML.safe_load(read_artifact(versioned_json_schema_file(2))) + expect(loaded_v2.dig("$defs", "Component", "properties", "name")).to eq( + json_schema_for_keyword_type("String", { + "ElasticGraph" => { + "type" => "String!", + "nameInIndex" => "name2" + } + }) + ) + expect(loaded_v2.dig("$defs", "Component", "properties", "another")).to eq( + json_schema_for_keyword_type("String", { + "ElasticGraph" => { + "type" => "String!", + "nameInIndex" => "another" + } + }) + ) + + # Here we add a different new field (`ordinal: Int!`), without bumping the version (and using `enforce_json_schema_version: false` + # to not have to bump the version)... + write_elastic_graph_schema_def_code(json_schema_version: 2, component_name_extras: "\nt.field 'ordinal', 'Int!'") + run_rake("schema_artifacts:dump", enforce_json_schema_version: false) + + # It should not be added to the v1 schema... + loaded_v1 = ::YAML.safe_load(read_artifact(versioned_json_schema_file(1))) + expect(loaded_v1.dig("$defs", "Component", "properties", "ordinal")).to eq(nil) + + # ...but it should be added to the v2 schema. + loaded_v2 = ::YAML.safe_load(read_artifact(versioned_json_schema_file(2))) + expect(loaded_v2.dig("$defs", "Component", "properties", "ordinal")).to eq({ + "$ref" => "#/$defs/Int", + "ElasticGraph" => {"type" => "Int!", "nameInIndex" => "ordinal"} + }) + end + + it "gives the user a clear error when there is ambiguity about what to do with a renamed or deleted field" do + # Verify the error message with 1 old JSON schema version (v8). + write_elastic_graph_schema_def_code(json_schema_version: 8) + run_rake("schema_artifacts:dump") + write_elastic_graph_schema_def_code(json_schema_version: 9, omit_component_name_field: true) + expect { run_rake("schema_artifacts:dump") }.to abort_with <<~EOS + The `Component.name` field (which existed in JSON schema version 8) no longer exists in the current schema definition. + ElasticGraph cannot guess what it should do with this field's data when ingesting events at this old version. + To continue, do one of the following: + + 1. If the `Component.name` field has been renamed, indicate this by calling `field.renamed_from "name"` on the renamed field. + 2. If the `Component.name` field has been dropped, indicate this by calling `type.deleted_field "name"` on the `Component` type. + 3. Alternately, if no publishers or in-flight events use JSON schema version 8, delete its file from `json_schemas_by_version`, and no further changes are required. + EOS + + # Verify the error message with 2 old JSON schema version (v8 and v9). + # The grammar/phrasing is adjusted slightly (e.g. "versions 8 and 9"). + write_elastic_graph_schema_def_code(json_schema_version: 9) + run_rake("schema_artifacts:dump") + write_elastic_graph_schema_def_code(json_schema_version: 10, omit_component_name_field: true) + expect { run_rake("schema_artifacts:dump") }.to abort_with <<~EOS + The `Component.name` field (which existed in JSON schema versions 8 and 9) no longer exists in the current schema definition. + ElasticGraph cannot guess what it should do with this field's data when ingesting events at these old versions. + To continue, do one of the following: + + 1. If the `Component.name` field has been renamed, indicate this by calling `field.renamed_from "name"` on the renamed field. + 2. If the `Component.name` field has been dropped, indicate this by calling `type.deleted_field "name"` on the `Component` type. + 3. Alternately, if no publishers or in-flight events use JSON schema versions 8 or 9, delete their files from `json_schemas_by_version`, and no further changes are required. + EOS + + # Verify the error message with 3 old JSON schema version (v8, v9, and v10). + # The grammar/phrasing is adjusted slightly (e.g. "versions 8, 9, and 10"). + write_elastic_graph_schema_def_code(json_schema_version: 10) + run_rake("schema_artifacts:dump") + write_elastic_graph_schema_def_code(json_schema_version: 11, omit_component_name_field: true) + expect { run_rake("schema_artifacts:dump") }.to abort_with <<~EOS + The `Component.name` field (which existed in JSON schema versions 8, 9, and 10) no longer exists in the current schema definition. + ElasticGraph cannot guess what it should do with this field's data when ingesting events at these old versions. + To continue, do one of the following: + + 1. If the `Component.name` field has been renamed, indicate this by calling `field.renamed_from "name"` on the renamed field. + 2. If the `Component.name` field has been dropped, indicate this by calling `type.deleted_field "name"` on the `Component` type. + 3. Alternately, if no publishers or in-flight events use JSON schema versions 8, 9, or 10, delete their files from `json_schemas_by_version`, and no further changes are required. + EOS + + # Demonstrate that these issues can be solved by each of the 3 options given. + # First, demonstrate indicating the field has been renamed. + write_elastic_graph_schema_def_code(json_schema_version: 11, omit_component_name_field: true, component_extras: "t.field('full_name', 'String') { |f| f.renamed_from 'name' }") + run_rake("schema_artifacts:dump") + delete_artifact(JSON_SCHEMAS_FILE) # so it doesn't force us to increment the version to 5 + + # Next, demonstrate indicating the field has been deleted. + write_elastic_graph_schema_def_code(json_schema_version: 11, omit_component_name_field: true, component_extras: "t.deleted_field 'name'") + run_rake("schema_artifacts:dump") + + # Finally, demonstrate deleting the old JSON schema version artifacts + delete_artifact(versioned_json_schema_file(8)) + delete_artifact(versioned_json_schema_file(9)) + delete_artifact(versioned_json_schema_file(10)) + write_elastic_graph_schema_def_code(json_schema_version: 11, omit_component_name_field: true) + run_rake("schema_artifacts:dump") + end + + it "gives the user a clear error when there is ambiguity about what to do with a renamed or deleted type" do + # Verify the error message with 1 old JSON schema version (v1). + write_elastic_graph_schema_def_code(json_schema_version: 1) + run_rake("schema_artifacts:dump") + write_elastic_graph_schema_def_code(json_schema_version: 2, component_suffix: "2") + expect { run_rake("schema_artifacts:dump") }.to abort_with <<~EOS + The `Component` type (which existed in JSON schema version 1) no longer exists in the current schema definition. + ElasticGraph cannot guess what it should do with this type's data when ingesting events at this old version. + To continue, do one of the following: + + 1. If the `Component` type has been renamed, indicate this by calling `type.renamed_from "Component"` on the renamed type. + 2. If the `Component` field has been dropped, indicate this by calling `schema.deleted_type "Component"` on the schema. + 3. Alternately, if no publishers or in-flight events use JSON schema version 1, delete its file from `json_schemas_by_version`, and no further changes are required. + EOS + + # Verify the error message with 2 old JSON schema version (v1 and v2). + # The grammar/phrasing is adjusted slightly (e.g. "versions 1 and 2"). + write_elastic_graph_schema_def_code(json_schema_version: 2) + run_rake("schema_artifacts:dump") + write_elastic_graph_schema_def_code(json_schema_version: 3, component_suffix: "2") + expect { run_rake("schema_artifacts:dump") }.to abort_with <<~EOS + The `Component` type (which existed in JSON schema versions 1 and 2) no longer exists in the current schema definition. + ElasticGraph cannot guess what it should do with this type's data when ingesting events at these old versions. + To continue, do one of the following: + + 1. If the `Component` type has been renamed, indicate this by calling `type.renamed_from "Component"` on the renamed type. + 2. If the `Component` field has been dropped, indicate this by calling `schema.deleted_type "Component"` on the schema. + 3. Alternately, if no publishers or in-flight events use JSON schema versions 1 or 2, delete their files from `json_schemas_by_version`, and no further changes are required. + EOS + + # Verify the error message with 3 old JSON schema version (v1, v2, and v3). + # The grammar/phrasing is adjusted slightly (e.g. "versions 1, 2, and 3"). + write_elastic_graph_schema_def_code(json_schema_version: 3) + run_rake("schema_artifacts:dump") + write_elastic_graph_schema_def_code(json_schema_version: 4, component_suffix: "2") + expect { run_rake("schema_artifacts:dump") }.to abort_with <<~EOS + The `Component` type (which existed in JSON schema versions 1, 2, and 3) no longer exists in the current schema definition. + ElasticGraph cannot guess what it should do with this type's data when ingesting events at these old versions. + To continue, do one of the following: + + 1. If the `Component` type has been renamed, indicate this by calling `type.renamed_from "Component"` on the renamed type. + 2. If the `Component` field has been dropped, indicate this by calling `schema.deleted_type "Component"` on the schema. + 3. Alternately, if no publishers or in-flight events use JSON schema versions 1, 2, or 3, delete their files from `json_schemas_by_version`, and no further changes are required. + EOS + + # Demonstrate that these issues can be solved by each of the 3 options given. + # First, demonstrate indicating the type has been renamed. + write_elastic_graph_schema_def_code(json_schema_version: 4, component_suffix: "2", component_extras: "t.renamed_from 'Component'") + run_rake("schema_artifacts:dump") + delete_artifact(JSON_SCHEMAS_FILE) # so it doesn't force us to increment the version to 5 + + # Next, demonstrate indicating the type has been deleted. + write_elastic_graph_schema_def_code(json_schema_version: 4, component_suffix: "2", component_extras: "schema.deleted_type 'Component'") + run_rake("schema_artifacts:dump") + + # Finally, demonstrate deleting the old JSON schema version artifacts + delete_artifact(versioned_json_schema_file(1)) + delete_artifact(versioned_json_schema_file(2)) + delete_artifact(versioned_json_schema_file(3)) + write_elastic_graph_schema_def_code(json_schema_version: 4, component_suffix: "2") + run_rake("schema_artifacts:dump") + end + + it "warns if there are `deleted_*` or `renamed_from` calls that are not needed so the user knows they can remove them" do + ::File.write("schema.rb", <<~EOS) + ElasticGraph.define_schema do |schema| + schema.json_schema_version 1 + schema.deleted_type "SomeType" + + schema.object_type "Widget" do |t| + t.renamed_from "Widget2" + t.deleted_field "name" + t.field "description", "String" do |f| + f.renamed_from "old_description" + end + t.renamed_from "Widget3" + end + end + EOS + + output = run_rake("schema_artifacts:dump") + expect(output.split("\n").first(9).join("\n")).to eq(<<~EOS.strip) + The schema definition has 5 unneeded reference(s) to deprecated schema elements. These can all be safely deleted: + + 1. `schema.deleted_type "SomeType"` at schema.rb:3 + 2. `type.renamed_from "Widget2"` at schema.rb:6 + 3. `type.deleted_field "name"` at schema.rb:7 + 4. `field.renamed_from "old_description"` at schema.rb:9 + 5. `type.renamed_from "Widget3"` at schema.rb:11 + + Dumped schema artifact to `config/schema/artifacts/datastore_config.yaml`. + EOS + end + + it "gives a clear error if excess `deleted_*` or `renamed_from` calls create a conflict" do + ::File.write("schema.rb", <<~EOS) + ElasticGraph.define_schema do |schema| + schema.json_schema_version 1 + schema.deleted_type "Widget" + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + + t.field "token", "ID" do |f| + f.renamed_from "id" + end + t.deleted_field "id" + end + end + EOS + + expect { + run_rake("schema_artifacts:dump") + }.to abort_with(<<~EOS) + The schema definition of `Widget` has conflicts. To resolve the conflict, remove the unneeded definitions from the following: + + 1. `schema.deleted_type "Widget"` at schema.rb:3 + + + The schema definition of `Widget.id` has conflicts. To resolve the conflict, remove the unneeded definitions from the following: + + 1. `field.renamed_from "id"` at schema.rb:10 + 2. `type.deleted_field "id"` at schema.rb:12 + EOS + end + + it "does not allow a routing or rollover field to be deleted since we cannot index documents without values for those fields" do + ::File.write("schema.rb", <<~EOS) + ElasticGraph.define_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Embedded" do |t| + t.field "workspace_id", "ID" + t.field "created_at", "DateTime" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "embedded", "Embedded" + t.index "widgets" do |i| + i.route_with "embedded.workspace_id" + i.rollover :yearly, "embedded.created_at" + end + end + end + EOS + + run_rake("schema_artifacts:dump") + + ::File.write("schema.rb", <<~EOS) + ElasticGraph.define_schema do |schema| + schema.json_schema_version 2 + + schema.object_type "Embedded" do |t| + t.field "workspace_id2", "ID", name_in_index: "workspace_id" + t.deleted_field "workspace_id" + + t.field "created_at2", "DateTime", name_in_index: "created_at" + t.deleted_field "created_at" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "embedded", "Embedded" + t.index "widgets" do |i| + i.route_with "embedded.workspace_id2" + i.rollover :yearly, "embedded.created_at2" + end + end + end + EOS + + expect { run_rake("schema_artifacts:dump") }.to abort_with(<<~EOS) + JSON schema version 1 has no field that maps to the routing field path of `Widget.embedded.workspace_id`. + Since the field path is required for routing, ElasticGraph cannot ingest events that lack it. To continue, do one of the following: + + 1. If the `Widget.embedded.workspace_id` field has been renamed, indicate this by calling `field.renamed_from "workspace_id"` on the renamed field rather than using `deleted_field`. + 2. Alternately, if no publishers or in-flight events use JSON schema version 1, delete its file from `json_schemas_by_version`, and no further changes are required. + + + JSON schema version 1 has no field that maps to the rollover field path of `Widget.embedded.created_at`. + Since the field path is required for rollover, ElasticGraph cannot ingest events that lack it. To continue, do one of the following: + + 1. If the `Widget.embedded.created_at` field has been renamed, indicate this by calling `field.renamed_from "created_at"` on the renamed field rather than using `deleted_field`. + 2. Alternately, if no publishers or in-flight events use JSON schema version 1, delete its file from `json_schemas_by_version`, and no further changes are required. + EOS + end + + it "does not change the formatting of the dumped artifacts in unexpected ways" do + config_dir = File.join(CommonSpecHelpers::REPO_ROOT, "config") + run_rake("schema_artifacts:dump", path_to_schema: File.join(config_dir, "schema.rb"), include_extension_module: false) + + # :nocov: -- some branches below depend on pass vs fail or local vs CI. + diff = `git diff --no-index #{File.join(config_dir, "schema", "artifacts")} config/schema/artifacts #{"--color" if $stdout.tty?}` + + unless diff == "" + RSpec.world.reporter.message("\n\nThe schema artifact diff:\n\n#{diff}") + + fail <<~EOS + Expected no formatting changes to the test/development schema artifacts, but there are some. If this is by design, + please delete and re-dump the artifacts with differences to bring our local artifacts up to date with the current + formatting. See "The schema artifact diff:" above for details. + EOS + end + # :nocov: + end + + it "retains `extend schema` in the dumped SDL if ElasticGraph includes it in the generated SDL string" do + write_elastic_graph_schema_def_code(json_schema_version: 1, extra_sdl: "") + run_rake("schema_artifacts:dump") + + # `extend` should not be added by default... + expect(read_artifact(GRAPHQL_SCHEMA_FILE)).not_to include("extend") + + write_elastic_graph_schema_def_code(json_schema_version: 1, extra_sdl: <<~EOS) + extend schema + @customDirective + + directive @customDirective repeatable on SCHEMA + EOS + run_rake("schema_artifacts:dump") + + # ...but it should be added when there's a schema that's been generated. + expect(read_artifact(GRAPHQL_SCHEMA_FILE).lines[3]).to eq("extend schema\n") + end + + it "omits unreferenced GraphQL types from the dumped runtime metadata" do + runtime_meta = runtime_metadata_for_elastic_graph_schema_def_code(include_date_time_fields: true) + expect(runtime_meta["scalar_types_by_name"].keys).to include("DateTime") + expect(runtime_meta["enum_types_by_name"].keys).to include("DateTimeGroupingTruncationUnitInput") + expect(runtime_meta["object_types_by_name"].keys).to include("DateTimeListFilterInput") + + runtime_meta = runtime_metadata_for_elastic_graph_schema_def_code(include_date_time_fields: false) + expect(runtime_meta["scalar_types_by_name"].keys).to exclude("DateTime") + expect(runtime_meta["enum_types_by_name"].keys).to exclude("DateTimeGroupingTruncationUnitInput") + expect(runtime_meta["object_types_by_name"].keys).to exclude("DateTimeListFilterInput") + end + + it "successfully checks schema artifacts when the rake task is run within a bundle that only includes the `elasticgraph-schema_definition` gem" do + # We want to ensure that `elasticgraph-schema_definition` gem declares (in its gemspec) all the + # dependencies necessary for the schema definition rake tasks. Unfortunately, it's test suite + # alone can't detect this, even when run via `script/run_gem_specs`, due to transitive dependencies + # of some of the test dependencies. For example, in January 2023, `elasticgraph-schema_definition` + # began needing parts of `elasticgraph-indexer` at run time, but we forgot to add it to the gemspec, + # and `elasticgraph-admin` is a test dependency, which transitively pulls in `elasticgraph-indexer`. + # + # Here we verify the dependencies by creating a standalone Gemfile and Rakefile in a tmp directory + # that just depends on the runtime deps of `elasticgraph-schema_definition` (and the runtime deps + # of those, recursively). + ::File.write("Gemfile", <<~EOS) + source "https://rubygems.org" + + gem "elasticgraph-schema_definition", path: "#{CommonSpecHelpers::REPO_ROOT}/elasticgraph-schema_definition" + + register_gemspec_gems_with_path = lambda do |eg_gem_name| + gemspec_contents = ::File.read("#{CommonSpecHelpers::REPO_ROOT}/\#{eg_gem_name}/\#{eg_gem_name}.gemspec") + eg_deps = gemspec_contents.scan(/^\\s+spec\\.add_dependency "((?:elasticgraph-)\\w+)"/).flatten + + eg_deps.each do |dep| + gem dep, path: "#{CommonSpecHelpers::REPO_ROOT}/\#{dep}" + register_gemspec_gems_with_path.call(dep) + end + end + + register_gemspec_gems_with_path.call("elasticgraph-schema_definition") + EOS + + ::File.write("Rakefile", <<~EOS) + project_root = "#{CommonSpecHelpers::REPO_ROOT}" + + require "elastic_graph/schema_definition/rake_tasks" + + ElasticGraph::SchemaDefinition::RakeTasks.new( + schema_element_name_form: :snake_case, + index_document_sizes: true, + path_to_schema: "\#{project_root}/config/schema.rb", + schema_artifacts_directory: "\#{project_root}/config/schema/artifacts", + enforce_json_schema_version: false + ) + EOS + + ::FileUtils.cp("#{CommonSpecHelpers::REPO_ROOT}/Gemfile.lock", "Gemfile.lock") + + expect_successful_run_of( + "bundle check || bundle install", + "bundle show", + "bundle exec rake schema_artifacts:check" + ) + end + + def expect_successful_run_of(*shell_commands) + outputs = [] + expect { + ::Bundler.with_original_env do + shell_commands.each do |command| + outputs << `#{command} 2>&1` + expect($?).to be_success, -> do + # :nocov: -- only covered when a test fails. + <<~EOS + Command `#{command}` failed with exit status #{$?.exitstatus}: + + #{outputs.join("\n\n")} + EOS + # :nocov: + end + end + end + }.to output(/Your Gemfile lists/).to_stderr_from_any_process + end + + let(:json_schema_version_setter_location_regex) do + # In `write_elastic_graph_schema_def_code` `json_schema_version` is called on the 2nd line of + # the file written to `schema.rb`. See below. + # + # Note: on Ruby 3.3, the path here winds up being slightly different; instead of just `schema.rb` it is something like: + # `../d20240216-23551-cvdjzo/schema.rb`. I think it's related to the temp directory we run these specs within. + /line 2 at `(\S*\/?)schema\.rb`/ + end + + def write_elastic_graph_schema_def_code(json_schema_version:, component_suffix: "", extra_sdl: "", component_name_extras: "", component_extras: "", omit_component_name_field: false) + code = <<~EOS + ElasticGraph.define_schema do |schema| + schema.json_schema_version #{json_schema_version} + schema.enum_type "Size" do |t| + t.values "SMALL", "MEDIUM", "LAGE" + end + + schema.object_type "MechanicalPart" do |t| + t.field "id", "ID!" do |f| + f.directive "fromExtensionModule" + end + + t.index "mechanical_parts" + end + + schema.object_type "ElectricalPart" do |t| + t.field "id", "ID!" + t.field "size", "Size" + t.index "electrical_parts" + end + + schema.union_type "Part" do |t| + t.subtypes %w[MechanicalPart ElectricalPart] + end + + schema.object_type "ComponentDesigner#{component_suffix}" do |t| + t.field "id", "ID!" + t.field "designed_component_names", "[String!]!" + t.index "component_designers#{component_suffix}" + end + + schema.object_type "Component#{component_suffix}" do |t| + t.field "id", "ID!" + #{%(t.field "name", "String!"#{component_name_extras}) unless omit_component_name_field} + t.field "designer_id", "ID" + t.index "components#{component_suffix}", number_of_shards: 5 + + t.derive_indexed_type_fields "ComponentDesigner#{component_suffix}", from_id: "designer_id" do |derive| + derive.append_only_set "designed_component_names", from: "name" + end + #{component_extras} + end + + schema.raw_sdl #{extra_sdl.inspect} + end + EOS + + ::File.write("schema.rb", code) + end + + def runtime_metadata_for_elastic_graph_schema_def_code(include_date_time_fields:) + ::File.write("schema.rb", <<~EOS) + ElasticGraph.define_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "MyType" do |t| + t.field "id", "ID!" + #{'t.field "timestamp", "DateTime"' if include_date_time_fields} + #{'t.field "timestamps", "[DateTime]"' if include_date_time_fields} + t.index "my_type" + end + end + EOS + + run_rake("schema_artifacts:dump", enforce_json_schema_version: false) + ::YAML.safe_load(read_artifact(RUNTIME_METADATA_FILE)) + end + + def expect_up_to_date_artifacts + output = nil + + expect { + output = run_rake("schema_artifacts:check") + }.not_to raise_error + + expect(output).to include(DATASTORE_CONFIG_FILE, JSON_SCHEMAS_FILE, "up to date") + end + + def expect_all_artifacts_out_of_date_because_they_havent_been_dumped + expect { + run_rake("schema_artifacts:check") + }.to abort_with { |error| + expect(error.message).to eq(<<~EOS.strip) + 5 schema artifact(s) are out of date. Run `rake schema_artifacts:dump` to update the following artifact(s): + + 1. config/schema/artifacts/datastore_config.yaml (file does not exist) + 2. config/schema/artifacts/json_schemas.yaml (file does not exist) + 3. config/schema/artifacts/json_schemas_by_version/v1.yaml (file does not exist) + 4. config/schema/artifacts/runtime_metadata.yaml (file does not exist) + 5. config/schema/artifacts/schema.graphql (file does not exist) + EOS + } + end + + def expect_missing_versioned_json_schema_artifact(version_file) + expect { + run_rake("schema_artifacts:check") + }.to abort_with { |error| + expect(error.message).to eq(<<~EOS.strip) + 1 schema artifact(s) are out of date. Run `rake schema_artifacts:dump` to update the following artifact(s): + + 1. config/schema/artifacts/json_schemas_by_version/#{version_file} (file does not exist) + EOS + } + end + + def expect_out_of_date_artifacts_with_details(example_diff, test_color: false) + expect { + run_rake("schema_artifacts:check", pretend_tty: test_color) + }.to abort_with { |error| + expect(error.message.lines.first(8).join).to eq(<<~EOS) + 6 schema artifact(s) are out of date. Run `rake schema_artifacts:dump` to update the following artifact(s): + + 1. config/schema/artifacts/datastore_config.yaml (see [1] below for the diff) + 2. config/schema/artifacts/json_schemas.yaml (see [2] below for the first 50 lines of the diff) + 3. config/schema/artifacts/json_schemas_by_version/v1.yaml (see [3] below for the diff) + 4. config/schema/artifacts/json_schemas_by_version/v2.yaml (file does not exist) + 5. config/schema/artifacts/runtime_metadata.yaml (see [4] below for the first 50 lines of the diff) + 6. config/schema/artifacts/schema.graphql (see [5] below for the first 50 lines of the diff) + EOS + + expect(error.message).to include(example_diff) + } + end + + def expect_out_of_date_artifacts + expect { + run_rake("schema_artifacts:check") + }.to abort_with a_string_including("out of date", DATASTORE_CONFIG_FILE, JSON_SCHEMAS_FILE) + end + + def read_artifact(name) + path = File.join("config", "schema", "artifacts", name) + File.exist?(path) && File.read(path) + end + + def delete_artifact(*name_parts) + ::File.delete(::File.join("config", "schema", "artifacts", *name_parts)) + end + + def versioned_json_schema_file(version) + ::File.join(JSON_SCHEMAS_BY_VERSION_DIRECTORY, "v#{version}.yaml") + end + end + + def run_rake( + *args, + enforce_json_schema_version: true, + pretend_tty: false, + path_to_schema: "schema.rb", + include_extension_module: true, + derived_type_name_formats: {}, + type_name_overrides: {}, + enum_value_overrides_by_type: {} + ) + if include_extension_module + extension_module = Module.new do + def as_active_instance + raw_sdl "directive @fromExtensionModule on FIELD_DEFINITION" + super + end + end + end + + super(*args) do |output| + allow(output).to receive(:tty?).and_return(true) if pretend_tty + + ElasticGraph::SchemaDefinition::RakeTasks.new( + schema_element_name_form: :snake_case, + index_document_sizes: true, + path_to_schema: path_to_schema, + schema_artifacts_directory: "config/schema/artifacts", + enforce_json_schema_version: enforce_json_schema_version, + extension_modules: [extension_module].compact, + derived_type_name_formats: derived_type_name_formats, + type_name_overrides: type_name_overrides, + enum_value_overrides_by_type: enum_value_overrides_by_type, + output: output + ) + end + end + + def json_schema_for_keyword_type(type, extras = {}) + { + "allOf" => [ + {"$ref" => "#/$defs/#{type}"}, + {"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH} + ] + }.merge(extras) + end + + def enum_types_in_dumped_graphql_schema + ::GraphQL::Schema.from_definition(read_artifact(GRAPHQL_SCHEMA_FILE)).types.filter_map do |name, type| + name if type.kind.enum? && !name.start_with?("__") + end.to_set + end + + def graphql_types_defined_in(schema_string) + ::GraphQL::Schema + .from_definition(schema_string) + .types + .keys + .reject { |t| t.start_with?("__") } + .sort + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/static_scripts_spec.rb b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/static_scripts_spec.rb new file mode 100644 index 00000000..a6020bd6 --- /dev/null +++ b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/static_scripts_spec.rb @@ -0,0 +1,27 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/schema_definition/results" +require "support/validate_script_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "Static scripts", :uses_datastore do + include ValidateScriptSupport + + Results::STATIC_SCRIPT_REPO.scripts.each do |script| + describe "the `#{script.scoped_name}` script" do + it "compiles in the datastore successfully" do + validate_script(script.id, script.to_artifact_payload) + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts/append_only_set_spec.rb b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts/append_only_set_spec.rb new file mode 100644 index 00000000..1a4e89a4 --- /dev/null +++ b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts/append_only_set_spec.rb @@ -0,0 +1,185 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/runtime_metadata_support" +require "support/script_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "Update scripts for append only set fields" do + include SchemaArtifacts::RuntimeMetadata::RuntimeMetadataSupport + + include_context "widget currency script support", expected_function_defs: [ + Indexing::DerivedFields::AppendOnlySet::IDEMPOTENTLY_INSERT_VALUE, + Indexing::DerivedFields::AppendOnlySet::IDEMPOTENTLY_INSERT_VALUES + ] + + it "produces a script when `derive_indexed_type_fields` is used with a single append only set field" do + script_id, payload, update_target = script_artifacts_for_widget_currency_from "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + + t.derive_indexed_type_fields "WidgetCurrency", from_id: "cost.currency" do |derive| + derive.append_only_set "workspace_ids", from: "workspace_id" + end + end + + expect_widget_currency_script(script_id, payload, <<~EOS.strip) + if (ctx._source.workspace_ids == null) { + ctx._source.workspace_ids = []; + } + + boolean workspace_ids_was_noop = !appendOnlySet_idempotentlyInsertValues(data["workspace_id"], ctx._source.workspace_ids); + + if (!scriptErrors.isEmpty()) { + throw new IllegalArgumentException("Derived index update failed due to bad input data: " + scriptErrors.join(" ")); + } + + // For records with no new values to index, only skip the update if the document itself doesn't already exist. + // Otherwise create an (empty) document to reflect the fact that the id has been seen. + if (ctx._source.id != null && workspace_ids_was_noop) { + ctx.op = 'none'; + } else { + // Here we set `_source.id` because if we don't, it'll never be set, making these docs subtly + // different from docs indexed the normal way. + // + // Note also that we MUST use `params.id` instead of `ctx._id`. The latter works on an update + // of an existing document, but is unavailable when we are inserting the document for the first time. + ctx._source.id = params.id; + } + EOS + + expect(update_target).to eq(derived_indexing_update_target_with( + type: "WidgetCurrency", + script_id: script_id, + id_source: "cost.currency", + routing_value_source: nil, + rollover_timestamp_value_source: nil, + data_params: {"workspace_id" => dynamic_param_with(source_path: "workspace_id", cardinality: :many)} + )) + end + + it "produces a script when `derive_indexed_type_fields` is used with multiple append only set fields" do + script_id, payload, update_target = script_artifacts_for_widget_currency_from "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + + t.derive_indexed_type_fields "WidgetCurrency", from_id: "cost.currency" do |derive| + derive.append_only_set "workspace_ids", from: "workspace_id" + derive.append_only_set "sizes", from: "options.size" + derive.append_only_set "colors", from: "options.color" + end + end + + expect_widget_currency_script(script_id, payload, <<~EOS.strip) + if (ctx._source.colors == null) { + ctx._source.colors = []; + } + if (ctx._source.sizes == null) { + ctx._source.sizes = []; + } + if (ctx._source.workspace_ids == null) { + ctx._source.workspace_ids = []; + } + + boolean colors_was_noop = !appendOnlySet_idempotentlyInsertValues(data["options.color"], ctx._source.colors); + boolean sizes_was_noop = !appendOnlySet_idempotentlyInsertValues(data["options.size"], ctx._source.sizes); + boolean workspace_ids_was_noop = !appendOnlySet_idempotentlyInsertValues(data["workspace_id"], ctx._source.workspace_ids); + + if (!scriptErrors.isEmpty()) { + throw new IllegalArgumentException("Derived index update failed due to bad input data: " + scriptErrors.join(" ")); + } + + // For records with no new values to index, only skip the update if the document itself doesn't already exist. + // Otherwise create an (empty) document to reflect the fact that the id has been seen. + if (ctx._source.id != null && colors_was_noop && sizes_was_noop && workspace_ids_was_noop) { + ctx.op = 'none'; + } else { + // Here we set `_source.id` because if we don't, it'll never be set, making these docs subtly + // different from docs indexed the normal way. + // + // Note also that we MUST use `params.id` instead of `ctx._id`. The latter works on an update + // of an existing document, but is unavailable when we are inserting the document for the first time. + ctx._source.id = params.id; + } + EOS + + expect(update_target).to eq(derived_indexing_update_target_with( + type: "WidgetCurrency", + script_id: script_id, + id_source: "cost.currency", + routing_value_source: nil, + rollover_timestamp_value_source: nil, + data_params: { + "workspace_id" => dynamic_param_with(source_path: "workspace_id", cardinality: :many), + "options.size" => dynamic_param_with(source_path: "options.size", cardinality: :many), + "options.color" => dynamic_param_with(source_path: "options.color", cardinality: :many) + } + )) + end + + context "with a nested destination field" do + it "defaults the parents of the nested field to an empty object, but avoids duplicating that initialization when one parent field has multiple derived subfields" do + script_id, payload, update_target = script_artifacts_for_widget_currency_from "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + + t.derive_indexed_type_fields "WidgetCurrency", from_id: "cost.currency" do |derive| + derive.append_only_set "widget_options.colors", from: "options.color" + derive.append_only_set "widget_options.sizes", from: "options.size" + end + end + + expect_widget_currency_script(script_id, payload, <<~EOS.strip) + if (ctx._source.widget_options == null) { + ctx._source.widget_options = [:]; + } + if (ctx._source.widget_options.colors == null) { + ctx._source.widget_options.colors = []; + } + if (ctx._source.widget_options.sizes == null) { + ctx._source.widget_options.sizes = []; + } + + boolean widget_options__colors_was_noop = !appendOnlySet_idempotentlyInsertValues(data["options.color"], ctx._source.widget_options.colors); + boolean widget_options__sizes_was_noop = !appendOnlySet_idempotentlyInsertValues(data["options.size"], ctx._source.widget_options.sizes); + + if (!scriptErrors.isEmpty()) { + throw new IllegalArgumentException("Derived index update failed due to bad input data: " + scriptErrors.join(" ")); + } + + // For records with no new values to index, only skip the update if the document itself doesn't already exist. + // Otherwise create an (empty) document to reflect the fact that the id has been seen. + if (ctx._source.id != null && widget_options__colors_was_noop && widget_options__sizes_was_noop) { + ctx.op = 'none'; + } else { + // Here we set `_source.id` because if we don't, it'll never be set, making these docs subtly + // different from docs indexed the normal way. + // + // Note also that we MUST use `params.id` instead of `ctx._id`. The latter works on an update + // of an existing document, but is unavailable when we are inserting the document for the first time. + ctx._source.id = params.id; + } + EOS + + expect(update_target).to eq(derived_indexing_update_target_with( + type: "WidgetCurrency", + script_id: script_id, + id_source: "cost.currency", + routing_value_source: nil, + rollover_timestamp_value_source: nil, + data_params: { + "options.size" => dynamic_param_with(source_path: "options.size", cardinality: :many), + "options.color" => dynamic_param_with(source_path: "options.color", cardinality: :many) + } + )) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts/immutable_value_spec.rb b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts/immutable_value_spec.rb new file mode 100644 index 00000000..9b1421b4 --- /dev/null +++ b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts/immutable_value_spec.rb @@ -0,0 +1,184 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/runtime_metadata_support" +require "support/script_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "Update scripts for `immutable_value` fields" do + include SchemaArtifacts::RuntimeMetadata::RuntimeMetadataSupport + + include_context "widget currency script support", expected_function_defs: [ + Indexing::DerivedFields::ImmutableValue::IDEMPOTENTLY_SET_VALUE + ] + + it "produces a script when `derive_indexed_type_fields` is used with a single `immutable_value` field" do + script_id, payload, update_target = script_artifacts_for_widget_currency_from "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + + t.derive_indexed_type_fields "WidgetCurrency", from_id: "cost.currency" do |derive| + derive.immutable_value "name", from: "cost_currency_name", nullable: true, can_change_from_null: false + end + end + + expect_widget_currency_script(script_id, payload, expected_code_for_name_field( + nullable: true, + can_change_from_null: false + )) + + expect(update_target).to eq(derived_indexing_update_target_with( + type: "WidgetCurrency", + script_id: script_id, + id_source: "cost.currency", + routing_value_source: nil, + rollover_timestamp_value_source: nil, + data_params: {"cost_currency_name" => dynamic_param_with(source_path: "cost_currency_name", cardinality: :many)} + )) + end + + it "defaults `nullable` to `true` and `can_change_from_null` to `false`" do + script_id, payload, _ = script_artifacts_for_widget_currency_from "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + + t.derive_indexed_type_fields "WidgetCurrency", from_id: "cost.currency" do |derive| + derive.immutable_value "name", from: "cost_currency_name" + end + end + + expect_widget_currency_script(script_id, payload, expected_code_for_name_field( + nullable: true, + can_change_from_null: false + )) + end + + it "allows `nullable` to be set to `false`" do + script_id, payload, _ = script_artifacts_for_widget_currency_from "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + + t.derive_indexed_type_fields "WidgetCurrency", from_id: "cost.currency" do |derive| + derive.immutable_value "name", from: "cost_currency_name", nullable: false, can_change_from_null: false + end + end + + expect_widget_currency_script(script_id, payload, expected_code_for_name_field( + nullable: false, + can_change_from_null: false + )) + end + + it "allows `can_change_from_null` to be set to `true`" do + script_id, payload, _ = script_artifacts_for_widget_currency_from "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + + t.derive_indexed_type_fields "WidgetCurrency", from_id: "cost.currency" do |derive| + derive.immutable_value "name", from: "cost_currency_name", nullable: true, can_change_from_null: true + end + end + + expect_widget_currency_script(script_id, payload, expected_code_for_name_field( + nullable: true, + can_change_from_null: true + )) + end + + it "does not allow `can_change_from_null` to be `true` when `nullable` is `false`" do + expect { + script_artifacts_for_widget_currency_from "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + + t.derive_indexed_type_fields "WidgetCurrency", from_id: "cost.currency" do |derive| + derive.immutable_value "name", from: "cost_currency_name", nullable: false, can_change_from_null: true + end + end + }.to raise_error Errors::SchemaError, a_string_including("nullable: false", "can_change_from_null: true") + end + + context "with a nested destination field" do + it "defaults the parents of the nested field to an empty object, but avoids duplicating that initialization when one parent field has multiple derived subfields" do + script_id, payload, update_target = script_artifacts_for_widget_currency_from "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + + t.derive_indexed_type_fields "WidgetCurrency", from_id: "cost.currency" do |derive| + derive.immutable_value "details.unit", from: "cost_currency_unit", nullable: true, can_change_from_null: false + derive.immutable_value "details.symbol", from: "cost_currency_symbol", nullable: true, can_change_from_null: false + end + end + + expect_widget_currency_script(script_id, payload, <<~EOS.strip) + if (ctx._source.details == null) { + ctx._source.details = [:]; + } + + boolean details__symbol_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_symbol"], ctx._source.details, "details.symbol", "symbol", true, false); + boolean details__unit_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_unit"], ctx._source.details, "details.unit", "unit", true, false); + + if (!scriptErrors.isEmpty()) { + throw new IllegalArgumentException("#{DERIVED_INDEX_FAILURE_MESSAGE_PREAMBLE}: " + scriptErrors.join(" ")); + } + + // For records with no new values to index, only skip the update if the document itself doesn't already exist. + // Otherwise create an (empty) document to reflect the fact that the id has been seen. + if (ctx._source.id != null && details__symbol_was_noop && details__unit_was_noop) { + ctx.op = 'none'; + } else { + // Here we set `_source.id` because if we don't, it'll never be set, making these docs subtly + // different from docs indexed the normal way. + // + // Note also that we MUST use `params.id` instead of `ctx._id`. The latter works on an update + // of an existing document, but is unavailable when we are inserting the document for the first time. + ctx._source.id = params.id; + } + EOS + + expect(update_target).to eq(derived_indexing_update_target_with( + type: "WidgetCurrency", + script_id: script_id, + id_source: "cost.currency", + routing_value_source: nil, + rollover_timestamp_value_source: nil, + data_params: { + "cost_currency_unit" => dynamic_param_with(source_path: "cost_currency_unit", cardinality: :many), + "cost_currency_symbol" => dynamic_param_with(source_path: "cost_currency_symbol", cardinality: :many) + } + )) + end + end + + def expected_code_for_name_field(nullable:, can_change_from_null:) + <<~EOS.chomp + + boolean name_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_name"], ctx._source, "name", "name", #{nullable}, #{can_change_from_null}); + + if (!scriptErrors.isEmpty()) { + throw new IllegalArgumentException("#{DERIVED_INDEX_FAILURE_MESSAGE_PREAMBLE}: " + scriptErrors.join(" ")); + } + + // For records with no new values to index, only skip the update if the document itself doesn't already exist. + // Otherwise create an (empty) document to reflect the fact that the id has been seen. + if (ctx._source.id != null && name_was_noop) { + ctx.op = 'none'; + } else { + // Here we set `_source.id` because if we don't, it'll never be set, making these docs subtly + // different from docs indexed the normal way. + // + // Note also that we MUST use `params.id` instead of `ctx._id`. The latter works on an update + // of an existing document, but is unavailable when we are inserting the document for the first time. + ctx._source.id = params.id; + } + EOS + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts/min_or_max_value_spec.rb b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts/min_or_max_value_spec.rb new file mode 100644 index 00000000..0e384a7a --- /dev/null +++ b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts/min_or_max_value_spec.rb @@ -0,0 +1,214 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/runtime_metadata_support" +require "support/script_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "Update scripts for `min_value`/`max_value` fields" do + include SchemaArtifacts::RuntimeMetadata::RuntimeMetadataSupport + + context "for just a `max_value` field" do + include_context "widget currency script support", expected_function_defs: [ + Indexing::DerivedFields::MinOrMaxValue.function_def(:max) + ] + + it "produces a script when `derive_indexed_type_fields` is used with a single max value field" do + script_id, payload, update_target = script_artifacts_for_widget_currency_from "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + + t.derive_indexed_type_fields "WidgetCurrency", from_id: "cost.currency" do |derive| + derive.max_value "newest_widget_created_at", from: "created_at" + end + end + + expect_widget_currency_script(script_id, payload, <<~EOS.chomp) + + boolean newest_widget_created_at_was_noop = !maxValue_idempotentlyUpdateValue(data["created_at"], ctx._source, "newest_widget_created_at"); + + if (!scriptErrors.isEmpty()) { + throw new IllegalArgumentException("#{DERIVED_INDEX_FAILURE_MESSAGE_PREAMBLE}: " + scriptErrors.join(" ")); + } + + // For records with no new values to index, only skip the update if the document itself doesn't already exist. + // Otherwise create an (empty) document to reflect the fact that the id has been seen. + if (ctx._source.id != null && newest_widget_created_at_was_noop) { + ctx.op = 'none'; + } else { + // Here we set `_source.id` because if we don't, it'll never be set, making these docs subtly + // different from docs indexed the normal way. + // + // Note also that we MUST use `params.id` instead of `ctx._id`. The latter works on an update + // of an existing document, but is unavailable when we are inserting the document for the first time. + ctx._source.id = params.id; + } + EOS + + expect(update_target).to eq(derived_indexing_update_target_with( + type: "WidgetCurrency", + script_id: script_id, + id_source: "cost.currency", + routing_value_source: nil, + rollover_timestamp_value_source: nil, + data_params: {"created_at" => dynamic_param_with(source_path: "created_at", cardinality: :many)} + )) + end + end + + context "for just a `min_value` field" do + include_context "widget currency script support", expected_function_defs: [ + Indexing::DerivedFields::MinOrMaxValue.function_def(:min) + ] + + it "produces a script when `derive_indexed_type_fields` is used with a single min value field" do + script_id, payload, update_target = script_artifacts_for_widget_currency_from "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + + t.derive_indexed_type_fields "WidgetCurrency", from_id: "cost.currency" do |derive| + derive.min_value "oldest_widget_created_at", from: "created_at" + end + end + + expect_widget_currency_script(script_id, payload, <<~EOS.chomp) + + boolean oldest_widget_created_at_was_noop = !minValue_idempotentlyUpdateValue(data["created_at"], ctx._source, "oldest_widget_created_at"); + + if (!scriptErrors.isEmpty()) { + throw new IllegalArgumentException("#{DERIVED_INDEX_FAILURE_MESSAGE_PREAMBLE}: " + scriptErrors.join(" ")); + } + + // For records with no new values to index, only skip the update if the document itself doesn't already exist. + // Otherwise create an (empty) document to reflect the fact that the id has been seen. + if (ctx._source.id != null && oldest_widget_created_at_was_noop) { + ctx.op = 'none'; + } else { + // Here we set `_source.id` because if we don't, it'll never be set, making these docs subtly + // different from docs indexed the normal way. + // + // Note also that we MUST use `params.id` instead of `ctx._id`. The latter works on an update + // of an existing document, but is unavailable when we are inserting the document for the first time. + ctx._source.id = params.id; + } + EOS + + expect(update_target).to eq(derived_indexing_update_target_with( + type: "WidgetCurrency", + script_id: script_id, + id_source: "cost.currency", + routing_value_source: nil, + rollover_timestamp_value_source: nil, + data_params: {"created_at" => dynamic_param_with(source_path: "created_at", cardinality: :many)} + )) + end + end + + context "for both a `min_value` and `max_value` field" do + include_context "widget currency script support", expected_function_defs: [ + Indexing::DerivedFields::MinOrMaxValue.function_def(:max), + Indexing::DerivedFields::MinOrMaxValue.function_def(:min) + ] + + it "produces a script when `derive_indexed_type_fields` is used with both min and max value fields on the same source field" do + script_id, payload, update_target = script_artifacts_for_widget_currency_from "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + + t.derive_indexed_type_fields "WidgetCurrency", from_id: "cost.currency" do |derive| + derive.max_value "newest_widget_created_at", from: "created_at" + derive.min_value "oldest_widget_created_at", from: "created_at" + end + end + + expect_widget_currency_script(script_id, payload, <<~EOS.chomp) + + boolean newest_widget_created_at_was_noop = !maxValue_idempotentlyUpdateValue(data["created_at"], ctx._source, "newest_widget_created_at"); + boolean oldest_widget_created_at_was_noop = !minValue_idempotentlyUpdateValue(data["created_at"], ctx._source, "oldest_widget_created_at"); + + if (!scriptErrors.isEmpty()) { + throw new IllegalArgumentException("#{DERIVED_INDEX_FAILURE_MESSAGE_PREAMBLE}: " + scriptErrors.join(" ")); + } + + // For records with no new values to index, only skip the update if the document itself doesn't already exist. + // Otherwise create an (empty) document to reflect the fact that the id has been seen. + if (ctx._source.id != null && newest_widget_created_at_was_noop && oldest_widget_created_at_was_noop) { + ctx.op = 'none'; + } else { + // Here we set `_source.id` because if we don't, it'll never be set, making these docs subtly + // different from docs indexed the normal way. + // + // Note also that we MUST use `params.id` instead of `ctx._id`. The latter works on an update + // of an existing document, but is unavailable when we are inserting the document for the first time. + ctx._source.id = params.id; + } + EOS + + expect(update_target).to eq(derived_indexing_update_target_with( + type: "WidgetCurrency", + script_id: script_id, + id_source: "cost.currency", + routing_value_source: nil, + rollover_timestamp_value_source: nil, + data_params: {"created_at" => dynamic_param_with(source_path: "created_at", cardinality: :many)} + )) + end + + context "with a nested destination field" do + it "defaults the parents of the nested field to an empty object, but avoids duplicating that initialization when one parent field has multiple derived subfields" do + script_id, payload, update_target = script_artifacts_for_widget_currency_from "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + + t.derive_indexed_type_fields "WidgetCurrency", from_id: "cost.currency" do |derive| + derive.max_value "widget_created_at.newest", from: "created_at" + derive.min_value "widget_created_at.oldest", from: "created_at" + end + end + + expect_widget_currency_script(script_id, payload, <<~EOS.chomp) + if (ctx._source.widget_created_at == null) { + ctx._source.widget_created_at = [:]; + } + + boolean widget_created_at__newest_was_noop = !maxValue_idempotentlyUpdateValue(data["created_at"], ctx._source.widget_created_at, "newest"); + boolean widget_created_at__oldest_was_noop = !minValue_idempotentlyUpdateValue(data["created_at"], ctx._source.widget_created_at, "oldest"); + + if (!scriptErrors.isEmpty()) { + throw new IllegalArgumentException("#{DERIVED_INDEX_FAILURE_MESSAGE_PREAMBLE}: " + scriptErrors.join(" ")); + } + + // For records with no new values to index, only skip the update if the document itself doesn't already exist. + // Otherwise create an (empty) document to reflect the fact that the id has been seen. + if (ctx._source.id != null && widget_created_at__newest_was_noop && widget_created_at__oldest_was_noop) { + ctx.op = 'none'; + } else { + // Here we set `_source.id` because if we don't, it'll never be set, making these docs subtly + // different from docs indexed the normal way. + // + // Note also that we MUST use `params.id` instead of `ctx._id`. The latter works on an update + // of an existing document, but is unavailable when we are inserting the document for the first time. + ctx._source.id = params.id; + } + EOS + + expect(update_target).to eq(derived_indexing_update_target_with( + type: "WidgetCurrency", + script_id: script_id, + id_source: "cost.currency", + routing_value_source: nil, + rollover_timestamp_value_source: nil, + data_params: {"created_at" => dynamic_param_with(source_path: "created_at", cardinality: :many)} + )) + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts_spec.rb b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts_spec.rb new file mode 100644 index 00000000..9d10814e --- /dev/null +++ b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts_spec.rb @@ -0,0 +1,111 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/runtime_metadata_support" +require "support/script_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "Update scripts" do + include SchemaArtifacts::RuntimeMetadata::RuntimeMetadataSupport + + describe "for a derived indexing type" do + include_context "widget currency script support", expected_function_defs: [ + Indexing::DerivedFields::AppendOnlySet::IDEMPOTENTLY_INSERT_VALUE, + Indexing::DerivedFields::AppendOnlySet::IDEMPOTENTLY_INSERT_VALUES, + Indexing::DerivedFields::ImmutableValue::IDEMPOTENTLY_SET_VALUE, + Indexing::DerivedFields::MinOrMaxValue.function_def(:max), + Indexing::DerivedFields::MinOrMaxValue.function_def(:min) + ] + + it "produces none when the `derive_indexed_type_fields` is not used in the schema definition" do + script_id, payload, update_target = script_artifacts_for_widget_currency_from "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + end + + expect(script_id).to eq nil + expect(payload).to eq nil + expect(update_target).to eq nil + end + + it "fails with a clear error when no derived fields are defined" do + expect { + script_artifacts_for_widget_currency_from "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + + t.derive_indexed_type_fields "WidgetCurrency", from_id: "cost.currency" do |derive| + end + end + }.to raise_error Errors::SchemaError, a_string_including("derive_indexed_type_fields", "Widget", "WidgetCurrency") + end + + it "produces a script when `derive_indexed_type_fields` is used with each type of derived field at the same time" do + script_id, payload, update_target = script_artifacts_for_widget_currency_from "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + + t.derive_indexed_type_fields( + "WidgetCurrency", + from_id: "cost.currency", + route_with: "cost_currency_name", + rollover_with: "currency_introduced_on" + ) do |derive| + derive.append_only_set "workspace_ids", from: "workspace_id" + derive.immutable_value "name", from: "cost_currency_name" + derive.min_value "oldest_widget_created_at", from: "created_at" + derive.max_value "newest_widget_created_at", from: "created_at" + end + end + + expect_widget_currency_script(script_id, payload, <<~EOS.strip) + if (ctx._source.workspace_ids == null) { + ctx._source.workspace_ids = []; + } + + boolean name_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_name"], ctx._source, "name", "name", true, false); + boolean newest_widget_created_at_was_noop = !maxValue_idempotentlyUpdateValue(data["created_at"], ctx._source, "newest_widget_created_at"); + boolean oldest_widget_created_at_was_noop = !minValue_idempotentlyUpdateValue(data["created_at"], ctx._source, "oldest_widget_created_at"); + boolean workspace_ids_was_noop = !appendOnlySet_idempotentlyInsertValues(data["workspace_id"], ctx._source.workspace_ids); + + if (!scriptErrors.isEmpty()) { + throw new IllegalArgumentException("#{DERIVED_INDEX_FAILURE_MESSAGE_PREAMBLE}: " + scriptErrors.join(" ")); + } + + // For records with no new values to index, only skip the update if the document itself doesn't already exist. + // Otherwise create an (empty) document to reflect the fact that the id has been seen. + if (ctx._source.id != null && name_was_noop && newest_widget_created_at_was_noop && oldest_widget_created_at_was_noop && workspace_ids_was_noop) { + ctx.op = 'none'; + } else { + // Here we set `_source.id` because if we don't, it'll never be set, making these docs subtly + // different from docs indexed the normal way. + // + // Note also that we MUST use `params.id` instead of `ctx._id`. The latter works on an update + // of an existing document, but is unavailable when we are inserting the document for the first time. + ctx._source.id = params.id; + } + EOS + + expect(update_target).to eq(derived_indexing_update_target_with( + type: "WidgetCurrency", + script_id: script_id, + id_source: "cost.currency", + routing_value_source: "cost_currency_name", + rollover_timestamp_value_source: "currency_introduced_on", + data_params: { + "workspace_id" => dynamic_param_with(source_path: "workspace_id", cardinality: :many), + "cost_currency_name" => dynamic_param_with(source_path: "cost_currency_name", cardinality: :many), + "created_at" => dynamic_param_with(source_path: "created_at", cardinality: :many) + } + )) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/spec_helper.rb b/elasticgraph-schema_definition/spec/spec_helper.rb new file mode 100644 index 00000000..1dbed77f --- /dev/null +++ b/elasticgraph-schema_definition/spec/spec_helper.rb @@ -0,0 +1,19 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-schema_definition`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. + +module ElasticGraph + module SchemaDefinition + module Scripting + FIXTURE_DIR = ::File.join(__dir__, "fixtures", "script_repos") + end + end +end diff --git a/elasticgraph-schema_definition/spec/support/example_extensions/indexing_preparer.rb b/elasticgraph-schema_definition/spec/support/example_extensions/indexing_preparer.rb new file mode 100644 index 00000000..cac93bab --- /dev/null +++ b/elasticgraph-schema_definition/spec/support/example_extensions/indexing_preparer.rb @@ -0,0 +1,12 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +class ExampleIndexingPreparer + def self.prepare_for_indexing(value) + end +end diff --git a/elasticgraph-schema_definition/spec/support/example_extensions/scalar_coercion_adapter.rb b/elasticgraph-schema_definition/spec/support/example_extensions/scalar_coercion_adapter.rb new file mode 100644 index 00000000..8bb040a8 --- /dev/null +++ b/elasticgraph-schema_definition/spec/support/example_extensions/scalar_coercion_adapter.rb @@ -0,0 +1,15 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +class ExampleScalarCoercionAdapter + def self.coerce_input(value, ctx) + end + + def self.coerce_result(value, ctx) + end +end diff --git a/elasticgraph-schema_definition/spec/support/json_schema_matcher.rb b/elasticgraph-schema_definition/spec/support/json_schema_matcher.rb new file mode 100644 index 00000000..59f8f36a --- /dev/null +++ b/elasticgraph-schema_definition/spec/support/json_schema_matcher.rb @@ -0,0 +1,142 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/json_schema/meta_schema_validator" +require "elastic_graph/json_schema/validator_factory" +require "json" + +RSpec::Matchers.define :have_json_schema_like do |type, expected_schema, options = {}| + # RSpec 3.13 has a regression related to keyword args that we work around here with an `options` hash. + # TODO: Switch back to an `include_typename` keyword arg once we upgrade to a version that fixes the regression. + # https://github.com/rspec/rspec-expectations/issues/1451 + include_typename = options.fetch(:include_typename, true) + + diffable + + attr_reader :actual, :expected + + chain :which_matches do |*expected_matches| + @expected_matches = expected_matches + end + + chain :and_fails_to_match do |*expected_non_matches| + @expected_non_matches = expected_non_matches + end + + match do |full_schema| + modified_expected_schema = if include_typename && expected_schema.key?("properties") + with_typename(type, expected_schema) + else + expected_schema + end + .then { |schema| normalize(schema) } + + @expected = JSON.pretty_generate(modified_expected_schema) + + actual_schema = normalize(full_schema.fetch("$defs").fetch(type)) + @actual = JSON.pretty_generate(actual_schema) + + @validator_factory = ElasticGraph::JSONSchema::ValidatorFactory.new(schema: full_schema, sanitize_pii: false) + + @meta_schema_validation_errors = ElasticGraph::JSONSchema.elastic_graph_internal_meta_schema_validator.validate(modified_expected_schema) + + if @meta_schema_validation_errors.empty? && actual_schema == modified_expected_schema + validator = @validator_factory.validator_for(type) + + @match_failures = (@expected_matches || []).filter_map.with_index do |payload, index| + if (failure = validator.validate_with_error_message(payload)) + match_failure_description(payload, index, failure) + end + end + + @non_match_failures = (@expected_non_matches || []).filter_map.with_index do |payload, index| + if validator.valid?(payload) + non_match_failure_description(payload, index) + end + end + + @match_failures.empty? && @non_match_failures.empty? + else + @match_failures = @non_match_failures = [] + false + end + end + + failure_message do |_actual_schema| + if @meta_schema_validation_errors.any? + <<~EOS + expected valid JSON schema[1] but got validation errors on the expected schema and got JSON schema[2]: + + #{@meta_schema_validation_errors.map { |e| JSON.pretty_generate(e) }.join("\n\n")} + + + [1] The expected schema: + #{expected} + + [2] Actual schema: + #{actual} + EOS + elsif @match_failures.any? + <<~EOS + expected given JSON payloads matched the JSON schema, but one or more did not. + + #{@match_failures.join("\n\n")} + EOS + elsif @non_match_failures.any? + <<~EOS + expected given JSON payloads to not match the JSON schema, but one or more did. + + #{@non_match_failures.join("\n\n")} + EOS + else + <<~EOS + expected valid JSON schema[1] but got JSON schema[2]. + + [1] Expected schema: + #{expected} + + [2] Actual schema: + #{actual} + EOS + end + end + + def match_failure_description(payload, index, failure) + <<~EOS + Failure at index #{index} from payload: + + #{JSON.pretty_generate(payload)} + + #{failure} + EOS + end + + def non_match_failure_description(payload, index) + <<~EOS + Failure at index #{index} from payload: + + #{JSON.pretty_generate(payload)} + EOS + end + + def normalize(schema) + ::JSON.parse(::JSON.generate(schema.sort.to_h)) + end + + def with_typename(type, schema) + new_schema = schema.dup + new_schema["properties"] = schema["properties"].merge({ + "__typename" => { + "type" => "string", + "const" => type, + "default" => type + } + }) + new_schema + end +end diff --git a/elasticgraph-schema_definition/spec/support/json_schema_matcher_spec.rb b/elasticgraph-schema_definition/spec/support/json_schema_matcher_spec.rb new file mode 100644 index 00000000..7463aff2 --- /dev/null +++ b/elasticgraph-schema_definition/spec/support/json_schema_matcher_spec.rb @@ -0,0 +1,190 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "support/json_schema_matcher" + +# Note: this spec exists to verify our custom JSON schema matcher works +# properly. It doesn't really validate ElasticGraph itself, and if it +# becomes a burden to maintain, consider deleting it. +RSpec.describe "JSON schema matcher", aggregate_failures: false do + it "passes when the schema is valid and the same" do + type = "MyType" + schema = { + "type" => "object", + "properties" => { + "id" => { + "anyOf" => [ + {"type" => "string"}, + {"type" => "null"} + ] + } + }, + "required" => %w[id] + } + + expect(schema_with(type, schema)).to have_json_schema_like(type, schema, include_typename: false) + end + + it "treats string and symbol keys as equivalent because they dump the same" do + string_key_schema = { + "type" => "object", + "properties" => { + "name" => {"type" => "string"} + } + } + + symbol_key_schema = { + type: "object", + properties: { + "name" => {type: "string"} + } + } + + schema = { + "$schema" => ::ElasticGraph::JSON_META_SCHEMA, + "$defs" => { + "StringType" => string_key_schema, + "SymbolType" => symbol_key_schema + } + } + + expect(schema).to have_json_schema_like("StringType", symbol_key_schema, include_typename: false) + expect(schema).to have_json_schema_like("SymbolType", string_key_schema, include_typename: false) + end + + it "fails when the expected schema has an invalid value" do + type = "InvalidType" + schema = { + "type" => "object", + "properties" => { + "is_happy" => { + "anyOf" => [ + {"type" => 7}, + {"type" => "null"} + ] + } + } + } + + expect { + expect(schema_with(type, schema)).to have_json_schema_like(type, schema) + }.to fail_with("but got validation errors") + end + + it "fails when the expected schema has an unknown field" do + type = "InvalidType" + schema = { + "type" => "object", + "properties" => { + "is_happy" => { + "anyOf" => [ + {"type" => "boolean", "foo" => 3}, + {"type" => "null"} + ] + } + } + } + + expect { + expect(schema_with(type, schema)).to have_json_schema_like(type, schema) + }.to fail_with("but got validation errors") + end + + it "fails when the expected and actual schemas are different but both valid" do + schema1 = { + "type" => "object", + "properties" => { + "is_happy" => { + "anyOf" => [ + {"type" => "boolean"}, + {"type" => "null"} + ] + } + } + } + + schema2 = { + "type" => "object", + "properties" => { + "is_happy" => { + "anyOf" => [ + {"type" => "string"}, + {"type" => "null"} + ] + } + } + } + + schema = { + "$schema" => ::ElasticGraph::JSON_META_SCHEMA, + "$defs" => { + "Type1" => schema1, + "Type2" => schema2 + } + } + + expect { + expect(schema).to have_json_schema_like("Type1", schema2) + }.to fail_with("but got JSON schema") + end + + it "uses the validator that allows extra `ElasticGraph` metadata in the JSON schema" do + type = "MyType" + schema = { + "type" => "object", + "properties" => { + "id" => { + "anyOf" => [ + {"type" => "string"}, + {"type" => "null"} + ], + "ElasticGraph" => { + "type" => "String", + "nameInIndex" => "id" + } + } + }, + "required" => %w[id] + } + + expect(schema_with(type, schema)).to have_json_schema_like(type, schema, include_typename: false) + end + + context "when `which_matches(...).and_fails_to_match(...)` is used" do + type = "Type" + schema = {"type" => "number"} + + it "passes when it correctly matches or fails to match as specified" do + expect(schema_with(type, schema)).to have_json_schema_like(type, schema) + .which_matches(1, 2, 3).and_fails_to_match("foo", "bar", nil) + end + + it "fails when one of the expected matches does not match" do + expect { + expect(schema_with(type, schema)).to have_json_schema_like(type, schema) + .which_matches(1, "bar", 3).and_fails_to_match("foo", "bazz", nil) + }.to fail_with("Failure at index 1 from payload", "bar") + end + + it "fails when one of the expected non matches does match" do + expect { + expect(schema_with(type, schema)).to have_json_schema_like(type, schema) + .which_matches(1, 2, 3).and_fails_to_match("foo", "bar", nil, 17) + }.to fail_with("Failure at index 3 from payload", "17") + end + end + + def schema_with(type, schema) + {"$schema" => ::ElasticGraph::JSON_META_SCHEMA, "$defs" => {type => schema}} + end + + def fail_with(*snippets) + raise_error(::RSpec::Expectations::ExpectationNotMetError, a_string_including(*snippets)) + end +end diff --git a/elasticgraph-schema_definition/spec/support/script_support.rb b/elasticgraph-schema_definition/spec/support/script_support.rb new file mode 100644 index 00000000..1ec104c7 --- /dev/null +++ b/elasticgraph-schema_definition/spec/support/script_support.rb @@ -0,0 +1,76 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "digest/md5" +require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" +require "elastic_graph/schema_definition/api" +require "elastic_graph/schema_definition/test_support" +require "support/validate_script_support" + +module ElasticGraph + module SchemaDefinition + ::RSpec.shared_context "ScriptSupport" do + include TestSupport + include ValidateScriptSupport + + def expect_script(script_id, prefix, payload, expected_source) + expect(script_id).to eq("#{prefix}_#{Digest::MD5.hexdigest(expected_source)}") + + expect(payload.fetch("context")).to eq("update") + expect(payload.fetch("script")).to match("lang" => "painless", "source" => an_instance_of(::String)) + expect(payload.dig("script", "source")).to eq(expected_source) + end + + def generate_script_artifacts(source_type, target_type, prefix: "update_#{target_type}_from_#{source_type}", &schema_definition) + results = define_schema(&schema_definition) + scripts = results.datastore_scripts.select { |id, payload| payload.fetch("context") == "update" } + scripts.each { |id, payload| validate_script(id, payload) } + + update_targets_by_type = results.runtime_metadata + .object_types_by_name + .transform_values(&:update_targets) + + # Filter scripts to just the one for the given destination_type and source_type, so that we ignore other scripts. + scripts = scripts.select { |k, v| k.start_with?(prefix) } + update_targets = update_targets_by_type.fetch(source_type).select { |ut| ut.type == target_type } + + expect(scripts.size).to be < 2 + expect(update_targets.size).to be < 2 + + script_id, script_contents = scripts.first + [script_id, script_contents, update_targets.first] + end + + def define_schema(&block) + super(schema_element_name_form: "snake_case") + end + end + + RSpec.shared_context "widget currency script support", :uses_datastore do |expected_function_defs: []| + include_context "ScriptSupport" + + define_method :expect_widget_currency_script do |script_id, payload, expected_source_except_functions| + parts = expected_function_defs + [Indexing::DerivedIndexedType::STATIC_SETUP_STATEMENTS + "\n" + expected_source_except_functions] + expected_source = parts.map(&:strip).join("\n\n") + expect_script(script_id, "update_WidgetCurrency_from_Widget", payload, expected_source) + end + + def script_artifacts_for_widget_currency_from(type_name, ...) + generate_script_artifacts("Widget", "WidgetCurrency") do |schema| + # Ensure `WidgetCurrency` is defined as the caller defines deriviation rules for it. + schema.object_type "WidgetCurrency" do |t| + t.field "id", "ID!" + t.index "widget_currencies" + end + + schema.object_type(type_name, ...) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/support/validate_script_support.rb b/elasticgraph-schema_definition/spec/support/validate_script_support.rb new file mode 100644 index 00000000..ca4817bc --- /dev/null +++ b/elasticgraph-schema_definition/spec/support/validate_script_support.rb @@ -0,0 +1,33 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" + +module ElasticGraph + module SchemaDefinition + module ValidateScriptSupport + def validate_script(id, payload) + main_datastore_client.put_script(id: id, body: {script: payload.fetch("script")}, context: payload.fetch("context")) + rescue Errors::BadDatastoreRequest => ex + # :nocov: -- only executed when we have a script that can't compile + message = JSON.pretty_generate(JSON.parse(ex.message.sub(/\A[^{]+/, ""))) + + raise <<~EOS + The script is invalid. + + #{payload.dig("script", "source")} + + #{"=" * 80} + + #{message} + EOS + # :nocov: + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_definition_spec_support.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_definition_spec_support.rb new file mode 100644 index 00000000..7fe73067 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_definition_spec_support.rb @@ -0,0 +1,37 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/schema_definition_helpers" + +module ElasticGraph + module SchemaDefinition + ::RSpec.shared_context "IndexDefinitionSpecSupport" do + include_context "SchemaDefinitionHelpers" + + def index_configs_for(*index_names, index_document_sizes: false, schema_element_name_form: "snake_case", &schema_definition) + config = define_schema( + index_document_sizes: index_document_sizes, + schema_element_name_form: schema_element_name_form, + &schema_definition + ).datastore_config + + index_names.map { |i| config.fetch("indices").fetch(i) } + end + + def index_template_configs_for(*index_names, index_document_sizes: false, schema_element_name_form: "snake_case", &schema_definition) + config = define_schema( + index_document_sizes: index_document_sizes, + schema_element_name_form: schema_element_name_form, + &schema_definition + ).datastore_config + + index_names.map { |i| config.fetch("index_templates").fetch(i) } + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/abstract_types_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/abstract_types_spec.rb new file mode 100644 index 00000000..6e299e30 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/abstract_types_spec.rb @@ -0,0 +1,319 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "index_mappings_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "Datastore config index mappings -- abstract types" do + include_context "IndexMappingsSpecSupport" + + shared_examples_for "a type with subtypes" do |type_def_method| + context "composed of 2 indexed types" do + it "generates separate mappings for the two subtypes" do + widget_mapping, component_mapping = index_mappings_for "widgets", "components" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "amount_cents", "Int" + link_subtype_to_supertype(t, "Thing") + t.index "widgets" + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "weight", "Int" + link_subtype_to_supertype(t, "Thing") + t.index "components" + end + + s.public_send type_def_method, "Thing" do |t| + link_supertype_to_subtypes(t, "Widget", "Component") + end + end + + expect(widget_mapping.dig("properties")).to include({ + "id" => {"type" => "keyword"}, + "name" => {"type" => "keyword"}, + "amount_cents" => {"type" => "integer"} + }).and exclude("weight") + + expect(component_mapping.dig("properties")).to include({ + "id" => {"type" => "keyword"}, + "name" => {"type" => "keyword"}, + "weight" => {"type" => "integer"} + }).and exclude("amount_cents") + end + end + + context "that is itself indexed" do + it "merges the subfields of the two types, and adds a __typename field to distinguish the subtype" do + mapping = index_mapping_for "things" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "amount_cents", "Int" + link_subtype_to_supertype(t, "Thing") + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "weight", "Int" + link_subtype_to_supertype(t, "Thing") + end + + s.public_send type_def_method, "Thing" do |t| + link_supertype_to_subtypes(t, "Widget", "Component") + t.index "things" + end + end + + expect(mapping.dig("properties")).to include({ + "id" => {"type" => "keyword"}, + "name" => {"type" => "keyword"}, + "amount_cents" => {"type" => "integer"}, + "weight" => {"type" => "integer"}, + "__typename" => {"type" => "keyword"} + }) + end + + it "handles the subtypes having no fields" do + mapping = index_mapping_for "things" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + link_subtype_to_supertype(t, "Thing") + end + + s.object_type "Component" do |t| + t.field "id", "ID" + link_subtype_to_supertype(t, "Thing") + end + + s.public_send type_def_method, "Thing" do |t| + link_supertype_to_subtypes(t, "Widget", "Component") + t.index "things" + end + end + + expect(mapping.dig("properties")).to include({ + "__typename" => {"type" => "keyword"} + }) + end + + it "raises an error if there is a common subfield with different mapping settings" do + expect { + index_mapping_for "things" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String!" do |f| + f.mapping null_value: "" + end + link_subtype_to_supertype(t, "Thing") + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.field "name", "String!" do |f| + f.mapping null_value: "[null]" + end + link_subtype_to_supertype(t, "Thing") + end + + s.object_type "Animal" do |t| + t.field "species", "String!" + link_subtype_to_supertype(t, "Thing") + end + + s.public_send type_def_method, "Thing" do |t| + link_supertype_to_subtypes(t, "Widget", "Component", "Animal") + t.index "things" + end + end + }.to raise_error(Errors::SchemaError, a_string_including("Conflicting definitions", "field `name`", "subtypes of `Thing`", "Widget", "Component").and(excluding("Animal"))) + end + + it "raises an error if there is a common subfield with different mapping types" do + expect { + index_mapping_for "things" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "owner_id", "String" + link_subtype_to_supertype(t, "Thing") + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.field "owner_id", "Int" + link_subtype_to_supertype(t, "Thing") + end + + s.object_type "Animal" do |t| + t.field "species", "String!" + link_subtype_to_supertype(t, "Thing") + end + + s.public_send type_def_method, "Thing" do |t| + link_supertype_to_subtypes(t, "Widget", "Component", "Animal") + t.index "things" + end + end + }.to raise_error(Errors::SchemaError, a_string_including("Conflicting definitions", "field `owner_id`", "subtypes of `Thing`", "Widget", "Component", '"type"=>"keyword"', '"type"=>"integer"').and(excluding("Animal"))) + end + + it "allows the different mapping setting issue to be resolved by configuring a different index field name for one field" do + mapping = index_mapping_for "things" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String!", name_in_index: "name_w" do |f| + f.mapping null_value: "" + end + link_subtype_to_supertype(t, "Thing") + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.field "name", "String!" do |f| + f.mapping null_value: "[null]" + end + link_subtype_to_supertype(t, "Thing") + end + + s.public_send type_def_method, "Thing" do |t| + link_supertype_to_subtypes(t, "Widget", "Component") + t.index "things" + end + end + + expect(mapping.dig("properties")).to include({ + "id" => {"type" => "keyword"}, + "name" => {"type" => "keyword", "null_value" => "[null]"}, + "name_w" => {"type" => "keyword", "null_value" => ""}, + "__typename" => {"type" => "keyword"} + }) + end + end + + context "that is an embedded type" do + it "merges the subfields of the two types, and adds a __typename field to distinguish the subtype" do + mapping = index_mapping_for "my_type" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "amount_cents", "Int" + link_subtype_to_supertype(t, "Thing") + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "weight", "Int" + link_subtype_to_supertype(t, "Thing") + end + + s.public_send type_def_method, "Thing" do |t| + link_supertype_to_subtypes(t, "Widget", "Component") + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "thing", "Thing" + t.index "my_type" + end + end + + expect(mapping.dig("properties", "thing")).to eq({ + "properties" => { + "id" => {"type" => "keyword"}, + "name" => {"type" => "keyword"}, + "amount_cents" => {"type" => "integer"}, + "weight" => {"type" => "integer"}, + "__typename" => {"type" => "keyword"} + } + }) + end + end + end + + context "on a type union" do + include_examples "a type with subtypes", :union_type do + def link_subtype_to_supertype(object_type, supertype_name) + # nothing to do; the linkage happens via a `subtypes` call on the supertype + end + + def link_supertype_to_subtypes(union_type, *subtype_names) + union_type.subtypes(*subtype_names) + end + end + end + + context "on an interface type" do + include_examples "a type with subtypes", :interface_type do + def link_subtype_to_supertype(object_type, interface_name) + object_type.implements interface_name + end + + def link_supertype_to_subtypes(interface_type, *subtype_names) + # nothing to do; the linkage happens via an `implements` call on the subtype + end + end + + it "ignores it if it has no subtypes" do + mappings = index_mappings_for do |s| + s.interface_type "Thing" do |t| + end + end + + expect(mappings).to be_empty + end + + it "supports a subtype recursion (e.g. an interface that implements an interface)" do + mapping = index_mapping_for "things" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "amount_cents", "Int!" + t.implements "WidgetOrComponent" + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "weight", "Int!" + t.implements "WidgetOrComponent" + end + + s.interface_type "WidgetOrComponent" do |t| + t.implements "Thing" + end + + s.object_type "Object" do |t| + t.field "id", "ID!" + t.field "description", "String!" + t.implements "Thing" + end + + s.interface_type "Thing" do |t| + t.index "things" + end + end + + expect(mapping.dig("properties")).to include({ + "id" => {"type" => "keyword"}, + "name" => {"type" => "keyword"}, + "amount_cents" => {"type" => "integer"}, + "weight" => {"type" => "integer"}, + "description" => {"type" => "keyword"}, + "__typename" => {"type" => "keyword"} + }) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/counts_field_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/counts_field_spec.rb new file mode 100644 index 00000000..8acad25c --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/counts_field_spec.rb @@ -0,0 +1,287 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "index_mappings_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "Datastore config index mappings -- `#{LIST_COUNTS_FIELD}` field" do + include_context "IndexMappingsSpecSupport" + + it "is omitted when there are no list fields" do + mapping = index_mapping_for "teams" do |schema| + # We have an embedded object field to demonstrate that its subfields aren't listed in the counts, either. + schema.object_type "TeamDetails" do |t| + t.field "first_year", "Int" + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "details", "TeamDetails" + t.index "teams" + end + end + + expect(mapping.dig("properties")).to exclude(LIST_COUNTS_FIELD) + end + + it "defines an integer subfield for each scalar list field" do + mapping = index_mapping_for "teams" do |schema| + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "past_names", "[String!]!" # non-null inside and out + t.field "home_cities", "[String!]" # non-null inside + t.field "seasons", "[Int]!" # non-null outside + t.field "games", "[String]" # nullable inside and out + t.index "teams" + end + end + + expect_list_counts_mappings(mapping, LIST_COUNTS_FIELD, %w[ + past_names + home_cities + seasons + games + ]) + end + + it "treats a `paginated_collection_field` as a list field since it is indexed that way" do + mapping = index_mapping_for "teams" do |schema| + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.paginated_collection_field "past_names", "String" + t.index "teams" + end + end + + expect_list_counts_mappings(mapping, LIST_COUNTS_FIELD, %w[past_names]) + end + + # TODO: consider dropping this test as part of disallowing `type: "nested"` on a list-of-scalar field. + it "does not attempt to find the subfields of a scalar list field that wrongly uses `type: nested`", :dont_validate_graphql_schema do + mapping = index_mapping_for "teams" do |schema| + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "past_names", "[String!]!" do |f| + f.mapping type: "nested" + end + t.index "teams" + end + end + + expect_list_counts_mappings(mapping, LIST_COUNTS_FIELD, %w[past_names]) + end + + it "uses pipe-separated paths for list fields embedded on object fields, since dots get interpreted as object nesting by the datastore" do + mapping = index_mapping_for "teams" do |schema| + schema.object_type "TeamHistory" do |t| + t.field "past_names", "[String!]!" + t.field "past_home_cities", "[String!]!" + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "history1", "TeamHistory" # nullable + t.field "history2", "TeamHistory!" # non-null + t.index "teams" + end + end + + expect_list_counts_mappings(mapping, LIST_COUNTS_FIELD, %w[ + history1.past_names + history1.past_home_cities + history2.past_names + history2.past_home_cities + ]) + end + + it "honors the configured `name_in_index`" do + mapping = index_mapping_for "teams" do |schema| + schema.object_type "TeamHistory" do |t| + t.field "past_names", "[String!]!", name_in_index: "past_names_in_index" + t.field "past_home_cities", "[String!]!" + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "history1", "TeamHistory", name_in_index: "history_in_index" + t.field "history2", "TeamHistory!" # non-null + t.index "teams" + end + end + + expect_list_counts_mappings(mapping, LIST_COUNTS_FIELD, %w[ + history_in_index.past_names_in_index + history_in_index.past_home_cities + history2.past_names_in_index + history2.past_home_cities + ]) + end + + it "ignores graphql-only fields" do + mapping = index_mapping_for "teams" do |schema| + schema.object_type "TeamHistory" do |t| + t.field "past_names", "[String!]!", graphql_only: true + t.field "past_home_cities", "[String!]!" + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "history1", "TeamHistory!" + t.field "history2", "TeamHistory!", graphql_only: true + t.index "teams" + end + end + + expect_list_counts_mappings(mapping, LIST_COUNTS_FIELD, %w[history1.past_home_cities]) + end + + context "when you have a list of embedded objects" do + it "defines an integer subfield for each field of the embedded object type, recursively, since the datastore will index a flat list of values" do + mapping = index_mapping_for "teams" do |schema| + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "seasons", "[TeamSeason!]!" do |f| + f.mapping type: "object" + end + t.index "teams" + end + + schema.object_type "TeamSeason" do |t| + t.field "year", "Int" + t.field "notes", "[String!]!" + t.field "players", "[Player!]!" do |f| + f.mapping type: "object" + end + end + + schema.object_type "Player" do |t| + t.field "name", "String" + t.field "nicknames", "[String!]!" + end + end + + expect_list_counts_mappings(mapping, LIST_COUNTS_FIELD, %w[ + seasons + seasons.notes + seasons.players + seasons.players.name + seasons.players.nicknames + seasons.year + ]) + end + end + + context "when you have a list of nested objects" do + it "defines a `#{LIST_COUNTS_FIELD}` subfield for the nested field, plus a separate `#{LIST_COUNTS_FIELD}` field under the nested field for its list fields" do + mapping = index_mapping_for "teams" do |schema| + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "seasons", "[TeamSeason!]!" do |f| + f.mapping type: "nested" + end + t.index "teams" + end + + schema.object_type "TeamSeason" do |t| + t.field "year", "Int" + t.field "notes", "[String!]!" + t.field "players", "[Player!]!" do |f| + f.mapping type: "nested" + end + end + + schema.object_type "Player" do |t| + t.field "name", "String" + t.field "nicknames", "[String!]!" + end + end + + expect_list_counts_mappings(mapping, LIST_COUNTS_FIELD, %w[seasons]) + expect_list_counts_mappings(mapping, "seasons.#{LIST_COUNTS_FIELD}", %w[notes players]) + expect_list_counts_mappings(mapping, "seasons.players.#{LIST_COUNTS_FIELD}", %w[nicknames]) + end + end + + it "lets you use a `nested` list under an `object` list and vice-versa" do + mapping = index_mapping_for "teams" do |schema| + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "current_name", "String" + t.field "past_names", "[String!]!" + t.field "current_players", "[Player!]!" do |f| + f.mapping type: "nested" + end + t.field "seasons", "[TeamSeason!]!" do |f| + f.mapping type: "object" + end + t.index "teams" + end + + schema.object_type "Player" do |t| + t.field "name", "String" + t.field "nicknames", "[String!]!" + t.field "seasons", "[PlayerSeason!]!" do |f| + f.mapping type: "object" + end + end + + schema.object_type "TeamSeason" do |t| + t.field "year", "Int" + t.field "notes", "[String!]!" + t.field "players", "[Player!]!" do |f| + f.mapping type: "nested" + end + end + + schema.object_type "PlayerSeason" do |t| + t.field "year", "Int" + t.field "games_played", "Int" + t.paginated_collection_field "awards", "String" + end + end + + expect_list_counts_mappings(mapping, LIST_COUNTS_FIELD, [ + "past_names", + "current_players", # `current_players` is a nested field so we stop there + "seasons", # `seasons` is an object field so we include it and its sub-lists + "seasons.notes", + "seasons.players", + "seasons.year" + ]) + + expect_list_counts_mappings(mapping, "current_players.#{LIST_COUNTS_FIELD}", %w[ + nicknames + seasons + seasons.awards + seasons.games_played + seasons.year + ]) + + expect_list_counts_mappings(mapping, "seasons.players.#{LIST_COUNTS_FIELD}", %w[ + nicknames + seasons + seasons.awards + seasons.games_played + seasons.year + ]) + end + + def expect_list_counts_mappings(mapping, path_to_counts_field, list_subpaths) + mapping_path = "properties.#{path_to_counts_field.gsub(".", ".properties.")}.properties".split(".") + actual_counts_mapping = mapping.dig(*mapping_path) + expected_counts_mapping = list_subpaths.to_h do |list_subpath| + [list_subpath.gsub(".", LIST_COUNTS_FIELD_PATH_KEY_SEPARATOR), {"type" => "integer"}] + end + + expect(actual_counts_mapping).to eq(expected_counts_mapping) + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/index_mappings_spec_support.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/index_mappings_spec_support.rb new file mode 100644 index 00000000..e2e91793 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/index_mappings_spec_support.rb @@ -0,0 +1,27 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../index_definition_spec_support" + +module ElasticGraph + module SchemaDefinition + ::RSpec.shared_context "IndexMappingsSpecSupport" do + include_context "IndexDefinitionSpecSupport" + + def index_mapping_for(index_name, **config_overrides, &schema_definition) + index_mappings_for(index_name, **config_overrides, &schema_definition).first + end + + def index_mappings_for(...) + index_configs_for(...).map do |config| + config.fetch("mappings") + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/indexing_only_fields_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/indexing_only_fields_spec.rb new file mode 100644 index 00000000..b3959b8c --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/indexing_only_fields_spec.rb @@ -0,0 +1,70 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "index_mappings_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "Datastore config index mappings -- indexing-only fields" do + include_context "IndexMappingsSpecSupport" + + it "allows indexing-only fields to specify their customized mapping" do + mapping = index_mapping_for "my_type" do |s| + s.object_type "MyType" do |t| + t.field "id", "ID!" + + t.field "date", "String", indexing_only: true do |f| + f.mapping type: "date" + end + + t.index "my_type" + end + end + + expect(mapping.dig("properties", "date")).to eq({"type" => "date"}) + end + + it "allows indexing-only fields to be objects with nested fields" do + mapping = index_mapping_for "my_type" do |s| + s.object_type "NestedType" do |t| + t.field "name", "String!" + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "nested", "NestedType!", indexing_only: true + + t.index "my_type" + end + end + + expect(mapping.dig("properties", "nested")).to eq({ + "properties" => { + "name" => {"type" => "keyword"} + } + }) + end + + it "raises an error when same mapping field is defined twice with different mapping types" do + expect { + index_mapping_for "cards" do |s| + s.object_type "Card" do |t| + t.field "id", "ID!" + t.index "cards" + t.field "meta", "Int" + + t.field "meta", "String", indexing_only: true do |f| + f.mapping type: "text" + end + end + end + }.to raise_error Errors::SchemaError, a_string_including("Duplicate indexing field", "Card", "meta", "graphql_only: true") + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/mapping_customizations_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/mapping_customizations_spec.rb new file mode 100644 index 00000000..f1f2ddef --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/mapping_customizations_spec.rb @@ -0,0 +1,211 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "index_mappings_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "Datastore config index mappings -- mapping customizations" do + include_context "IndexMappingsSpecSupport" + + it "respects `mapping` customizations set on a field definition, allowing them to augment or replace the mapping of the base type" do + mapping = index_mapping_for "my_type" do |s| + s.scalar_type "MyText" do |t| + t.json_schema type: "string" + t.mapping type: "text" + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + + t.field "built_in_scalar_augmented", "String!" do |f| + f.mapping enabled: true + end + + t.field "built_in_scalar_replaced", "String!" do |f| + f.mapping type: "text" + end + + t.field "built_in_scalar_augmented_and_replaced", "String!" do |f| + f.mapping type: "text", enabled: true + end + + t.field "custom_scalar", "MyText!" + + t.field "custom_scalar_augmented", "MyText!" do |f| + f.mapping analyzer: "hyper_text_analyzer" + end + + t.field "custom_scalar_replaced", "MyText!" do |f| + f.mapping type: "keyword" + end + + t.field "custom_scalar_augmented_and_replaced", "MyText!" do |f| + f.mapping type: "keyword", enabled: true + end + + t.index "my_type" + end + end + + expect(mapping.fetch("properties")).to include( + "id" => {"type" => "keyword"}, + "built_in_scalar_augmented" => {"type" => "keyword", "enabled" => true}, + "built_in_scalar_replaced" => {"type" => "text"}, + "built_in_scalar_augmented_and_replaced" => {"type" => "text", "enabled" => true}, + "custom_scalar" => {"type" => "text"}, + "custom_scalar_augmented" => {"type" => "text", "analyzer" => "hyper_text_analyzer"}, + "custom_scalar_replaced" => {"type" => "keyword"}, + "custom_scalar_augmented_and_replaced" => {"type" => "keyword", "enabled" => true} + ) + end + + it "allows custom mapping options to be built up over multiple `mapping` calls" do + mapping = index_mapping_for "my_type" do |s| + s.object_type "MyType" do |t| + t.field "id", "ID!" + + t.field "name", "String!" do |f| + f.mapping type: "keyword" + f.mapping enabled: true + f.mapping type: "text" # demonstrate that the last value for an option wins + end + + t.index "my_type" + end + end + + expect(mapping.dig("properties", "name")).to eq({"type" => "text", "enabled" => true}) + end + + it "prevents `mapping` on a field definition from overriding the mapping params in an unsupported way" do + index_mapping_for "my_type" do |s| + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.index "my_type" + + t.field "description", "String!" do |f| + expect { + f.mapping unsupported_customizable: "abc123" + }.to raise_error Errors::SchemaError, a_string_including("unsupported_customizable") + + expect(f.mapping_options).to be_empty + end + end + end + end + + it "allows the mapping type to be customized from a defined object type, omitting `properties` in that case" do + define_point = lambda do |s| + s.object_type "Point" do |t| + t.field "x", "Float" + t.field "y", "Float" + t.mapping type: "point" + end + end + + define_my_type = lambda do |s| + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "location", "Point" + t.index "my_type" + end + end + + # We should get the same mapping regardless of which type is defined first. + type_before_reference_mapping = index_mapping_for "my_type" do |s| + define_point.call(s) + define_my_type.call(s) + end + + type_after_reference_mapping = index_mapping_for "my_type" do |s| + define_my_type.call(s) + define_point.call(s) + end + + expect(type_before_reference_mapping).to eq(type_after_reference_mapping) + expect(type_before_reference_mapping.dig("properties", "location")).to eq({"type" => "point"}) + end + + it "does not consider `mapping: type` to be a custom mapping type since that is the default for an object" do + define_point = lambda do |s| + s.object_type "Point" do |t| + t.field "x", "Float" + t.field "y", "Float" + t.mapping type: "object" + end + end + + define_my_type = lambda do |s| + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "location", "Point" + t.index "my_type" + end + end + + # We should get the same mapping regardless of which type is defined first. + type_before_reference_mapping = index_mapping_for "my_type" do |s| + define_point.call(s) + define_my_type.call(s) + end + + type_after_reference_mapping = index_mapping_for "my_type" do |s| + define_my_type.call(s) + define_point.call(s) + end + + expect(type_before_reference_mapping).to eq(type_after_reference_mapping) + expect(type_before_reference_mapping.dig("properties", "location")).to eq({ + "type" => "object", + "properties" => {"x" => {"type" => "double"}, "y" => {"type" => "double"}} + }) + end + + it "merges in any custom mapping options into the underlying generated mapping" do + define_point = lambda do |s| + s.object_type "Point" do |t| + t.field "x", "Float" + t.field "y", "Float" + t.mapping meta: {defined_by: "ElasticGraph"} + end + end + + define_my_type = lambda do |s| + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "location", "Point" + t.index "my_type" + end + end + + # We should get the same mapping regardless of which type is defined first. + type_before_reference_mapping = index_mapping_for "my_type" do |s| + define_point.call(s) + define_my_type.call(s) + end + + type_after_reference_mapping = index_mapping_for "my_type" do |s| + define_my_type.call(s) + define_point.call(s) + end + + expect(type_before_reference_mapping).to eq(type_after_reference_mapping) + expect(type_before_reference_mapping.dig("properties", "location")).to eq({ + "properties" => { + "x" => {"type" => "double"}, + "y" => {"type" => "double"} + }, + "meta" => { + "defined_by" => "ElasticGraph" + } + }) + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/miscellaneous_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/miscellaneous_spec.rb new file mode 100644 index 00000000..b22235a8 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/miscellaneous_spec.rb @@ -0,0 +1,398 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "index_mappings_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "Datastore config index mappings -- miscellaneous" do + include_context "IndexMappingsSpecSupport" + + it "puts `dynamic` before `properties` in the returned hash, because it makes the dumped YAML more readable" do + mapping = index_mapping_for "my_type" do |s| + s.object_type "MyType" do |t| + t.field "id", "ID" + t.index "my_type" + end + end + + # We care about putting `dynamic` before `properties` because this: + # + # mappings: + # dynamic: strict + # properties: + # id: + # type: keyword + # name: + # type: keyword + # created_at: + # type: date + # format: strict_date_time + # part_ids: + # type: keyword + # + # ...is more readable than this: + # + # mappings: + # properties: + # id: + # type: keyword + # name: + # type: keyword + # created_at: + # type: date + # format: strict_date_time + # part_ids: + # type: keyword + # dynamic: strict + # + # In the latter case, `dynamic: strict` is pushed way down in the YAML file where it's less + # clear what index it applies to. Since `dynamic: string` is always a one-liner in the YAML + # while `properties` can have many lines, it's helpful to put `dynamic` first. + expect(mapping.keys).to eq %w[dynamic properties] + end + + it "sets `dynamic: strict` on the index to disallow new fields from being created dynamically" do + mapping = index_mapping_for "my_type" do |s| + s.object_type "MyType" do |t| + t.field "id", "ID" + t.index "my_type" + end + end + + expect(mapping).to include("dynamic" => "strict") + end + + it "set runtime script which will include the runtime fields" do + mapping = index_mapping_for "my_type" do |s| + s.object_type "MyType" do |t| + t.field "id", "ID" do |f| + f.runtime_script "example test script" + end + t.index "my_type" + end + end + + expect(mapping).to include("runtime" => {"id" => {"type" => "keyword", "script" => {"source" => "example test script"}}}) + end + + it "requires an `id` field on every indexed type" do + expect { + index_mapping_for "my_type" do |s| + s.object_type "MyType" do |t| + t.index "my_type" + end + end + }.to raise_error a_string_including("Field `MyType.id` cannot be resolved, but indexed types must have an `id` field.") + end + + it "allows the required `id` field to be an indexing-only field, since we require it for indexing but it need not be exposed to GraphQL clients" do + expect { + index_mapping_for "my_type" do |s| + s.object_type "MyType" do |t| + t.field "id", "ID!", indexing_only: true + t.index "my_type" + end + end + }.not_to raise_error + end + + it "includes a `__versions` property (so update scripts can maintain the versions) and a `__sources` property (so that we can filter on present sources)" do + mapping = index_mapping_for "my_type" do |s| + s.object_type "MyType" do |t| + t.field "id", "ID" + t.index "my_type" + end + end + + expect(mapping.dig("properties")).to include({ + "id" => {"type" => "keyword"}, + "__versions" => {"dynamic" => "false", "type" => "object"}, + "__sources" => {"type" => "keyword"} + }) + end + + it "keeps `source_from` fields in the mapping so that indexed documents support the field even though it comes from an alternate source" do + mapping = index_mapping_for "components" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String!" + + t.index "widgets" + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.relates_to_one "widget", "Widget", via: "component_ids", dir: :in + + t.field "widget_name", "String!" do |f| + f.sourced_from "widget", "name" + end + + t.index "components" + end + end + + expect(mapping.fetch("properties")).to include( + "widget_name" => {"type" => "keyword"} + ) + end + + it "can generate a simple mapping for a type with only primitives (of each built-in GraphQL scalar type)" do + mapping = index_mapping_for "my_type" do |s| + s.object_type "MyType" do |t| + t.field "id", "ID" + t.field "color", "[Int!]" + t.field "name", "String!" + t.field "is_happy", "Boolean" + t.field "dimension", "[Float!]" + t.index "my_type" + end + end + + expect(mapping.dig("properties")).to include({ + "id" => {"type" => "keyword"}, + "color" => {"type" => "integer"}, + "name" => {"type" => "keyword"}, + "is_happy" => {"type" => "boolean"}, + "dimension" => {"type" => "double"} + }) + end + + it "respects the configured `name_in_index`" do + mapping = index_mapping_for "my_type" do |s| + s.object_type "Options" do |t| + t.field "size", "String!" + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "name", "String!", name_in_index: "name2" + t.field "options", "Options!", name_in_index: "options2" + t.index "my_type" + end + end + + expect(mapping.dig("properties")).to include({ + "id" => {"type" => "keyword"}, + "name2" => {"type" => "keyword"}, + "options2" => { + "properties" => { + "size" => {"type" => "keyword"} + } + } + }) + end + + it "does not allow a `name_in_index` to be a path to a child field unless it is `graphql_only: true` since we have not yet made the indexing side work for that" do + generate_mapping = lambda do |**field_options| + index_mapping_for "my_type" do |s| + s.object_type "Options" do |t| + t.field "size", "String!" + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "size", "String!", name_in_index: "options.size", **field_options + t.field "options", "Options!" + t.index "my_type" + end + end + end + + expect(&generate_mapping).to raise_error Errors::SchemaError, a_string_including( + "MyType.size: String!", "invalid `name_in_index`", "Only `graphql_only: true`" + ) + + mapping = generate_mapping.call(graphql_only: true) + + # Verify that it does not have a property for `size` or `options.size` + expect(mapping.fetch("properties").keys).to contain_exactly("id", "options", "__sources", "__versions") + expect(mapping.fetch("properties")).to include({ + "id" => {"type" => "keyword"}, + "options" => { + "properties" => { + "size" => {"type" => "keyword"} + } + } + }) + end + + it "supports the datastore `geo_point` type via a GraphQL `GeoLocation` type" do + mapping = index_mapping_for "my_type" do |s| + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "location", "GeoLocation" + t.index "my_type" + end + end + + expect(mapping.dig("properties", "location")).to eq({"type" => "geo_point"}) + end + + it "includes `{ _routing: { required: true } }` in the mapping if index is using custom shard routing" do + mapping_with_routing = index_mapping_for "my_type" do |s| + s.object_type "MyType" do |t| + t.field "id", "ID" + t.field "name", "String!" + t.index "my_type" do |i| + i.route_with "name" + end + end + end + + mapping_without_routing = index_mapping_for "my_type" do |s| + s.object_type "MyType" do |t| + t.field "id", "ID" + t.field "name", "String!" + t.index "my_type" + end + end + + expect(mapping_with_routing).to include({"_routing" => {"required" => true}}) + expect(mapping_without_routing.keys).to exclude("_routing") + end + + it "includes `{ _size: { enabled: true } }` in the mapping if `index_document_sizes` is set to true" do + mapping_with_sizes = index_mapping_for "my_type", index_document_sizes: true do |s| + s.object_type "MyType" do |t| + t.field "id", "ID" + t.index "my_type" + end + end + + mapping_without_sizes = index_mapping_for "my_type", index_document_sizes: false do |s| + s.object_type "MyType" do |t| + t.field "id", "ID" + t.index "my_type" + end + end + + expect(mapping_with_sizes).to include({"_size" => {"enabled" => true}}) + expect(mapping_without_sizes.keys).to exclude("_size") + end + + it "returns a simple mapping for a type with enums" do + mapping = index_mapping_for "widgets" do |s| + s.enum_type "Color" do |t| + t.values "RED", "BLUE", "GREEN" + end + + s.enum_type "Size" do |t| + t.values "SMALL", "MEDIUM", "LARGE" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "amount_cents", "Int!" + t.field "name", "String!" + t.field "size", "Size" + t.field "color", "Color" + t.index "widgets" + end + end + + expect(mapping.dig("properties")).to include({ + "id" => {"type" => "keyword"}, + "amount_cents" => {"type" => "integer"}, + "name" => {"type" => "keyword"}, + "size" => {"type" => "keyword"}, + "color" => {"type" => "keyword"} + }) + end + + it "returns a mapping for a type with embedded objects" do + mapping = index_mapping_for "widgets" do |s| + s.object_type "Color" do |t| + t.field "red", "Int!" + t.field "green", "Int!" + t.field "blue", "Int!" + end + + s.enum_type "Size" do |t| + t.values "SMALL", "MEDIUM", "LARGE" + end + + s.object_type "WidgetOptions" do |t| + t.field "size", "Size" + t.field "color", "String!" + t.field "color_breakdown", "Color!" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "amount_cents", "Int!" + t.field "name", "String!" + t.field "options", "WidgetOptions" + t.index "widgets" + end + end + + expect(mapping.dig("properties", "options")).to eq({ + "properties" => { + "size" => {"type" => "keyword"}, + "color" => {"type" => "keyword"}, + "color_breakdown" => { + "properties" => { + "red" => {"type" => "integer"}, + "green" => {"type" => "integer"}, + "blue" => {"type" => "integer"} + } + } + } + }) + end + + it "replaces an empty `properties` hash with `type => object` on a nested empty type, since the datastore normalizes it that way" do + mapping = index_mapping_for "empty" do |s| + # We need to define `EmptyTypeFilterInput` by hand to avoid schema parsing errors. + s.raw_sdl "input EmptyTypeFilterInput { foo: Int }" + + s.object_type "EmptyType" do |t| + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "empty", "EmptyType" + t.index "empty" + end + end + + expect(mapping.dig("properties", "empty")).to eq({"type" => "object"}) + end + + it "does not allow an enum value to be longer than `DEFAULT_MAX_KEYWORD_LENGTH` since we use a `keyword` mapping" do + expect(index_mapping_for_enum_with_value("A" * DEFAULT_MAX_KEYWORD_LENGTH)).to eq({"type" => "keyword"}) + + too_long_value = "A" * (DEFAULT_MAX_KEYWORD_LENGTH + 1) + expect { + index_mapping_for_enum_with_value(too_long_value) + }.to raise_error Errors::SchemaError, a_string_including( + "Enum value `SomeEnum.#{too_long_value}` is too long: it is #{DEFAULT_MAX_KEYWORD_LENGTH + 1} characters", + "cannot exceed #{DEFAULT_MAX_KEYWORD_LENGTH} characters" + ) + end + + def index_mapping_for_enum_with_value(value) + mapping = index_mapping_for "my_type" do |s| + s.enum_type "SomeEnum" do |t| + t.value value + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "some_enum", "SomeEnum" + t.index "my_type" + end + end + + mapping.dig("properties", "some_enum") + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/paginated_collection_field_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/paginated_collection_field_spec.rb new file mode 100644 index 00000000..b7971af7 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/paginated_collection_field_spec.rb @@ -0,0 +1,72 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "index_mappings_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "Datastore config index mappings -- `paginated_collection_field`" do + include_context "IndexMappingsSpecSupport" + + it "generates the mapping for a `paginated_collection_field`" do + mapping = index_mapping_for "widgets" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.paginated_collection_field "names", "String" + t.index "widgets" + end + end + + expect(mapping.dig("properties", "names")).to eq({"type" => "keyword"}) + end + + it "honors the `name_in_index` passed to `paginated_collection_field`" do + mapping = index_mapping_for "widgets" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.paginated_collection_field "names", "String", name_in_index: "names2" + t.index "widgets" + end + end + + expect(mapping.dig("properties")).not_to include("names") + expect(mapping.dig("properties", "names2")).to eq({"type" => "keyword"}) + end + + it "honors the configured mapping type for a `paginated_collection_field`" do + mapping = index_mapping_for "widgets" do |s| + s.object_type "Color" do |t| + t.field "red", "Int" + t.field "green", "Int" + t.field "blue", "Int" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + + t.paginated_collection_field "colors_nested", "Color" do |f| + f.mapping type: "nested" + end + + t.paginated_collection_field "colors_object", "Color" do |f| + f.mapping type: "object" + end + + t.index "widgets" + end + end + + color_mapping = {"properties" => {"blue" => {"type" => "integer"}, "green" => {"type" => "integer"}, "red" => {"type" => "integer"}}} + expect(mapping.dig("properties").select { |k| k =~ /color/ }).to eq({ + "colors_nested" => color_mapping.merge("type" => "nested"), + "colors_object" => color_mapping.merge("type" => "object") + }) + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/relation_fields_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/relation_fields_spec.rb new file mode 100644 index 00000000..b31223cc --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/relation_fields_spec.rb @@ -0,0 +1,73 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "index_mappings_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "Datastore config index mappings -- relation fields" do + include_context "IndexMappingsSpecSupport" + + context "on a relation with an outbound foreign key" do + it "includes a foreign key field for a GraphQL relation field" do + one, many = index_mappings_for "my_type_one", "my_type_many" do |s| + s.object_type "OtherType" do |t| + t.field "id", "ID!" + t.index "other_type" + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.relates_to_one "other", "OtherType!", via: "other_id", dir: :out + t.index "my_type_one" + end + + s.object_type "MyType2" do |t| + t.field "id", "ID!" + t.relates_to_many "other", "OtherType", via: "other_id", dir: :out, singular: "other" + t.index "my_type_many" + end + end.map { |h| h.dig("properties") } + + expect([one, many]).to all include({ + "id" => {"type" => "keyword"}, + "other_id" => {"type" => "keyword"} + }) + end + end + + context "on a relation with an inbound foreign key" do + it "includes the foreign key field when the relation is self-referential, regardless of the details of the relation (one or many)" do + one, many = index_mappings_for "my_type_one", "my_type_many" do |s| + s.object_type "MyTypeOne" do |t| + t.field "id", "ID" + t.relates_to_one "parent", "MyTypeOne", via: "children_ids", dir: :in + t.index "my_type_one" + end + + s.object_type "MyTypeMany" do |t| + t.field "id", "ID" + t.relates_to_many "children", "MyTypeMany", via: "parent_id", dir: :in, singular: "child" + t.index "my_type_many" + end + end + + expect(one.dig("properties")).to include({ + "id" => {"type" => "keyword"}, + "children_ids" => {"type" => "keyword"} + }) + + expect(many.dig("properties")).to include({ + "id" => {"type" => "keyword"}, + "parent_id" => {"type" => "keyword"} + }) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_overview_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_overview_spec.rb new file mode 100644 index 00000000..172ea8c3 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_overview_spec.rb @@ -0,0 +1,167 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "index_definition_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "Datastore config -- index overview" do + include_context "IndexDefinitionSpecSupport" + + it "orders the indices alphabetically for consistent dump output" do + config1 = all_index_configs_for do |s| + s.object_type "AType" do |t| + t.field "id", "ID!" + t.index "a_type" + end + + s.object_type "BType" do |t| + t.field "id", "ID!" + t.index "b_type" + end + end + + config2 = all_index_configs_for do |s| + s.object_type "BType" do |t| + t.field "id", "ID!" + t.index "b_type" + end + + s.object_type "AType" do |t| + t.field "id", "ID!" + t.index "a_type" + end + end + + expect(config1.keys).to eq ["a_type", "b_type"] + expect(config2.keys).to eq ["a_type", "b_type"] + end + + it "dumps an index with `rollover` as a template with and `index_pattern`" do + widgets = index_template_configs_for "widgets" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime!" + t.index "widgets" do |i| + i.rollover :monthly, "created_at" + end + end + end.first + + components = index_configs_for "components" do |s| + s.object_type "Component" do |t| + t.field "id", "ID!" + t.index "components" + end + end.first + + expect(widgets).to match( + "template" => { + "settings" => an_instance_of(Hash), + "mappings" => an_instance_of(Hash), + "aliases" => {} + }, + "index_patterns" => ["widgets_rollover__*"] + ) + + expect(components).to match( + "settings" => an_instance_of(Hash), + "mappings" => an_instance_of(Hash), + "aliases" => {} + ) + end + + it "dumps each index definition under either `indices` or `index_templates` based on if it has rollover config" do + datastore_config = define_schema(schema_element_name_form: :snake_case) do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime!" + t.index "widgets" do |i| + i.rollover :monthly, "created_at" + end + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.index "components" + end + end.datastore_config + + expect(datastore_config.fetch("indices").keys).to contain_exactly("components") + expect(datastore_config.fetch("index_templates").keys).to contain_exactly("widgets") + end + + it "raises a clear exception if an embedded type is recursively self-referential without using a relation" do + expect { + all_index_configs_for do |s| + s.object_type "Type1" do |t| + t.field "t2", "Type2" + end + + s.object_type "Type2" do |t| + t.field "t3", "Type3" + end + + s.object_type "Type3" do |t| + t.field "t1", "Type1" + end + end + }.to raise_error(Errors::SchemaError, a_string_including("self-referential", "Type1", "Type2", "Type3")) + end + + it "allows an embedded type to have a relation to its parent type (which would otherwise form a cycle)" do + expect { + all_index_configs_for do |s| + s.object_type "Type1" do |t| + t.field "t2", "Type2" + end + + s.object_type "Type2" do |t| + t.field "t3", "Type3" + end + + s.object_type "Type3" do |t| + t.relates_to_one "t1", "Type1", via: "t1_id", dir: :out + end + end + }.not_to raise_error + end + + it "does not allow an index to have the infix follover marker in the name, since ElasticGraph uses that to mark and parse rollover index names" do + expect { + index_template_configs_for "widgets" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.index "widget#{ROLLOVER_INDEX_INFIX_MARKER}" + end + end + }.to raise_error Errors::SchemaError, a_string_including("invalid index definition name", ROLLOVER_INDEX_INFIX_MARKER) + end + + it "ignores interface types" do + configs = all_index_configs_for do |s| + s.interface_type "MyType" do |t| + t.field "id", "ID!" + end + end + + expect(configs.keys).to eq [] + end + + def all_index_configs_for(index_document_sizes: false, schema_element_name_form: "snake_case", &schema_definition) + datastore_config = define_schema( + index_document_sizes: index_document_sizes, + schema_element_name_form: schema_element_name_form, + &schema_definition + ).datastore_config + + datastore_config.fetch("indices") + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_settings_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_settings_spec.rb new file mode 100644 index 00000000..3d08a5ef --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_settings_spec.rb @@ -0,0 +1,82 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "index_definition_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "Datastore config -- index settings" do + include_context "IndexDefinitionSpecSupport" + + it "returns reasonable default settings that we want to use for an index" do + settings = index_settings_for "my_type" do |s| + s.object_type "MyType" do |t| + t.field "id", "ID" + t.index "my_type" + end + end + + expect(settings).to include( + "index.mapping.ignore_malformed" => false, + "index.mapping.coerce" => false, + "index.number_of_replicas" => 1, + "index.number_of_shards" => 1 + ) + end + + it "allows specific settings to be overridden via the `index` options in the schema definitionAPI" do + settings = index_settings_for "my_type" do |s| + s.object_type "MyType" do |t| + t.field "id", "ID" + t.index "my_type", number_of_replicas: 2, mapping: {coerce: true}, some: {other_setting: false} + end + end + + expect(settings).to include( + "index.mapping.ignore_malformed" => false, + "index.mapping.coerce" => true, + "index.number_of_replicas" => 2, + "index.number_of_shards" => 1, + "index.some.other_setting" => false + ) + end + + it "does not include `route_with` or `rollover` options in the dumped settings, regardless of the schema form, since they are not used by the datastore itself" do + camel_settings, snake_settings = %w[snake_case camelCase].map do |form| + index_template_settings_for "my_type", schema_element_name_form: form do |s| + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime!" + t.field "name", "String!" + t.index "my_type" do |i| + i.route_with "name" + i.rollover :monthly, "created_at" + end + end + end + end + + expect(camel_settings).to eq(Indexing::Index::DEFAULT_SETTINGS) + expect(snake_settings).to eq(Indexing::Index::DEFAULT_SETTINGS) + end + + def index_settings_for(index_name, **config_overrides, &schema_definition) + index_configs_for(index_name, **config_overrides, &schema_definition) + .first + .fetch("settings") + end + + def index_template_settings_for(index_name, **config_overrides, &schema_definition) + index_template_configs_for(index_name, **config_overrides, &schema_definition) + .first + .fetch("template") + .fetch("settings") + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/aggregated_values_type_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/aggregated_values_type_spec.rb new file mode 100644 index 00000000..995ad865 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/aggregated_values_type_spec.rb @@ -0,0 +1,516 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "graphql_schema_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "GraphQL schema generation", "an `*AggregatedValues` type" do + include_context "GraphQL schema spec support" + + with_both_casing_forms do + it "includes a field for each aggregatable numeric field" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID", aggregatable: false + t.field "cost", "Int!" do |f| + f.documentation "The cost of the widget." + end + t.field "cost_float", "Float!" + t.field "cost_json_safe_long", "JsonSafeLong" + t.field "cost_long_string", "LongString" + t.field "cost_byte", "Int!" do |f| + f.mapping type: "byte" + end + t.field "cost_short", "Int" do |f| + f.mapping type: "short" + end + t.field "size", "Int", aggregatable: false # opting out of being aggregatable + t.index "widgets" + end + end + + expect(aggregated_values_type_from(results, "Widget", include_docs: true)).to eq(<<~EOS.strip) + """ + Type used to perform aggregation computations on `Widget` fields. + """ + type WidgetAggregatedValues { + """ + Computed aggregate values for the `cost` field: + + > The cost of the widget. + """ + cost: IntAggregatedValues + """ + Computed aggregate values for the `cost_float` field. + """ + cost_float: FloatAggregatedValues + """ + Computed aggregate values for the `cost_json_safe_long` field. + """ + cost_json_safe_long: JsonSafeLongAggregatedValues + """ + Computed aggregate values for the `cost_long_string` field. + """ + cost_long_string: LongStringAggregatedValues + """ + Computed aggregate values for the `cost_byte` field. + """ + cost_byte: IntAggregatedValues + """ + Computed aggregate values for the `cost_short` field. + """ + cost_short: IntAggregatedValues + } + EOS + end + + it "includes a field for each aggregatable date field (of any sort)" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID", aggregatable: false + t.field "date", "Date" + t.field "date_time", "DateTime" + t.field "local_time", "LocalTime" + t.index "widgets" + end + end + + expect(aggregated_values_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregatedValues { + date: DateAggregatedValues + date_time: DateTimeAggregatedValues + local_time: LocalTimeAggregatedValues + } + EOS + end + + it "does not care if the numeric fields are lists or scalars or nullable or not" do + results = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!", groupable: false + t.field "some_string", "String" + t.field "some_string2", "[String!]" + t.field "some_int", "Int" + t.field "some_int2", "[Int]" + t.field "some_float", "Float!" + t.field "some_float2", "[Float!]" + t.field "some_long", "JsonSafeLong" + t.field "some_long2", "[JsonSafeLong!]!" + t.index "widgets" + end + end + + expect(aggregated_values_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregatedValues { + id: NonNumericAggregatedValues + some_string: NonNumericAggregatedValues + some_string2: NonNumericAggregatedValues + some_int: IntAggregatedValues + some_int2: IntAggregatedValues + some_float: FloatAggregatedValues + some_float2: FloatAggregatedValues + some_long: JsonSafeLongAggregatedValues + some_long2: JsonSafeLongAggregatedValues + } + EOS + end + + it "does not generate the type if no fields are aggregatable" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID", aggregatable: false + t.field "size", "Int", aggregatable: false + t.index "widgets" + end + end + + expect(aggregated_values_type_from(results, "Widget")).to eq nil + end + + it "does not generate a field that references an `AggregatedValues` type when there are no aggregatable fields on it" do + results = define_schema do |schema| + schema.object_type "WidgetOptions" do |t| + t.field "size", "Int", aggregatable: false + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "options", "WidgetOptions" + t.index "widgets" + end + end + + expect(aggregated_values_type_from(results, "WidgetOptions")).to eq nil + expect(aggregated_values_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregatedValues { + id: NonNumericAggregatedValues + } + EOS + end + + it "generates the `AggregatedValues` based on the element type of a `paginated_collection_field` rather than the connection type" do + results = define_schema do |schema| + schema.object_type "WidgetOptions" do |t| + t.field "size", "Int" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.paginated_collection_field "options", "WidgetOptions" do |f| + f.mapping type: "object" + end + t.index "widgets" + end + end + + expect(aggregated_values_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregatedValues { + id: NonNumericAggregatedValues + options: WidgetOptionsAggregatedValues + } + EOS + end + + it "hides `nested` fields on the `AggregatedValues` type since we don't yet support them" do + results = define_schema do |schema| + schema.object_type "WidgetOptions" do |t| + t.field "size", "Int" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.paginated_collection_field "options", "WidgetOptions" do |f| + f.mapping type: "nested" + end + t.index "widgets" + end + end + + expect(aggregated_values_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregatedValues { + id: NonNumericAggregatedValues + } + EOS + end + + it "avoids making `mapping type: 'text'` fields aggregatable since Elasticsearch and OpenSearch don't support it" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "name", "String" + t.field "description", "String" do |f| + f.mapping type: "text" + end + t.index "widgets" + end + end + + expect(aggregated_values_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregatedValues { + id: NonNumericAggregatedValues + name: NonNumericAggregatedValues + } + EOS + end + + it "references a nested `*AggregatedValues` type for embedded object fields" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "cost", "Int" + t.field "widget_options", "WidgetOptions" + t.index "widgets" + end + + schema.object_type "WidgetOptions" do |t| + t.field "size", "Int" + end + end + + expect(aggregated_values_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregatedValues { + id: NonNumericAggregatedValues + cost: IntAggregatedValues + widget_options: WidgetOptionsAggregatedValues + } + EOS + + expect(aggregated_values_type_from(results, "WidgetOptions")).to eq(<<~EOS.strip) + type WidgetOptionsAggregatedValues { + size: IntAggregatedValues + } + EOS + end + + it "uses `NonNumericAggregatedValues` for object fields with custom mapping types" do + results = define_schema do |schema| + schema.object_type "Point1" do |t| + t.field "x", "Float" + t.field "y", "Float" + t.mapping type: "point" + end + + schema.object_type "Point2" do |t| + t.field "x", "Float" + t.field "y", "Float" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "point1", "Point1" + t.field "point2", "Point2" + + t.index "widgets" + end + end + + expect(aggregated_values_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregatedValues { + id: NonNumericAggregatedValues + point1: NonNumericAggregatedValues + point2: Point2AggregatedValues + } + EOS + + expect(aggregated_values_type_from(results, "Point1")).to eq(nil) + + expect(aggregated_values_type_from(results, "Point2")).to eq(<<~EOS.strip) + type Point2AggregatedValues { + x: FloatAggregatedValues + y: FloatAggregatedValues + } + EOS + end + + it "makes object fields with custom mapping options aggregatable so long as the `type` hasn't been customized" do + results = define_schema do |schema| + schema.object_type "Point" do |t| + t.field "x", "Float" + t.field "y", "Float" + t.mapping meta: {defined_by: "ElasticGraph"} + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "point", "Point" + + t.index "widgets" + end + end + + expect(aggregated_values_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregatedValues { + id: NonNumericAggregatedValues + point: PointAggregatedValues + } + EOS + + expect(aggregated_values_type_from(results, "Point")).to eq(<<~EOS.strip) + type PointAggregatedValues { + x: FloatAggregatedValues + y: FloatAggregatedValues + } + EOS + end + + it "does not make relation fields aggregatable (but still makes a non-relation field of the same type aggregatable)" do + results = define_schema do |schema| + schema.object_type "Component" do |t| + t.field "id", "ID" + t.field "cost", "Int" + + t.index "components" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.relates_to_one "related_component", "Component", via: "component_id", dir: :out + t.field "embedded_component", "Component" + + t.index "widgets" + end + end + + expect(aggregated_values_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregatedValues { + id: NonNumericAggregatedValues + embedded_component: ComponentAggregatedValues + } + EOS + end + + it "allows the aggregated values fields to be customized" do + result = define_schema do |api| + api.raw_sdl "directive @external on FIELD_DEFINITION" + + api.object_type "WidgetOptions" do |t| + t.field "size", "Int" + end + + api.object_type "Widget" do |t| + t.field "cost", "Int" do |f| + f.customize_aggregated_values_field do |avf| + avf.directive "deprecated" + end + + f.customize_aggregated_values_field do |avf| + avf.directive "external" + end + end + + t.field "options", "WidgetOptions" do |f| + f.customize_aggregated_values_field do |avf| + avf.directive "deprecated" + end + end + + t.field "size", "Int" do |f| + f.customize_aggregated_values_field do |avf| + avf.directive "external" + end + end + end + end + + expect(aggregated_values_type_from(result, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregatedValues { + cost: IntAggregatedValues @deprecated @external + options: WidgetOptionsAggregatedValues @deprecated + size: IntAggregatedValues @external + } + EOS + end + + shared_examples_for "a type with subtypes" do |type_def_method| + it "defines a field for an abstract type if that abstract type has aggregatable fields" do + results = define_schema do |api| + api.object_type "Person" do |t| + link_subtype_to_supertype(t, "Inventor") + t.field "name", "String" + t.field "age", "Int" + t.field "nationality", "String" + end + + api.object_type "Company" do |t| + link_subtype_to_supertype(t, "Inventor") + t.field "name", "String" + t.field "age", "Int" + t.field "stock_ticker", "String" + end + + api.public_send type_def_method, "Inventor" do |t| + link_supertype_to_subtypes(t, "Person", "Company") + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "inventor", "Inventor" + t.index "widgets" + end + end + + expect(aggregated_values_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregatedValues { + id: NonNumericAggregatedValues + inventor: InventorAggregatedValues + } + EOS + end + + it "defines the type using the set union of the fields of the subtypes" do + result = define_schema do |api| + api.object_type "Person" do |t| + link_subtype_to_supertype(t, "Inventor") + t.field "age", "Int" + t.field "income", "Float" + end + + api.object_type "Company" do |t| + link_subtype_to_supertype(t, "Inventor") + t.field "age", "Int" + t.field "share_value", "Float" + end + + api.public_send type_def_method, "Inventor" do |t| + link_supertype_to_subtypes(t, "Person", "Company") + end + end + + expect(aggregated_values_type_from(result, "Inventor")).to eq(<<~EOS.strip) + type InventorAggregatedValues { + age: IntAggregatedValues + income: FloatAggregatedValues + share_value: FloatAggregatedValues + } + EOS + end + end + + context "on a type union" do + include_examples "a type with subtypes", :union_type do + def link_subtype_to_supertype(object_type, supertype_name) + # nothing to do; the linkage happens via a `subtypes` call on the supertype + end + + def link_supertype_to_subtypes(union_type, *subtype_names) + union_type.subtypes(*subtype_names) + end + end + end + + context "on an interface type" do + include_examples "a type with subtypes", :interface_type do + def link_subtype_to_supertype(object_type, interface_name) + object_type.implements interface_name + end + + def link_supertype_to_subtypes(interface_type, *subtype_names) + # nothing to do; the linkage happens via an `implements` call on the subtype + end + end + + it "recursively resolves the union of fields, to support type hierarchies" do + result = define_schema do |api| + api.object_type "Person" do |t| + t.implements "Human" + t.field "age", "Int" + t.field "income", "Float" + end + + api.object_type "Company" do |t| + t.implements "Organization" + t.field "age", "Int" + t.field "share_value", "Float" + end + + api.interface_type "Human" do |t| + t.implements "Inventor" + end + + api.interface_type "Organization" do |t| + t.implements "Inventor" + end + + api.interface_type "Inventor" do |t| + end + end + + expect(aggregated_values_type_from(result, "Inventor")).to eq(<<~EOS.strip) + type InventorAggregatedValues { + age: IntAggregatedValues + income: FloatAggregatedValues + share_value: FloatAggregatedValues + } + EOS + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/aggregation_grouped_by_type_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/aggregation_grouped_by_type_spec.rb new file mode 100644 index 00000000..485f1294 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/aggregation_grouped_by_type_spec.rb @@ -0,0 +1,867 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "graphql_schema_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "GraphQL schema generation", "an `*GroupedBy` type" do + include_context "GraphQL schema spec support" + + with_both_casing_forms do + it "includes a field for each groupable field" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID", groupable: true + t.field "name", "String" do |f| + f.documentation "The widget's name." + end + t.field "size", "String" + t.field "cost", "Int", groupable: false + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget", include_docs: true)).to eq(<<~EOS.strip) + """ + Type used to specify the `Widget` fields to group by for aggregations. + """ + type WidgetGroupedBy { + """ + The `id` field value for this group. + """ + id: ID + """ + The `name` field value for this group: + + > The widget's name. + """ + name: String + """ + The `size` field value for this group. + """ + size: String + } + EOS + end + + it "omits `id` on an indexed type by default since grouping on it would yield buckets of 1 document each" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "name", "String" + t.field "size", "String" + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetGroupedBy { + name: String + size: String + } + EOS + end + + it "includes `id` on a non-indexed type since it is not necessarily a unique primary key" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "name", "String" + t.field "size", "String" + end + end + + expect(grouped_by_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetGroupedBy { + id: ID + name: String + size: String + } + EOS + end + + it "does not define grouped by field for a list or paginated collection field by default since it has odd semantics" do + results = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!", groupable: false + t.field "some_string", "String" + t.field "some_string2", "[String!]" + t.field "some_int", "Int" + t.field "some_int2", "[Int]" + t.field "some_float", "Float!" + t.field "some_float2", "[Float!]" + t.field "some_long", "JsonSafeLong" + t.field "some_long2", "[JsonSafeLong!]!" + t.paginated_collection_field "tags", "String" + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetGroupedBy { + some_string: String + some_int: Int + some_float: Float + some_long: JsonSafeLong + } + EOS + end + + it "allows a scalar list field to be grouped by specifying its `singular` name" do + results = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!", groupable: false + t.field "tags", "[String!]", singular: "tag" + t.field "categories", "[String!]", singular: "category" do |f| + f.documentation "The category of the widget." + end + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget", include_docs: true)).to eq(<<~EOS.strip) + """ + Type used to specify the `Widget` fields to group by for aggregations. + """ + type WidgetGroupedBy { + """ + The individual value from `tags` for this group. + + Note: `tags` is a collection field, but selecting this field will group on individual values of `tags`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `tags` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `tags` multiple times for a single document, that document will only be included in the group + once. + """ + tag: String + """ + The individual value from `categories` for this group: + + > The category of the widget. + + Note: `categories` is a collection field, but selecting this field will group on individual values of `categories`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `categories` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `categories` multiple times for a single document, that document will only be included in the group + once. + """ + category: String + } + EOS + end + + it "allows a paginated collection field to be grouped by specifying its `singular` name" do + results = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!", groupable: false + t.paginated_collection_field "tags", "String", singular: "tag" + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget", include_docs: true)).to eq(<<~EOS.strip) + """ + Type used to specify the `Widget` fields to group by for aggregations. + """ + type WidgetGroupedBy { + """ + The individual value from `tags` for this group. + + Note: `tags` is a collection field, but selecting this field will group on individual values of `tags`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `tags` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `tags` multiple times for a single document, that document will only be included in the group + once. + """ + tag: String + } + EOS + end + + it "respects type name overrides when generating the `singular` grouping field for a paginated collection field" do + results = define_schema(type_name_overrides: {LocalTime: "TimeOfDay"}) do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!", groupable: false + t.paginated_collection_field "times", "LocalTime", singular: "time" + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetGroupedBy { + time: TimeOfDay + } + EOS + end + + it "allows a list-of-objects-of-scalars to be grouped on so the scalar subfields can be grouped on" do + results = define_schema do |api| + api.object_type "WidgetOptions" do |t| + t.field "size", "Int" + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "options", "[WidgetOptions!]!" do |f| + f.mapping type: "object" + end + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget", include_docs: true)).to eq(<<~EOS.strip) + """ + Type used to specify the `Widget` fields to group by for aggregations. + """ + type WidgetGroupedBy { + """ + The `options` field value for this group. + + Note: `options` is a collection field, but selecting this field will group on individual values of the selected subfields of `options`. + That means that a document may be grouped into multiple aggregation groupings (i.e. when its `options` + field has multiple values) leading to some data duplication in the response. However, if a value shows + up in `options` multiple times for a single document, that document will only be included in the group + once. + """ + options: WidgetOptionsGroupedBy + } + EOS + + expect(grouped_by_type_from(results, "WidgetOptions")).to eq(<<~EOS.strip) + type WidgetOptionsGroupedBy { + size: Int + } + EOS + end + + it "makes a list-of-nested-objects field ungroupable since nested fields require special aggregation operations in the datastore query to work properly" do + results = define_schema do |api| + api.object_type "WidgetOptions" do |t| + t.field "size", "Int" + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "options", "[WidgetOptions!]!" do |f| + f.mapping type: "nested" + end + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetGroupedBy { + name: String + } + EOS + end + + it "does not generate the type if no fields are groupable" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" # id is automatically not groupable, because it uniquely identifies each document! + t.field "cost", "Int", groupable: false + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget")).to eq nil + end + + it "references a nested `GroupedBy` type for embedded object fields" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "name", "String" + t.field "widget_options", "WidgetOptions" + t.index "widgets" + end + + schema.object_type "WidgetOptions" do |t| + t.field "size", "String" + t.field "color", "String" + end + end + + expect(grouped_by_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetGroupedBy { + name: String + widget_options: WidgetOptionsGroupedBy + } + EOS + + expect(grouped_by_type_from(results, "WidgetOptions")).to eq(<<~EOS.strip) + type WidgetOptionsGroupedBy { + size: String + color: String + } + EOS + end + + context "with `legacy_grouping_schema: true`" do + it "defines legacy grouping arguments for Date and DateTime fields with full documentation" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID", groupable: true + t.field "created_on", "Date", legacy_grouping_schema: true + t.field "created_at", "DateTime", legacy_grouping_schema: true + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget", include_docs: true)).to eq(<<~EOS.strip) + """ + Type used to specify the `Widget` fields to group by for aggregations. + """ + type WidgetGroupedBy { + """ + The `id` field value for this group. + """ + id: ID + """ + The `created_on` field value for this group. + """ + created_on( + """ + Determines the grouping granularity for this field. + """ + #{schema_elements.granularity}: DateGroupingGranularityInput! + """ + Number of days (positive or negative) to shift the `Date` boundaries of each date grouping bucket. + + For example, when grouping by `YEAR`, this can be used to align the buckets with fiscal or school years instead of calendar years. + """ + #{schema_elements.offset_days}: Int): Date + """ + The `created_at` field value for this group. + """ + created_at( + """ + Determines the grouping granularity for this field. + """ + #{schema_elements.granularity}: DateTimeGroupingGranularityInput! + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + #{schema_elements.time_zone}: TimeZone = "UTC" + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 24 hours to change what day-of-week weeks are considered to start on. + """ + offset: DateTimeGroupingOffsetInput): DateTime + } + EOS + end + + it "makes fields of all leaf types groupable except when it has a `text` mapping since that can't be grouped on efficiently" do + results = define_schema do |schema| + schema.enum_type "Color" do |t| + t.values "RED", "GREEN", "YELLOW" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "name", "String!" + t.field "workspace_id", "ID" + t.field "size", "String!" + t.field "cost", "Int!" + t.field "cost_byte", "Int" do |f| + f.mapping type: "byte" + end + t.field "cost_short", "Int" do |f| + f.mapping type: "short" + end + t.field "cost_float", "Float!" + t.field "cost_json_safe_long", "JsonSafeLong" + t.field "cost_long_string", "LongString" + t.field "metadata", "Untyped" + t.field "zone", "TimeZone!" + t.field "sold", "Boolean" + t.field "color", "Color" + t.field "created_on", "Date!", legacy_grouping_schema: true + t.field "created_at", "DateTime", legacy_grouping_schema: true + t.field "description", "String" do |f| + f.mapping type: "text" + end + + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetGroupedBy { + name: String + workspace_id: ID + size: String + cost: Int + cost_byte: Int + cost_short: Int + cost_float: Float + cost_json_safe_long: JsonSafeLong + cost_long_string: LongString + metadata: Untyped + zone: TimeZone + sold: Boolean + color: Color + created_on( + granularity: DateGroupingGranularityInput! + #{schema_elements.offset_days}: Int): Date + created_at( + granularity: DateTimeGroupingGranularityInput! + #{schema_elements.time_zone}: TimeZone = "UTC" + offset: DateTimeGroupingOffsetInput): DateTime + } + EOS + end + end + + context "with `legacy_grouping_schema: false`" do + it "defines grouping arguments for Date and DateTime fields with full documentation" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID", groupable: true + t.field "created_on", "Date", legacy_grouping_schema: false + t.field "created_at", "DateTime", legacy_grouping_schema: false + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget", include_docs: true)).to eq(<<~EOS.strip) + """ + Type used to specify the `Widget` fields to group by for aggregations. + """ + type WidgetGroupedBy { + """ + The `id` field value for this group. + """ + id: ID + """ + Offers the different grouping options for the `created_on` value within this group. + """ + created_on: DateGroupedBy + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + } + EOS + end + + it "makes fields of all leaf types groupable except when it has a `text` mapping since that can't be grouped on efficiently" do + results = define_schema do |schema| + schema.enum_type "Color" do |t| + t.values "RED", "GREEN", "YELLOW" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "name", "String!" + t.field "workspace_id", "ID" + t.field "size", "String!" + t.field "cost", "Int!" + t.field "cost_byte", "Int" do |f| + f.mapping type: "byte" + end + t.field "cost_short", "Int" do |f| + f.mapping type: "short" + end + t.field "cost_float", "Float!" + t.field "cost_json_safe_long", "JsonSafeLong" + t.field "cost_long_string", "LongString" + t.field "metadata", "Untyped" + t.field "zone", "TimeZone!" + t.field "sold", "Boolean" + t.field "color", "Color" + t.field "created_on", "Date!", legacy_grouping_schema: false + t.field "created_at", "DateTime", legacy_grouping_schema: false + t.field "description", "String" do |f| + f.mapping type: "text" + end + + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetGroupedBy { + name: String + workspace_id: ID + size: String + cost: Int + cost_byte: Int + cost_short: Int + cost_float: Float + cost_json_safe_long: JsonSafeLong + cost_long_string: LongString + metadata: Untyped + zone: TimeZone + sold: Boolean + color: Color + created_on: DateGroupedBy + created_at: DateTimeGroupedBy + } + EOS + end + end + + context "with `legacy_grouping_schema` not specified" do + it "defines grouping arguments for Date and DateTime fields with full documentation" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID", groupable: true + t.field "created_on", "Date" + t.field "created_at", "DateTime" + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget", include_docs: true)).to eq(<<~EOS.strip) + """ + Type used to specify the `Widget` fields to group by for aggregations. + """ + type WidgetGroupedBy { + """ + The `id` field value for this group. + """ + id: ID + """ + Offers the different grouping options for the `created_on` value within this group. + """ + created_on: DateGroupedBy + """ + Offers the different grouping options for the `created_at` value within this group. + """ + created_at: DateTimeGroupedBy + } + EOS + end + + it "makes fields of all leaf types groupable except when it has a `text` mapping since that can't be grouped on efficiently" do + results = define_schema do |schema| + schema.enum_type "Color" do |t| + t.values "RED", "GREEN", "YELLOW" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "name", "String!" + t.field "workspace_id", "ID" + t.field "size", "String!" + t.field "cost", "Int!" + t.field "cost_byte", "Int" do |f| + f.mapping type: "byte" + end + t.field "cost_short", "Int" do |f| + f.mapping type: "short" + end + t.field "cost_float", "Float!" + t.field "cost_json_safe_long", "JsonSafeLong" + t.field "cost_long_string", "LongString" + t.field "metadata", "Untyped" + t.field "zone", "TimeZone!" + t.field "sold", "Boolean" + t.field "color", "Color" + t.field "created_on", "Date!" + t.field "created_at", "DateTime" + t.field "description", "String" do |f| + f.mapping type: "text" + end + + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetGroupedBy { + name: String + workspace_id: ID + size: String + cost: Int + cost_byte: Int + cost_short: Int + cost_float: Float + cost_json_safe_long: JsonSafeLong + cost_long_string: LongString + metadata: Untyped + zone: TimeZone + sold: Boolean + color: Color + created_on: DateGroupedBy + created_at: DateTimeGroupedBy + } + EOS + end + end + + it "does not make object fields with a custom mapping type groupable" do + results = define_schema do |schema| + schema.object_type "Point1" do |t| + t.field "x", "Float" + t.field "y", "Float" + t.mapping type: "point" + end + + schema.object_type "Point2" do |t| + t.field "x", "Float" + t.field "y", "Float" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "point1", "Point1" + t.field "point2", "Point2" + + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetGroupedBy { + point2: Point2GroupedBy + } + EOS + + expect(grouped_by_type_from(results, "Point1")).to eq(nil) + + expect(grouped_by_type_from(results, "Point2")).to eq(<<~EOS.strip) + type Point2GroupedBy { + x: Float + y: Float + } + EOS + end + + it "makes object fields with custom mapping options groupable so long as the `type` hasn't been customized" do + results = define_schema do |schema| + schema.object_type "Point" do |t| + t.field "x", "Float" + t.field "y", "Float" + t.mapping meta: {defined_by: "ElasticGraph"} + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "point", "Point" + + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetGroupedBy { + point: PointGroupedBy + } + EOS + + expect(grouped_by_type_from(results, "Point")).to eq(<<~EOS.strip) + type PointGroupedBy { + x: Float + y: Float + } + EOS + end + + it "does not make relation fields groupable (but still makes a non-relation field of the same type groupable)" do + results = define_schema do |schema| + schema.object_type "Component" do |t| + t.field "id", "ID" + t.field "name", "String" + + t.index "components" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.relates_to_one "related_component", "Component", via: "component_id", dir: :out + t.field "embedded_component", "Component" + + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetGroupedBy { + embedded_component: ComponentGroupedBy + } + EOS + end + + it "allows the grouped by fields to be customized" do + result = define_schema do |api| + api.raw_sdl "directive @external on FIELD_DEFINITION" + + api.object_type "WidgetOptions" do |t| + t.field "color", "String" + end + + api.object_type "Widget" do |t| + t.field "name", "String" do |f| + f.customize_grouped_by_field do |gbf| + gbf.directive "deprecated" + end + + f.customize_grouped_by_field do |gbf| + gbf.directive "external" + end + end + + t.field "options", "WidgetOptions" do |f| + f.customize_grouped_by_field do |gbf| + gbf.directive "deprecated" + end + end + + t.field "size", "Int" do |f| + f.customize_grouped_by_field do |gbf| + gbf.directive "external" + end + end + end + end + + expect(grouped_by_type_from(result, "Widget")).to eq(<<~EOS.strip) + type WidgetGroupedBy { + name: String @deprecated @external + options: WidgetOptionsGroupedBy @deprecated + size: Int @external + } + EOS + end + + shared_examples_for "a type with subtypes" do |type_def_method| + it "defines a field for an abstract type if that abstract type has aggregatable fields" do + results = define_schema do |api| + api.object_type "Person" do |t| + link_subtype_to_supertype(t, "Inventor") + t.field "name", "String" + t.field "age", "Int" + t.field "nationality", "String" + end + + api.object_type "Company" do |t| + link_subtype_to_supertype(t, "Inventor") + t.field "name", "String" + t.field "age", "Int" + t.field "stock_ticker", "String" + end + + api.public_send type_def_method, "Inventor" do |t| + link_supertype_to_subtypes(t, "Person", "Company") + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "inventor", "Inventor" + t.index "widgets" + end + end + + expect(grouped_by_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetGroupedBy { + inventor: InventorGroupedBy + } + EOS + end + + it "defines the type using the set union of the fields of the subtypes" do + result = define_schema do |api| + api.object_type "Person" do |t| + link_subtype_to_supertype(t, "Inventor") + t.field "name", "String" + t.field "age", "Int" + t.field "nationality", "String" + end + + api.object_type "Company" do |t| + link_subtype_to_supertype(t, "Inventor") + t.field "name", "String" + t.field "age", "Int" + t.field "stock_ticker", "String" + end + + api.public_send type_def_method, "Inventor" do |t| + link_supertype_to_subtypes(t, "Person", "Company") + end + end + + expect(grouped_by_type_from(result, "Inventor")).to eq(<<~EOS.strip) + type InventorGroupedBy { + name: String + age: Int + nationality: String + stock_ticker: String + } + EOS + end + end + + context "on a type union" do + include_examples "a type with subtypes", :union_type do + def link_subtype_to_supertype(object_type, supertype_name) + # nothing to do; the linkage happens via a `subtypes` call on the supertype + end + + def link_supertype_to_subtypes(union_type, *subtype_names) + union_type.subtypes(*subtype_names) + end + end + end + + context "on an interface type" do + include_examples "a type with subtypes", :interface_type do + def link_subtype_to_supertype(object_type, interface_name) + object_type.implements interface_name + end + + def link_supertype_to_subtypes(interface_type, *subtype_names) + # nothing to do; the linkage happens via an `implements` call on the subtype + end + end + + it "recursively resolves the union of fields, to support type hierarchies" do + result = define_schema do |api| + api.object_type "Person" do |t| + t.implements "Human" + t.field "age", "Int" + t.field "income", "Float" + end + + api.object_type "Company" do |t| + t.implements "Organization" + t.field "age", "Int" + t.field "share_value", "Float" + end + + api.interface_type "Human" do |t| + t.implements "Inventor" + end + + api.interface_type "Organization" do |t| + t.implements "Inventor" + end + + api.interface_type "Inventor" do |t| + end + end + + expect(grouped_by_type_from(result, "Inventor")).to eq(<<~EOS.strip) + type InventorGroupedBy { + age: Int + income: Float + share_value: Float + } + EOS + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/built_in_types_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/built_in_types_spec.rb new file mode 100644 index 00000000..3561827a --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/built_in_types_spec.rb @@ -0,0 +1,1629 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "graphql_schema_spec_support" +require "elastic_graph/schema_definition/schema_elements/type_namer" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "GraphQL schema generation", "built-in types" do + include_context "GraphQL schema spec support" + + with_both_casing_forms do |form| + if form == :camelCase # Our docs are written assuming `camelCase` since that's the common GraphQL convention. + it "fully documents all primary built in types" do + derived_type_regexes = SchemaElements::TypeNamer.new.regexes.values + + primary_built_in_types_and_docs = ::GraphQL::Schema.from_definition(@result).types.filter_map do |name, type| + if %w[Query].include?(name) || name.start_with?("__") || derived_type_regexes.any? { |r| r.match(name) } + # skip it as it's not a primary type. + else + [name, type.description.gsub(/\s+/, " ")] + end + end.to_h + + built_in_types_code = ::File.read($LOADED_FEATURES.grep(/schema_elements\/built_in_types/).first) + class_comment = built_in_types_code[/\n\s+module SchemaElements\n(.*)\n\s+class BuiltInTypes/m, 1] + + # https://rubular.com/r/uyl4hdk96uMXDC + documented_types_and_docs = class_comment.scan(/\n\s+# (\w+)\n\s+# : (.*?)(?=\n\s+# (?:\w+\n\s+# :|##|@!))/m).to_h do |type, doc| + normalized_doc = doc.gsub(/\n\s*#/m, "").gsub(/\s+/, " ") + [type, normalized_doc] + end + + # Surface any missing types + expect(documented_types_and_docs.keys).to match_array(primary_built_in_types_and_docs.keys) + + # For types that are in both, verify the documentation matches. + (documented_types_and_docs.keys & primary_built_in_types_and_docs.keys).each do |type| + expected_doc = primary_built_in_types_and_docs.fetch(type) + actual_doc = documented_types_and_docs.fetch(type) + + expect("- `#{type}`: #{actual_doc}").to eq("- `#{type}`: #{expected_doc}") + end + end + end + + it "defines an `@eg_latency_slo` directive" do + expect(type_named("@#{schema_elements.eg_latency_slo}", include_docs: true)).to eq(<<~EOS.strip) + """ + Indicates an upper bound on how quickly a query must respond to meet the service-level objective. + ElasticGraph will log a "good event" message if the query latency is less than or equal to this value, + and a "bad event" message if the query latency is greater than this value. These messages can be used + to drive an SLO dashboard. + + Note that the latency compared against this only contains processing time within ElasticGraph itself. + Any time spent on sending the request or response over the network is not included in the comparison. + """ + directive @#{schema_elements.eg_latency_slo}(#{schema_elements.ms}: Int!) on QUERY + EOS + end + + it "defines a `Cursor` scalar" do + expect(type_named("Cursor", include_docs: true)).to eq(<<~EOS.strip) + """ + An opaque string value representing a specific location in a paginated connection type. + Returned cursors can be passed back in the next query via the `before` or `after` + arguments to continue paginating from that point. + """ + scalar Cursor + EOS + end + + it "defines a `GeoLocation` object type and related filter types" do + expect(type_named("GeoLocation", include_docs: true)).to eq(<<~EOS.strip) + """ + Geographic coordinates representing a location on the Earth's surface. + """ + type GeoLocation { + """ + Angular distance north or south of the Earth's equator, measured in degrees from -90 to +90. + """ + latitude: Float + """ + Angular distance east or west of the Prime Meridian at Greenwich, UK, measured in degrees from -180 to +180. + """ + longitude: Float + } + EOS + + expect(type_named("GeoLocationFilterInput", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type used to specify filters on `GeoLocation` fields. + + Will be ignored if passed as an empty object (or as `null`). + """ + input GeoLocationFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + #{schema_elements.any_of}: [GeoLocationFilterInput!] + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + not: GeoLocationFilterInput + """ + Matches records where the field's geographic location is within a specified distance from the + location identified by `latitude` and `longitude`. + + Will be ignored when `null` or an empty object is passed. + """ + near: GeoLocationDistanceFilterInput + } + EOS + + expect(type_named("GeoLocationListElementFilterInput", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type used to specify filters on elements of a `[GeoLocation]` field. + + Will be ignored if passed as an empty object (or as `null`). + """ + input GeoLocationListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + #{schema_elements.any_of}: [GeoLocationListElementFilterInput!] + """ + Matches records where the field's geographic location is within a specified distance from the + location identified by `latitude` and `longitude`. + + Will be ignored when `null` or an empty object is passed. + """ + near: GeoLocationDistanceFilterInput + } + EOS + + expect(type_named("GeoLocationDistanceFilterInput", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type used to specify distance filtering parameters on `GeoLocation` fields. + """ + input GeoLocationDistanceFilterInput { + """ + Angular distance north or south of the Earth's equator, measured in degrees from -90 to +90. + """ + latitude: Float! + """ + Angular distance east or west of the Prime Meridian at Greenwich, UK, measured in degrees from -180 to +180. + """ + longitude: Float! + """ + Maximum distance (of the provided `unit`) to consider "near" the location identified + by `latitude` and `longitude`. + """ + #{schema_elements.max_distance}: Float! + """ + Determines the unit of the specified `#{schema_elements.max_distance}`. + """ + unit: DistanceUnitInput! + } + EOS + end + + it "defines a `Date` scalar and filter types" do + expect(type_named("Date", include_docs: true)).to eq(<<~EOS.strip) + """ + A date, represented as an [ISO 8601 date string](https://en.wikipedia.org/wiki/ISO_8601). + """ + scalar Date + EOS + + expect(type_named("DateFilterInput")).to eq(<<~EOS.strip) + input DateFilterInput { + #{schema_elements.any_of}: [DateFilterInput!] + #{schema_elements.not}: DateFilterInput + #{schema_elements.equal_to_any_of}: [Date] + #{schema_elements.gt}: Date + #{schema_elements.gte}: Date + #{schema_elements.lt}: Date + #{schema_elements.lte}: Date + } + EOS + + expect(type_named("DateListElementFilterInput")).to eq(<<~EOS.strip) + input DateListElementFilterInput { + #{schema_elements.any_of}: [DateListElementFilterInput!] + #{schema_elements.equal_to_any_of}: [Date!] + #{schema_elements.gt}: Date + #{schema_elements.gte}: Date + #{schema_elements.lt}: Date + #{schema_elements.lte}: Date + } + EOS + end + + it "defines a `DateTime` scalar and filter types and a `DateTimeTimeOfDayFilterInput` to filter on a DateTime's time of day" do + expect(type_named("DateTime", include_docs: true)).to eq(<<~EOS.strip) + """ + A timestamp, represented as an [ISO 8601 time string](https://en.wikipedia.org/wiki/ISO_8601). + """ + scalar DateTime + EOS + + expect(type_named("DateTimeFilterInput")).to eq(<<~EOS.strip) + input DateTimeFilterInput { + #{schema_elements.any_of}: [DateTimeFilterInput!] + #{schema_elements.not}: DateTimeFilterInput + #{schema_elements.equal_to_any_of}: [DateTime] + #{schema_elements.gt}: DateTime + #{schema_elements.gte}: DateTime + #{schema_elements.lt}: DateTime + #{schema_elements.lte}: DateTime + #{schema_elements.time_of_day}: DateTimeTimeOfDayFilterInput + } + EOS + + expect(type_named("DateTimeListElementFilterInput")).to eq(<<~EOS.strip) + input DateTimeListElementFilterInput { + #{schema_elements.any_of}: [DateTimeListElementFilterInput!] + #{schema_elements.equal_to_any_of}: [DateTime!] + #{schema_elements.gt}: DateTime + #{schema_elements.gte}: DateTime + #{schema_elements.lt}: DateTime + #{schema_elements.lte}: DateTime + #{schema_elements.time_of_day}: DateTimeTimeOfDayFilterInput + } + EOS + + expect(type_named("DateTimeTimeOfDayFilterInput", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type used to specify filters on the time-of-day of `DateTime` fields. + + Will be ignored if passed as an empty object (or as `null`). + """ + input DateTimeTimeOfDayFilterInput { + """ + Matches records where the time of day of the `DateTime` field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + #{schema_elements.equal_to_any_of}: [LocalTime!] + """ + Matches records where the time of day of the `DateTime` field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + #{schema_elements.gt}: LocalTime + """ + Matches records where the time of day of the `DateTime` field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + #{schema_elements.gte}: LocalTime + """ + Matches records where the time of day of the `DateTime` field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + #{schema_elements.lt}: LocalTime + """ + Matches records where the time of day of the `DateTime` field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + lte: LocalTime + """ + TimeZone to use when comparing the `DateTime` values against the provided `LocalTime` values. + """ + #{schema_elements.time_zone}: TimeZone! = "UTC" + } + EOS + end + + it "respects a type name override for the `DateTime` type" do + result = define_schema(type_name_overrides: {DateTime: "Timestamp"}) + + expect(type_def_from(result, "Timestamp")).to eq("scalar Timestamp") + + expect(type_def_from(result, "TimestampFilterInput")).to eq(<<~EOS.strip) + input TimestampFilterInput { + #{schema_elements.any_of}: [TimestampFilterInput!] + #{schema_elements.not}: TimestampFilterInput + #{schema_elements.equal_to_any_of}: [Timestamp] + #{schema_elements.gt}: Timestamp + #{schema_elements.gte}: Timestamp + #{schema_elements.lt}: Timestamp + #{schema_elements.lte}: Timestamp + #{schema_elements.time_of_day}: TimestampTimeOfDayFilterInput + } + EOS + + expect(type_def_from(result, "TimestampListElementFilterInput")).to eq(<<~EOS.strip) + input TimestampListElementFilterInput { + #{schema_elements.any_of}: [TimestampListElementFilterInput!] + #{schema_elements.equal_to_any_of}: [Timestamp!] + #{schema_elements.gt}: Timestamp + #{schema_elements.gte}: Timestamp + #{schema_elements.lt}: Timestamp + #{schema_elements.lte}: Timestamp + #{schema_elements.time_of_day}: TimestampTimeOfDayFilterInput + } + EOS + + expect(type_def_from(result, "TimestampTimeOfDayFilterInput")).to eq(<<~EOS.strip) + input TimestampTimeOfDayFilterInput { + #{schema_elements.equal_to_any_of}: [LocalTime!] + #{schema_elements.gt}: LocalTime + #{schema_elements.gte}: LocalTime + #{schema_elements.lt}: LocalTime + #{schema_elements.lte}: LocalTime + #{schema_elements.time_zone}: TimeZone! = "UTC" + } + EOS + end + + it "respects type name overrides for all of the `DateTime` filter types" do + result = define_schema(type_name_overrides: { + DateTimeFilterInput: "TimestampFilterInput", + DateTimeListElementFilterInput: "TimestampListElementFilterInput" + }) + + expect(type_def_from(result, "TimestampFilterInput")).to eq(<<~EOS.strip) + input TimestampFilterInput { + #{schema_elements.any_of}: [TimestampFilterInput!] + #{schema_elements.not}: TimestampFilterInput + #{schema_elements.equal_to_any_of}: [DateTime] + #{schema_elements.gt}: DateTime + #{schema_elements.gte}: DateTime + #{schema_elements.lt}: DateTime + #{schema_elements.lte}: DateTime + #{schema_elements.time_of_day}: DateTimeTimeOfDayFilterInput + } + EOS + + expect(type_def_from(result, "TimestampListElementFilterInput")).to eq(<<~EOS.strip) + input TimestampListElementFilterInput { + #{schema_elements.any_of}: [TimestampListElementFilterInput!] + #{schema_elements.equal_to_any_of}: [DateTime!] + #{schema_elements.gt}: DateTime + #{schema_elements.gte}: DateTime + #{schema_elements.lt}: DateTime + #{schema_elements.lte}: DateTime + #{schema_elements.time_of_day}: DateTimeTimeOfDayFilterInput + } + EOS + end + + it "defines a `LocalTime` scalar and filter types" do + expect(type_named("LocalTime", include_docs: true)).to eq(<<~EOS.strip) + """ + A local time such as `"23:59:33"` or `"07:20:47.454"` without a time zone or offset, formatted based on the + [partial-time portion of RFC3339](https://datatracker.ietf.org/doc/html/rfc3339#section-5.6). + """ + scalar LocalTime + EOS + + expect(type_named("LocalTimeFilterInput")).to eq(<<~EOS.strip) + input LocalTimeFilterInput { + #{schema_elements.any_of}: [LocalTimeFilterInput!] + #{schema_elements.not}: LocalTimeFilterInput + #{schema_elements.equal_to_any_of}: [LocalTime] + #{schema_elements.gt}: LocalTime + #{schema_elements.gte}: LocalTime + #{schema_elements.lt}: LocalTime + #{schema_elements.lte}: LocalTime + } + EOS + + expect(type_named("LocalTimeListElementFilterInput")).to eq(<<~EOS.strip) + input LocalTimeListElementFilterInput { + #{schema_elements.any_of}: [LocalTimeListElementFilterInput!] + #{schema_elements.equal_to_any_of}: [LocalTime!] + #{schema_elements.gt}: LocalTime + #{schema_elements.gte}: LocalTime + #{schema_elements.lt}: LocalTime + #{schema_elements.lte}: LocalTime + } + EOS + end + + it "defines a `TimeZone` scalar and filter types" do + expect(type_named("TimeZone", include_docs: true)).to eq(<<~EOS.strip) + """ + An [IANA time zone identifier](https://www.iana.org/time-zones), such as `America/Los_Angeles` or `UTC`. + + For a full list of valid identifiers, see the [wikipedia article](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List). + """ + scalar TimeZone + EOS + + expect(type_named("TimeZoneFilterInput")).to eq(<<~EOS.strip) + input TimeZoneFilterInput { + #{schema_elements.any_of}: [TimeZoneFilterInput!] + #{schema_elements.not}: TimeZoneFilterInput + #{schema_elements.equal_to_any_of}: [TimeZone] + } + EOS + + expect(type_named("TimeZoneListElementFilterInput")).to eq(<<~EOS.strip) + input TimeZoneListElementFilterInput { + #{schema_elements.any_of}: [TimeZoneListElementFilterInput!] + #{schema_elements.equal_to_any_of}: [TimeZone!] + } + EOS + end + + it "defines a `DateTimeUnit` enum and filter types" do + expect(type_named("DateTimeUnit", include_docs: true)).to eq(<<~EOS.strip) + """ + Enumeration of `DateTime` units. + """ + enum DateTimeUnit { + """ + The time period of a full rotation of the Earth with respect to the Sun. + """ + DAY + """ + 1/24th of a day. + """ + HOUR + """ + 1/60th of an hour. + """ + MINUTE + """ + 1/60th of a minute. + """ + SECOND + """ + 1/1000th of a second. + """ + MILLISECOND + } + EOS + + expect(type_named("DateTimeUnitFilterInput")).to eq(<<~EOS.strip) + input DateTimeUnitFilterInput { + #{schema_elements.any_of}: [DateTimeUnitFilterInput!] + #{schema_elements.not}: DateTimeUnitFilterInput + #{schema_elements.equal_to_any_of}: [DateTimeUnitInput] + } + EOS + + expect(type_named("DateTimeUnitListElementFilterInput")).to eq(<<~EOS.strip) + input DateTimeUnitListElementFilterInput { + #{schema_elements.any_of}: [DateTimeUnitListElementFilterInput!] + #{schema_elements.equal_to_any_of}: [DateTimeUnitInput!] + } + EOS + end + + it "defines a `DateTimeGroupingOffsetInput` input type" do + expect(type_named("DateTimeGroupingOffsetInput", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type offered when grouping on `DateTime` fields, representing the amount of offset + (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 1 day to change + what day-of-week weeks are considered to start on. + """ + input DateTimeGroupingOffsetInput { + """ + Number (positive or negative) of the given `unit` to offset the boundaries of the `DateTime` groupings. + """ + amount: Int! + """ + Unit of offsetting to apply to the boundaries of the `DateTime` groupings. + """ + unit: DateTimeUnitInput! + } + EOS + end + + it "defines a `JsonSafeLong` scalar and filter types" do + expect(type_named("JsonSafeLong", include_docs: true)).to eq(<<~EOS.strip) + """ + A numeric type for large integer values that can serialize safely as JSON. + + While JSON itself has no hard limit on the size of integers, the RFC-7159 spec + mentions that values outside of the range -9,007,199,254,740,991 (-(2^53) + 1) + to 9,007,199,254,740,991 (2^53 - 1) may not be interopable with all JSON + implementations. As it turns out, the number implementation used by JavaScript + has this issue. When you parse a JSON string that contains a numeric value like + `4693522397653681111`, the parsed result will contain a rounded value like + `4693522397653681000`. + + While this is entirely a client-side problem, we want to preserve maximum compatibility + with common client languages. Given the ubiquity of GraphiQL as a GraphQL client, + we want to avoid this problem. + + Our solution is to support two separate types: + + - This type (`JsonSafeLong`) is serialized as a number, but limits values to the safely + serializable range. + - The `LongString` type supports long values that use all 64 bits, but serializes as a + string rather than a number, avoiding the JavaScript compatibility problems. + + For more background, see the [JavaScript `Number.MAX_SAFE_INTEGER` + docs](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER). + """ + scalar JsonSafeLong + EOS + + expect(type_named("JsonSafeLongFilterInput")).to eq(<<~EOS.strip) + input JsonSafeLongFilterInput { + #{schema_elements.any_of}: [JsonSafeLongFilterInput!] + #{schema_elements.not}: JsonSafeLongFilterInput + #{schema_elements.equal_to_any_of}: [JsonSafeLong] + #{schema_elements.gt}: JsonSafeLong + #{schema_elements.gte}: JsonSafeLong + #{schema_elements.lt}: JsonSafeLong + #{schema_elements.lte}: JsonSafeLong + } + EOS + + expect(type_named("JsonSafeLongListElementFilterInput")).to eq(<<~EOS.strip) + input JsonSafeLongListElementFilterInput { + #{schema_elements.any_of}: [JsonSafeLongListElementFilterInput!] + #{schema_elements.equal_to_any_of}: [JsonSafeLong!] + #{schema_elements.gt}: JsonSafeLong + #{schema_elements.gte}: JsonSafeLong + #{schema_elements.lt}: JsonSafeLong + #{schema_elements.lte}: JsonSafeLong + } + EOS + end + + it "defines a `LongString` scalar and filter types" do + expect(type_named("LongString", include_docs: true)).to eq(<<~EOS.strip) + """ + A numeric type for large integer values in the inclusive range -2^63 + (-9,223,372,036,854,775,808) to (2^63 - 1) (9,223,372,036,854,775,807). + + Note that `LongString` values are serialized as strings within JSON, to avoid + interopability problems with JavaScript. If you want a large integer type that + serializes within JSON as a number, use `JsonSafeLong`. + """ + scalar LongString + EOS + + expect(type_named("LongStringListElementFilterInput")).to eq(<<~EOS.strip) + input LongStringListElementFilterInput { + #{schema_elements.any_of}: [LongStringListElementFilterInput!] + #{schema_elements.equal_to_any_of}: [LongString!] + #{schema_elements.gt}: LongString + #{schema_elements.gte}: LongString + #{schema_elements.lt}: LongString + #{schema_elements.lte}: LongString + } + EOS + end + + %w[Int Float].each do |type| + it "defines `#{type}FilterInput` and `#{type}ListElementFilterInput` inputs (but not a `#{type}` scalar, since it's built in)" do + expect(type_named(type)).to eq nil + + expect(type_named("#{type}FilterInput")).to eq(<<~EOS.strip) + input #{type}FilterInput { + #{schema_elements.any_of}: [#{type}FilterInput!] + #{schema_elements.not}: #{type}FilterInput + #{schema_elements.equal_to_any_of}: [#{type}] + #{schema_elements.gt}: #{type} + #{schema_elements.gte}: #{type} + #{schema_elements.lt}: #{type} + #{schema_elements.lte}: #{type} + } + EOS + + expect(type_named("#{type}ListElementFilterInput")).to eq(<<~EOS.strip) + input #{type}ListElementFilterInput { + #{schema_elements.any_of}: [#{type}ListElementFilterInput!] + #{schema_elements.equal_to_any_of}: [#{type}!] + #{schema_elements.gt}: #{type} + #{schema_elements.gte}: #{type} + #{schema_elements.lt}: #{type} + #{schema_elements.lte}: #{type} + } + EOS + end + end + + %w[String ID Boolean].each do |type| + it "defines `#{type}FilterInput` and `#{type}ListElementFilterInput` inputs (but not a `#{type}` scalar, since it's built in)" do + expect(type_named(type)).to eq nil + + expect(type_named("#{type}FilterInput")).to eq(<<~EOS.strip) + input #{type}FilterInput { + #{schema_elements.any_of}: [#{type}FilterInput!] + #{schema_elements.not}: #{type}FilterInput + #{schema_elements.equal_to_any_of}: [#{type}] + } + EOS + + expect(type_named("#{type}ListElementFilterInput")).to eq(<<~EOS.strip) + input #{type}ListElementFilterInput { + #{schema_elements.any_of}: [#{type}ListElementFilterInput!] + #{schema_elements.equal_to_any_of}: [#{type}!] + } + EOS + end + end + + it "defines `TextFilterInput` and `TextListElementFilterInput` input (but not a `Text` scalar, since it's a specialized string filter)" do + expect(type_named("Text")).to eq nil + + # The `TextFilterInput` has customizations compared to other scalar filters, so it is worth verifying the generated docs. + expect(type_named("TextFilterInput", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type used to specify filters on `String` fields that have been indexed for full text search. + + Will be ignored if passed as an empty object (or as `null`). + """ + input TextFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + #{schema_elements.any_of}: [TextFilterInput!] + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + #{schema_elements.not}: TextFilterInput + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + #{schema_elements.equal_to_any_of}: [String] + """ + Matches records where the field value matches the provided value using full text search. + + Will be ignored when `null` is passed. + """ + #{schema_elements.matches}: String @deprecated(reason: "Use `#{schema_elements.matches_query}` instead.") + """ + Matches records where the field value matches the provided query using full text search. + This is more lenient than `#{schema_elements.matches_phrase}`: the order of terms is ignored, and, + by default, only one search term is required to be in the field value. + + Will be ignored when `null` is passed. + """ + #{schema_elements.matches_query}: MatchesQueryFilterInput + """ + Matches records where the field value has a phrase matching the provided phrase using + full text search. This is stricter than `#{schema_elements.matches_query}`: all terms must match + and be in the same order as the provided phrase. + + Will be ignored when `null` is passed. + """ + #{schema_elements.matches_phrase}: MatchesPhraseFilterInput + } + EOS + + expect(type_named("TextListElementFilterInput", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type used to specify filters on `String` fields that have been indexed for full text search. + + Will be ignored if passed as an empty object (or as `null`). + """ + input TextListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + #{schema_elements.any_of}: [TextListElementFilterInput!] + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + #{schema_elements.equal_to_any_of}: [String!] + """ + Matches records where the field value matches the provided value using full text search. + + Will be ignored when `null` is passed. + """ + #{schema_elements.matches}: String @deprecated(reason: "Use `#{schema_elements.matches_query}` instead.") + """ + Matches records where the field value matches the provided query using full text search. + This is more lenient than `#{schema_elements.matches_phrase}`: the order of terms is ignored, and, + by default, only one search term is required to be in the field value. + + Will be ignored when `null` is passed. + """ + #{schema_elements.matches_query}: MatchesQueryFilterInput + """ + Matches records where the field value has a phrase matching the provided phrase using + full text search. This is stricter than `#{schema_elements.matches_query}`: all terms must match + and be in the same order as the provided phrase. + + Will be ignored when `null` is passed. + """ + #{schema_elements.matches_phrase}: MatchesPhraseFilterInput + } + EOS + end + + describe "`*ListFilterInput` types" do + %w[Boolean ID String Untyped].each do |scalar| + it "defines a `#{scalar}ListFilterInput` to support filtering on lists of `#{scalar}` values" do + expect_list_filter(scalar) + end + end + + %w[DistanceUnit].each do |enum| + it "defines a `#{enum}ListFilterInput` to support filtering on lists of `#{enum}` values" do + expect_list_filter(enum) + end + end + + %w[Date DateTime Float Int JsonSafeLong LocalTime LongString].each do |scalar| + it "defines a `#{scalar}ListFilterInput` to support filtering on lists of `#{scalar}` values" do + expect_list_filter(scalar) + end + end + + %w[GeoLocation].each do |scalar| + it "defines a `#{scalar}ListFilterInput` to support filtering on lists of `#{scalar}` values" do + expect_list_filter(scalar) + end + end + + %w[Text].each do |scalar| + it "defines a `#{scalar}ListFilterInput` to support filtering on lists of `#{scalar}` values" do + expect_list_filter(scalar, fields_description: "`[String]` fields that have been indexed for full text search") + end + end + + def expect_list_filter(scalar, fields_description: "`[#{scalar}]` fields") + expect(type_named("#{scalar}ListFilterInput", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type used to specify filters on #{fields_description}. + + Will be ignored if passed as an empty object (or as `null`). + """ + input #{scalar}ListFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + #{schema_elements.any_of}: [#{scalar}ListFilterInput!] + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + #{schema_elements.not}: #{scalar}ListFilterInput + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + #{schema_elements.any_satisfy}: #{scalar}ListElementFilterInput + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `#{scalar}ListFilterInput` input because of collisions between key names. For example, if you want to provide + multiple `#{schema_elements.any_satisfy}: ...` filters, you could do `#{schema_elements.all_of}: [{#{schema_elements.any_satisfy}: ...}, {#{schema_elements.any_satisfy}: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + #{schema_elements.all_of}: [#{scalar}ListFilterInput!] + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + #{schema_elements.count}: IntFilterInput + } + EOS + end + end + + it "defines a `MatchesQueryFilterInput` type" do + expect(type_named("MatchesQueryFilterInput", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type used to specify parameters for the `#{schema_elements.matches_query}` filtering operator. + + Will be ignored if passed as `null`. + """ + input MatchesQueryFilterInput { + """ + The input query to search for. + """ + #{schema_elements.query}: String! + """ + Number of allowed modifications per term to arrive at a match. For example, if set to 'ONE', the input + term 'glue' would match 'blue' but not 'clued', since the latter requires two modifications. + """ + #{schema_elements.allowed_edits_per_term}: MatchesQueryAllowedEditsPerTermInput! = "DYNAMIC" + """ + Set to `true` to match only if all terms in `#{schema_elements.query}` are found, or + `false` to only require one term to be found. + """ + #{schema_elements.require_all_terms}: Boolean! = false + } + EOS + end + + it "defines a `MatchesPhraseFilterInput` type" do + expect(type_named("MatchesPhraseFilterInput", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type used to specify parameters for the `#{schema_elements.matches_phrase}` filtering operator. + + Will be ignored if passed as `null`. + """ + input MatchesPhraseFilterInput { + """ + The input phrase to search for. + """ + #{schema_elements.phrase}: String! + } + EOS + end + + it "defines a `PageInfo` type" do + expect(type_named("PageInfo", include_docs: true)).to eq(<<~EOS.strip) + """ + Provides information about the specific fetched page. This implements the `PageInfo` + specification from the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-undefined.PageInfo). + """ + type PageInfo { + """ + Indicates if there is another page of results available after the current one. + """ + #{schema_elements.has_next_page}: Boolean! + """ + Indicates if there is another page of results available before the current one. + """ + #{schema_elements.has_previous_page}: Boolean! + """ + The `Cursor` of the first edge of the current page. This can be passed in the next query as + a `before` argument to paginate backwards. + """ + #{schema_elements.start_cursor}: Cursor + """ + The `Cursor` of the last edge of the current page. This can be passed in the next query as + a `after` argument to paginate forwards. + """ + #{schema_elements.end_cursor}: Cursor + } + EOS + end + + ["FloatAggregatedValues"].each do |expected_type| + it "defines an `#{expected_type}` type" do + expect(type_named(expected_type, include_docs: true)).to eq(<<~EOS.strip) + """ + A return type used from aggregations to provided aggregated values over `Float` fields. + """ + type #{expected_type} { + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + #{schema_elements.approximate_distinct_value_count}: JsonSafeLong + """ + The sum of the field values within this grouping. + + As with all double-precision `Float` values, operations are subject to floating-point loss + of precision, so the value may be approximate. + """ + #{schema_elements.approximate_sum}: Float! + """ + The minimum of the field values within this grouping. + + The value will be "exact" in that the aggregation computation will return + the exact value of the smallest float that has been indexed, without + introducing any new imprecision. However, floats by their nature are + naturally imprecise since they cannot precisely represent all real numbers. + """ + #{schema_elements.exact_min}: Float + """ + The maximum of the field values within this grouping. + + The value will be "exact" in that the aggregation computation will return + the exact value of the largest float that has been indexed, without + introducing any new imprecision. However, floats by their nature are + naturally imprecise since they cannot precisely represent all real numbers. + """ + #{schema_elements.exact_max}: Float + """ + The average (mean) of the field values within this grouping. + + The computation of this value may introduce additional imprecision (on top of the + natural imprecision of floats) when it deals with intermediary values that are + outside the `JsonSafeLong` range (-9,007,199,254,740,991 to 9,007,199,254,740,991). + """ + #{schema_elements.approximate_avg}: Float + } + EOS + end + end + + %w[Date DateTime LocalTime].each do |scalar_type| + it "defines a `#{scalar_type}AggregatedValues` type" do + expect(type_named("#{scalar_type}AggregatedValues", include_docs: true)).to eq(<<~EOS.strip) + """ + A return type used from aggregations to provided aggregated values over `#{scalar_type}` fields. + """ + type #{scalar_type}AggregatedValues { + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + #{schema_elements.approximate_distinct_value_count}: JsonSafeLong + """ + The minimum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + #{schema_elements.exact_min}: #{scalar_type} + """ + The maximum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + #{schema_elements.exact_max}: #{scalar_type} + """ + The average (mean) of the field values within this grouping. + The returned value will be rounded to the nearest `#{scalar_type}` value. + """ + #{schema_elements.approximate_avg}: #{scalar_type} + } + EOS + end + end + + it "defines a `NonNumericAggregatedValues` type" do + expect(type_named("NonNumericAggregatedValues", include_docs: true)).to eq(<<~EOS.strip) + """ + A return type used from aggregations to provided aggregated values over non-numeric fields. + """ + type NonNumericAggregatedValues { + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + #{schema_elements.approximate_distinct_value_count}: JsonSafeLong + } + EOS + end + + it "defines an `AggregationCountDetail` type" do + expect(type_named("AggregationCountDetail", include_docs: true)).to eq(<<~EOS.strip) + """ + Provides detail about an aggregation `count`. + """ + type AggregationCountDetail { + """ + The (approximate) count of documents in this aggregation bucket. + + When documents in an aggregation bucket are sourced from multiple shards, the count may be only + approximate. The `#{schema_elements.upper_bound}` indicates the maximum value of the true count, but usually + the true count is much closer to this approximate value (which also provides a lower bound on the + true count). + + When this approximation is known to be exact, the same value will be available from `#{schema_elements.exact_value}` + and `#{schema_elements.upper_bound}`. + """ + #{schema_elements.approximate_value}: JsonSafeLong! + """ + The exact count of documents in this aggregation bucket, if an exact value can be determined. + + When documents in an aggregation bucket are sourced from multiple shards, it may not be possible to + efficiently determine an exact value. When no exact value can be determined, this field will be `null`. + The `#{schema_elements.approximate_value}` field--which will never be `null`--can be used to get an approximation + for the count. + """ + #{schema_elements.exact_value}: JsonSafeLong + """ + An upper bound on how large the true count of documents in this aggregation bucket could be. + + When documents in an aggregation bucket are sourced from multiple shards, it may not be possible to + efficiently determine an exact value. The `#{schema_elements.approximate_value}` field provides an approximation, + and this field puts an upper bound on the true count. + """ + #{schema_elements.upper_bound}: JsonSafeLong! + } + EOS + end + + %w[Int JsonSafeLong].each do |scalar_type| + long_type = "JsonSafeLong" + ["#{scalar_type}AggregatedValues"].each do |expected_type| + it "defines an `#{expected_type}` type" do + expect(type_named(expected_type, include_docs: true)).to eq(<<~EOS.strip) + """ + A return type used from aggregations to provided aggregated values over `#{scalar_type}` fields. + """ + type #{expected_type} { + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + #{schema_elements.approximate_distinct_value_count}: JsonSafeLong + """ + The (approximate) sum of the field values within this grouping. + + Sums of large `#{scalar_type}` values can result in overflow, where the exact sum cannot + fit in a `#{long_type}` return value. This field, as a double-precision `Float`, can + represent larger sums, but the value may only be approximate. + """ + #{schema_elements.approximate_sum}: Float! + """ + The exact sum of the field values within this grouping, if it fits in a `#{long_type}`. + + Sums of large `#{scalar_type}` values can result in overflow, where the exact sum cannot + fit in a `#{long_type}`. In that case, `null` will be returned, and `#{schema_elements.approximate_sum}` + can be used to get an approximate value. + """ + #{schema_elements.exact_sum}: #{long_type} + """ + The minimum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + #{schema_elements.exact_min}: #{scalar_type} + """ + The maximum of the field values within this grouping. + + So long as the grouping contains at least one non-null value for the + underlying indexed field, this will return an exact non-null value. + """ + #{schema_elements.exact_max}: #{scalar_type} + """ + The average (mean) of the field values within this grouping. + + Note that the returned value is approximate. Imprecision can be introduced by the computation if + any intermediary values fall outside the `JsonSafeLong` range (-9,007,199,254,740,991 + to 9,007,199,254,740,991). + """ + #{schema_elements.approximate_avg}: Float + } + EOS + end + end + end + + ["LongStringAggregatedValues"].each do |expected_type| + it "defines an `#{expected_type}` type" do + expect(type_named(expected_type, include_docs: true)).to eq(<<~EOS.strip) + """ + A return type used from aggregations to provided aggregated values over `LongString` fields. + """ + type #{expected_type} { + """ + An approximation of the number of unique values for this field within this grouping. + + The approximation uses the HyperLogLog++ algorithm from the [HyperLogLog in Practice](https://research.google.com/pubs/archive/40671.pdf) + paper. The accuracy of the returned value varies based on the specific dataset, but + it usually differs from the true distinct value count by less than 7%. + """ + #{schema_elements.approximate_distinct_value_count}: JsonSafeLong + """ + The (approximate) sum of the field values within this grouping. + + Sums of large `LongString` values can result in overflow, where the exact sum cannot + fit in a `LongString` return value. This field, as a double-precision `Float`, can + represent larger sums, but the value may only be approximate. + """ + #{schema_elements.approximate_sum}: Float! + """ + The exact sum of the field values within this grouping, if it fits in a `JsonSafeLong`. + + Sums of large `LongString` values can result in overflow, where the exact sum cannot + fit in a `JsonSafeLong`. In that case, `null` will be returned, and `#{schema_elements.approximate_sum}` + can be used to get an approximate value. + """ + #{schema_elements.exact_sum}: JsonSafeLong + """ + The minimum of the field values within this grouping. + + So long as the grouping contains at least one non-null value, and no values exceed the + `JsonSafeLong` range in the underlying indexed field, this will return an exact non-null value. + + If no non-null values are available, or if the minimum value is outside the `JsonSafeLong` + range, `null` will be returned. `#{schema_elements.approximate_min}` can be used to differentiate between these + cases and to get an approximate value. + """ + #{schema_elements.exact_min}: JsonSafeLong + """ + The maximum of the field values within this grouping. + + So long as the grouping contains at least one non-null value, and no values exceed the + `JsonSafeLong` range in the underlying indexed field, this will return an exact non-null value. + + If no non-null values are available, or if the maximum value is outside the `JsonSafeLong` + range, `null` will be returned. `#{schema_elements.approximate_max}` can be used to differentiate between these + cases and to get an approximate value. + """ + #{schema_elements.exact_max}: JsonSafeLong + """ + The minimum of the field values within this grouping. + + The aggregation computation performed to identify the smallest value is not able + to maintain exact precision when dealing with values that are outside the `JsonSafeLong` + range (-9,007,199,254,740,991 to 9,007,199,254,740,991). + In that case, the `#{schema_elements.exact_min}` field will return `null`, but this field will provide + a value which may be approximate. + """ + #{schema_elements.approximate_min}: LongString + """ + The maximum of the field values within this grouping. + + The aggregation computation performed to identify the largest value is not able + to maintain exact precision when dealing with values that are outside the `JsonSafeLong` + range (-9,007,199,254,740,991 to 9,007,199,254,740,991). + In that case, the `#{schema_elements.exact_max}` field will return `null`, but this field will provide + a value which may be approximate. + """ + #{schema_elements.approximate_max}: LongString + """ + The average (mean) of the field values within this grouping. + + Note that the returned value is approximate. Imprecision can be introduced by the computation if + any intermediary values fall outside the `JsonSafeLong` range (-9,007,199,254,740,991 + to 9,007,199,254,740,991). + """ + #{schema_elements.approximate_avg}: Float + } + EOS + end + end + + it "correctly defines `Cursor` when our schema does not reference it, but references `PageInfo`, which does" do + result = define_schema do |api| + api.raw_sdl <<~EOS + type MyType { + page_info: PageInfo + } + EOS + end + + expect(result).to include("scalar Cursor") + end + + it "creates the built in types after extension modules are applied to allow factory extensions to apply to them" do + deprecatable = Module.new do + def deprecated! + directive "deprecated" + end + end + + factory_extension = Module.new do + define_method :new_scalar_type do |name, &block| + super(name) do |t| + t.extend deprecatable + block.call(t) + end + end + end + + api_extension = Module.new do + define_singleton_method :extended do |api| + api.factory.extend factory_extension + end + end + + result = define_schema(extension_modules: [api_extension]) do |api| + api.on_built_in_types do |t| + t.deprecated! if t.name == "DateTime" + end + end + + expect(type_def_from(result, "DateTime")).to eq "scalar DateTime @deprecated" + end + + describe "#on_built_in_types" do + it "can tag built in types" do + result = define_schema do |api| + api.raw_sdl <<~SDL + directive @tag(name: String!) repeatable on ARGUMENT_DEFINITION | ENUM | ENUM_VALUE | FIELD_DEFINITION | INPUT_FIELD_DEFINITION | INPUT_OBJECT | INTERFACE | OBJECT | SCALAR | UNION + SDL + + api.object_type "NotABuiltInType" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index "widgets" + end + + api.on_built_in_types do |t| + t.directive("tag", name: "tag1") + end + + api.on_built_in_types do |t| + t.directive("tag", name: "tag2") + end + end + + all_type_names = types_defined_in(result) + categorized_type_names = all_type_names.group_by do |type_name| + if type_name.start_with?("__") || STOCK_GRAPHQL_SCALARS.include?(type_name) + :not_explicitly_defined + elsif type_name.include?("NotABuiltInType") + :expect_no_tags + else + :expect_tags + end + end + + # Verify that we have types in all 3 categories as expected. + expect(categorized_type_names).to include(:expect_no_tags, :expect_tags, :not_explicitly_defined) + expect(categorized_type_names[:expect_no_tags]).not_to be_empty + expect(categorized_type_names[:expect_tags]).not_to be_empty + expect(categorized_type_names[:not_explicitly_defined]).not_to be_empty + + type_defs_by_name = all_type_names.to_h { |type| [type, type_def_from(result, type)] } + expect(type_defs_by_name.select { |k, type_def| type_def.nil? }.keys).to match_array(categorized_type_names[:not_explicitly_defined]) + + categorized_type_names[:expect_tags].each do |type| + expect(type_defs_by_name[type]).to include("@tag(name: \"tag1\")", "@tag(name: \"tag2\")") + end + + categorized_type_names[:expect_no_tags].each do |type| + expect(type_defs_by_name[type]).not_to include("@tag") + end + end + end + + describe "standard enum types" do + it "generates a `DateGroupingGranularity` enum" do + expect(type_named("DateGroupingGranularity", include_docs: true)).to eq(<<~EOS.strip) + """ + Enumerates the supported granularities of a `Date`. + """ + enum DateGroupingGranularity { + """ + The year a `Date` falls in. + """ + YEAR + """ + The quarter a `Date` falls in. + """ + QUARTER + """ + The month a `Date` falls in. + """ + MONTH + """ + The week, beginning on Monday, a `Date` falls in. + """ + WEEK + """ + The exact day of a `Date`. + """ + DAY + } + EOS + end + + it "generates a `DateGroupingTruncationUnit` enum" do + expect(type_named("DateGroupingTruncationUnit", include_docs: true)).to eq(<<~EOS.strip) + """ + Enumerates the supported truncation units of a `Date`. + """ + enum DateGroupingTruncationUnit { + """ + The year a `Date` falls in. + """ + YEAR + """ + The quarter a `Date` falls in. + """ + QUARTER + """ + The month a `Date` falls in. + """ + MONTH + """ + The week, beginning on Monday, a `Date` falls in. + """ + WEEK + """ + The exact day of a `Date`. + """ + DAY + } + EOS + end + + it "generates a `DateTimeGroupingGranularity` enum" do + expect(type_named("DateTimeGroupingGranularity", include_docs: true)).to eq(<<~EOS.strip) + """ + Enumerates the supported granularities of a `DateTime`. + """ + enum DateTimeGroupingGranularity { + """ + The year a `DateTime` falls in. + """ + YEAR + """ + The quarter a `DateTime` falls in. + """ + QUARTER + """ + The month a `DateTime` falls in. + """ + MONTH + """ + The week, beginning on Monday, a `DateTime` falls in. + """ + WEEK + """ + The day a `DateTime` falls in. + """ + DAY + """ + The hour a `DateTime` falls in. + """ + HOUR + """ + The minute a `DateTime` falls in. + """ + MINUTE + """ + The second a `DateTime` falls in. + """ + SECOND + } + EOS + end + + it "generates a `DateTimeGroupingTruncationUnit` enum" do + expect(type_named("DateTimeGroupingTruncationUnit", include_docs: true)).to eq(<<~EOS.strip) + """ + Enumerates the supported truncation units of a `DateTime`. + """ + enum DateTimeGroupingTruncationUnit { + """ + The year a `DateTime` falls in. + """ + YEAR + """ + The quarter a `DateTime` falls in. + """ + QUARTER + """ + The month a `DateTime` falls in. + """ + MONTH + """ + The week, beginning on Monday, a `DateTime` falls in. + """ + WEEK + """ + The day a `DateTime` falls in. + """ + DAY + """ + The hour a `DateTime` falls in. + """ + HOUR + """ + The minute a `DateTime` falls in. + """ + MINUTE + """ + The second a `DateTime` falls in. + """ + SECOND + } + EOS + end + + it "generates a `DistanceUnit` enum for use in geo location filtering" do + expect(type_named("DistanceUnit", include_docs: true)).to eq(<<~EOS.strip) + """ + Enumerates the supported distance units. + """ + enum DistanceUnit { + """ + A United States customary unit of 5,280 feet. + """ + MILE + """ + A United States customary unit of 3 feet. + """ + YARD + """ + A United States customary unit of 12 inches. + """ + FOOT + """ + A United States customary unit equal to 1/12th of a foot. + """ + INCH + """ + A metric system unit equal to 1,000 meters. + """ + KILOMETER + """ + The base unit of length in the metric system. + """ + METER + """ + A metric system unit equal to 1/100th of a meter. + """ + CENTIMETER + """ + A metric system unit equal to 1/1,000th of a meter. + """ + MILLIMETER + """ + An international unit of length used for air, marine, and space navigation. Equivalent to 1,852 meters. + """ + NAUTICAL_MILE + } + EOS + end + end + + describe "date and time grouped by types" do + it "generates a `DateTimeGroupedBy` type" do + expect(type_named("DateTimeGroupedBy", include_docs: true)).to eq(<<~EOS.strip) + """ + Allows for grouping `DateTime` values based on the desired return type. + """ + type DateTimeGroupedBy { + """ + Used when grouping on the full `DateTime` value. + """ + #{schema_elements.as_date_time}( + """ + Amount of offset (positive or negative) to shift the `DateTime` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 1 day to change what day-of-week weeks are considered to start on. + """ + #{schema_elements.offset}: DateTimeGroupingOffsetInput + """ + The time zone to use when determining which grouping a `DateTime` value falls in. + """ + #{schema_elements.time_zone}: TimeZone! = "UTC" + """ + Determines the grouping truncation unit for this field. + """ + #{schema_elements.truncation_unit}: DateTimeGroupingTruncationUnitInput!): DateTime + """ + An alternative to `#{schema_elements.as_date_time}` for when grouping on just the date is desired. + """ + #{schema_elements.as_date}( + """ + Amount of offset (positive or negative) to shift the `Date` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 1 day to change what day-of-week weeks are considered to start on. + """ + #{schema_elements.offset}: DateGroupingOffsetInput + """ + The time zone to use when determining which grouping a `Date` value falls in. + """ + #{schema_elements.time_zone}: TimeZone! = "UTC" + """ + Determines the grouping truncation unit for this field. + """ + #{schema_elements.truncation_unit}: DateGroupingTruncationUnitInput!): Date + """ + An alternative to `#{schema_elements.as_date_time}` for when grouping on just the time-of-day is desired. + """ + #{schema_elements.as_time_of_day}( + """ + Amount of offset (positive or negative) to shift the `LocalTime` boundaries of each grouping bucket. + + For example, when grouping by `HOUR`, you can apply an offset of -5 minutes to shift `LocalTime` + values to the prior hour when they fall between the the top of an hour and 5 after. + """ + #{schema_elements.offset}: LocalTimeGroupingOffsetInput + """ + The time zone to use when determining which grouping a `LocalTime` value falls in. + """ + #{schema_elements.time_zone}: TimeZone! = "UTC" + """ + Determines the grouping truncation unit for this field. + """ + #{schema_elements.truncation_unit}: LocalTimeGroupingTruncationUnitInput!): LocalTime + """ + An alternative to `#{schema_elements.as_date_time}` for when grouping on the day-of-week is desired. + """ + #{schema_elements.as_day_of_week}( + """ + Amount of offset (positive or negative) to shift the `DayOfWeek` boundaries of each grouping bucket. + + For example, you can apply an offset of -2 hours to shift `DateTime` values to the prior `DayOfWeek` + when they fall between midnight and 2 AM. + """ + #{schema_elements.offset}: DayOfWeekGroupingOffsetInput + """ + The time zone to use when determining which grouping a `DayOfWeek` value falls in. + """ + #{schema_elements.time_zone}: TimeZone! = "UTC"): DayOfWeek + } + EOS + end + + it "generates a `DateGroupedBy` type" do + expect(type_named("DateGroupedBy", include_docs: true)).to eq(<<~EOS.strip) + """ + Allows for grouping `Date` values based on the desired return type. + """ + type DateGroupedBy { + """ + Used when grouping on the full `Date` value. + """ + #{schema_elements.as_date}( + """ + Amount of offset (positive or negative) to shift the `Date` boundaries of each grouping bucket. + + For example, when grouping by `WEEK`, you can shift by 1 day to change what day-of-week weeks are considered to start on. + """ + #{schema_elements.offset}: DateGroupingOffsetInput + """ + Determines the grouping truncation unit for this field. + """ + #{schema_elements.truncation_unit}: DateGroupingTruncationUnitInput!): Date + """ + An alternative to `#{schema_elements.as_date}` for when grouping on the day-of-week is desired. + """ + #{schema_elements.as_day_of_week}( + """ + Amount of offset (positive or negative) to shift the `DayOfWeek` boundaries of each grouping bucket. + + For example, you can apply an offset of -2 hours to shift `DateTime` values to the prior `DayOfWeek` + when they fall between midnight and 2 AM. + """ + #{schema_elements.offset}: DayOfWeekGroupingOffsetInput): DayOfWeek + } + EOS + end + + it "generates a `DayOfWeek` enum" do + expect(type_named("DayOfWeek", include_docs: true)).to eq(<<~EOS.strip) + """ + Indicates the specific day of the week. + """ + enum DayOfWeek { + """ + Monday. + """ + MONDAY + """ + Tuesday. + """ + TUESDAY + """ + Wednesday. + """ + WEDNESDAY + """ + Thursday. + """ + THURSDAY + """ + Friday. + """ + FRIDAY + """ + Saturday. + """ + SATURDAY + """ + Sunday. + """ + SUNDAY + } + EOS + end + end + + before(:context) { @result = define_schema } + + def type_named(name, include_docs: false) + type_def_from(@result, name, include_docs: include_docs) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/define_schema_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/define_schema_spec.rb new file mode 100644 index 00000000..c921c823 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/define_schema_spec.rb @@ -0,0 +1,205 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/spec_support/have_readable_to_s_and_inspect_output" +require_relative "graphql_schema_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "ElasticGraph.define_schema" do + include_context "GraphQL schema spec support" + + with_both_casing_forms do + it "evaluates the given block against the active API instance, allowing `ElasticGraph.define_schema` to be used many times" do + api = API.new(schema_elements, true) + + api.as_active_instance do + ElasticGraph.define_schema do |schema| + schema.json_schema_version 1 + schema.object_type("Person") do |t| + t.field "id", "ID" + end + end + + ElasticGraph.define_schema do |schema| + schema.object_type("Place") do |t| + t.field "id", "ID" + end + end + end + + result = api.results.graphql_schema_string + + expect(type_def_from(result, "Person")).to eq(<<~EOS.strip) + type Person { + id: ID + } + EOS + + expect(type_def_from(result, "Place")).to eq(<<~EOS.strip) + type Place { + id: ID + } + EOS + end + + it "raises a clear error when there is no active API instance" do + expect { + ElasticGraph.define_schema { |schema| } + }.to raise_error Errors::SchemaError, a_string_including("Let ElasticGraph load") + end + + it "does not leak the active API instance, even when an error occurs" do + api = API.new(schema_elements, true) + + expect { + api.as_active_instance do + ElasticGraph.define_schema do |schema| + raise "boom" + end + end + }.to raise_error(/boom/) + + expect { + ElasticGraph.define_schema { |schema| } + }.to raise_error Errors::SchemaError, a_string_including("Let ElasticGraph load") + end + + it "does not allow `user_defined_field_references_by_type_name` to be accessed on `state` before the schema definition is done" do + expect { + define_schema do |schema| + schema.state.user_defined_field_references_by_type_name + end + }.to raise_error( + Errors::SchemaError, + "Cannot access `user_defined_field_references_by_type_name` until the schema definition is complete." + ) + end + + it "produces the same GraphQL output, regardless of the order the types are defined in" do + object_type_definitions = { + "Component" => lambda do |t| + t.field "id", "ID!" + t.relates_to_one "widget", "Widget", via: "widget_id", dir: :out + t.index "components" + end, + + "Widget" => lambda do |t| + t.field "id", "ID!" + t.field "versions", "[WidgetVersion]" do |f| + f.mapping type: "nested" + end + t.relates_to_many "components", "Component", via: "widget_id", dir: :in, singular: "component" + t.index "widgets" + end, + + "WidgetVersion" => lambda do |t| + t.field "version", "Int!" + end + } + + all_definition_orderings = [ + # Note: when this spec was written, these first 2 orderings (where `Component` came first) caused infinite recursion. + # At the time, having `Component` come first caused an issue because it caused `ComponentEdge` (with it's + # `node: Component` field) and `ComponentConnection` (with its `nodes: [Component!]!` field) to be generated + # before, `Widget.components` was processed, leading to additional field references to the `Component` type. + %w[Component Widget WidgetVersion], + %w[Component WidgetVersion Widget], + # In contrast, these last 4 orderings did not cause that problem and always produced the same output. + %w[Widget WidgetVersion Component], + %w[Widget Component WidgetVersion], + %w[WidgetVersion Widget Component], + %w[WidgetVersion Component Widget] + ] + + uniq_results_for_each_ordering = all_definition_orderings.map do |type_names_in_order| + define_schema do |schema| + type_names_in_order.each do |type_name| + schema.object_type(type_name, &object_type_definitions.fetch(type_name)) + end + end + end.uniq + + expect(uniq_results_for_each_ordering.size).to eq 1 + + # Also compare the first and last, so that if there are multiple we get a diff showing how they differ. + expect(uniq_results_for_each_ordering.first).to eq uniq_results_for_each_ordering.last + end + + it "returns reasonably-sized strings from `#inspect` and `#to_s` for all objects exposed to users so that the exception output if the user misspells a method name is readable" do + define_schema do |schema| + expect(schema).to have_readable_to_s_and_inspect_output + + schema.on_built_in_types do |t| + expect(t).to have_readable_to_s_and_inspect_output + end + + schema.scalar_type "MyScalar" do |t| + expect(t).to have_readable_to_s_and_inspect_output.including("MyScalar") + t.mapping type: "keyword" + t.json_schema type: "string" + end + + schema.enum_type "Color" do |t| + expect(t).to have_readable_to_s_and_inspect_output.including("Color") + t.value "RED" do |v| + expect(v).to have_readable_to_s_and_inspect_output.including("RED") + end + end + + schema.interface_type "Identifiable" do |t| + expect(t).to have_readable_to_s_and_inspect_output.including("Identifiable") + + t.field "id", "ID" do |f| + expect(f).to have_readable_to_s_and_inspect_output.including("id: ID") + end + end + + schema.union_type "Entity" do |t| + expect(t).to have_readable_to_s_and_inspect_output.including("Entity") + t.subtype "Widget" + end + + schema.object_type "Widget" do |t| + expect(t).to have_readable_to_s_and_inspect_output.including("Widget") + + t.field "name", "String!" do |f| + f.documentation "the field docs" + expect(f).to have_readable_to_s_and_inspect_output.including("Widget.name: String!").and_excluding("the field docs") + + f.argument "int", "Int" do |a| + a.documentation "the arg docs" + expect(a).to have_readable_to_s_and_inspect_output.including("Widget.name(int: Int)").and_excluding("the arg docs") + end + + f.customize_filter_field do |ff| + expect(ff).to have_readable_to_s_and_inspect_output + end + + f.on_each_generated_schema_element do |se| + expect(se).to have_readable_to_s_and_inspect_output + end + end + + t.field "id", "ID!" + + t.index "widgets" do |i| + expect(i).to have_readable_to_s_and_inspect_output.including("widgets") + end + end + + expect(schema.factory).to have_readable_to_s_and_inspect_output + expect(schema.state).to have_readable_to_s_and_inspect_output + expect(schema.results).to have_readable_to_s_and_inspect_output + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/derived_graphql_type_customizations_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/derived_graphql_type_customizations_spec.rb new file mode 100644 index 00000000..d8e07a51 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/derived_graphql_type_customizations_spec.rb @@ -0,0 +1,387 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "graphql_schema_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "GraphQL schema generation", "derived graphql type customizations" do + include_context "GraphQL schema spec support" + + with_both_casing_forms do + it "allows `customize_derived_types` to be used to customize specific types derived from indexed types" do + result = define_schema do |api| + api.raw_sdl "directive @external on OBJECT | INPUT_OBJECT | ENUM" + api.raw_sdl "directive @derived on OBJECT | INPUT_OBJECT | ENUM" + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "cost", "Int" + # Some derived types are only generated when a type has nested fields. + t.field "is_nested", "[IsNested!]!" do |f| + f.mapping type: "nested" + end + t.index "widgets" + + # Here we are trying to exhaustively cover all possible derived graphql types to ensure that our "invalid derived graphql type" detection + # doesn't wrongly consider any of these to be invalid. + t.customize_derived_types( + "WidgetConnection", "WidgetAggregatedValues", "WidgetGroupedBy", + "WidgetAggregation", "WidgetAggregationEdge", "WidgetAggregationConnection", + "HasNestedWidgetSubAggregation", "HasNestedWidgetSubAggregationConnection", + "HasNestedWidgetSubAggregationSubAggregations", "WidgetAggregationSubAggregations" + ) do |dt| + dt.directive "deprecated" + end + + t.customize_derived_types "WidgetAggregation", "WidgetEdge", "WidgetFilterInput", "WidgetListFilterInput", "WidgetFieldsListFilterInput", "WidgetSortOrderInput" do |dt| + dt.directive "external" + end + + t.customize_derived_types :all do |dt| + dt.directive "derived" + end + end + + api.object_type "IsNested" do |t| + t.field "something", "String" + end + + api.object_type "HasNested" do |t| + t.field "id", "ID" + # Some derived types are only generated when a type is used by a nested field. + t.field "widgets", "[Widget!]!" do |f| + f.mapping type: "nested" + end + t.index "has_nested" + end + end + + expect(type_def_from(result, "Widget")).not_to include("@deprecated", "@external", "@derived") + + expect(connection_type_from(result, "Widget")).to include("WidgetConnection @deprecated @derived {") + expect(aggregated_values_type_from(result, "Widget")).to include("WidgetAggregatedValues @deprecated @derived {") + expect(grouped_by_type_from(result, "Widget")).to include("WidgetGroupedBy @deprecated @derived {") + expect(edge_type_from(result, "Widget")).to include("WidgetEdge @external @derived {") + expect(filter_type_from(result, "Widget")).to include("WidgetFilterInput @external @derived {") + expect(list_filter_type_from(result, "Widget")).to include("WidgetListFilterInput @external @derived {") + expect(fields_list_filter_type_from(result, "Widget")).to include("WidgetFieldsListFilterInput @external @derived {") + expect(sort_order_type_from(result, "Widget")).to include("WidgetSortOrderInput @external @derived {") + expect(aggregation_type_from(result, "Widget")).to include("WidgetAggregation @deprecated @external @derived {") + expect(aggregation_connection_type_from(result, "Widget")).to include("WidgetAggregationConnection @deprecated @derived {") + expect(aggregation_edge_type_from(result, "Widget")).to include("WidgetAggregationEdge @deprecated @derived {") + + expect(sub_aggregation_type_from(result, "HasNestedWidget")).to include("HasNestedWidgetSubAggregation @deprecated @derived {") + expect(sub_aggregation_connection_type_from(result, "HasNestedWidget")).to include("HasNestedWidgetSubAggregationConnection @deprecated @derived {") + expect(sub_aggregation_sub_aggregations_type_from(result, "HasNestedWidget")).to include("HasNestedWidgetSubAggregationSubAggregations @deprecated @derived {") + expect(aggregation_sub_aggregations_type_from(result, "Widget")).to include("WidgetAggregationSubAggregations @deprecated @derived {") + end + + it "respects `customize_derived_types` with every derived graphql type" do + known_derived_graphql_type_names = %w[ + WidgetAggregatedValues + WidgetGroupedBy + WidgetAggregation + WidgetAggregationConnection + WidgetAggregationEdge + WidgetConnection + WidgetEdge + WidgetFilterInput + WidgetListFilterInput + WidgetFieldsListFilterInput + WidgetSortOrderInput + HasNestedWidgetSubAggregation + HasNestedWidgetSubAggregationConnection + HasNestedWidgetSubAggregationSubAggregations + WidgetAggregationSubAggregations + ] + + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "cost", "Int" + t.index "widgets" + + # Some derived types are only generated when a type has nested fields. + t.field "is_nested", "[IsNested!]!" do |f| + f.mapping type: "nested" + end + + t.customize_derived_types(*known_derived_graphql_type_names) do |dt| + dt.directive "deprecated" + end + end + + api.object_type "IsNested" do |t| + t.field "something", "String" + end + + api.object_type "HasNested" do |t| + t.field "id", "ID" + # Some derived types are only generated when a type is used by a nested field. + t.field "widgets", "[Widget!]!" do |f| + f.mapping type: "nested" + end + t.index "has_nested" + end + end + + all_widget_type_names = ( + result.scan(/(?:type|input|enum|union|interface|scalar) (\w*Widget[^\s]+)/).flatten - + # These types are derived from the `IsNested` type, not from `Widget`, so they should not be impacted or covered here. + %w[WidgetIsNestedSubAggregation WidgetIsNestedSubAggregationConnection HasNestedWidgetIsNestedSubAggregation HasNestedWidgetIsNestedSubAggregationConnection] + ) + expect(all_widget_type_names).to match_array(known_derived_graphql_type_names), + "This test is meant to cover all derived graphql types but is missing some. Please update the test to include them: " \ + "#{all_widget_type_names - known_derived_graphql_type_names}" + + expect(connection_type_from(result, "Widget")).to include("WidgetConnection @deprecated") + expect(aggregated_values_type_from(result, "Widget")).to include("WidgetAggregatedValues @deprecated") + expect(grouped_by_type_from(result, "Widget")).to include("WidgetGroupedBy @deprecated") + expect(filter_type_from(result, "Widget")).to include("WidgetFilterInput @deprecated") + expect(list_filter_type_from(result, "Widget")).to include("WidgetListFilterInput @deprecated") + expect(fields_list_filter_type_from(result, "Widget")).to include("WidgetFieldsListFilterInput @deprecated") + expect(edge_type_from(result, "Widget")).to include("WidgetEdge @deprecated") + expect(sort_order_type_from(result, "Widget")).to include("WidgetSortOrderInput @deprecated") + expect(aggregation_type_from(result, "Widget")).to include("WidgetAggregation @deprecated") + expect(aggregation_connection_type_from(result, "Widget")).to include("WidgetAggregationConnection @deprecated") + expect(aggregation_edge_type_from(result, "Widget")).to include("WidgetAggregationEdge @deprecated") + + expect(sub_aggregation_type_from(result, "HasNestedWidget")).to include("HasNestedWidgetSubAggregation @deprecated") + expect(sub_aggregation_connection_type_from(result, "HasNestedWidget")).to include("HasNestedWidgetSubAggregationConnection @deprecated") + expect(sub_aggregation_sub_aggregations_type_from(result, "HasNestedWidget")).to include("HasNestedWidgetSubAggregationSubAggregations @deprecated") + expect(aggregation_sub_aggregations_type_from(result, "Widget")).to include("WidgetAggregationSubAggregations @deprecated") + + expect(type_def_from(result, "Widget")).not_to include("@deprecated") + end + + it "allows `customize_derived_types` to be used on relay types generated for paginated collection fields" do + result = define_schema do |api| + api.raw_sdl "directive @external on OBJECT" + + api.scalar_type "Url" do |t| + t.json_schema type: "string" + t.mapping type: "keyword" + + t.customize_derived_types "UrlEdge", "UrlConnection" do |dt| + dt.directive "external" + end + end + + api.object_type "Business" do |t| + t.field "id", "ID" + t.paginated_collection_field "urls", "Url" + t.index "businesses" + end + end + + expect(connection_type_from(result, "Url")).to start_with "type UrlConnection @external {" + expect(edge_type_from(result, "Url")).to start_with "type UrlEdge @external {" + end + + it "allows `customize_derived_type_fields` to be used to customize specific fields on a specific derived graphql type" do + result = define_schema do |api| + api.raw_sdl "directive @external on FIELD_DEFINITION" + api.raw_sdl "directive @internal on FIELD_DEFINITION" + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "cost", "Int" + t.index "widgets" + + # Some derived types are only generated when a type has nested fields. + t.field "is_nested1", "[IsNested!]!" do |f| + f.mapping type: "nested" + end + + t.field "is_nested2", "[IsNested!]!" do |f| + f.mapping type: "nested" + end + + t.customize_derived_type_fields "WidgetConnection", schema_elements.edges, schema_elements.page_info do |dt| + dt.directive "internal" + end + + t.customize_derived_type_fields "WidgetConnection", schema_elements.edges, schema_elements.nodes, schema_elements.total_edge_count do |dt| + dt.directive "external" + end + + t.customize_derived_type_fields "WidgetAggregationSubAggregations", "is_nested2" do |dt| + dt.directive "external" + end + end + + api.object_type "IsNested" do |t| + t.field "something", "String" + end + + api.object_type "HasNested" do |t| + t.field "id", "ID" + # Some derived types are only generated when a type is used by a nested field. + t.field "widgets", "[Widget!]!" do |f| + f.mapping type: "nested" + end + t.index "has_nested" + end + end + + expect(connection_type_from(result, "Widget")).to eq(<<~EOS.strip) + type WidgetConnection { + edges: [WidgetEdge!]! @internal @external + nodes: [Widget!]! @external + #{schema_elements.page_info}: PageInfo! @internal + #{schema_elements.total_edge_count}: JsonSafeLong! @external + } + EOS + + expect(type_def_from(result, "Widget")).not_to include("@internal") + expect(filter_type_from(result, "Widget")).not_to include("@internal") + expect(list_filter_type_from(result, "Widget")).not_to include("@internal") + expect(fields_list_filter_type_from(result, "Widget")).not_to include("@internal") + expect(edge_type_from(result, "Widget")).not_to include("@internal") + expect(sort_order_type_from(result, "Widget")).not_to include("@internal") + expect(aggregated_values_type_from(result, "Widget")).not_to include("@internal") + expect(aggregation_type_from(result, "Widget")).not_to include("@internal") + expect(aggregation_connection_type_from(result, "Widget")).not_to include("@internal") + expect(aggregation_edge_type_from(result, "Widget")).not_to include("@internal") + + expect(sub_aggregation_type_from(result, "HasNestedWidget")).not_to include("@internal") + expect(sub_aggregation_connection_type_from(result, "HasNestedWidget")).not_to include("@internal") + expect(sub_aggregation_sub_aggregations_type_from(result, "HasNestedWidget")).not_to include("@internal") + expect(aggregation_sub_aggregations_type_from(result, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregationSubAggregations { + is_nested1( + #{schema_elements.filter}: IsNestedFilterInput + #{schema_elements.first}: Int): WidgetIsNestedSubAggregationConnection + is_nested2( + #{schema_elements.filter}: IsNestedFilterInput + #{schema_elements.first}: Int): WidgetIsNestedSubAggregationConnection @external + } + EOS + end + + it "notifies the user of an invalid derived graphql type name passed to `customize_derived_types`" do + expect { + define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index "widgets" + + # WidgetConnection is misspelled. + t.customize_derived_types("WidgetConection", "WidgetGroupedBy") {} + end + end + }.to raise_error Errors::SchemaError, a_string_including("customize_derived_types", "WidgetConection") + end + + it "does not consider a valid derived graphql type suffix passed to `customize_derived_types` to be valid" do + expect { + define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index "widgets" + + # The derived graphql type is `WidgetConnection`, not `Connection` + t.customize_derived_types("Connection") {} + end + end + }.to raise_error Errors::SchemaError, a_string_including("customize_derived_types", "Connection") + end + + it "notifies the user of a derived graphql type passed to `customize_derived_types` that winds up not existing but could exist if the type was defined differently" do + expect { + define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!", groupable: false + t.field "name", "String" + + t.customize_derived_types("WidgetFilterInput", "WidgetConnection", "WidgetEdge") {} + end + end + }.to raise_error Errors::SchemaError, a_string_including("customize_derived_types", "WidgetConnection", "WidgetEdge").and(excluding("WidgetFilterInput")) + end + + it "notifies the user of an invalid derived graphql type name passed to `customize_derived_type_fields`" do + expect { + define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index "widgets" + + # WidgetConnection is misspelled. + t.customize_derived_type_fields("WidgetConection", "edges") {} + end + end + }.to raise_error Errors::SchemaError, a_string_including("customize_derived_type_fields", "WidgetConection") + end + + it "does not consider a valid derived graphql type suffix passed to `customize_derived_type_fields` to be valid" do + expect { + define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index "widgets" + + # The derived graphql type is `WidgetConnection`, not `Connection` + t.customize_derived_type_fields("Connection", "edges") {} + end + end + }.to raise_error Errors::SchemaError, a_string_including("customize_derived_type_fields", "Connection") + end + + it "notifies the user of a derived graphql type passed to `customize_derived_type_fields` that winds up not existing but could exist if the type was defined differently" do + expect { + define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + + t.customize_derived_type_fields("WidgetConnection", "edges") {} + end + end + }.to raise_error Errors::SchemaError, a_string_including("customize_derived_type_fields", "WidgetConnection") + end + + it "notifies the user of an invalid derived field name passed to `customize_derived_type_fields`" do + expect { + define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index "widgets" + + # edge is misspelled (missing an `s`). + t.customize_derived_type_fields("WidgetConnection", "edge") {} + end + end + }.to raise_error Errors::SchemaError, a_string_including("customize_derived_type_fields", "WidgetConnection", "edge") + end + + it "notifies the user if `customize_derived_type_fields` is used with a type that has no fields (e.g. an Enum type)" do + expect { + define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index "widgets" + + t.customize_derived_type_fields("WidgetSortOrderInput", "edges") {} + end + end + }.to raise_error Errors::SchemaError, a_string_including("customize_derived_type_fields", "WidgetSortOrderInput") + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/enum_type_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/enum_type_spec.rb new file mode 100644 index 00000000..ff60c8a2 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/enum_type_spec.rb @@ -0,0 +1,309 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "graphql_schema_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "GraphQL schema generation", "#enum_type" do + include_context "GraphQL schema spec support" + + with_both_casing_forms do + it "can generate a simple enum type" do + result = enum_type "Color" do |e| + e.value "RED" + e.value "GREEN" + e.value "BLUE" + end + + expect(result).to eq(<<~EOS) + enum Color { + RED + GREEN + BLUE + } + EOS + end + + it "allows many values to be defined in one call for convenience (but value directives are not supported)" do + result = enum_type "Color" do |e| + # they can be individually listed + e.values "RED", "GREEN" + # ...or passed as a single array + e.values %w[YELLOW ORANGE] + end + + expect(result).to eq(<<~EOS) + enum Color { + RED + GREEN + YELLOW + ORANGE + } + EOS + end + + it "can generate directives on the type" do + result = define_schema do |schema| + schema.raw_sdl "directive @foo(size: Int = null) repeatable on ENUM" + + schema.enum_type "Color" do |e| + e.directive "foo", size: 1 + e.directive "foo", size: 3 + e.value "RED" + e.value "GREEN" + e.value "BLUE" + end + end + + expect(type_def_from(result, "Color")).to eq(<<~EOS.strip) + enum Color @foo(size: 1) @foo(size: 3) { + RED + GREEN + BLUE + } + EOS + end + + it "respects a configured type name override" do + result = define_schema(type_name_overrides: {"Color" => "Hue"}) do |schema| + schema.object_type "Widget" do |t| + t.paginated_collection_field "hues", "Color" + end + + schema.enum_type "Color" do |t| + t.value "RED" + end + end + + expect(type_def_from(result, "Color")).to eq nil + expect(type_def_from(result, "Hue")).to eq(<<~EOS.strip) + enum Hue { + RED + } + EOS + + expect(type_def_from(result, "HueInput")).to eq(<<~EOS.strip) + enum HueInput { + RED + } + EOS + + expect(type_def_from(result, "HueFilterInput")).not_to eq nil + expect(type_def_from(result, "HueConnection")).not_to eq nil + expect(type_def_from(result, "HueEdge")).not_to eq nil + + # Verify that there are _no_ `Color` types defined + expect(result.lines.grep(/Color/)).to be_empty + end + + it "respects a configured enum value override" do + result = define_schema(enum_value_overrides_by_type: {Color: {RED: "PINK"}}) do |schema| + schema.enum_type "Color" do |t| + t.value "RED" + end + end + + expect(type_def_from(result, "Color")).to eq(<<~EOS.strip) + enum Color { + PINK + } + EOS + end + + it "allows the input variant to be renamed to the same name as the output variant via a type name override" do + result = define_schema(type_name_overrides: {"ColorInput" => "Color"}) do |schema| + schema.object_type "Widget" do |t| + t.paginated_collection_field "colors", "Color" + end + + schema.enum_type "Color" do |t| + t.value "RED" + end + end + + expect(type_def_from(result, "Color")).to eq(<<~EOS.strip) + enum Color { + RED + } + EOS + + expect(type_def_from(result, "ColorInput")).to eq nil + + # Verify that `ColorInput` isn't referenced anywhere in the schema. + expect(result.lines.grep(/ColorInput/)).to be_empty + end + + it "allows the input variant to be customized using `customize_derived_types`" do + result = define_schema do |schema| + schema.enum_type "Color" do |t| + t.value "RED" + + t.customize_derived_types "ColorInput" do |dt| + dt.directive "deprecated" + end + end + end + + expect(type_def_from(result, "Color")).to eq(<<~EOS.strip) + enum Color { + RED + } + EOS + + expect(type_def_from(result, "ColorInput")).to eq(<<~EOS.strip) + enum ColorInput @deprecated { + RED + } + EOS + end + + it "can generate directives on the values" do + result = define_schema do |schema| + schema.raw_sdl "directive @foo(size: Int = null) repeatable on ENUM_VALUE" + + schema.enum_type "Color" do |e| + e.value "RED" do |v| + v.directive "foo", size: 1 + v.directive "foo", size: 3 + end + e.value "GREEN" do |v| + v.directive "foo", size: 5 + end + e.value "BLUE" + end + end + + expect(type_def_from(result, "Color")).to eq(<<~EOS.strip) + enum Color { + RED @foo(size: 1) @foo(size: 3) + GREEN @foo(size: 5) + BLUE + } + EOS + end + + it "supports doc comments on the enum type and enum values" do + result = enum_type "Color" do |e| + e.documentation "The set of valid colors." + e.value "RED" do |v| + v.documentation "The color red." + end + e.value "GREEN" do |v| + v.documentation <<~EOS + The color green. + (This is multiline.) + EOS + end + e.value "BLUE" + end + + expect(result).to eq(<<~EOS) + """ + The set of valid colors. + """ + enum Color { + """ + The color red. + """ + RED + """ + The color green. + (This is multiline.) + """ + GREEN + BLUE + } + EOS + end + + it "raises a clear error when the enum type name is not formatted correctly" do + expect { + define_schema do |api| + api.enum_type("Invalid.Name") {} + end + }.to raise_invalid_graphql_name_error_for("Invalid.Name") + end + + it "raises a clear error when an enum value name is not formatted correctly" do + expect { + define_schema do |api| + api.enum_type "Color" do |e| + e.value "INVALID.NAME" + end + end + }.to raise_invalid_graphql_name_error_for("INVALID.NAME") + end + + it "raises a clear error when the type name has the type wrapping characters" do + expect { + define_schema do |api| + api.enum_type "[InvalidName!]!" do |e| + e.value "INVALID" + end + end + }.to raise_invalid_graphql_name_error_for("[InvalidName!]!") + end + + it "raises a clear error when the same type is defined multiple times" do + expect { + define_schema do |api| + api.enum_type "Color" do |e| + e.value "RED" + e.value "GREEN" + e.value "BLUE" + end + + api.enum_type "Color" do |e| + e.value "RED2" + e.value "GREEN2" + e.value "BLUE2" + end + end + }.to raise_error Errors::SchemaError, a_string_including("Duplicate", "Color") + end + + it "raises a clear error when the same value is defined multiple times" do + expect { + enum_type "Color" do |e| + e.value "RED" + e.value "GREEN" + e.value "RED" + end + }.to raise_error Errors::SchemaError, a_string_including("Duplicate", "Color", "RED") + end + + it "raises a clear error when no enum values are defined" do + expect { + enum_type "Color" do |e| + end + }.to raise_error Errors::SchemaError, a_string_including("Color", "has no values") + end + + it "raises a clear error when values are defined with spaces in them" do + expect { + enum_type "Color" do |e| + # Notice this uses `%[ ]`, not `%w[ ]`, which makes it a string, and not an array--woops. + # It therefore has unintended whitespace and is invalid. + e.values %(RED GREEN) + end + }.to raise_invalid_graphql_name_error_for("RED GREEN") + end + + def enum_type(name, *args, **options, &block) + result = define_schema do |api| + api.enum_type(name, *args, **options, &block) + end + + # We add a line break to match the expectations which use heredocs. + type_def_from(result, name, include_docs: true) + "\n" + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/field_arguments_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/field_arguments_spec.rb new file mode 100644 index 00000000..7abcba83 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/field_arguments_spec.rb @@ -0,0 +1,133 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "graphql_schema_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "GraphQL schema generation", "field arguments" do + include_context "GraphQL schema spec support" + + with_both_casing_forms do + it "allows field arguments to be defined with documentation and directives" do + result = define_schema do |api| + api.object_type "MyType" do |t| + t.field "negate", "Int!" do |f| + f.argument "x", "Int" do |a| + a.documentation "The value to negate." + a.directive "deprecated", reason: "This arg will stop being supported." + end + end + end + end + + expect(type_def_from(result, "MyType", include_docs: true)).to eq(<<~EOS.strip) + type MyType { + negate( + """ + The value to negate. + """ + x: Int @deprecated(reason: "This arg will stop being supported.")): Int! + } + EOS + end + + it "allows field arguments to be defined with default values" do + result = define_schema do |api| + api.object_type "MyType" do |t| + t.field "foo", "Int!" do |f| + f.argument "no_default", "Int!" + f.argument "default_of_null", "Int" do |a| + a.default nil + end + + f.argument "default_of_3", "Int!" do |a| + a.default 3 + end + + f.argument "default_with_directive", "Int" do |a| + a.directive "deprecated", reason: "unused" + a.default 4 + end + end + end + end + + expect(type_def_from(result, "MyType")).to eq(<<~EOS.strip) + type MyType { + foo( + no_default: Int! + default_of_null: Int = null + default_of_3: Int! = 3 + default_with_directive: Int = 4 @deprecated(reason: "unused")): Int! + } + EOS + end + + it "allows input fields to be defined with default values" do + result = define_schema do |api| + filter = api.factory.new_filter_input_type("InputWithDefaults") do |t| + t.field "no_default", "Int!" + t.field "default_of_null", "Int" do |f| + f.default nil + end + + t.field "default_of_3", "Int!" do |f| + f.default 3 + end + + t.field "default_with_directive", "Int" do |f| + f.directive "deprecated", reason: "unused" + f.default 4 + end + + t.graphql_only true + end + + api.state.register_input_type(filter) + end + + expect(type_def_from(result, "InputWithDefaultsFilterInput")).to eq(<<~EOS.strip) + input InputWithDefaultsFilterInput { + #{schema_elements.any_of}: [InputWithDefaultsFilterInput!] + not: InputWithDefaultsFilterInput + no_default: Int! + default_of_null: Int = null + default_of_3: Int! = 3 + default_with_directive: Int = 4 @deprecated(reason: "unused") + } + EOS + end + + it "does not offer the `.default` API on non-input fields (since it only makes sense on inputs)" do + expect { + define_schema do |api| + api.object_type "MyType" do |t| + t.field "default_of_null", "Int" do |f| + f.default nil + end + end + end + }.to raise_error NoMethodError, a_string_including("default", ElasticGraph::SchemaDefinition::SchemaElements::Field.name) + end + + it "raises a clear error when a field argument name is not formatted correctly" do + expect { + define_schema do |api| + api.object_type "MyType" do |t| + t.field "foo", "Int!" do |f| + f.argument "invalid.name", "Int!" + end + end + end + }.to raise_invalid_graphql_name_error_for("invalid.name") + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/filters_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/filters_spec.rb new file mode 100644 index 00000000..38f2b0ed --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/filters_spec.rb @@ -0,0 +1,1260 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "graphql_schema_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "GraphQL schema generation", "filters" do + include_context "GraphQL schema spec support" + + with_both_casing_forms do + it "defines filter types for indexed types with docs" do + result = define_schema do |api| + api.object_type "WidgetOptions" do |t| + t.field "size", "Int" + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" do |f| + f.documentation <<~EOD + The identifier. + + Another paragraph. + EOD + end + + t.field "the_options", "WidgetOptions" + t.field "cost", "Int" + + t.index "widgets" + end + end + + expect(filter_type_from(result, "Widget", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type used to specify filters on `Widget` fields. + + Will be ignored if passed as an empty object (or as `null`). + """ + input WidgetFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + #{schema_elements.any_of}: [WidgetFilterInput!] + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + #{schema_elements.not}: WidgetFilterInput + """ + Used to filter on the `id` field: + + > The identifier. + > + > Another paragraph. + + Will be ignored if `null` or an empty object is passed. + """ + id: IDFilterInput + """ + Used to filter on the `the_options` field. + + Will be ignored if `null` or an empty object is passed. + """ + the_options: WidgetOptionsFilterInput + """ + Used to filter on the `cost` field. + + Will be ignored if `null` or an empty object is passed. + """ + cost: IntFilterInput + } + EOS + end + + it "defines filter types for embedded types" do + result = define_schema do |api| + api.object_type "WidgetOptions" do |t| + t.field "size", "Int!" do |f| + f.documentation "The size of the widget." + end + + t.field "main_color", "String" + end + end + + expect(filter_type_from(result, "WidgetOptions", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type used to specify filters on `WidgetOptions` fields. + + Will be ignored if passed as an empty object (or as `null`). + """ + input WidgetOptionsFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + #{schema_elements.any_of}: [WidgetOptionsFilterInput!] + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + #{schema_elements.not}: WidgetOptionsFilterInput + """ + Used to filter on the `size` field: + + > The size of the widget. + + Will be ignored if `null` or an empty object is passed. + """ + size: IntFilterInput + """ + Used to filter on the `main_color` field. + + Will be ignored if `null` or an empty object is passed. + """ + main_color: StringFilterInput + } + EOS + + expect(list_filter_type_from(result, "WidgetOptions", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type used to specify filters on `[WidgetOptions]` fields. + + Will be ignored if passed as an empty object (or as `null`). + """ + input WidgetOptionsListFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + #{schema_elements.any_of}: [WidgetOptionsListFilterInput!] + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + #{schema_elements.not}: WidgetOptionsListFilterInput + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + #{schema_elements.any_satisfy}: WidgetOptionsFilterInput + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `WidgetOptionsListFilterInput` input because of collisions between key names. For example, if you want to provide + multiple `#{schema_elements.any_satisfy}: ...` filters, you could do `#{schema_elements.all_of}: [{#{schema_elements.any_satisfy}: ...}, {#{schema_elements.any_satisfy}: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + #{schema_elements.all_of}: [WidgetOptionsListFilterInput!] + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + #{schema_elements.count}: IntFilterInput + } + EOS + end + + it "does not define or reference a filter type for embedded object types that have no filterable fields" do + result = define_schema do |api| + api.object_type "WidgetOptions" do |t| + t.field "size", "Int", filterable: false + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "nullable_options", "WidgetOptions" + t.field "non_nullable_options", "WidgetOptions!" + t.index "widgets" + end + end + + expect(filter_type_from(result, "WidgetOptions")).to be nil + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.strip) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + id: IDFilterInput + } + EOS + end + + it "does not define or reference a filter type (or list filter type) for embedded object types that have a custom mapping type" do + result = define_schema do |api| + api.object_type "PointWithCustomMapping" do |t| + t.field "x", "Float" + t.field "y", "Float" + t.mapping type: "point" + end + + api.object_type "Point" do |t| + t.field "x", "Float" + t.field "y", "Float" + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "nullable_point_with_custom_mapping", "PointWithCustomMapping" + t.field "non_null_point_with_custom_mapping", "PointWithCustomMapping!" + t.field "nullable_point", "Point" + t.field "non_null_point", "Point!" + t.index "widgets" + end + end + + expect(filter_type_from(result, "PointWithCustomMapping")).to be nil + expect(list_filter_type_from(result, "PointWithCustomMapping")).to be nil + expect(list_element_filter_type_from(result, "PointWithCustomMapping")).to be nil + expect(filter_type_from(result, "Point")).not_to be nil + expect(list_filter_type_from(result, "Point")).not_to be nil + expect(list_element_filter_type_from(result, "Point")).to be nil + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.strip) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + id: IDFilterInput + nullable_point: PointFilterInput + non_null_point: PointFilterInput + } + EOS + end + + it "does not consider `type: object` to be a custom mapping type for an object (since that is the default)" do + result = define_schema do |api| + api.object_type "Point" do |t| + t.field "x", "Float" + t.field "y", "Float" + t.mapping type: "object" + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "nullable_point", "Point" + t.field "non_null_point", "Point!" + t.index "widgets" + end + end + + expect(filter_type_from(result, "Point")).not_to be nil + expect(list_filter_type_from(result, "Point")).not_to be nil + expect(list_element_filter_type_from(result, "Point")).to be nil + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.strip) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + id: IDFilterInput + nullable_point: PointFilterInput + non_null_point: PointFilterInput + } + EOS + end + + it "makes object fields with custom mapping options filterable so long as the `type` hasn't been customized" do + result = define_schema do |api| + api.object_type "Point" do |t| + t.field "x", "Float" + t.field "y", "Float" + t.mapping meta: {defined_by: "ElasticGraph"} + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "nullable_point", "Point" + t.field "non_null_point", "Point!" + t.index "widgets" + end + end + + expect(filter_type_from(result, "Point")).not_to be nil + expect(list_filter_type_from(result, "Point")).not_to be nil + expect(list_element_filter_type_from(result, "Point")).to be nil + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.strip) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + id: IDFilterInput + nullable_point: PointFilterInput + non_null_point: PointFilterInput + } + EOS + end + + it "does not define or reference a filter field for a relation field" do + result = define_schema do |api| + api.object_type "Component" do |t| + t.field "id", "ID" + t.relates_to_one "widget", "Widget", via: "widget_id", dir: :out + t.index "components" + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.relates_to_many "components", "Component", via: "widget_id", dir: :in, singular: "component" + t.index "widgets" + end + end + + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.strip) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + id: IDFilterInput + } + EOS + end + + it "does not recurse infinitely when dealing with self-referential relations" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID", filterable: false + t.relates_to_one "parent_widget", "Widget", via: "parent_widget_id", dir: :in + t.index "widgets" + end + end + + expect(filter_type_from(result, "Widget")).to be nil + end + + it "does not copy custom directives from the source field to the filter field because we can't be sure the directive is valid on an input field" do + result = define_schema do |api| + api.raw_sdl "directive @foo(bar: Int) on FIELD_DEFINITION" + api.raw_sdl "directive @bar on FIELD_DEFINITION" + + api.object_type "WidgetOptions" do |t| + t.field "size", "Int!" do |f| + f.directive "foo", bar: 1 + end + + t.field "main_color", "String" do |f| + f.directive "bar" + end + end + end + + # Directives are expected to define what elements they are valid on, e.g.: + # directive @key(fields: _FieldSet!) repeatable on OBJECT | INTERFACE + # + # Input objects are different enum values for the `on`, and we can't count on + # directives that are valid on object fields necessarily being valid on input + # fields, so we do not copy them to the filter fields. + # + # If there's a need to copy them forward, we can revisit this, but we'll have to + # interpret the directive definitions to know if it's safe to copy them forward, + # or allow users to configure if they get copied onto the filter. + expect(filter_type_from(result, "WidgetOptions")).to eq(<<~EOS.strip) + input WidgetOptionsFilterInput { + #{schema_elements.any_of}: [WidgetOptionsFilterInput!] + #{schema_elements.not}: WidgetOptionsFilterInput + size: IntFilterInput + main_color: StringFilterInput + } + EOS + end + + it "defines filter types for enum types with docs" do + result = define_schema do |api| + api.enum_type "Color" do |e| + e.values "RED", "GREEN", "BLUE" + end + end + + expect(filter_type_from(result, "Color", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type used to specify filters on `Color` fields. + + Will be ignored if passed as an empty object (or as `null`). + """ + input ColorFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + #{schema_elements.any_of}: [ColorFilterInput!] + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + #{schema_elements.not}: ColorFilterInput + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + #{schema_elements.equal_to_any_of}: [ColorInput] + } + EOS + + expect(list_element_filter_type_from(result, "Color", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type used to specify filters on elements of a `[Color]` field. + + Will be ignored if passed as an empty object (or as `null`). + """ + input ColorListElementFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + #{schema_elements.any_of}: [ColorListElementFilterInput!] + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + #{schema_elements.equal_to_any_of}: [ColorInput!] + } + EOS + + expect(list_filter_type_from(result, "Color", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type used to specify filters on `[Color]` fields. + + Will be ignored if passed as an empty object (or as `null`). + """ + input ColorListFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + #{schema_elements.any_of}: [ColorListFilterInput!] + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + #{schema_elements.not}: ColorListFilterInput + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + #{schema_elements.any_satisfy}: ColorListElementFilterInput + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `ColorListFilterInput` input because of collisions between key names. For example, if you want to provide + multiple `#{schema_elements.any_satisfy}: ...` filters, you could do `#{schema_elements.all_of}: [{#{schema_elements.any_satisfy}: ...}, {#{schema_elements.any_satisfy}: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + #{schema_elements.all_of}: [ColorListFilterInput!] + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + #{schema_elements.count}: IntFilterInput + } + EOS + end + + it "skips defining filters for `relates_to_one` fields" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "cost", "Int" + t.relates_to_one "inventor", "Person", via: "inventor_id", dir: :out + t.index "widgets" + end + + api.object_type "Person" do |t| + t.field "id", "ID" + t.index "people" + end + end + + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.strip) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + id: IDFilterInput + cost: IntFilterInput + } + EOS + end + + it "allows the user to opt-out a field from being filterable" do + result = define_schema do |api| + api.object_type "WidgetOptions" do |t| + t.field "size", "Int!" + t.field "main_color", "String", filterable: false + end + end + + expect(filter_type_from(result, "WidgetOptions")).to eq(<<~EOS.strip) + input WidgetOptionsFilterInput { + #{schema_elements.any_of}: [WidgetOptionsFilterInput!] + #{schema_elements.not}: WidgetOptionsFilterInput + size: IntFilterInput + } + EOS + end + + it "provides a filter field for `GeoLocation` fields in spite of their using a custom mapping since we have support for the `GeoLocationFilterInput`" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "location", "GeoLocation" + t.index "widgets" + end + end + + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.strip) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + not: WidgetFilterInput + id: IDFilterInput + location: GeoLocationFilterInput + } + EOS + end + + it "uses `TextFilterInput` instead of `StringFilterInput` for text fields (even though they are strings in GraphQL)" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "name", "String" + t.field "description", "String" do |f| + f.mapping type: "text" + end + end + end + + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.strip) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + name: StringFilterInput + description: TextFilterInput + } + EOS + end + + it "avoids generating the filter type for an indexed type that has no filterable fields" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!", filterable: false + t.index "widgets" + end + end + + expect(filter_type_from(result, "Widget")).to eq nil + end + + it "avoids generating the filter type for an embedded type that has no filterable fields" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!", filterable: false + end + end + + expect(filter_type_from(result, "Widget")).to eq nil + end + + it "allows filtering fields to be customized using a block" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" do |f| + f.documentation "id field" + + f.customize_filter_field do |ff| + ff.directive "deprecated" + end + + f.customize_filter_field do |ff| + ff.documentation "Custom filter field documentation!" + end + end + end + end + + # Demonstrate that the filtering customizations don't impact the `Widget.id` field. + expect(type_def_from(result, "Widget", include_docs: true)).to eq(<<~EOS.strip) + type Widget { + """ + id field + """ + id: ID! + } + EOS + + expect(filter_type_from(result, "Widget", include_docs: true)).to include(<<~EOS.strip) + """ + Custom filter field documentation! + """ + id: IDFilterInput @deprecated + } + EOS + end + + it "references a `*ListFilterInput` from a list-of-text-strings field on the filter type" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "tags", "[String!]!" + end + end + + expect(filter_type_from(result, "Widget", include_docs: true).lines.last(7).join).to eq(<<~EOS.chomp) + """ + Used to filter on the `tags` field. + + Will be ignored if `null` or an empty object is passed. + """ + tags: StringListFilterInput + } + EOS + end + + it "ignores nullability when deciding whether to references a `*ListFilterInput` from a list-of-scalars field on the filter type" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "tags1", "[String!]!" + t.field "tags2", "[String!]" + t.field "tags3", "[String]!" + t.field "tags4", "[String]" + end + end + + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.chomp) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + tags1: StringListFilterInput + tags2: StringListFilterInput + tags3: StringListFilterInput + tags4: StringListFilterInput + } + EOS + end + + it "references a `*ListFilterInput` from a paginated list-of-scalars field on the filter type" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.paginated_collection_field "tags", "String" + end + end + + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.chomp) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + tags: StringListFilterInput + } + EOS + end + + it "respects a configured type name override when generating the filter field from a `paginated_collection_field`" do + result = define_schema(type_name_overrides: {LocalTime: "TimeOfDay"}) do |api| + api.object_type "Widget" do |t| + t.paginated_collection_field "times", "LocalTime" + end + end + + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.chomp) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + times: TimeOfDayListFilterInput + } + EOS + end + + it "references a `*ListFilterInput` from a list-of-enums field on the filter type" do + result = define_schema do |api| + api.enum_type "Color" do |t| + t.values "RED", "GREEN", "BLUE" + end + + api.object_type "Widget" do |t| + t.field "colors", "[Color!]!" + end + end + + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.chomp) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + colors: ColorListFilterInput + } + EOS + end + + it "references a `*ListFilterInput` from a list-of-text-strings field on the filter type" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "tags", "[String!]!" do |f| + f.mapping type: "text" + end + end + end + + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.chomp) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + tags: TextListFilterInput + } + EOS + end + + it "references a `*ListFilterInput` from a list-of-geo-locations field on the filter type since it is a leaf field type in the index" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "locations", "[GeoLocation!]!" + end + end + + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.chomp) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + locations: GeoLocationListFilterInput + } + EOS + end + + it "omits a filter for a list-of-custom-object-mapping field since we don't know how to support filtering on it" do + result = define_schema do |api| + api.object_type "Shape" do |t| + t.field "type", "String" + t.field "coordinates", "[Float!]!" + t.mapping type: "geo_shape" + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "shapes", "[Shape!]!" + end + end + + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.chomp) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + id: IDFilterInput + } + EOS + end + + it "references a `*ListFilterInput` from a list-of-nested-objects field on the filter type since nested documents get separately indexed" do + result = define_schema do |api| + api.object_type "Shape" do |t| + t.field "type", "String" + t.field "coordinates", "[Float!]!" + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "shapes", "[Shape!]!" do |f| + f.mapping type: "nested" + end + end + end + + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.chomp) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + id: IDFilterInput + shapes: ShapeListFilterInput + } + EOS + end + + it "references the `*FieldsListFilterInput` from a list-of-embedded objects field on the filter type to make `any_satisfy` show up where we want it to" do + result = define_schema do |api| + api.object_type "Shape" do |t| + t.field "type", "String" + t.field "coordinates", "[Float!]!" + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "shapes", "[Shape!]!" do |f| + f.mapping type: "object" + end + end + end + + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.chomp) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + id: IDFilterInput + shapes: ShapeFieldsListFilterInput + } + EOS + end + + it "references a `*ListFilterInput` from a `paginated_collection_field` that is mapped with `nested`" do + result = define_schema do |api| + api.object_type "PlayerSeason" do |t| + t.field "year", "Int" + end + + api.object_type "Player" do |t| + t.paginated_collection_field "seasons", "PlayerSeason" do |f| + f.mapping type: "nested" + end + end + + api.object_type "Team" do |t| + t.field "id", "ID!" + t.field "players", "[Player]" do |f| + f.mapping type: "object" + end + end + end + + expect(fields_list_filter_type_from(result, "Player")).to eq(<<~EOS.strip) + input PlayerFieldsListFilterInput { + #{schema_elements.any_of}: [PlayerFieldsListFilterInput!] + #{schema_elements.not}: PlayerFieldsListFilterInput + seasons: PlayerSeasonListFilterInput + #{schema_elements.count}: IntFilterInput + } + EOS + end + + it "respects a type name override when generating the `*ListFilterInput` field from a `paginated_collection_field` that is mapped with `nested`" do + result = define_schema(type_name_overrides: {PlayerSeason: "SeasonForAPlayer"}) do |api| + api.object_type "PlayerSeason" do |t| + t.field "year", "Int" + end + + api.object_type "Player" do |t| + t.paginated_collection_field "seasons", "PlayerSeason" do |f| + f.mapping type: "nested" + end + end + + api.object_type "Team" do |t| + t.field "id", "ID!" + t.field "players", "[Player]" do |f| + f.mapping type: "object" + end + end + end + + expect(fields_list_filter_type_from(result, "Player")).to eq(<<~EOS.strip) + input PlayerFieldsListFilterInput { + #{schema_elements.any_of}: [PlayerFieldsListFilterInput!] + #{schema_elements.not}: PlayerFieldsListFilterInput + seasons: SeasonForAPlayerListFilterInput + #{schema_elements.count}: IntFilterInput + } + EOS + end + + describe "`*FieldsListFilterInput` types" do + it "documents how it differs from other filter types" do + result = define_schema do |api| + api.object_type "WidgetOptions" do |t| + t.field "int", "Int" + end + end + + expect(fields_list_filter_type_from(result, "WidgetOptions", include_docs: true).lines.first(8).join).to eq(<<~EOS) + """ + Input type used to specify filters on a `WidgetOptions` object referenced directly + or transitively from a list field that has been configured to index each leaf field as + its own flattened list of values. + + Will be ignored if passed as an empty object (or as `null`). + """ + input WidgetOptionsFieldsListFilterInput { + EOS + end + + it "defines a `*ListFilterInput` field for each scalar or enum field, regardless of it is a list or singleton value field, and regardless of nullability" do + result = define_schema do |api| + api.enum_type "Color" do |t| + t.values "RED", "BLUE", "GREEN" + end + + api.object_type "WidgetOptions" do |t| + t.field "single_color1", "Color" + t.field "single_color2", "Color!" + t.field "colors1", "[Color]" + t.field "colors2", "[Color!]" + t.field "colors3", "[Color]!" + t.field "colors4", "[Color!]!" + t.field "single_int1", "Int" + t.field "single_int2", "Int!" + t.field "ints1", "[Int]" + t.field "ints2", "[Int!]" + t.field "ints3", "[Int]!" + t.field "ints4", "[Int!]!" + end + end + + expect(fields_list_filter_type_from(result, "WidgetOptions")).to eq(<<~EOS.strip) + input WidgetOptionsFieldsListFilterInput { + #{schema_elements.any_of}: [WidgetOptionsFieldsListFilterInput!] + #{schema_elements.not}: WidgetOptionsFieldsListFilterInput + single_color1: ColorListFilterInput + single_color2: ColorListFilterInput + colors1: ColorListFilterInput + colors2: ColorListFilterInput + colors3: ColorListFilterInput + colors4: ColorListFilterInput + single_int1: IntListFilterInput + single_int2: IntListFilterInput + ints1: IntListFilterInput + ints2: IntListFilterInput + ints3: IntListFilterInput + ints4: IntListFilterInput + #{schema_elements.count}: IntFilterInput + } + EOS + end + + it "defines a `*ListFilterInput` field for each object, regardless of it is a list or singleton value field, and regardless of nullability" do + result = define_schema do |api| + api.object_type "WidgetOptions" do |t| + t.field "size", "Int" + end + + api.object_type "Widget" do |t| + t.field "options", "WidgetOptions" + t.field "embedded_options_list", "[WidgetOptions]" do |f| + f.mapping type: "object" + end + t.field "nested_options_list", "[WidgetOptions]" do |f| + f.mapping type: "nested" + end + end + end + + expect(fields_list_filter_type_from(result, "Widget")).to eq(<<~EOS.strip) + input WidgetFieldsListFilterInput { + #{schema_elements.any_of}: [WidgetFieldsListFilterInput!] + #{schema_elements.not}: WidgetFieldsListFilterInput + options: WidgetOptionsFieldsListFilterInput + embedded_options_list: WidgetOptionsFieldsListFilterInput + nested_options_list: WidgetOptionsListFilterInput + #{schema_elements.count}: IntFilterInput + } + EOS + end + + it "treats `GeoLocation` fields like a scalar field rather than an object field, since it is a leaf field in the index" do + result = define_schema do |api| + api.object_type "WidgetOptions" do |t| + t.field "geo_location", "GeoLocation" + t.field "geo_locations", "[GeoLocation]" + end + end + + expect(fields_list_filter_type_from(result, "WidgetOptions")).to eq(<<~EOS.strip) + input WidgetOptionsFieldsListFilterInput { + #{schema_elements.any_of}: [WidgetOptionsFieldsListFilterInput!] + #{schema_elements.not}: WidgetOptionsFieldsListFilterInput + geo_location: GeoLocationListFilterInput + geo_locations: GeoLocationListFilterInput + #{schema_elements.count}: IntFilterInput + } + EOS + end + + it "omits fields which are not filterable" do + result = define_schema do |api| + api.object_type "WidgetOptions" do |t| + t.field "int1", "Int", filterable: false + t.field "int2", "Int" + end + end + + expect(fields_list_filter_type_from(result, "WidgetOptions")).to eq(<<~EOS.strip) + input WidgetOptionsFieldsListFilterInput { + #{schema_elements.any_of}: [WidgetOptionsFieldsListFilterInput!] + #{schema_elements.not}: WidgetOptionsFieldsListFilterInput + int2: IntListFilterInput + #{schema_elements.count}: IntFilterInput + } + EOS + end + end + + it "forces the user to decide if they want to use `object` or `nested` for the mapping type of a list-of-objects field" do + expect { + define_schema do |api| + api.object_type "WidgetOptions" do |t| + t.field "int", "Int" + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "options", "[WidgetOptions]" + t.index "widgets" + end + end + }.to raise_error Errors::SchemaError, a_string_including( + "`Widget.options` is a list-of-objects field, but the mapping type has not been explicitly specified" + ) + end + + shared_examples_for "a type with subtypes" do |type_def_method| + it "defines a filter using the set union of the fields of the subtypes" do + result = define_schema do |api| + api.object_type "Person" do |t| + link_subtype_to_supertype(t, "Inventor") + t.field "name", "String" + t.field "nationality", "String" + end + + api.object_type "Company" do |t| + link_subtype_to_supertype(t, "Inventor") + t.field "name", "String" + t.field "stock_ticker", "String" + end + + api.public_send type_def_method, "Inventor" do |t| + link_supertype_to_subtypes(t, "Person", "Company") + end + end + + # Note: we would like to support filtering on `__typename` but this is invalid according to the + # GraphQL Spec: http://spec.graphql.org/June2018/#sec-Input-Objects + # > For each input field of an Input Object type: + # > 2. The input field must not have a name which begins with the characters "__" (two underscores). + expect(filter_type_from(result, "Inventor")).to eq(<<~EOS.strip) + input InventorFilterInput { + #{schema_elements.any_of}: [InventorFilterInput!] + #{schema_elements.not}: InventorFilterInput + name: StringFilterInput + nationality: StringFilterInput + stock_ticker: StringFilterInput + } + EOS + end + + it "raises a clear error if overlapping fields have different types (meaning we can't define a shared filter field for it)" do + expect { + define_schema do |api| + api.enum_type "ShirtSize" do |t| + t.values "S", "M", "L" + end + + api.enum_type "PantsSize" do |t| + t.values "M", "L", "XL" + end + + api.object_type "Shirt" do |t| + link_subtype_to_supertype(t, "ClothingItem") + t.field "size", "ShirtSize" + t.field "shirt_color", "String" + end + + api.object_type "Pants" do |t| + link_subtype_to_supertype(t, "ClothingItem") + t.field "size", "PantsSize" + t.field "pants_color", "String" + end + + api.public_send type_def_method, "ClothingItem" do |t| + link_supertype_to_subtypes(t, "Shirt", "Pants") + end + end + }.to raise_error(Errors::SchemaError, a_string_including("Conflicting definitions", "field `size`", "subtypes of `ClothingItem`", "Shirt", "Pants")) + end + + it "does not raise an error if the difference in type for overlapping fields is just nullable vs non-nullable since filter fields are all nullable anyway" do + result = define_schema do |api| + api.enum_type "Size" do |t| + t.values "S", "M", "L" + end + + api.object_type "Shirt" do |t| + link_subtype_to_supertype(t, "ClothingItem") + t.field "size", "Size!" + t.field "shirt_color", "String" + end + + api.object_type "Pants" do |t| + link_subtype_to_supertype(t, "ClothingItem") + t.field "size", "Size" + t.field "pants_color", "String" + end + + api.public_send type_def_method, "ClothingItem" do |t| + link_supertype_to_subtypes(t, "Shirt", "Pants") + end + end + + expect(filter_type_from(result, "ClothingItem")).to eq(<<~EOS.strip) + input ClothingItemFilterInput { + #{schema_elements.any_of}: [ClothingItemFilterInput!] + #{schema_elements.not}: ClothingItemFilterInput + size: SizeFilterInput + shirt_color: StringFilterInput + pants_color: StringFilterInput + } + EOS + end + + it "still raises an error if the difference in type is list vs scalar" do + expect { + define_schema do |api| + api.enum_type "Size" do |t| + t.values "S", "M", "L" + end + + api.object_type "Shirt" do |t| + link_subtype_to_supertype(t, "ClothingItem") + t.field "size", "Size" + t.field "shirt_color", "String" + end + + api.object_type "Pants" do |t| + link_subtype_to_supertype(t, "ClothingItem") + t.field "size", "[Size]" + t.field "pants_color", "String" + end + + api.public_send type_def_method, "ClothingItem" do |t| + link_supertype_to_subtypes(t, "Shirt", "Pants") + end + end + }.to raise_error(Errors::SchemaError, a_string_including("Conflicting definitions", "field `size`", "subtypes of `ClothingItem`", "Shirt", "Pants")) + end + + it "still excludes `filterable: false` fields from the generated filter type" do + result = define_schema do |api| + api.object_type "Person" do |t| + link_subtype_to_supertype(t, "Inventor") + t.field "name", "String" + t.field "nationality", "String", filterable: false + end + + api.object_type "Company" do |t| + link_subtype_to_supertype(t, "Inventor") + t.field "name", "String" + t.field "stock_ticker", "String" + end + + api.public_send type_def_method, "Inventor" do |t| + link_supertype_to_subtypes(t, "Person", "Company") + end + end + + expect(filter_type_from(result, "Inventor")).to eq(<<~EOS.strip) + input InventorFilterInput { + #{schema_elements.any_of}: [InventorFilterInput!] + #{schema_elements.not}: InventorFilterInput + name: StringFilterInput + stock_ticker: StringFilterInput + } + EOS + end + end + + context "on a type union" do + include_examples "a type with subtypes", :union_type do + def link_subtype_to_supertype(object_type, supertype_name) + # nothing to do; the linkage happens via a `subtypes` call on the supertype + end + + def link_supertype_to_subtypes(union_type, *subtype_names) + union_type.subtypes(*subtype_names) + end + end + end + + context "on an interface type" do + include_examples "a type with subtypes", :interface_type do + def link_subtype_to_supertype(object_type, interface_name) + object_type.implements interface_name + end + + def link_supertype_to_subtypes(interface_type, *subtype_names) + # nothing to do; the linkage happens via an `implements` call on the subtype + end + end + + it "recursively resolves the union of fields, to support type hierarchies" do + result = define_schema do |api| + api.object_type "Person" do |t| + t.implements "Human" + t.field "name", "String" + t.field "nationality", "String" + end + + api.object_type "Company" do |t| + t.implements "Organization" + t.field "name", "String" + t.field "stock_ticker", "String" + end + + api.interface_type "Human" do |t| + t.implements "Inventor" + end + + api.interface_type "Organization" do |t| + t.implements "Inventor" + end + + api.interface_type "Inventor" do |t| + end + end + + expect(filter_type_from(result, "Inventor")).to eq(<<~EOS.strip) + input InventorFilterInput { + #{schema_elements.any_of}: [InventorFilterInput!] + #{schema_elements.not}: InventorFilterInput + name: StringFilterInput + nationality: StringFilterInput + stock_ticker: StringFilterInput + } + EOS + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/graphql_schema_spec_support.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/graphql_schema_spec_support.rb new file mode 100644 index 00000000..bf2b3f0c --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/graphql_schema_spec_support.rb @@ -0,0 +1,122 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" +require "elastic_graph/spec_support/schema_definition_helpers" +require "graphql" + +module ElasticGraph + module SchemaDefinition + ::RSpec.shared_context "GraphQL schema spec support" do + include_context "SchemaDefinitionHelpers" + + def self.with_both_casing_forms(&block) + context "with schema elements configured to use camelCase" do + before(:context) { @schema_elements = SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: "camelCase") } + attr_reader :schema_elements + module_exec(:camelCase, &block) + end + + context "with schema elements configured to use snake_case" do + before(:context) { @schema_elements = SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: "snake_case") } + attr_reader :schema_elements + module_exec(:snake_case, &block) + end + end + + def raise_invalid_graphql_name_error_for(name) + raise_error Errors::InvalidGraphQLNameError, a_string_including("Not a valid GraphQL name: `#{name}`", GRAPHQL_NAME_VALIDITY_DESCRIPTION) + end + + def define_schema(**options, &block) + define_schema_with_schema_elements(schema_elements, **options, &block).graphql_schema_string + end + + def correctly_cased(name) + schema_elements.normalize_case(name) + end + + def types_defined_in(schema_string) + ::GraphQL::Schema.from_definition(schema_string).types.keys + end + + def grouped_by_type_from(sdl, source_type, include_docs: false) + derived_graphql_type_def_from(sdl, source_type, "GroupedBy", include_docs: include_docs) + end + + def aggregated_values_type_from(sdl, source_type, include_docs: false) + derived_graphql_type_def_from(sdl, source_type, "AggregatedValues", include_docs: include_docs) + end + + def connection_type_from(sdl, source_type, include_docs: false) + derived_graphql_type_def_from(sdl, source_type, "Connection", include_docs: include_docs) + end + + def edge_type_from(sdl, source_type, include_docs: false) + derived_graphql_type_def_from(sdl, source_type, "Edge", include_docs: include_docs) + end + + def list_filter_type_from(sdl, source_type, include_docs: false) + filter_type_from(sdl, "#{source_type}List", include_docs: include_docs) + end + + def list_element_filter_type_from(sdl, source_type, include_docs: false) + filter_type_from(sdl, "#{source_type}ListElement", include_docs: include_docs) + end + + def fields_list_filter_type_from(sdl, source_type, include_docs: false) + filter_type_from(sdl, "#{source_type}FieldsList", include_docs: include_docs) + end + + def filter_type_from(sdl, source_type, include_docs: false) + derived_graphql_type_def_from(sdl, source_type, "FilterInput", include_docs: include_docs) + end + + def aggregation_type_from(sdl, source_type, include_docs: false) + derived_graphql_type_def_from(sdl, source_type, "Aggregation", include_docs: include_docs) + end + + def sub_aggregation_type_from(sdl, source_type, include_docs: false) + derived_graphql_type_def_from(sdl, source_type, "SubAggregation", include_docs: include_docs) + end + + def aggregation_sub_aggregations_type_from(sdl, source_type, under: nil, include_docs: false) + derived_graphql_type_def_from(sdl, source_type, "Aggregation#{under}SubAggregations", include_docs: include_docs) + end + + def sub_aggregation_sub_aggregations_type_from(sdl, source_type, under: nil, include_docs: false) + derived_graphql_type_def_from(sdl, source_type, "SubAggregation#{under}SubAggregations", include_docs: include_docs) + end + + def aggregation_connection_type_from(sdl, source_type, include_docs: false) + connection_type_from(sdl, "#{source_type}Aggregation", include_docs: include_docs) + end + + def sub_aggregation_connection_type_from(sdl, source_type, include_docs: false) + connection_type_from(sdl, "#{source_type}SubAggregation", include_docs: include_docs) + end + + def aggregation_edge_type_from(sdl, source_type, include_docs: false) + edge_type_from(sdl, "#{source_type}Aggregation", include_docs: include_docs) + end + + def sub_aggregation_edge_type_from(sdl, source_type, include_docs: false) + edge_type_from(sdl, "#{source_type}SubAggregation", include_docs: include_docs) + end + + def sort_order_type_from(sdl, source_type, include_docs: false) + derived_graphql_type_def_from(sdl, source_type, "SortOrderInput", include_docs: include_docs) + end + + def derived_graphql_type_def_from(sdl, source_type, derived_graphql_type_suffix, include_docs: false) + type_def_from(sdl, "#{source_type}#{derived_graphql_type_suffix}", include_docs: include_docs) + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/implements_shared_examples.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/implements_shared_examples.rb new file mode 100644 index 00000000..2ae13034 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/implements_shared_examples.rb @@ -0,0 +1,364 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaDefinition + RSpec.shared_examples_for "#implements" do |graphql_definition_keyword:, ruby_definition_method:| + it "generates the correct `implements` syntax in the GraphQL SDL" do + result = define_schema do |schema| + schema.interface_type "HasID" do |t| + t.field "id", "ID!" + end + + schema.interface_type "HasName" do |t| + t.field "name", "String" + end + + schema.interface_type "HasColor" do |t| + t.field "color", "String" + end + + schema.public_send ruby_definition_method, "Thing" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "color", "String" + + t.implements "HasID", "HasName" + t.implements "HasColor" + end + end + + expect(type_def_from(result, "Thing")).to eq(<<~EOS.strip) + #{graphql_definition_keyword} Thing implements HasID & HasName & HasColor { + id: ID! + name: String + color: String + } + EOS + end + + it "allows the `implements` call to come before the interface definition or the field implementations" do + result = define_schema do |schema| + schema.public_send ruby_definition_method, "Thing" do |t| + t.implements "HasID" + t.field "id", "ID!" + t.field "name", "String" + end + + schema.interface_type "HasID" do |t| + t.field "id", "ID!" + end + end + + expect(type_def_from(result, "Thing")).to eq(<<~EOS.strip) + #{graphql_definition_keyword} Thing implements HasID { + id: ID! + name: String + } + EOS + end + + it "raises a clear error when the type name is not formatted correctly" do + expect { + define_schema do |schema| + schema.public_send(ruby_definition_method, "Invalid.Name") {} + end + }.to raise_invalid_graphql_name_error_for("Invalid.Name") + end + + it "raises a clear error when the type name is not formatted correctly" do + expect { + define_schema do |schema| + schema.public_send ruby_definition_method, "Thing" do |t| + t.field "foo", "Invalid.Type!" + end + end + }.to raise_invalid_graphql_name_error_for("Invalid.Type") + end + + it "only allows wrapping characters ([]!) at their appropriate positions" do + expect { + define_schema do |schema| + schema.public_send ruby_definition_method, "Thing" do |t| + t.field "foo", "Invalid[Type!" + end + end + }.to raise_invalid_graphql_name_error_for("Invalid[Type") + + expect { + define_schema do |schema| + schema.public_send ruby_definition_method, "Thing" do |t| + t.field "foo", "Invalid]Type!" + end + end + }.to raise_invalid_graphql_name_error_for("Invalid]Type") + + expect { + define_schema do |schema| + schema.public_send ruby_definition_method, "Thing" do |t| + t.field "foo", "Invalid!Type!" + end + end + }.to raise_invalid_graphql_name_error_for("Invalid!Type") + end + + it "raises a clear error when the named type does not exist" do + expect { + define_schema do |schema| + schema.public_send ruby_definition_method, "Thing" do |t| + t.field "id", "ID!" + t.implements "HasID" + end + end + }.to raise_error Errors::SchemaError, a_string_including("Thing", "`HasID` is not defined") + end + + it "raises a clear error when the named type is not an interface" do + expect { + define_schema do |schema| + schema.enum_type "HasID" do |t| + t.value "FOO" + end + + schema.public_send ruby_definition_method, "Thing" do |t| + t.field "id", "ID!" + t.implements "HasID" + end + end + }.to raise_error Errors::SchemaError, a_string_including("Thing", "`HasID` is not an interface") + end + + it "raises a clear error when an object type does not implement one of the interface fields" do + expect { + define_schema do |schema| + schema.interface_type "HasID" do |t| + t.field "id", "ID!" + end + + schema.public_send ruby_definition_method, "Thing" do |t| + t.field "name", "String" + t.field "color", "String" + + t.implements "HasID" + end + end + }.to raise_error Errors::SchemaError, a_string_including("Thing", "HasID", "missing `id`") + end + + it "raises a clear error when the object type implements one of the interface fields with the wrong type" do + expect { + define_schema do |schema| + schema.interface_type "HasName" do |t| + t.field "name", "String" + end + + schema.public_send ruby_definition_method, "Thing" do |t| + t.field "name", "Int" + t.field "color", "String" + + t.implements "HasName" + end + end + }.to raise_error Errors::SchemaError, a_string_including("Thing", "HasName", "`name: String` vs `name: Int`") + end + + it "raises a clear error if the object and interface fields have different argument names" do + expect { + define_schema do |schema| + schema.interface_type "HasName" do |t| + t.field "name", "String" do |f| + f.argument "truncate_at", "Int" + end + end + + schema.public_send ruby_definition_method, "Thing" do |t| + t.field "name", "String" do |f| + f.argument "truncate_after", "Int" + end + t.field "color", "String" + + t.implements "HasName" + end + end + }.to raise_error Errors::SchemaError, a_string_including("Thing", "HasName", "`name(truncate_at: Int): String` vs `name(truncate_after: Int): String") + end + + it "raises a clear error if the object and interface fields have different argument values" do + expect { + define_schema do |schema| + schema.interface_type "HasName" do |t| + t.field "name", "String" do |f| + f.argument "truncate_at", "Int" + end + end + + schema.public_send ruby_definition_method, "Thing" do |t| + t.field "name", "String" do |f| + f.argument "truncate_at", "String" + end + t.field "color", "String" + + t.implements "HasName" + end + end + }.to raise_error Errors::SchemaError, a_string_including("Thing", "HasName", "`name(truncate_at: Int): String` vs `name(truncate_at: String): String") + end + + it "does not care if the interface and object fields have different documentation" do + result = define_schema do |schema| + schema.public_send ruby_definition_method, "Thing" do |t| + t.implements "HasID" + t.field "id", "ID!" do |f| + f.documentation "Thing docs" + end + t.field "name", "String" + end + + schema.interface_type "HasID" do |t| + t.field "id", "ID!" do |f| + f.documentation "HasID docs" + end + end + end + + expect(type_def_from(result, "Thing")).to eq(<<~EOS.strip) + #{graphql_definition_keyword} Thing implements HasID { + id: ID! + name: String + } + EOS + end + + it "does not care if the interface and object fields have different directives" do + result = define_schema do |schema| + schema.public_send ruby_definition_method, "Thing" do |t| + t.implements "HasID" + t.field "id", "ID!" + t.field "name", "String" + end + + schema.interface_type "HasID" do |t| + t.field "id", "ID!" do |f| + f.directive "deprecated" + end + end + end + + expect(type_def_from(result, "Thing")).to eq(<<~EOS.strip) + #{graphql_definition_keyword} Thing implements HasID { + id: ID! + name: String + } + EOS + end + + it "does not care if the interface and object fields have different JSON schema" do + result = define_schema do |schema| + schema.public_send ruby_definition_method, "Thing" do |t| + t.implements "HasID" + t.field "id", "ID!" do |f| + f.json_schema maxLength: 40 + end + t.field "name", "String" + end + + schema.interface_type "HasID" do |t| + t.field "id", "ID!" do |f| + f.json_schema maxLength: 30 + end + end + end + + expect(type_def_from(result, "Thing")).to eq(<<~EOS.strip) + #{graphql_definition_keyword} Thing implements HasID { + id: ID! + name: String + } + EOS + end + + it "does not care if the interface and object fields have different index mappings" do + result = define_schema do |schema| + schema.public_send ruby_definition_method, "Thing" do |t| + t.implements "HasID" + t.field "id", "ID!" do |f| + f.mapping type: "text" + end + t.field "name", "String" + end + + schema.interface_type "HasID" do |t| + t.field "id", "ID!" do |f| + f.mapping type: "keyword" + end + end + end + + expect(type_def_from(result, "Thing")).to eq(<<~EOS.strip) + #{graphql_definition_keyword} Thing implements HasID { + id: ID! + name: String + } + EOS + end + + it "does not care if the interface and object fields have different ElasticGraph abilities" do + result = define_schema do |schema| + schema.public_send ruby_definition_method, "Thing" do |t| + t.implements "HasName" + t.field "name", "String", sortable: false, groupable: false, filterable: false + end + + schema.interface_type "HasName" do |t| + t.field "name", "String", sortable: true, groupable: true, filterable: true + end + end + + expect(type_def_from(result, "Thing")).to eq(<<~EOS.strip) + #{graphql_definition_keyword} Thing implements HasName { + name: String + } + EOS + end + + it "does not care if one subtype has extra fields that another subtypes lacks" do + result = define_schema do |schema| + schema.public_send ruby_definition_method, "Thing1" do |t| + t.implements "HasName" + t.field "name", "String", sortable: false, groupable: false, filterable: false + t.field "thing1", "Int" + end + + schema.public_send ruby_definition_method, "Thing2" do |t| + t.implements "HasName" + t.field "name", "String", sortable: false, groupable: false, filterable: false + t.field "thing2", "Int" + end + + schema.interface_type "HasName" do |t| + t.field "name", "String", sortable: true, groupable: true, filterable: true + end + end + + expect(type_def_from(result, "Thing1")).to eq(<<~EOS.strip) + #{graphql_definition_keyword} Thing1 implements HasName { + name: String + thing1: Int + } + EOS + + expect(type_def_from(result, "Thing2")).to eq(<<~EOS.strip) + #{graphql_definition_keyword} Thing2 implements HasName { + name: String + thing2: Int + } + EOS + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/indexed_aggregation_type_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/indexed_aggregation_type_spec.rb new file mode 100644 index 00000000..d91d5877 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/indexed_aggregation_type_spec.rb @@ -0,0 +1,287 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "graphql_schema_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "GraphQL schema generation", "an indexed `*Aggregation` type" do + include_context "GraphQL schema spec support" + + with_both_casing_forms do + it "is defined for an indexed object type" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "name", "String" + t.field "cost", "Int" + t.index "widgets" + end + end + + expect(aggregation_type_from(results, "Widget", include_docs: true)).to eq(<<~EOS.strip) + """ + Return type representing a bucket of `Widget` documents for an aggregations query. + """ + type WidgetAggregation { + """ + Used to specify the `Widget` fields to group by. The returned values identify each aggregation bucket. + """ + #{schema_elements.grouped_by}: WidgetGroupedBy + """ + The count of `Widget` documents in an aggregation bucket. + """ + count: JsonSafeLong! + """ + Provides computed aggregated values over all `Widget` documents in an aggregation bucket. + """ + #{schema_elements.aggregated_values}: WidgetAggregatedValues + } + EOS + end + + it "includes a `sub_aggregations` field when the indexed type has `nested` fields" do + results = define_schema do |schema| + schema.object_type "Player" do |t| + t.field "name", "String" + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "players", "[Player!]!" do |f| + f.mapping type: "nested" + end + t.index "teams" + end + end + + expect(aggregation_type_from(results, "Team", include_docs: true)).to eq(<<~EOS.strip) + """ + Return type representing a bucket of `Team` documents for an aggregations query. + """ + type TeamAggregation { + """ + Used to specify the `Team` fields to group by. The returned values identify each aggregation bucket. + """ + #{schema_elements.grouped_by}: TeamGroupedBy + """ + The count of `Team` documents in an aggregation bucket. + """ + #{schema_elements.count}: JsonSafeLong! + """ + Provides computed aggregated values over all `Team` documents in an aggregation bucket. + """ + #{schema_elements.aggregated_values}: TeamAggregatedValues + """ + Used to perform sub-aggregations of `TeamAggregation` data. + """ + #{schema_elements.sub_aggregations}: TeamAggregationSubAggregations + } + EOS + end + + it "is defined for an indexed union type" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "name", "String" + t.field "cost", "Int" + end + + schema.union_type "Entity" do |t| + t.subtype "Widget" + t.index "entities" + end + end + + expect(aggregation_type_from(results, "Entity")).to eq(<<~EOS.strip) + type EntityAggregation { + #{schema_elements.grouped_by}: EntityGroupedBy + count: JsonSafeLong! + #{schema_elements.aggregated_values}: EntityAggregatedValues + } + EOS + end + + it "is defined for an indexed interface type" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.implements "Named" + t.field "id", "ID" + t.field "name", "String" + t.field "cost", "Int" + end + + schema.interface_type "Named" do |t| + t.field "name", "String" + t.index "named" + end + end + + expect(aggregation_type_from(results, "Named")).to eq(<<~EOS.strip) + type NamedAggregation { + #{schema_elements.grouped_by}: NamedGroupedBy + count: JsonSafeLong! + #{schema_elements.aggregated_values}: NamedAggregatedValues + } + EOS + end + + it "is not defined for an object type that is not indexed" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "name", "String" + t.field "cost", "Int" + end + end + + expect(aggregation_type_from(results, "Widget")).to eq nil + end + + it "is not defined for a union type that is not indexed" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "name", "String" + t.field "cost", "Int" + end + + schema.union_type "Entity" do |t| + t.subtype "Widget" + end + end + + expect(aggregation_type_from(results, "Entity")).to eq(nil) + end + + it "is not defined for an interface type that is not indexed" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.implements "Named" + t.field "id", "ID" + t.field "name", "String" + t.field "cost", "Int" + end + + schema.interface_type "Named" do |t| + t.field "name", "String" + end + end + + expect(aggregation_type_from(results, "Named")).to eq(nil) + end + + it "omits the `grouped_by` field if there are no fields to group by" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" # id is automatically not groupable, because it uniquely identifies each document! + t.field "cost", "Int", groupable: false + t.index "widgets" + end + end + + expect(aggregation_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregation { + count: JsonSafeLong! + #{schema_elements.aggregated_values}: WidgetAggregatedValues + } + EOS + end + + it "omits the `aggregated_values` field if there are no fields to aggregate" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID", aggregatable: false + t.field "name", "String", aggregatable: false + t.index "widgets" + end + end + + expect(aggregation_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregation { + #{schema_elements.grouped_by}: WidgetGroupedBy + count: JsonSafeLong! + } + EOS + end + + it "has only the `count` field if no fields are groupable or aggregatable" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID", aggregatable: false, groupable: false + t.index "widgets" + end + end + + expect(aggregation_type_from(results, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregation { + count: JsonSafeLong! + } + EOS + end + + it "also defines the `AggregationConnection` and `AggregationEdge` types to support relay pagination" do + results = define_schema do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "name", "String" + t.field "cost", "Int" + t.index "widgets" + end + end + + expect(aggregation_connection_type_from(results, "Widget", include_docs: true)).to eq(<<~EOS.strip) + """ + Represents a paginated collection of `WidgetAggregation` results. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. + """ + type WidgetAggregationConnection { + """ + Wraps a specific `WidgetAggregation` to pair it with its pagination cursor. + """ + edges: [WidgetAggregationEdge!]! + """ + The list of `WidgetAggregation` results. + """ + nodes: [WidgetAggregation!]! + """ + Provides pagination-related information. + """ + #{schema_elements.page_info}: PageInfo! + } + EOS + + expect(aggregation_edge_type_from(results, "Widget", include_docs: true)).to eq(<<~EOS.strip) + """ + Represents a specific `WidgetAggregation` in the context of a `WidgetAggregationConnection`, + providing access to both the `WidgetAggregation` and a pagination `Cursor`. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. + """ + type WidgetAggregationEdge { + """ + The `WidgetAggregation` of this edge. + """ + node: WidgetAggregation + """ + The `Cursor` of this `WidgetAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `WidgetAggregation`. + """ + cursor: Cursor + } + EOS + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/interface_type_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/interface_type_spec.rb new file mode 100644 index 00000000..4b580e13 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/interface_type_spec.rb @@ -0,0 +1,118 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "graphql_schema_spec_support" +require_relative "implements_shared_examples" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "GraphQL schema generation", "#interface_type" do + include_context "GraphQL schema spec support" + + with_both_casing_forms do + it "acts like `object_type` but defines a GraphQL `interface` instead of a GraphQL `type`" do + result = define_schema do |schema| + schema.interface_type "Named" do |t| + t.documentation "A type that has a name." + t.field "name", "String" do |f| + f.documentation "The name of the object." + end + end + end + + expect(type_def_from(result, "Named", include_docs: true)).to eq(<<~EOS.strip) + """ + A type that has a name. + """ + interface Named { + """ + The name of the object. + """ + name: String + } + EOS + end + + it "raises a clear error if some subtypes are indexed and others are not" do + expect { + define_schema do |api| + api.object_type("Person") do |t| + t.field "id", "ID" + t.field "name", "String" + t.implements "Inventor" + t.index "people" + end + + api.object_type("Company") do |t| + t.field "name", "String" + t.implements "Inventor" + end + + api.interface_type "Inventor" do |t| + t.field "name", "String" + end + end + }.to raise_error Errors::SchemaError, a_string_including("Inventor", "indexed") + end + + it "respects a configured type name override" do + result = define_schema(type_name_overrides: {"Named" => "Nameable"}) do |schema| + schema.interface_type "Named" do |t| + t.field "name", "String" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "name", "String" + t.implements "Named" + t.index "widgets" + end + end + + expect(type_def_from(result, "Named")).to eq nil + expect(type_def_from(result, "Nameable")).to eq(<<~EOS.strip) + interface Nameable { + name: String + } + EOS + expect(type_def_from(result, "Widget")).to eq(<<~EOS.strip) + type Widget implements Nameable { + id: ID + name: String + } + EOS + + expect(type_def_from(result, "NameableFilterInput")).not_to eq nil + expect(type_def_from(result, "NameableConnection")).not_to eq nil + expect(type_def_from(result, "NameableEdge")).not_to eq nil + + # Verify that there are _no_ `Named` types defined + expect(result.lines.grep(/Named/)).to be_empty + end + + %w[not all_of any_of any_satisfy].each do |field_name| + it "produces a clear error if a `#{field_name}` field is defined since that will conflict with the filtering operators" do + expect { + define_schema do |schema| + schema.interface_type "WidgetOptions" do |t| + t.field schema_elements.public_send(field_name), "String" + end + end + }.to raise_error Errors::SchemaError, a_string_including("WidgetOptions.#{schema_elements.public_send(field_name)}", "reserved") + end + end + + describe "#implements" do + include_examples "#implements", + graphql_definition_keyword: "interface", + ruby_definition_method: :interface_type + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/object_type_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/object_type_spec.rb new file mode 100644 index 00000000..ad60f78e --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/object_type_spec.rb @@ -0,0 +1,650 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "graphql_schema_spec_support" +require_relative "implements_shared_examples" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "GraphQL schema generation", "#object_type" do + include_context "GraphQL schema spec support" + + with_both_casing_forms do + it "can generate a simple embedded type (no directives)" do + result = object_type "WidgetOptions" do |t| + t.field "size", "String!" + t.field "main_color", "String" + end + + expect(result).to eq(<<~EOS) + type WidgetOptions { + size: String! + main_color: String + } + EOS + end + + it "generates documentation comments when the caller calls `documentation` on a type or field" do + result = object_type "WidgetOptions" do |t| + t.documentation "Options for a widget." + + t.field "size", "String!" do |f| + f.documentation "The size of the widget." + end + + t.field "color", "String!" do |f| + f.documentation <<~EOS + Multiline strings + are also formatted correctly! + EOS + end + end + + expect(result).to eq(<<~EOS) + """ + Options for a widget. + """ + type WidgetOptions { + """ + The size of the widget. + """ + size: String! + """ + Multiline strings + are also formatted correctly! + """ + color: String! + } + EOS + end + + it "respects a configured type name override" do + result = define_schema(type_name_overrides: {"Widget" => "Gadget"}) do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "name", "String" + t.index "widgets" + end + end + + expect(type_def_from(result, "Widget")).to eq nil + expect(type_def_from(result, "Gadget")).to eq(<<~EOS.strip) + type Gadget { + id: ID + name: String + } + EOS + + expect(type_def_from(result, "GadgetFilterInput")).not_to eq nil + expect(type_def_from(result, "GadgetConnection")).not_to eq nil + expect(type_def_from(result, "GadgetEdge")).not_to eq nil + + # Verify that there are _no_ `Widget` types defined + expect(result.lines.grep(/Widgedt/)).to be_empty + end + + it "raises a clear error when the type name is not formatted correctly" do + expect { + object_type("Invalid.Name") {} + }.to raise_invalid_graphql_name_error_for("Invalid.Name") + end + + it "raises a clear error when the type name is not formatted correctly" do + expect { + object_type "WidgetOptions" do |t| + t.field "invalid.name", "String!" + end + }.to raise_invalid_graphql_name_error_for("invalid.name") + end + + it "fails with a clear error when a field is defined with unrecognized options" do + expect { + object_type "WidgetOptions" do |t| + t.field "size", "String!", invalid_option: 3 + end + }.to raise_error(a_string_including("invalid_option")) + end + + it "raises a clear exception if an embedded type is recursively self-referential without using a relation" do + expect { + define_schema do |s| + s.object_type "Type1" do |t| + t.field "t2", "Type2" + end + + s.object_type "Type2" do |t| + t.field "t3", "Type3" + end + + s.object_type "Type3" do |t| + t.field "t1", "Type1" + end + end + }.to raise_error Errors::SchemaError, a_string_including('The set of ["Type2", "Type3", "Type1"] forms a circular reference chain') + end + + it "allows multiple types to be defined in one evaluation block" do + result = define_schema do |api| + api.object_type "T1" do |t| + t.field "size", "String!" + end + + api.object_type "T2" do |t| + t.field "size", "String!" + end + end + + expect(type_def_from(result, "T1")).to eq(<<~EOS.chomp) + type T1 { + size: String! + } + EOS + + expect(type_def_from(result, "T2")).to eq(<<~EOS.chomp) + type T2 { + size: String! + } + EOS + end + + it "produces a clear error when the same field is defined multiple times" do + expect { + object_type "WidgetOptions" do |t| + t.field "size", "String!" + t.field "size", "String" + end + }.to raise_error Errors::SchemaError, a_string_including("Duplicate", "WidgetOptions", "field", "size") + end + + %w[not all_of any_of any_satisfy].each do |field_name| + it "produces a clear error if a `#{field_name}` field is defined since that will conflict with the filtering operators" do + expect { + object_type "WidgetOptions" do |t| + t.field schema_elements.public_send(field_name), "String" + end + }.to raise_error Errors::SchemaError, a_string_including("WidgetOptions.#{schema_elements.public_send(field_name)}", "reserved") + end + end + + it "raises a clear error when the same type is defined multiple times" do + expect { + define_schema do |api| + api.object_type "WidgetOptions" do |t| + t.field "size", "String!" + t.field "main_color", "String" + end + + api.object_type "WidgetOptions" do |t| + t.field "size2", "String!" + t.field "main_color2", "String" + end + end + }.to raise_error Errors::SchemaError, a_string_including("Duplicate", "WidgetOptions") + end + + it "allows a type to be defined with no fields, since the GraphQL gem allows it" do + result = object_type("WidgetOptions") + + expect(result.delete("\n")).to eq("type WidgetOptions {}") + end + + describe "#implements" do + include_examples "#implements", + graphql_definition_keyword: "type", + ruby_definition_method: :object_type + end + + describe "directives" do + it "raises a clear error when the directive name is not formatted correctly" do + expect { + object_type "WidgetOptions" do |t| + t.directive "invalid.name" + end + }.to raise_invalid_graphql_name_error_for("invalid.name") + end + + it "can be added to the type with no arguments" do + result = define_schema do |schema| + schema.raw_sdl "directive @foo on OBJECT" + schema.raw_sdl "directive @bar on OBJECT" + + schema.object_type "WidgetOptions" do |t| + t.directive "foo" + t.directive "bar" + t.field "size", "String!" + t.field "main_color", "String" + end + end + + expect(type_def_from(result, "WidgetOptions")).to eq(<<~EOS.strip) + type WidgetOptions @foo @bar { + size: String! + main_color: String + } + EOS + end + + it "can be added to the type with arguments" do + result = define_schema do |schema| + schema.raw_sdl "directive @foo(size: Int) on OBJECT" + schema.raw_sdl "directive @bar(color: String) on OBJECT" + + schema.object_type "WidgetOptions" do |t| + t.directive "foo", size: 1 + t.directive "foo", size: 3 + t.directive "bar", color: "red" + t.field "size", "String!" + t.field "main_color", "String" + end + end + + expect(type_def_from(result, "WidgetOptions")).to eq(<<~EOS.strip) + type WidgetOptions @foo(size: 1) @foo(size: 3) @bar(color: "red") { + size: String! + main_color: String + } + EOS + end + + it "can be added to fields, with no directive arguments" do + result = define_schema do |schema| + schema.raw_sdl "directive @foo on FIELD_DEFINITION" + schema.raw_sdl "directive @bar on FIELD_DEFINITION" + + schema.object_type "WidgetOptions" do |t| + t.field "size", "String!" do |f| + f.directive "foo" + f.directive "bar" + end + end + end + + expect(type_def_from(result, "WidgetOptions")).to eq(<<~EOS.strip) + type WidgetOptions { + size: String! @foo @bar + } + EOS + end + + it "can be added to fields, with directive arguments" do + result = define_schema do |schema| + schema.raw_sdl "directive @foo(size: Int) on FIELD_DEFINITION" + schema.raw_sdl "directive @bar(color: String) on FIELD_DEFINITION" + + schema.object_type "WidgetOptions" do |t| + t.field "size", "String!" do |f| + f.directive "foo", size: 1 + f.directive "foo", size: 3 + f.directive "bar", color: "red" + end + end + end + + expect(type_def_from(result, "WidgetOptions")).to eq(<<~EOS.strip) + type WidgetOptions { + size: String! @foo(size: 1) @foo(size: 3) @bar(color: "red") + } + EOS + end + end + + describe "#on_each_generated_schema_element" do + it "applies the given block to each schema element generated for this field, supporting customizations across all of them" do + result = define_schema do |api| + api.raw_sdl "directive @external on FIELD_DEFINITION | ENUM_VALUE | INPUT_FIELD_DEFINITION" + + api.object_type "Money" do |t| + t.field "amount", "Int" + t.field "currency", "String" + end + + api.object_type "Widget" do |t| + t.field "id", "ID", groupable: false, aggregatable: false + t.field "cost", "Int" do |f| + f.on_each_generated_schema_element do |gse| + gse.directive "deprecated" + end + + f.on_each_generated_schema_element do |gse| + gse.directive "external" + end + end + + t.field "costs", "[Money]" do |f| + f.mapping type: "nested" + f.on_each_generated_schema_element do |gse| + gse.directive "deprecated" + end + end + + t.index "widgets" + end + end + + expect(type_def_from(result, "Widget")).to eq(<<~EOS.strip) + type Widget { + id: ID + cost: Int @deprecated @external + costs: [Money] @deprecated + } + EOS + + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.strip) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + not: WidgetFilterInput + id: IDFilterInput + cost: IntFilterInput @deprecated @external + costs: MoneyListFilterInput @deprecated + } + EOS + + expect(aggregated_values_type_from(result, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregatedValues { + cost: IntAggregatedValues @deprecated @external + } + EOS + + expect(grouped_by_type_from(result, "Widget")).to eq(<<~EOS.strip) + type WidgetGroupedBy { + cost: Int @deprecated @external + } + EOS + + expect(sort_order_type_from(result, "Widget")).to eq(<<~EOS.strip) + enum WidgetSortOrderInput { + id_ASC + id_DESC + cost_ASC @deprecated @external + cost_DESC @deprecated @external + } + EOS + + expect(aggregation_sub_aggregations_type_from(result, "Widget")).to eq(<<~EOS.strip) + type WidgetAggregationSubAggregations { + costs( + #{schema_elements.filter}: MoneyFilterInput + #{schema_elements.first}: Int): WidgetMoneySubAggregationConnection @deprecated + } + EOS + end + end + + describe "relation fields" do + it "can define a has-one relation using `relates_to_one`" do + result = define_schema do |schema| + schema.object_type "Widget" do |t| + t.relates_to_one "inventor", "Person", via: "inventor_id", dir: :out do |f| + f.documentation "The inventor of this Widget." + end + end + + schema.object_type "Person" do |t| + t.field "id", "ID" + t.index "people" + end + end + + expect(type_def_from(result, "Widget", include_docs: true)).to eq(<<~EOS.strip) + type Widget { + """ + The inventor of this Widget. + """ + inventor: Person + } + EOS + end + + it "can define a has-many relation using `relates_to_many`" do + pre_def = ->(api) { + api.object_type "Person" do |t| + t.field "id", "ID" + t.field "name", "String", filterable: true, sortable: true + t.index "people" + end + } + + result = object_type "Widget", pre_def: pre_def do |t| + t.relates_to_many "inventors", "Person", via: "inventor_ids", dir: :out, singular: "inventor" do |f| + f.documentation "The collection of inventors of this Widget." + end + end + + expect(result).to eq(<<~EOS) + type Widget { + """ + The collection of inventors of this Widget. + """ + inventors( + """ + Used to filter the returned `inventors` based on the provided criteria. + """ + #{schema_elements.filter}: PersonFilterInput + """ + Used to specify how the returned `inventors` should be sorted. + """ + #{schema_elements.order_by}: [PersonSortOrderInput!] + """ + Used in conjunction with the `after` argument to forward-paginate through the `inventors`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `inventors`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + #{schema_elements.first}: Int + """ + Used to forward-paginate through the `inventors`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + #{schema_elements.after}: Cursor + """ + Used in conjunction with the `before` argument to backward-paginate through the `inventors`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `inventors`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + #{schema_elements.last}: Int + """ + Used to backward-paginate through the `inventors`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + #{schema_elements.before}: Cursor): PersonConnection + """ + Aggregations over the `inventors` data: + + > The collection of inventors of this Widget. + """ + #{correctly_cased "inventor_aggregations"}( + """ + Used to filter the `Person` documents that get aggregated over based on the provided criteria. + """ + filter: PersonFilterInput + """ + Used in conjunction with the `after` argument to forward-paginate through the `#{correctly_cased "inventor_aggregations"}`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `#{correctly_cased "inventor_aggregations"}`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + """ + Used to forward-paginate through the `#{correctly_cased "inventor_aggregations"}`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + """ + Used in conjunction with the `before` argument to backward-paginate through the `#{correctly_cased "inventor_aggregations"}`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `#{correctly_cased "inventor_aggregations"}`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + """ + Used to backward-paginate through the `#{correctly_cased "inventor_aggregations"}`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor): PersonAggregationConnection + } + EOS + end + + it "respects a type name override of the related type when generating the fields for a `relates_to_many`" do + results = define_schema(type_name_overrides: {Component: "Part"}) do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.relates_to_many "components", "Component", via: "widget_id", dir: :in, singular: "component" + t.index "widgets" + end + + schema.object_type "Component" do |t| + t.field "id", "ID!" + t.index "widgets" + end + end + + expect(type_def_from(results, "Widget")).to eq(<<~EOS.strip) + type Widget { + id: ID! + components( + #{schema_elements.filter}: PartFilterInput + #{schema_elements.order_by}: [PartSortOrderInput!] + #{schema_elements.first}: Int + #{schema_elements.after}: Cursor + #{schema_elements.last}: Int + #{schema_elements.before}: Cursor): PartConnection + #{correctly_cased "component_aggregations"}( + #{schema_elements.filter}: PartFilterInput + #{schema_elements.first}: Int + #{schema_elements.after}: Cursor + #{schema_elements.last}: Int + #{schema_elements.before}: Cursor): PartAggregationConnection + } + EOS + end + + it "omits the `order_by` arg on a has-many when the type is not sortable" do + pre_def = ->(api) { + api.object_type "Person" do |t| + t.field "id", "ID", sortable: false + t.index "people" + t.field "name", "String", filterable: true, sortable: false + end + } + + result = object_type "Widget", pre_def: pre_def, include_docs: false do |t| + t.relates_to_many "inventors", "Person", via: "inventor_ids", dir: :out, singular: "inventor" + end + + expect(result).to start_with(<<~EOS) + type Widget { + inventors( + #{schema_elements.filter}: PersonFilterInput + #{schema_elements.first}: Int + #{schema_elements.after}: Cursor + #{schema_elements.last}: Int + #{schema_elements.before}: Cursor): PersonConnection + EOS + end + + it "omits the `filter` arg on a has-many when the type is not filterable" do + pre_def = ->(api) { + api.object_type "Person" do |t| + t.field "id", "ID", filterable: false + t.index "people" + t.field "name", "String", filterable: false, sortable: true + end + } + + result = object_type "Widget", pre_def: pre_def, include_docs: false do |t| + t.relates_to_many "inventors", "Person", via: "inventor_ids", dir: :out, singular: "inventor" + end + + expect(result).to start_with(<<~EOS) + type Widget { + inventors( + #{schema_elements.order_by}: [PersonSortOrderInput!] + #{schema_elements.first}: Int + #{schema_elements.after}: Cursor + #{schema_elements.last}: Int + #{schema_elements.before}: Cursor): PersonConnection + EOS + end + end + + it "can generate a simple indexed type (with just `name`)" do + result = object_type "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + end + + expect(result).to eq(<<~EOS) + type Widget { + id: ID + } + EOS + end + + describe "custom shard routing options" do + it "documents the importance of filtering on the custom routing field on the parent type" do + result = object_type "Widget", include_docs: true do |t| + t.documentation "A widget." + t.field "id", "ID" + t.field "user_id", "ID" do |f| + f.json_schema nullable: false + end + + t.index "widgets" do |i| + i.route_with "user_id" + end + end + + expect(result).to eq(<<~EOS) + """ + A widget. + + For more performant queries on this type, please filter on `user_id` if possible. + """ + type Widget { + id: ID + user_id: ID + } + EOS + end + end + + def object_type(name, *args, pre_def: nil, include_docs: true, &block) + result = define_schema do |api| + pre_def&.call(api) + api.object_type(name, *args, &block) + end + + # We add a line break to match the expectations which use heredocs. + type_def_from(result, name, include_docs: include_docs) + "\n" + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/paginated_collection_field_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/paginated_collection_field_spec.rb new file mode 100644 index 00000000..64de4663 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/paginated_collection_field_spec.rb @@ -0,0 +1,242 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "graphql_schema_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "GraphQL schema generation", "#paginated_collection_field" do + include_context "GraphQL schema spec support" + + with_both_casing_forms do + it "defines the field as a `Connection` type" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.paginated_collection_field "names", "String" + end + end + + expect(type_def_from(result, "Widget")).to eq(<<~EOS.chomp) + type Widget { + names( + first: Int + after: Cursor + last: Int + before: Cursor): StringConnection + } + EOS + end + + it "causes the `Connection` type of the provided element type to be generated" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.paginated_collection_field "names", "String" + t.index "widgets" + end + end + + expect(connection_type_from(result, "String")).to eq(<<~EOS.chomp) + type StringConnection { + #{schema_elements.edges}: [StringEdge!]! + #{schema_elements.nodes}: [String!]! + #{schema_elements.page_info}: PageInfo! + #{schema_elements.total_edge_count}: JsonSafeLong! + } + EOS + end + + it "causes the `Edge` type of the provided element type to be generated" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.paginated_collection_field "names", "String" + end + end + + expect(edge_type_from(result, "String")).to eq(<<~EOS.chomp) + type StringEdge { + node: String + cursor: Cursor + } + EOS + end + + it "supports collections of objects in addition to scalars" do + result = define_schema do |api| + api.object_type "WidgetOptions" do |t| + t.field "size", "Int" + end + + api.object_type "Widget" do |t| + t.field "id", "ID" + t.paginated_collection_field "optionses", "WidgetOptions" do |f| + f.mapping type: "object" + end + t.index "widgets" + end + end + + expect(type_def_from(result, "Widget")).to eq(<<~EOS.chomp) + type Widget { + id: ID + optionses( + first: Int + after: Cursor + last: Int + before: Cursor): WidgetOptionsConnection + } + EOS + + expect(connection_type_from(result, "WidgetOptions")).to eq(<<~EOS.chomp) + type WidgetOptionsConnection { + #{schema_elements.edges}: [WidgetOptionsEdge!]! + #{schema_elements.nodes}: [WidgetOptions!]! + #{schema_elements.page_info}: PageInfo! + #{schema_elements.total_edge_count}: JsonSafeLong! + } + EOS + + expect(edge_type_from(result, "WidgetOptions")).to eq(<<~EOS.chomp) + type WidgetOptionsEdge { + node: WidgetOptions + cursor: Cursor + } + EOS + end + + it "avoids referencing a `*ConnectionFilterInput` type that won't exist when defining the corresponding filter type, and avoids defining the pagination args on the filter's field" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.paginated_collection_field "names", "String" + end + end + + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.chomp) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + names: StringListFilterInput + } + EOS + end + + it "respects a type name override used for the type passed to `paginated_collection_field`" do + result = define_schema(type_name_overrides: {LocalTime: "TimeOfDay"}) do |api| + api.object_type "Widget" do |t| + t.paginated_collection_field "times", "LocalTime" + end + end + + expect(type_def_from(result, "Widget")).to eq(<<~EOS.chomp) + type Widget { + times( + #{schema_elements.first}: Int + #{schema_elements.after}: Cursor + #{schema_elements.last}: Int + #{schema_elements.before}: Cursor): TimeOfDayConnection + } + EOS + + expect(connection_type_from(result, "TimeOfDay")).to eq(<<~EOS.chomp) + type TimeOfDayConnection { + #{schema_elements.edges}: [TimeOfDayEdge!]! + #{schema_elements.nodes}: [TimeOfDay!]! + #{schema_elements.page_info}: PageInfo! + #{schema_elements.total_edge_count}: JsonSafeLong! + } + EOS + + expect(edge_type_from(result, "TimeOfDay")).to eq(<<~EOS.chomp) + type TimeOfDayEdge { + #{schema_elements.node}: TimeOfDay + #{schema_elements.cursor}: Cursor + } + EOS + + expect(filter_type_from(result, "Widget")).to eq(<<~EOS.chomp) + input WidgetFilterInput { + #{schema_elements.any_of}: [WidgetFilterInput!] + #{schema_elements.not}: WidgetFilterInput + times: TimeOfDayListFilterInput + } + EOS + end + + it "avoids generating a sort order enum value for the field, just as it would for a list field" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.paginated_collection_field "names", "String" + t.index "widgets" + end + end + + expect(sort_order_type_from(result, "Widget")).to eq(<<~EOS.chomp) + enum WidgetSortOrderInput { + id_ASC + id_DESC + } + EOS + end + + it "allows the field to be documented just like with `.field`" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.paginated_collection_field "names", "String" do |f| + f.documentation "Paginated names." + end + end + end + + expect(type_def_from(result, "Widget", include_docs: true)).to eq(<<~EOS.chomp) + type Widget { + """ + Paginated names. + """ + names( + """ + Used in conjunction with the `after` argument to forward-paginate through the `names`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `names`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + """ + Used to forward-paginate through the `names`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + """ + Used in conjunction with the `before` argument to backward-paginate through the `names`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `names`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + """ + Used to backward-paginate through the `names`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor): StringConnection + } + EOS + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/relay_types_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/relay_types_spec.rb new file mode 100644 index 00000000..a656259d --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/relay_types_spec.rb @@ -0,0 +1,382 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "graphql_schema_spec_support" +require "elastic_graph/schema_definition/test_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "GraphQL schema generation", "relay types" do + include_context "GraphQL schema spec support" + + with_both_casing_forms do + it "defines `*Edge` types for each defined `object_type`" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.index "widgets" + end + end + + expect(edge_type_from(result, "Widget", include_docs: true)).to eq(<<~EOS.strip) + """ + Represents a specific `Widget` in the context of a `WidgetConnection`, + providing access to both the `Widget` and a pagination `Cursor`. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. + """ + type WidgetEdge { + """ + The `Widget` of this edge. + """ + node: Widget + """ + The `Cursor` of this `Widget`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `Widget`. + """ + cursor: Cursor + } + EOS + end + + it "defines Edge types for indexed union types" do + result = define_schema do |api| + api.object_type "Person" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "age", "Int" + t.field "nationality", "String" + t.index "people" + end + + api.object_type "Company" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "age", "Int" + t.field "stock_ticker", "String" + t.index "companies" + end + + api.union_type "Inventor" do |t| + t.subtypes "Person", "Company" + end + end + + expect(edge_type_from(result, "Inventor")).to eq(<<~EOS.strip) + type InventorEdge { + node: Inventor + cursor: Cursor + } + EOS + end + + it "defines `*Edge` types for each indexed aggregation type" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "int", "Int", aggregatable: true + t.index "widgets" + end + end + + expect(aggregation_edge_type_from(result, "Widget", include_docs: true)).to eq(<<~EOS.strip) + """ + Represents a specific `WidgetAggregation` in the context of a `WidgetAggregationConnection`, + providing access to both the `WidgetAggregation` and a pagination `Cursor`. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Edge-Types) for more info. + """ + type WidgetAggregationEdge { + """ + The `WidgetAggregation` of this edge. + """ + node: WidgetAggregation + """ + The `Cursor` of this `WidgetAggregation`. This can be passed in the next query as + a `before` or `after` argument to continue paginating from this `WidgetAggregation`. + """ + cursor: Cursor + } + EOS + end + + it "does not define Edge types for embedded union types" do + result = define_schema do |api| + api.object_type "Person" do |t| + t.field "name", "String" + t.field "age", "Int" + t.field "nationality", "String" + end + + api.object_type "Company" do |t| + t.field "name", "String" + t.field "age", "Int" + t.field "stock_ticker", "String" + end + + api.union_type "Inventor" do |t| + t.subtypes "Person", "Company" + end + end + + expect(edge_type_from(result, "Inventor")).to eq(nil) + end + + it "defines `*Connection` types for each defined `object_type`" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "cost_amount", "Int" + t.field "created_at", "DateTime" + t.index "widgets" + end + end + + expect(connection_type_from(result, "Widget", include_docs: true)).to eq(<<~EOS.strip) + """ + Represents a paginated collection of `Widget` results. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. + """ + type WidgetConnection { + """ + Wraps a specific `Widget` to pair it with its pagination cursor. + """ + #{schema_elements.edges}: [WidgetEdge!]! + """ + The list of `Widget` results. + """ + #{schema_elements.nodes}: [Widget!]! + """ + Provides pagination-related information. + """ + #{schema_elements.page_info}: PageInfo! + """ + The total number of edges available in this connection to paginate over. + """ + #{schema_elements.total_edge_count}: JsonSafeLong! + } + EOS + end + + it "mentions the possible efficiency improvement of querying a derived index in the aggregation comments when it applies" do + result = define_schema do |api| + api.object_type "WidgetWorkspace" do |t| + t.field "id", "ID!" + t.field "currencies", "[String!]!" + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "workspace_id", "ID" + t.field "cost_currency", "String" + t.index "widgets" + t.derive_indexed_type_fields "WidgetWorkspace", from_id: "workspace_id" do |derive| + derive.append_only_set "currencies", from: "cost_currency" + end + end + end + + expect(aggregation_efficency_hint_for(result, "widget_aggregations")).to eq(<<~EOS.strip) + """ + Aggregations over the `widgets` data: + + > Fetches `Widget`s based on the provided arguments. + + Note: aggregation queries are relatively expensive, and some fields have been pre-aggregated to allow + more efficient queries for some common aggregation cases: + + - The root `#{correctly_cased "widget_workspaces"}` field groups by `workspace_id` + """ + EOS + end + + it "respects a type name override when generating the aggregation efficiency hints" do + result = define_schema(type_name_overrides: {WidgetWorkspace: "WorkspaceOfWidget"}) do |api| + api.object_type "WidgetWorkspace" do |t| + t.field "id", "ID!" + t.field "currencies", "[String!]!" + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "workspace_id", "ID" + t.field "cost_currency", "String" + t.index "widgets" + t.derive_indexed_type_fields "WidgetWorkspace", from_id: "workspace_id" do |derive| + derive.append_only_set "currencies", from: "cost_currency" + end + end + end + + expect(aggregation_efficency_hint_for(result, "widget_aggregations")).to eq(<<~EOS.strip) + """ + Aggregations over the `widgets` data: + + > Fetches `Widget`s based on the provided arguments. + + Note: aggregation queries are relatively expensive, and some fields have been pre-aggregated to allow + more efficient queries for some common aggregation cases: + + - The root `#{correctly_cased "workspace_of_widgets"}` field groups by `workspace_id` + """ + EOS + end + + it "defines Connection types for indexed union types" do + result = define_schema do |api| + api.object_type "Person" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "age", "Int" + t.field "nationality", "String", groupable: true + t.index "people" + end + + api.object_type "Company" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "age", "Int" + t.field "stock_ticker", "String" + t.index "companies" + end + + api.union_type "Inventor" do |t| + t.subtypes "Person", "Company" + end + end + + expect(connection_type_from(result, "Inventor")).to eq(<<~EOS.strip) + type InventorConnection { + #{schema_elements.edges}: [InventorEdge!]! + #{schema_elements.nodes}: [Inventor!]! + #{schema_elements.page_info}: PageInfo! + #{schema_elements.total_edge_count}: JsonSafeLong! + } + EOS + end + + it "defines Connection types for indexed interface types" do + result = define_schema do |api| + api.object_type "Person" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "age", "Int" + t.field "nationality", "String", groupable: true + t.index "people" + end + + api.object_type "Company" do |t| + t.implements "Inventor" + t.field "id", "ID!" + t.field "name", "String" + t.field "age", "Int" + t.field "stock_ticker", "String" + t.index "companies" + end + + api.interface_type "Inventor" do |t| + t.field "name", "String" + end + end + + expect(connection_type_from(result, "Inventor")).to eq(<<~EOS.strip) + type InventorConnection { + #{schema_elements.edges}: [InventorEdge!]! + #{schema_elements.nodes}: [Inventor!]! + #{schema_elements.page_info}: PageInfo! + #{schema_elements.total_edge_count}: JsonSafeLong! + } + EOS + end + + it "defines `*Connection` types for each indexed aggregation type" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "int", "Int", aggregatable: true + t.index "widgets" + end + end + + expect(aggregation_connection_type_from(result, "Widget", include_docs: true)).to eq(<<~EOS.strip) + """ + Represents a paginated collection of `WidgetAggregation` results. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Connection-Types) for more info. + """ + type WidgetAggregationConnection { + """ + Wraps a specific `WidgetAggregation` to pair it with its pagination cursor. + """ + #{schema_elements.edges}: [WidgetAggregationEdge!]! + """ + The list of `WidgetAggregation` results. + """ + #{schema_elements.nodes}: [WidgetAggregation!]! + """ + Provides pagination-related information. + """ + #{schema_elements.page_info}: PageInfo! + } + EOS + end + + it "does not define Connection types for embedded union types" do + result = define_schema do |api| + api.object_type "Person" do |t| + t.field "name", "String" + t.field "age", "Int" + t.field "nationality", "String" + end + + api.object_type "Company" do |t| + t.field "name", "String" + t.field "age", "Int" + t.field "stock_ticker", "String" + end + + api.union_type "Inventor" do |t| + t.subtypes "Person", "Company" + end + end + + expect(connection_type_from(result, "Inventor")).to eq(nil) + end + + it "avoids defining an `aggregations` field on a Connection type when there is no `Aggregation` type" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!", aggregatable: false, groupable: false + t.index "widgets" + end + end + + expect(connection_type_from(result, "Widget")).to eq(<<~EOS.strip) + type WidgetConnection { + #{schema_elements.edges}: [WidgetEdge!]! + #{schema_elements.nodes}: [Widget!]! + #{schema_elements.page_info}: PageInfo! + #{schema_elements.total_edge_count}: JsonSafeLong! + } + EOS + end + + def aggregation_efficency_hint_for(result, query_field) + query_def = type_def_from(result, "Query", include_docs: true) + aggregations_comments = query_def[/(#{TestSupport::DOC_COMMENTS})\s*#{correctly_cased(query_field)}\(/, 1] + aggregations_comments.split("\n").map { |l| l.delete_prefix(" ") }.join("\n") + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/root_query_type_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/root_query_type_spec.rb new file mode 100644 index 00000000..ff96979b --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/root_query_type_spec.rb @@ -0,0 +1,391 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "graphql_schema_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "GraphQL schema generation", "root Query type" do + include_context "GraphQL schema spec support" + + with_both_casing_forms do + it "generates a document search field and an aggregations field for each indexed type" do + result = define_schema do |api| + api.object_type "Person" do |t| + t.implements "NamedEntity" + t.field "id", "ID" + t.field "name", "String" + t.field "nationality", "String", filterable: false + t.index "people" + end + + api.object_type "Company" do |t| + t.implements "NamedEntity" + t.field "id", "ID" + t.field "name", "String" + t.field "stock_ticker", "String" + t.index "companies" + end + + api.interface_type "NamedEntity" do |t| + t.field "id", "ID" + t.field "name", "String" + end + + api.union_type "Inventor" do |t| + t.subtypes "Person", "Company" + end + + api.object_type "Foo" do |t| + t.field "id", "ID" + t.field "size", "String" + end + + api.object_type "Bar" do |t| + t.field "id", "ID" + t.field "length", "Int" + end + + api.object_type "Class" do |t| + t.field "id", "ID" + t.field "name", "String" + t.index "classes" + end + + api.union_type "FooOrBar" do |t| + t.subtypes "Foo", "Bar" + t.index "foos_or_bars" + end + end + + expect(type_def_from(result, "Query")).to eq(<<~EOS.strip) + type Query { + classes( + filter: ClassFilterInput + #{correctly_cased "order_by"}: [ClassSortOrderInput!] + first: Int + after: Cursor + last: Int + before: Cursor): ClassConnection + #{correctly_cased "class_aggregations"}( + filter: ClassFilterInput + first: Int + after: Cursor + last: Int + before: Cursor): ClassAggregationConnection + companys( + filter: CompanyFilterInput + #{correctly_cased "order_by"}: [CompanySortOrderInput!] + first: Int + after: Cursor + last: Int + before: Cursor): CompanyConnection + #{correctly_cased "company_aggregations"}( + filter: CompanyFilterInput + first: Int + after: Cursor + last: Int + before: Cursor): CompanyAggregationConnection + #{correctly_cased "foo_or_bars"}( + filter: FooOrBarFilterInput + #{correctly_cased "order_by"}: [FooOrBarSortOrderInput!] + first: Int + after: Cursor + last: Int + before: Cursor): FooOrBarConnection + #{correctly_cased "foo_or_bar_aggregations"}( + filter: FooOrBarFilterInput + first: Int + after: Cursor + last: Int + before: Cursor): FooOrBarAggregationConnection + inventors( + filter: InventorFilterInput + #{correctly_cased "order_by"}: [InventorSortOrderInput!] + first: Int + after: Cursor + last: Int + before: Cursor): InventorConnection + #{correctly_cased "inventor_aggregations"}( + filter: InventorFilterInput + first: Int + after: Cursor + last: Int + before: Cursor): InventorAggregationConnection + #{correctly_cased "named_entitys"}( + filter: NamedEntityFilterInput + #{correctly_cased "order_by"}: [NamedEntitySortOrderInput!] + first: Int + after: Cursor + last: Int + before: Cursor): NamedEntityConnection + #{correctly_cased "named_entity_aggregations"}( + filter: NamedEntityFilterInput + first: Int + after: Cursor + last: Int + before: Cursor): NamedEntityAggregationConnection + persons( + filter: PersonFilterInput + #{correctly_cased "order_by"}: [PersonSortOrderInput!] + first: Int + after: Cursor + last: Int + before: Cursor): PersonConnection + #{correctly_cased "person_aggregations"}( + filter: PersonFilterInput + first: Int + after: Cursor + last: Int + before: Cursor): PersonAggregationConnection + } + EOS + end + + it "allows the Query field names and directives to be customized on the indexed type definitions" do + result = define_schema do |api| + api.object_type "Person" do |t| + t.implements "NamedEntity" + t.root_query_fields plural: "people", singular: "human" do |f| + f.directive "deprecated" + end + t.field "id", "ID" + t.field "name", "String" + t.field "nationality", "String", filterable: false + t.index "people" + end + + api.union_type "Inventor" do |t| + t.root_query_fields plural: "inventorees" + t.subtypes "Person" + end + + api.interface_type "NamedEntity" do |t| + t.root_query_fields plural: "named_entities" + t.field "id", "ID" + t.field "name", "String" + end + + api.object_type "Widget" do |t| + t.implements "NamedEntity" + t.field "id", "ID" + t.field "name", "String" + t.index "widgets" + end + end + + expect(type_def_from(result, "Query")).to eq(<<~EOS.strip) + type Query { + inventorees( + filter: InventorFilterInput + #{correctly_cased "order_by"}: [InventorSortOrderInput!] + first: Int + after: Cursor + last: Int + before: Cursor): InventorConnection + #{correctly_cased "inventor_aggregations"}( + filter: InventorFilterInput + first: Int + after: Cursor + last: Int + before: Cursor): InventorAggregationConnection + named_entities( + filter: NamedEntityFilterInput + #{correctly_cased "order_by"}: [NamedEntitySortOrderInput!] + first: Int + after: Cursor + last: Int + before: Cursor): NamedEntityConnection + #{correctly_cased "named_entity_aggregations"}( + filter: NamedEntityFilterInput + first: Int + after: Cursor + last: Int + before: Cursor): NamedEntityAggregationConnection + people( + filter: PersonFilterInput + #{correctly_cased "order_by"}: [PersonSortOrderInput!] + first: Int + after: Cursor + last: Int + before: Cursor): PersonConnection @deprecated + #{correctly_cased "human_aggregations"}( + filter: PersonFilterInput + first: Int + after: Cursor + last: Int + before: Cursor): PersonAggregationConnection @deprecated + widgets( + filter: WidgetFilterInput + #{correctly_cased "order_by"}: [WidgetSortOrderInput!] + first: Int + after: Cursor + last: Int + before: Cursor): WidgetConnection + #{correctly_cased "widget_aggregations"}( + filter: WidgetFilterInput + first: Int + after: Cursor + last: Int + before: Cursor): WidgetAggregationConnection + } + EOS + end + + it "documents each field" do + result = define_schema do |api| + api.object_type "Person" do |t| + t.root_query_fields plural: "people" + t.field "id", "ID" + t.field "name", "String" + t.field "nationality", "String", filterable: false + t.index "people" + end + end + + expect(type_def_from(result, "Query", include_docs: true)).to eq(<<~EOS.strip) + """ + The query entry point for the entire schema. + """ + type Query { + """ + Fetches `Person`s based on the provided arguments. + """ + people( + """ + Used to filter the returned `people` based on the provided criteria. + """ + filter: PersonFilterInput + """ + Used to specify how the returned `people` should be sorted. + """ + #{correctly_cased "order_by"}: [PersonSortOrderInput!] + """ + Used in conjunction with the `after` argument to forward-paginate through the `people`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `people`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + """ + Used to forward-paginate through the `people`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + """ + Used in conjunction with the `before` argument to backward-paginate through the `people`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `people`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + """ + Used to backward-paginate through the `people`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor): PersonConnection + """ + Aggregations over the `people` data: + + > Fetches `Person`s based on the provided arguments. + """ + #{correctly_cased "person_aggregations"}( + """ + Used to filter the `Person` documents that get aggregated over based on the provided criteria. + """ + filter: PersonFilterInput + """ + Used in conjunction with the `after` argument to forward-paginate through the `#{correctly_cased "person_aggregations"}`. + When provided, limits the number of returned results to the first `n` after the provided + `after` cursor (or from the start of the `#{correctly_cased "person_aggregations"}`, if no `after` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + first: Int + """ + Used to forward-paginate through the `#{correctly_cased "person_aggregations"}`. When provided, the next page after the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + after: Cursor + """ + Used in conjunction with the `before` argument to backward-paginate through the `#{correctly_cased "person_aggregations"}`. + When provided, limits the number of returned results to the last `n` before the provided + `before` cursor (or from the end of the `#{correctly_cased "person_aggregations"}`, if no `before` cursor is provided). + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + last: Int + """ + Used to backward-paginate through the `#{correctly_cased "person_aggregations"}`. When provided, the previous page before the + provided cursor will be returned. + + See the [Relay GraphQL Cursor Connections + Specification](https://relay.dev/graphql/connections.htm#sec-Arguments) for more info. + """ + before: Cursor): PersonAggregationConnection + } + EOS + end + + it "does not include field arguments that would provide unsupported capabilities" do + result = define_schema do |api| + api.object_type "Person" do |t| + t.root_query_fields plural: "people" + t.field "id", "ID!", sortable: false, filterable: false + t.index "people" + end + end + + expect(type_def_from(result, "Query")).to eq(<<~EOS.strip) + type Query { + people( + first: Int + after: Cursor + last: Int + before: Cursor): PersonConnection + #{correctly_cased "person_aggregations"}( + first: Int + after: Cursor + last: Int + before: Cursor): PersonAggregationConnection + } + EOS + end + + it "can be overridden via `raw_sdl` to support ElasticGraph tests that require a custom `Query` type" do + query_type_def = <<~EOS.strip + type Query { + foo: Int + } + EOS + + result = define_schema do |schema| + schema.raw_sdl query_type_def + end + + expect(type_def_from(result, "Query")).to eq(query_type_def) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/scalar_type_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/scalar_type_spec.rb new file mode 100644 index 00000000..63badb8d --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/scalar_type_spec.rb @@ -0,0 +1,294 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "graphql_schema_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "GraphQL schema generation", "#scalar_type" do + include_context "GraphQL schema spec support" + + with_both_casing_forms do + it "generates the SDL for a custom scalar type" do + result = scalar_type "BigInt" do |t| + t.mapping type: "long" + t.json_schema type: "integer" + end + + expect(type_def_from(result, "BigInt")).to eq(<<~EOS.strip) + scalar BigInt + EOS + end + + it "requires the `mapping` to be specified so we know how to index it in the datastore" do + expect { + scalar_type "BigInt" do |t| + t.json_schema type: "integer" + end + }.to raise_error Errors::SchemaError, a_string_including("BigInt", "lacks `mapping`") + end + + it "requires the `json_schema` to be specified so we know how it should be encoded in an ingested event" do + expect { + scalar_type "BigInt" do |t| + t.mapping type: "long" + end + }.to raise_error Errors::SchemaError, a_string_including("BigInt", "lacks `json_schema`") + end + + it "requires a `type` be specified on the `mapping` since we can't guess what the mapping type should be" do + expect { + scalar_type "BigInt" do |t| + t.json_schema type: "integer" + t.mapping null_value: 0 + end + }.to raise_error Errors::SchemaError, a_string_including("BigInt", "mapping", "type:") + end + + it "respects a configured type name override" do + result = define_schema(type_name_overrides: {"BigInt" => "LargeNumber"}) do |schema| + schema.object_type "Widget" do |t| + t.paginated_collection_field "nums", "BigInt" + end + + schema.scalar_type "BigInt" do |t| + t.mapping type: "long" + t.json_schema type: "integer" + end + end + + expect(type_def_from(result, "BigInt")).to eq nil + expect(type_def_from(result, "LargeNumber")).to eq("scalar LargeNumber") + + expect(type_def_from(result, "LargeNumberFilterInput")).not_to eq nil + expect(type_def_from(result, "LargeNumberConnection")).not_to eq nil + expect(type_def_from(result, "LargeNumberEdge")).not_to eq nil + + # Verify that there are _no_ `BigInt` types defined + expect(result.lines.grep(/BigInt/)).to be_empty + end + + it "allows additional directives to be defined on the scalar" do + result = define_schema do |schema| + schema.raw_sdl "directive @meta(since_date: String = null, author: String = null) on SCALAR" + + schema.scalar_type "BigInt" do |t| + t.mapping type: "long" + t.json_schema type: "integer" + t.directive "meta", since_date: "2021-08-01" + t.directive "meta", author: "John" + end + end + + expect(type_def_from(result, "BigInt")).to eq(<<~EOS.strip) + scalar BigInt @meta(since_date: "2021-08-01") @meta(author: "John") + EOS + end + + it "allows documentation to be defined on the scalar" do + result = scalar_type "BigInt" do |t| + t.mapping type: "long" + t.json_schema type: "integer" + t.documentation "A number that exceeds the normal `Int` max." + end + + expect(type_def_from(result, "BigInt", include_docs: true)).to eq(<<~EOS.strip) + """ + A number that exceeds the normal `Int` max. + """ + scalar BigInt + EOS + end + + it "defines a filter type with `any_of` and `equal_to_any_of` for a mapping type that can't efficiently support range queries" do + result = scalar_type "FullText" do |t| + t.mapping type: "text" + t.json_schema type: "string" + end + + expect(filter_type_from(result, "FullText")).to eq(<<~EOS.strip) + input FullTextFilterInput { + #{schema_elements.any_of}: [FullTextFilterInput!] + #{schema_elements.not}: FullTextFilterInput + #{schema_elements.equal_to_any_of}: [FullText] + } + EOS + end + + it "defines a filter type with `any_of`, `equal_to_any_of`, and comparison operators for a numeric mapping type that can efficiently support range queries" do + result = scalar_type "Short" do |t| + t.mapping type: "short" + t.json_schema type: "integer" + end + + expect(filter_type_from(result, "Short")).to eq(<<~EOS.strip) + input ShortFilterInput { + #{schema_elements.any_of}: [ShortFilterInput!] + #{schema_elements.not}: ShortFilterInput + #{schema_elements.equal_to_any_of}: [Short] + #{schema_elements.gt}: Short + #{schema_elements.gte}: Short + #{schema_elements.lt}: Short + #{schema_elements.lte}: Short + } + EOS + end + + it "defines a filter type with `any_of`, `equal_to_any_of`, and comparison operators for a date mapping type that can efficiently support range queries" do + result = scalar_type "CalendarDate" do |t| + t.mapping type: "date" + t.json_schema type: "string" + end + + expect(filter_type_from(result, "CalendarDate")).to eq(<<~EOS.strip) + input CalendarDateFilterInput { + #{schema_elements.any_of}: [CalendarDateFilterInput!] + #{schema_elements.not}: CalendarDateFilterInput + #{schema_elements.equal_to_any_of}: [CalendarDate] + #{schema_elements.gt}: CalendarDate + #{schema_elements.gte}: CalendarDate + #{schema_elements.lt}: CalendarDate + #{schema_elements.lte}: CalendarDate + } + EOS + end + + it "defines a `*ListFilterInput` type so that lists of the custom scalar type can be filtered on" do + result = scalar_type "Short" do |t| + t.mapping type: "short" + t.json_schema type: "integer" + end + + expect(list_filter_type_from(result, "Short", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type used to specify filters on `[Short]` fields. + + Will be ignored if passed as an empty object (or as `null`). + """ + input ShortListFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + #{schema_elements.any_of}: [ShortListFilterInput!] + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + #{schema_elements.not}: ShortListFilterInput + """ + Matches records where any of the list elements match the provided sub-filter. + + Will be ignored when `null` or an empty object is passed. + """ + #{schema_elements.any_satisfy}: ShortListElementFilterInput + """ + Matches records where all of the provided sub-filters evaluate to true. This works just like an AND operator in SQL. + + Note: multiple filters are automatically ANDed together. This is only needed when you have multiple filters that can't + be provided on a single `ShortListFilterInput` input because of collisions between key names. For example, if you want to provide + multiple `#{schema_elements.any_satisfy}: ...` filters, you could do `#{schema_elements.all_of}: [{#{schema_elements.any_satisfy}: ...}, {#{schema_elements.any_satisfy}: ...}]`. + + Will be ignored when `null` or an empty list is passed. + """ + #{schema_elements.all_of}: [ShortListFilterInput!] + """ + Used to filter on the number of non-null elements in this list field. + + Will be ignored when `null` or an empty object is passed. + """ + count: IntFilterInput + } + EOS + end + + it "documents each filter field" do + result = scalar_type "Byte" do |t| + t.mapping type: "byte" + t.json_schema type: "integer" + end + + expect(filter_type_from(result, "Byte", include_docs: true)).to eq(<<~EOS.strip) + """ + Input type used to specify filters on `Byte` fields. + + Will be ignored if passed as an empty object (or as `null`). + """ + input ByteFilterInput { + """ + Matches records where any of the provided sub-filters evaluate to true. + This works just like an OR operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. + """ + #{schema_elements.any_of}: [ByteFilterInput!] + """ + Matches records where the provided sub-filter evaluates to false. + This works just like a NOT operator in SQL. + + Will be ignored when `null` or an empty object is passed. + """ + #{schema_elements.not}: ByteFilterInput + """ + Matches records where the field value is equal to any of the provided values. + This works just like an IN operator in SQL. + + Will be ignored when `null` is passed. When an empty list is passed, will cause this + part of the filter to match no documents. When `null` is passed in the list, will + match records where the field value is `null`. + """ + #{schema_elements.equal_to_any_of}: [Byte] + """ + Matches records where the field value is greater than (>) the provided value. + + Will be ignored when `null` is passed. + """ + #{schema_elements.gt}: Byte + """ + Matches records where the field value is greater than or equal to (>=) the provided value. + + Will be ignored when `null` is passed. + """ + #{schema_elements.gte}: Byte + """ + Matches records where the field value is less than (<) the provided value. + + Will be ignored when `null` is passed. + """ + #{schema_elements.lt}: Byte + """ + Matches records where the field value is less than or equal to (<=) the provided value. + + Will be ignored when `null` is passed. + """ + #{schema_elements.lte}: Byte + } + EOS + end + + it "raises a clear error when the type name is not formatted correctly" do + expect { + scalar_type("Invalid.Name") {} + }.to raise_invalid_graphql_name_error_for("Invalid.Name") + end + + def scalar_type(...) + define_schema do |api| + api.scalar_type(...) + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/sort_order_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/sort_order_spec.rb new file mode 100644 index 00000000..d4fd5cff --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/sort_order_spec.rb @@ -0,0 +1,487 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "graphql_schema_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "GraphQL schema generation", "sort order enum types" do + include_context "GraphQL schema spec support" + + with_both_casing_forms do + it "defines ASC and DESC enum values for each scalar field (boolean, and text fields) for each indexed type" do + result = define_schema do |api| + # demonstrate that it doesn't try to query the `Color` type for its subfields even though + # it's a type defined via our type definition API. + api.enum_type "Color" do |e| + e.values "RED", "GREEN", "BLUE" + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "another_id", "ID" + t.field "some_string", "String" + t.field "some_int", "Int!" + t.field "some_ints", "[Int]" + t.field "some_ints2", "[Int!]" + t.field "some_ints3", "[Int]!" + t.field "some_ints4", "[Int!]!" + t.field "some_float", "Float" + t.field "some_date", "Date" + t.field "some_date_time", "DateTime!" + t.field "color", "Color" + t.field "some_boolean", "Boolean" + t.field "another_boolean", "Boolean!" + t.field "some_text", "String" do |f| + f.mapping type: "text" + end + t.relates_to_one "parent", "Widget", via: "parent_id", dir: :out + t.relates_to_many "children", "Widget", via: "children_ids", dir: :in, singular: "child" + t.index "widgets" + end + end + + expect(sort_order_type_from(result, "Widget", include_docs: true)).to eq(<<~EOS.strip) + """ + Enumerates the ways `Widget`s can be sorted. + """ + enum WidgetSortOrderInput { + """ + Sorts ascending by the `id` field. + """ + id_ASC + """ + Sorts descending by the `id` field. + """ + id_DESC + """ + Sorts ascending by the `another_id` field. + """ + another_id_ASC + """ + Sorts descending by the `another_id` field. + """ + another_id_DESC + """ + Sorts ascending by the `some_string` field. + """ + some_string_ASC + """ + Sorts descending by the `some_string` field. + """ + some_string_DESC + """ + Sorts ascending by the `some_int` field. + """ + some_int_ASC + """ + Sorts descending by the `some_int` field. + """ + some_int_DESC + """ + Sorts ascending by the `some_float` field. + """ + some_float_ASC + """ + Sorts descending by the `some_float` field. + """ + some_float_DESC + """ + Sorts ascending by the `some_date` field. + """ + some_date_ASC + """ + Sorts descending by the `some_date` field. + """ + some_date_DESC + """ + Sorts ascending by the `some_date_time` field. + """ + some_date_time_ASC + """ + Sorts descending by the `some_date_time` field. + """ + some_date_time_DESC + """ + Sorts ascending by the `color` field. + """ + color_ASC + """ + Sorts descending by the `color` field. + """ + color_DESC + } + EOS + end + + it "allows the sortability of a field to be set explicitly" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!", sortable: false + t.field "another_id", "ID", sortable: true + t.field "some_string", "String", sortable: true + t.field "some_int", "Int!", sortable: false + t.field "some_float", "Float" + t.index "widgets" + end + end + + expect(sort_order_type_from(result, "Widget")).to eq(<<~EOS.strip) + enum WidgetSortOrderInput { + another_id_ASC + another_id_DESC + some_string_ASC + some_string_DESC + some_float_ASC + some_float_DESC + } + EOS + end + + it "defines enum values for scalar fields of singleton embedded object types" do + result = define_schema do |api| + api.object_type "Color" do |t| + t.field "red", "Int!" + t.field "green", "Int!", name_in_index: "grn" + t.field "blue", "Int!" + end + + api.object_type "WidgetOptions" do |t| + t.field "the_size", "Int" + t.field "the_color", "Color", name_in_index: "clr" + t.field "some_id", "ID" + t.field "colors", "[Color]" do |f| + f.mapping type: "object" + end + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "the_options", "WidgetOptions" + t.field "options_array", "[WidgetOptions]" do |f| + f.mapping type: "object" + end + t.field "some_float", "Float" + t.index "widgets" + end + end + + expect(sort_order_type_from(result, "Widget")).to eq(<<~EOS.strip) + enum WidgetSortOrderInput { + id_ASC + id_DESC + the_options_the_size_ASC + the_options_the_size_DESC + the_options_the_color_red_ASC + the_options_the_color_red_DESC + the_options_the_color_green_ASC + the_options_the_color_green_DESC + the_options_the_color_blue_ASC + the_options_the_color_blue_DESC + the_options_some_id_ASC + the_options_some_id_DESC + some_float_ASC + some_float_DESC + } + EOS + end + + it "allows the derived enum values to be customized" do + enum_field_paths = [] + + result = define_schema do |api| + api.raw_sdl "directive @external on ENUM_VALUE" + + api.object_type "WidgetOptions" do |t| + t.field "the_size", "Int" + t.field "some_id", "ID" do |f| + f.customize_sort_order_enum_values do |v| + enum_field_paths << v.sort_order_field_path + v.directive "external" + end + end + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "the_options", "WidgetOptions" do |f| + f.customize_sort_order_enum_values do |v| + enum_field_paths << v.sort_order_field_path + v.directive "deprecated" + end + end + + t.field "some_float", "Float" do |f| + f.customize_sort_order_enum_values do |v| + enum_field_paths << v.sort_order_field_path + v.directive "deprecated" + end + + f.customize_sort_order_enum_values do |v| + enum_field_paths << v.sort_order_field_path + v.directive "external" + end + end + + t.index "widgets" + end + end + + expect(sort_order_type_from(result, "Widget")).to eq(<<~EOS.strip) + enum WidgetSortOrderInput { + id_ASC + id_DESC + the_options_the_size_ASC @deprecated + the_options_the_size_DESC @deprecated + the_options_some_id_ASC @deprecated @external + the_options_some_id_DESC @deprecated @external + some_float_ASC @deprecated @external + some_float_DESC @deprecated @external + } + EOS + + expect(enum_field_paths.flatten).to all be_a(SchemaElements::Field) + expect(enum_field_paths.map { |p| p.map(&:name).join(".") }).to match_array( + # this field has 1 directive for ASC, 1 for DESC = 2 total + (["the_options.the_size"] * 2) + + # this field has 2 directives for ASC, 2 for DESC = 4 total + (["the_options.some_id"] * 4) + + # this field has 2 directives for ASC, 2 for DESC = 4 total + (["some_float"] * 4) + ) + end + + it "generates the `SortOrderInput` type for the `id` field even if no other fields are sortable" do + result = define_schema do |api| + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.index "widgets" + end + end + + expect(sort_order_type_from(result, "Widget")).to eq(<<~EOS.strip) + enum WidgetSortOrderInput { + id_ASC + id_DESC + } + EOS + end + + it "does not define enum values for object fields that have a custom mapping type since we do not know if we can sort by the custom mapping" do + result = define_schema do |api| + api.object_type "Point" do |t| + t.field "x", "Float" + t.field "y", "Float" + t.mapping type: "point" + end + + api.object_type "Point2" do |t| + t.field "x", "Float" + t.field "y", "Float" + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "nullable_point", "Point" + t.field "non_null_point", "Point!" + t.field "nullable_point2", "Point2" + t.field "non_null_point2", "Point2!" + t.index "widgets" + end + end + + expect(sort_order_type_from(result, "Widget")).to eq(<<~EOS.strip) + enum WidgetSortOrderInput { + id_ASC + id_DESC + nullable_point2_x_ASC + nullable_point2_x_DESC + nullable_point2_y_ASC + nullable_point2_y_DESC + non_null_point2_x_ASC + non_null_point2_x_DESC + non_null_point2_y_ASC + non_null_point2_y_DESC + } + EOS + end + + it "makes object fields with custom mapping options sortable so long as the `type` hasn't been customized" do + result = define_schema do |api| + api.object_type "Point" do |t| + t.field "x", "Float" + t.field "y", "Float" + t.mapping meta: {defined_by: "ElasticGraph"} + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "nullable_point", "Point" + t.field "non_null_point", "Point!" + t.index "widgets" + end + end + + expect(sort_order_type_from(result, "Widget")).to eq(<<~EOS.strip) + enum WidgetSortOrderInput { + id_ASC + id_DESC + nullable_point_x_ASC + nullable_point_x_DESC + nullable_point_y_ASC + nullable_point_y_DESC + non_null_point_x_ASC + non_null_point_x_DESC + non_null_point_y_ASC + non_null_point_y_DESC + } + EOS + end + + shared_examples_for "a type with subtypes" do |type_def_method| + it "is generated for a directly indexed type" do + result = define_schema do |api| + api.object_type "Person" do |t| + link_subtype_to_supertype(t, "Inventor") + t.field "id", "ID!" + t.field "name", "String" + t.field "age", "Int" + t.field "nationality", "String" + t.index "people" + end + + api.object_type "Company" do |t| + link_subtype_to_supertype(t, "Inventor") + t.field "id", "ID!" + t.field "name", "String" + t.field "age", "Int" + t.field "stock_ticker", "String" + t.index "companies" + end + + api.public_send type_def_method, "Inventor" do |t| + link_supertype_to_subtypes(t, "Person", "Company") + end + end + + expect(sort_order_type_from(result, "Inventor")).to eq(<<~EOS.strip) + enum InventorSortOrderInput { + id_ASC + id_DESC + name_ASC + name_DESC + age_ASC + age_DESC + nationality_ASC + nationality_DESC + stock_ticker_ASC + stock_ticker_DESC + } + EOS + end + + it "is not generated for an embedded type" do + result = define_schema do |api| + api.object_type "Person" do |t| + link_subtype_to_supertype(t, "Inventor") + t.field "name", "String" + t.field "age", "Int" + t.field "nationality", "String" + end + + api.object_type "Company" do |t| + link_subtype_to_supertype(t, "Inventor") + t.field "name", "String" + t.field "age", "Int" + t.field "stock_ticker", "String" + end + + api.public_send type_def_method, "Inventor" do |t| + link_supertype_to_subtypes(t, "Person", "Company") + end + end + + expect(sort_order_type_from(result, "Inventor")).to eq nil + end + end + + context "on a type union" do + include_examples "a type with subtypes", :union_type do + def link_subtype_to_supertype(object_type, supertype_name) + # nothing to do; the linkage happens via a `subtypes` call on the supertype + end + + def link_supertype_to_subtypes(union_type, *subtype_names) + union_type.subtypes(*subtype_names) + end + end + end + + context "on an interface type" do + include_examples "a type with subtypes", :interface_type do + def link_subtype_to_supertype(object_type, interface_name) + object_type.implements interface_name + end + + def link_supertype_to_subtypes(interface_type, *subtype_names) + # nothing to do; the linkage happens via an `implements` call on the subtype + end + end + end + + it "recursively resolves the union of fields, to support type hierarchies" do + result = define_schema do |api| + api.object_type "Person" do |t| + t.implements "Human" + t.field "id", "ID!" + t.field "name", "String" + t.field "age", "Int" + t.field "nationality", "String" + t.index "people" + end + + api.object_type "Company" do |t| + t.implements "Organization" + t.field "id", "ID!" + t.field "name", "String" + t.field "age", "Int" + t.field "stock_ticker", "String" + t.index "companies" + end + + api.interface_type "Human" do |t| + t.implements "Inventor" + end + + api.interface_type "Organization" do |t| + t.implements "Inventor" + end + + api.interface_type "Inventor" do |t| + end + end + + expect(sort_order_type_from(result, "Inventor")).to eq(<<~EOS.strip) + enum InventorSortOrderInput { + id_ASC + id_DESC + name_ASC + name_DESC + age_ASC + age_DESC + nationality_ASC + nationality_DESC + stock_ticker_ASC + stock_ticker_DESC + } + EOS + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/sub_aggregation_type_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/sub_aggregation_type_spec.rb new file mode 100644 index 00000000..509322be --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/sub_aggregation_type_spec.rb @@ -0,0 +1,851 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "graphql_schema_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "GraphQL schema generation", "a `*SubAggregation` type" do + include_context "GraphQL schema spec support" + + shared_examples_for "sub-aggregation types" do + describe "`*SubAggregation` (singular) types" do + it "defines one and related relay types for each type referenced from a `nested` field" do + results = define_schema do |schema| + schema.object_type "Player" do |t| + t.field "id", "ID!" + t.field "name", "String" + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "name", "String" + define_collection_field t, "players", "Player" do |f| + f.mapping type: "nested" + end + t.index "teams" + end + end + + expect(sub_aggregation_type_from(results, "TeamPlayer", include_docs: true)).to eq(<<~EOS.strip) + """ + Return type representing a bucket of `Player` objects for a sub-aggregation within each `TeamAggregation`. + """ + type TeamPlayerSubAggregation { + """ + Details of the count of `Player` documents in a sub-aggregation bucket. + """ + #{schema_elements.count_detail}: AggregationCountDetail + """ + Used to specify the `Player` fields to group by. The returned values identify each sub-aggregation bucket. + """ + #{schema_elements.grouped_by}: PlayerGroupedBy + """ + Provides computed aggregated values over all `Player` documents in a sub-aggregation bucket. + """ + #{schema_elements.aggregated_values}: PlayerAggregatedValues + } + EOS + + expect(sub_aggregation_connection_type_from(results, "TeamPlayer", include_docs: true)).to eq(<<~EOS.strip) + """ + Represents a collection of `TeamPlayerSubAggregation` results. + """ + type TeamPlayerSubAggregationConnection { + """ + The list of `TeamPlayerSubAggregation` results. + """ + nodes: [TeamPlayerSubAggregation!]! + } + EOS + + # We should not have an Edge type since we don't support pagination at this time. + expect(sub_aggregation_edge_type_from(results, "TeamPlayer")).to eq(nil) + end + + it "correctly names them when dealing with a nested-field-in-a-nested-field" do + results = define_schema do |schema| + schema.object_type "Season" do |t| + t.field "year", "Int" + end + + schema.object_type "Player" do |t| + t.field "id", "ID!" + t.field "name", "String" + define_collection_field t, "seasons", "Season" do |f| + f.mapping type: "nested" + end + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "name", "String" + define_collection_field t, "players", "Player" do |f| + f.mapping type: "nested" + end + t.index "teams" + end + end + + expect(sub_aggregation_type_from(results, "TeamPlayerSeason", include_docs: true)).to eq(<<~EOS.strip) + """ + Return type representing a bucket of `Season` objects for a sub-aggregation within each `TeamPlayerSubAggregation`. + """ + type TeamPlayerSeasonSubAggregation { + """ + Details of the count of `Season` documents in a sub-aggregation bucket. + """ + #{schema_elements.count_detail}: AggregationCountDetail + """ + Used to specify the `Season` fields to group by. The returned values identify each sub-aggregation bucket. + """ + #{schema_elements.grouped_by}: SeasonGroupedBy + """ + Provides computed aggregated values over all `Season` documents in a sub-aggregation bucket. + """ + #{schema_elements.aggregated_values}: SeasonAggregatedValues + } + EOS + + expect(sub_aggregation_connection_type_from(results, "TeamPlayerSeason", include_docs: true)).to eq(<<~EOS.strip) + """ + Represents a collection of `TeamPlayerSeasonSubAggregation` results. + """ + type TeamPlayerSeasonSubAggregationConnection { + """ + The list of `TeamPlayerSeasonSubAggregation` results. + """ + nodes: [TeamPlayerSeasonSubAggregation!]! + } + EOS + + # We should not have an Edge type since we don't support pagination at this time. + expect(sub_aggregation_edge_type_from(results, "TeamPlayerSeason")).to eq(nil) + end + + it "defines separate contextual `*SubAggregation` types for each context a nested type lives in" do + results = define_schema do |schema| + schema.object_type "Season" do |t| + t.field "year", "Int" + define_collection_field t, "players", "Player" do |f| + f.mapping type: "nested" + end + define_collection_field t, "comments", "Comment" do |f| + f.mapping type: "nested" + end + end + + schema.object_type "Player" do |t| + t.field "id", "ID!" + t.field "name", "String" + define_collection_field t, "comments", "Comment" do |f| + f.mapping type: "nested" + end + end + + schema.object_type "Comment" do |t| + t.field "author", "String" + t.field "text", "String" + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "name", "String" + define_collection_field t, "current_players", "Player" do |f| + f.mapping type: "nested" + end + + define_collection_field t, "seasons", "Season" do |f| + f.mapping type: "nested" + end + t.index "teams" + end + end + + expect(sub_aggregation_type_from(results, "TeamPlayer").split("\n").first).to eq("type TeamPlayerSubAggregation {") + expect(sub_aggregation_type_from(results, "TeamSeasonPlayer").split("\n").first).to eq("type TeamSeasonPlayerSubAggregation {") + expect(sub_aggregation_type_from(results, "TeamPlayerComment").split("\n").first).to eq("type TeamPlayerCommentSubAggregation {") + expect(sub_aggregation_type_from(results, "TeamSeasonPlayerComment").split("\n").first).to eq("type TeamSeasonPlayerCommentSubAggregation {") + expect(sub_aggregation_type_from(results, "TeamSeasonConnectionPlayer")).to be nil + expect(sub_aggregation_type_from(results, "TeamSeasonConnectionComment")).to be nil + expect(sub_aggregation_type_from(results, "TeamSeasonConnectionPlayerComment")).to be nil + expect(sub_aggregation_type_from(results, "TeamSeasonConnectionPlayerConnectionComment")).to be nil + expect(sub_aggregation_type_from(results, "TeamPlayerConnectionComment")).to be nil + end + + it "defines one and related relay types when there are extra non-nested object layers in the definition" do + results = define_schema do |schema| + schema.object_type "Player" do |t| + t.field "id", "ID!" + t.field "name", "String" + end + + schema.object_type "TeamPayroll" do |t| + t.field "personnel", "TeamPersonnel" + end + + schema.object_type "TeamPersonnel" do |t| + define_collection_field t, "players", "Player" do |f| + f.mapping type: "nested" + end + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "payroll", "TeamPayroll" + t.index "teams" + end + end + + expect(sub_aggregation_type_from(results, "TeamPlayer", include_docs: true)).to eq(<<~EOS.strip) + """ + Return type representing a bucket of `Player` objects for a sub-aggregation within each `TeamAggregation`. + """ + type TeamPlayerSubAggregation { + """ + Details of the count of `Player` documents in a sub-aggregation bucket. + """ + #{schema_elements.count_detail}: AggregationCountDetail + """ + Used to specify the `Player` fields to group by. The returned values identify each sub-aggregation bucket. + """ + #{schema_elements.grouped_by}: PlayerGroupedBy + """ + Provides computed aggregated values over all `Player` documents in a sub-aggregation bucket. + """ + #{schema_elements.aggregated_values}: PlayerAggregatedValues + } + EOS + + expect(sub_aggregation_connection_type_from(results, "TeamPlayer", include_docs: true)).to eq(<<~EOS.strip) + """ + Represents a collection of `TeamPlayerSubAggregation` results. + """ + type TeamPlayerSubAggregationConnection { + """ + The list of `TeamPlayerSubAggregation` results. + """ + nodes: [TeamPlayerSubAggregation!]! + } + EOS + + # We should not have an Edge type since we don't support pagination at this time. + expect(sub_aggregation_edge_type_from(results, "TeamPlayer")).to eq(nil) + end + + it "does not define one or related relay types for types which are not referenced from a `nested` field" do + results = define_schema do |schema| + schema.object_type "Player" do |t| + t.field "id", "ID!" + t.field "name", "String" + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + define_collection_field t, "players", "Player" do |f| + f.mapping type: "object" + end + t.index "teams" + end + end + + expect(sub_aggregation_type_from(results, "Player")).to eq nil + expect(sub_aggregation_connection_type_from(results, "Player")).to eq(nil) + expect(sub_aggregation_edge_type_from(results, "Player")).to eq(nil) + end + + it "omits the `grouped_by` field if no fields are groupable" do + results = define_schema do |schema| + schema.object_type "Player" do |t| + t.field "id", "ID!", groupable: false + t.field "name", "String", groupable: false + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "name", "String" + define_collection_field t, "players", "Player" do |f| + f.mapping type: "nested" + end + t.index "teams" + end + end + + expect(sub_aggregation_type_from(results, "TeamPlayer")).to eq(<<~EOS.strip) + type TeamPlayerSubAggregation { + #{schema_elements.count_detail}: AggregationCountDetail + #{schema_elements.aggregated_values}: PlayerAggregatedValues + } + EOS + end + + it "omits the `aggregated_values` field if no fields are aggregatable" do + results = define_schema do |schema| + schema.object_type "Player" do |t| + t.field "id", "ID!", aggregatable: false + t.field "name", "String", aggregatable: false + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "name", "String" + define_collection_field t, "players", "Player" do |f| + f.mapping type: "nested" + end + t.index "teams" + end + end + + expect(sub_aggregation_type_from(results, "TeamPlayer")).to eq(<<~EOS.strip) + type TeamPlayerSubAggregation { + #{schema_elements.count_detail}: AggregationCountDetail + #{schema_elements.grouped_by}: PlayerGroupedBy + } + EOS + end + end + + describe "`*SubAggregations` (plural) types" do + it "defines one with a field for each `nested` field of the source type" do + results = define_schema do |schema| + schema.object_type "Player" do |t| + t.field "name", "String" + end + + schema.object_type "Season" do |t| + t.field "year", "Int" + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "name", "String" + define_collection_field t, "players_nested", "Player" do |f| + f.documentation "The players on the team." + f.mapping type: "nested" + end + define_collection_field t, "seasons_nested", "Season" do |f| + f.documentation "The seasons the team has played." + f.mapping type: "nested" + end + define_collection_field t, "players_object", "Player" do |f| + f.mapping type: "object" + end + define_collection_field t, "seasons_object", "Season" do |f| + f.mapping type: "object" + end + t.index "teams" + end + end + + expect(aggregation_sub_aggregations_type_from(results, "Team", include_docs: true)).to eq(<<~EOS.strip) + """ + Provides access to the `#{schema_elements.sub_aggregations}` within each `TeamAggregation`. + """ + type TeamAggregationSubAggregations { + """ + Used to perform a sub-aggregation of `players_nested`: + + > The players on the team. + """ + players_nested( + """ + Used to filter the `Player` documents included in this sub-aggregation based on the provided criteria. + """ + #{schema_elements.filter}: PlayerFilterInput + """ + Determines how many sub-aggregation buckets should be returned. + """ + #{schema_elements.first}: Int): TeamPlayerSubAggregationConnection + """ + Used to perform a sub-aggregation of `seasons_nested`: + + > The seasons the team has played. + """ + seasons_nested( + """ + Used to filter the `Season` documents included in this sub-aggregation based on the provided criteria. + """ + #{schema_elements.filter}: SeasonFilterInput + """ + Determines how many sub-aggregation buckets should be returned. + """ + #{schema_elements.first}: Int): TeamSeasonSubAggregationConnection + } + EOS + end + + it "allows the sub-aggregation fields to be customized" do + results = define_schema do |schema| + schema.raw_sdl "directive @external on FIELD_DEFINITION" + + schema.object_type "Player" do |t| + t.field "name", "String" + end + + schema.object_type "Season" do |t| + t.field "year", "Int" + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "name", "String" + + define_collection_field t, "players_nested", "Player" do |f| + f.mapping type: "nested" + f.customize_sub_aggregations_field do |saf| + saf.directive "deprecated" + end + end + + define_collection_field t, "seasons_nested", "Season" do |f| + f.mapping type: "nested" + f.customize_sub_aggregations_field do |saf| + saf.directive "external" + end + end + + t.index "teams" + end + end + + expect(aggregation_sub_aggregations_type_from(results, "Team")).to eq(<<~EOS.strip) + type TeamAggregationSubAggregations { + players_nested( + #{schema_elements.filter}: PlayerFilterInput + #{schema_elements.first}: Int): TeamPlayerSubAggregationConnection @deprecated + seasons_nested( + #{schema_elements.filter}: SeasonFilterInput + #{schema_elements.first}: Int): TeamSeasonSubAggregationConnection @external + } + EOS + end + + it "defines extra types and fields when there is an additional object layer in the definition" do + results = define_schema do |schema| + schema.object_type "Player" do |t| + t.field "name", "String" + end + + schema.object_type "Season" do |t| + t.field "year", "Int" + end + + schema.object_type "TeamCollections" do |t| + define_collection_field t, "players_nested", "Player" do |f| + f.mapping type: "nested" + end + define_collection_field t, "seasons_nested", "Season" do |f| + f.mapping type: "nested" + end + define_collection_field t, "players_object", "Player" do |f| + f.mapping type: "object" + end + define_collection_field t, "seasons_object", "Season" do |f| + f.mapping type: "object" + end + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "collections", "TeamCollections" + t.index "teams" + end + end + + expect(aggregation_sub_aggregations_type_from(results, "Team", include_docs: true)).to eq(<<~EOS.strip) + """ + Provides access to the `#{schema_elements.sub_aggregations}` within each `TeamAggregation`. + """ + type TeamAggregationSubAggregations { + """ + Used to perform a sub-aggregation of `collections`. + """ + collections: TeamAggregationCollectionsSubAggregations + } + EOS + + expect(aggregation_sub_aggregations_type_from(results, "Team", under: "Collections", include_docs: true)).to eq(<<~EOS.strip) + """ + Provides access to the `#{schema_elements.sub_aggregations}` under `collections` within each `TeamAggregation`. + """ + type TeamAggregationCollectionsSubAggregations { + """ + Used to perform a sub-aggregation of `players_nested`. + """ + players_nested( + """ + Used to filter the `Player` documents included in this sub-aggregation based on the provided criteria. + """ + #{schema_elements.filter}: PlayerFilterInput + """ + Determines how many sub-aggregation buckets should be returned. + """ + #{schema_elements.first}: Int): TeamPlayerSubAggregationConnection + """ + Used to perform a sub-aggregation of `seasons_nested`. + """ + seasons_nested( + """ + Used to filter the `Season` documents included in this sub-aggregation based on the provided criteria. + """ + #{schema_elements.filter}: SeasonFilterInput + """ + Determines how many sub-aggregation buckets should be returned. + """ + #{schema_elements.first}: Int): TeamSeasonSubAggregationConnection + } + EOS + end + + it "defines extra types and fields when there are multiple additional object layers in the definition" do + results = define_schema do |schema| + schema.object_type "Player" do |t| + t.field "name", "String" + end + + schema.object_type "Season" do |t| + t.field "year", "Int" + end + + schema.object_type "TeamCollectionsInner" do |t| + define_collection_field t, "players_nested", "Player" do |f| + f.mapping type: "nested" + end + define_collection_field t, "seasons_nested", "Season" do |f| + f.mapping type: "nested" + end + define_collection_field t, "players_object", "Player" do |f| + f.mapping type: "object" + end + define_collection_field t, "seasons_object", "Season" do |f| + f.mapping type: "object" + end + end + + schema.object_type "TeamCollectionsOuter" do |t| + t.field "inner", "TeamCollectionsInner" + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "outer_collections", "TeamCollectionsOuter" + t.index "teams" + end + end + + expect(aggregation_sub_aggregations_type_from(results, "Team", include_docs: true)).to eq(<<~EOS.strip) + """ + Provides access to the `#{schema_elements.sub_aggregations}` within each `TeamAggregation`. + """ + type TeamAggregationSubAggregations { + """ + Used to perform a sub-aggregation of `outer_collections`. + """ + outer_collections: TeamAggregationOuterCollectionsSubAggregations + } + EOS + + expect(aggregation_sub_aggregations_type_from(results, "Team", under: "OuterCollections", include_docs: true)).to eq(<<~EOS.strip) + """ + Provides access to the `#{schema_elements.sub_aggregations}` under `outer_collections` within each `TeamAggregation`. + """ + type TeamAggregationOuterCollectionsSubAggregations { + """ + Used to perform a sub-aggregation of `inner`. + """ + inner: TeamAggregationOuterCollectionsInnerSubAggregations + } + EOS + + expect(aggregation_sub_aggregations_type_from(results, "Team", under: "OuterCollectionsInner", include_docs: true)).to eq(<<~EOS.strip) + """ + Provides access to the `#{schema_elements.sub_aggregations}` under `outer_collections.inner` within each `TeamAggregation`. + """ + type TeamAggregationOuterCollectionsInnerSubAggregations { + """ + Used to perform a sub-aggregation of `players_nested`. + """ + players_nested( + """ + Used to filter the `Player` documents included in this sub-aggregation based on the provided criteria. + """ + #{schema_elements.filter}: PlayerFilterInput + """ + Determines how many sub-aggregation buckets should be returned. + """ + #{schema_elements.first}: Int): TeamPlayerSubAggregationConnection + """ + Used to perform a sub-aggregation of `seasons_nested`. + """ + seasons_nested( + """ + Used to filter the `Season` documents included in this sub-aggregation based on the provided criteria. + """ + #{schema_elements.filter}: SeasonFilterInput + """ + Determines how many sub-aggregation buckets should be returned. + """ + #{schema_elements.first}: Int): TeamSeasonSubAggregationConnection + } + EOS + end + + it "generates the expected types when dealing with a nested-field-in-a-nested-field" do + results = define_schema do |schema| + schema.object_type "Season" do |t| + t.field "year", "Int" + end + + schema.object_type "Player" do |t| + t.field "id", "ID!" + t.field "name", "String" + define_collection_field t, "seasons", "Season" do |f| + f.mapping type: "nested" + end + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "name", "String" + define_collection_field t, "players", "Player" do |f| + f.mapping type: "nested" + end + t.index "teams" + end + end + + expect(aggregation_sub_aggregations_type_from(results, "Team", include_docs: true)).to eq(<<~EOS.strip) + """ + Provides access to the `#{schema_elements.sub_aggregations}` within each `TeamAggregation`. + """ + type TeamAggregationSubAggregations { + """ + Used to perform a sub-aggregation of `players`. + """ + players( + """ + Used to filter the `Player` documents included in this sub-aggregation based on the provided criteria. + """ + #{schema_elements.filter}: PlayerFilterInput + """ + Determines how many sub-aggregation buckets should be returned. + """ + #{schema_elements.first}: Int): TeamPlayerSubAggregationConnection + } + EOS + + expect(sub_aggregation_type_from(results, "TeamPlayer", include_docs: true)).to eq(<<~EOS.strip) + """ + Return type representing a bucket of `Player` objects for a sub-aggregation within each `TeamAggregation`. + """ + type TeamPlayerSubAggregation { + """ + Details of the count of `Player` documents in a sub-aggregation bucket. + """ + #{schema_elements.count_detail}: AggregationCountDetail + """ + Used to specify the `Player` fields to group by. The returned values identify each sub-aggregation bucket. + """ + #{schema_elements.grouped_by}: PlayerGroupedBy + """ + Provides computed aggregated values over all `Player` documents in a sub-aggregation bucket. + """ + #{schema_elements.aggregated_values}: PlayerAggregatedValues + """ + Used to perform sub-aggregations of `TeamPlayerSubAggregation` data. + """ + #{schema_elements.sub_aggregations}: TeamPlayerSubAggregationSubAggregations + } + EOS + + expect(sub_aggregation_sub_aggregations_type_from(results, "TeamPlayer", include_docs: true)).to eq(<<~EOS.strip) + """ + Provides access to the `#{schema_elements.sub_aggregations}` within each `TeamPlayerSubAggregation`. + """ + type TeamPlayerSubAggregationSubAggregations { + """ + Used to perform a sub-aggregation of `seasons`. + """ + seasons( + """ + Used to filter the `Season` documents included in this sub-aggregation based on the provided criteria. + """ + #{schema_elements.filter}: SeasonFilterInput + """ + Determines how many sub-aggregation buckets should be returned. + """ + #{schema_elements.first}: Int): TeamPlayerSeasonSubAggregationConnection + } + EOS + end + + it "supports aggregations for a type that's both a root indexed type and embedded, when we have object and nested fields" do + results = define_schema do |schema| + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "name", "String" + + define_collection_field t, "current_players", "Player" do |f| + f.mapping type: "object" + end + + t.index "teams" + end + + schema.object_type "Player" do |t| + t.field "id", "ID" + t.field "name", "String" + + define_collection_field t, "seasons", "Season" do |f| + f.mapping type: "nested" + end + + t.index "players" + end + + schema.object_type "Season" do |t| + t.field "year", "Int" + end + end + + expect(aggregation_sub_aggregations_type_from(results, "Team")).to eq(<<~EOS.strip) + type TeamAggregationSubAggregations { + current_players: TeamAggregationCurrentPlayersSubAggregations + } + EOS + + expect(aggregation_sub_aggregations_type_from(results, "Team", under: "CurrentPlayers")).to eq(<<~EOS.strip) + type TeamAggregationCurrentPlayersSubAggregations { + seasons( + #{schema_elements.filter}: SeasonFilterInput + #{schema_elements.first}: Int): TeamSeasonSubAggregationConnection + } + EOS + + expect(aggregation_sub_aggregations_type_from(results, "Player")).to eq(<<~EOS.strip) + type PlayerAggregationSubAggregations { + seasons( + #{schema_elements.filter}: SeasonFilterInput + #{schema_elements.first}: Int): PlayerSeasonSubAggregationConnection + } + EOS + end + + it "does not define any for an indexed type that has no nested fields" do + results = define_schema do |schema| + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index "teams" + end + end + + expect(aggregation_sub_aggregations_type_from(results, "Team")).to eq nil + end + + it "does not get stuck in infinite recursion when a nested field is referenced from a type in a circular relationship with another type" do + results = define_schema do |schema| + schema.object_type "Component" do |t| + t.field "id", "ID!" + t.relates_to_one "widget", "Widget", via: "widgetId", dir: :out + t.index "components" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.relates_to_one "part", "Part", via: "partId", dir: :out + t.relates_to_one "component", "Component", via: "widgetId", dir: :out + t.index "widgets" + end + + schema.object_type "Material" do |t| + t.field "type", "String" + end + + schema.object_type "Part" do |t| + t.field "id", "ID!" + define_collection_field t, "materials", "Material" do |f| + f.mapping type: "nested" + end + t.index "parts" + end + end + + expect(aggregation_sub_aggregations_type_from(results, "Part")).to eq <<~EOS.strip + type PartAggregationSubAggregations { + materials( + #{schema_elements.filter}: MaterialFilterInput + #{schema_elements.first}: Int): PartMaterialSubAggregationConnection + } + EOS + end + end + + it "avoids generating sub-aggregation types for type unions that have indexed sub-types because it is hard to do correctly and we do not need it yet" do + results = define_schema do |schema| + schema.object_type "Options" do |t| + t.field "color", "String" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "nested_options", "[Options!]!" do |f| + # This mapping field is what triggers sub-aggregation types to be generated at all. + f.mapping type: "nested" + end + t.index "widgets" + end + + schema.object_type "Component" do |t| + t.field "id", "ID" + t.index "components" + end + + schema.union_type "WidgetOrComponent" do |t| + t.subtypes "Widget", "Component" + end + end + + # They should be generated for `Widget`... + expect(results.lines.grep(/WidgetOptions\w*SubAgg/)).not_to be_empty + # ...but not for `WidgetOrComponent`. + expect(results.lines.grep(/WidgetOrComponentOptions\w*SubAgg/)).to be_empty + + # And its aggregation type should not have a `sub_aggregations` field. + expect(aggregation_type_from(results, "WidgetOrComponent")).to eq(<<~EOS.strip) + type WidgetOrComponentAggregation { + #{schema_elements.count}: JsonSafeLong! + #{schema_elements.aggregated_values}: WidgetOrComponentAggregatedValues + } + EOS + end + end + + with_both_casing_forms do + context "when collection fields are defined as a GraphQL list" do + include_examples "sub-aggregation types" + + def define_collection_field(parent_type, field_name, element_type_name, &block) + parent_type.field(field_name, "[#{element_type_name}!]!", &block) + end + end + + context "when collection fields are defined as a relay connection using `paginated_connection_field`" do + include_examples "sub-aggregation types" + + def define_collection_field(parent_type, field_name, element_type_name, &block) + parent_type.paginated_collection_field(field_name, element_type_name, &block) + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/union_type_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/union_type_spec.rb new file mode 100644 index 00000000..d3d4b237 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/union_type_spec.rb @@ -0,0 +1,297 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "graphql_schema_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "GraphQL schema generation", "#union_type" do + include_context "GraphQL schema spec support" + + with_both_casing_forms do + it "can generate a union of a single type" do + pre_def = ->(api) { + api.object_type("Person") do |t| + t.field "id", "ID" + end + } + + result = union_type "Inventor", pre_def: pre_def do |t| + t.subtype "Person" + end + + expect(result).to eq(<<~EOS) + union Inventor = Person + EOS + end + + it "can generate a union of a multiple types" do + pre_def = ->(api) { + %w[Person Company].each do |name| + api.object_type(name) do |t| + t.field "id", "ID" + # Verify `relates_to_many` doesn't cause problems -- originally, our field argument + # implementation wrongly caused it to consider the same `relates_to_many` field on + # both subtypes to be different when they were the same. + t.relates_to_many "inventions", "Inventor", via: "inventor_id", dir: :in, singular: "invention" + t.index name.downcase + end + end + } + + result = union_type "Inventor", pre_def: pre_def do |t| + t.subtype "Person" + t.subtype "Company" + end + + expect(result).to eq(<<~EOS) + union Inventor = Person | Company + EOS + end + + it "allows many subtypes to be defined in one call for convenience" do + pre_def = ->(api) { + %w[Red Green Yellow Orange].each do |name| + api.object_type(name) do |t| + t.field "id", "ID" + end + end + } + + result = union_type "Color", pre_def: pre_def do |e| + # they can be individually listed + e.subtypes "Red", "Green" + # ...or passed as a single array + e.subtypes %w[Yellow Orange] + end + + expect(result).to eq(<<~EOS) + union Color = Red | Green | Yellow | Orange + EOS + end + + it "can generate directives on the type" do + pre_def = ->(api) { + api.raw_sdl "directive @foo(size: Int = null) repeatable on UNION" + + %w[Red Green Blue].each do |name| + api.object_type(name) do |t| + t.field "id", "ID" + end + end + } + + result = union_type "Color", pre_def: pre_def do |t| + t.directive "foo", size: 1 + t.directive "foo", size: 3 + t.subtype "Red" + t.subtype "Green" + t.subtype "Blue" + end + + expect(result).to eq(<<~EOS) + union Color @foo(size: 1) @foo(size: 3) = Red | Green | Blue + EOS + end + + it "supports doc comments on the type" do + pre_def = ->(api) { + api.object_type("Person") do |t| + t.field "id", "ID" + end + } + + result = union_type "Inventor", pre_def: pre_def do |t| + t.documentation "A person who has invented something." + t.subtype "Person" + end + + expect(result).to eq(<<~EOS) + """ + A person who has invented something. + """ + union Inventor = Person + EOS + end + + it "respects configured type name overrides in both the supertype and subtype names" do + result = define_schema(type_name_overrides: {"Thing" => "Entity", "Widget" => "Gadget"}) do |schema| + schema.object_type "Component" do |t| + t.field "id", "ID" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + end + + schema.union_type "Thing" do |t| + t.subtype "Component" + t.subtype "Widget" + t.index "things" + end + end + + expect(type_def_from(result, "Thing")).to eq nil + expect(type_def_from(result, "Entity")).to eq("union Entity = Component | Gadget") + + expect(type_def_from(result, "EntityFilterInput")).not_to eq nil + expect(type_def_from(result, "EntityConnection")).not_to eq nil + expect(type_def_from(result, "EntityEdge")).not_to eq nil + + # Verify that there are _no_ `Thing` types defined + expect(result.lines.grep(/Thing/)).to be_empty + end + + it "raises a clear error when the union type name is invalid" do + expect { + define_schema do |api| + api.object_type("Person") do |t| + t.field "id", "ID" + end + + api.union_type("Invalid.Name") {} + end + }.to raise_invalid_graphql_name_error_for("Invalid.Name") + end + + it "raises a clear error when the same type union is defined multiple times" do + expect { + define_schema do |api| + %w[Red Red2].each do |name| + api.object_type name do |t| + t.field "id", "ID" + end + end + + api.union_type "Color" do |t| + t.subtype "Red" + end + + api.union_type "Color" do |t| + t.subtype "Red2" + end + end + }.to raise_error Errors::SchemaError, a_string_including("Duplicate", "Color") + end + + it "raises a clear error when the same subtype is defined multiple times" do + expect { + define_schema do |api| + %w[Red Green Blue].each do |name| + api.object_type(name) do |t| + t.field "id", "ID" + end + end + + api.union_type "Color" do |t| + t.subtype "Red" + t.subtype "Green" + t.subtype "Red" + end + end + }.to raise_error Errors::SchemaError, a_string_including("Duplicate", "Union", "Red") + end + + it "raises a clear error when no subtypes are defined" do + expect { + union_type "Color" do |e| + end + }.to raise_error Errors::SchemaError, a_string_including("Color", "has no subtypes") + end + + it "raises a clear error if one of the subtypes is undefined" do + expect { + union_type "Inventor" do |t| + t.subtype "Person" + end + }.to raise_error Errors::SchemaError, a_string_including("Person", "not a defined object type") + end + + it "raises a clear error when the type name has the type wrapping characters" do + expect { + define_schema do |api| + api.object_type("Person") do |t| + t.field "id", "ID" + end + + api.union_type "[InvalidName!]!" do |e| + e.subtype "Person" + end + end + }.to raise_invalid_graphql_name_error_for("[InvalidName!]!") + end + + it "raises a clear error if one of the subtypes is an enum type" do + expect { + define_schema do |api| + api.enum_type("Person") do |t| + t.value "Bob" + end + + api.union_type "Inventor" do |t| + t.subtype "Person" + end + end + }.to raise_error Errors::SchemaError, a_string_including("Person", "not a defined object type") + end + + it "raises a clear error if some subtypes are indexed and others are not" do + expect { + define_schema do |api| + api.object_type("Person") do |t| + t.field "id", "ID" + t.index "people" + end + + api.object_type("Company") do |t| + end + + api.union_type "Inventor" do |t| + t.subtypes "Person", "Company" + end + end + }.to raise_error Errors::SchemaError, a_string_including("Inventor", "indexed") + end + + it "allows the same field on two subtypes to have different documentation" do + result = define_schema do |api| + api.object_type "Person" do |t| + t.field "name", "String" do |f| + f.documentation "The person's name." + end + t.field "nationality", "String" + end + + api.object_type "Company" do |t| + t.field "name", "String" do |f| + f.documentation "The company's name." + end + t.field "stock_ticker", "String" + end + + api.union_type "Inventor" do |t| + t.subtypes "Person", "Company" + end + end + + expect(type_def_from(result, "Inventor")).to eq("union Inventor = Person | Company") + end + + def union_type(name, *args, pre_def: nil, **options, &block) + result = define_schema do |api| + pre_def&.call(api) + api.union_type(name, *args, **options, &block) + end + + # We add a line break to match the expectations which use heredocs. + type_def_from(result, name, include_docs: true) + "\n" + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/json_schema_with_metadata_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/json_schema_with_metadata_spec.rb new file mode 100644 index 00000000..fad8dbee --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/json_schema_with_metadata_spec.rb @@ -0,0 +1,1070 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/schema_definition_helpers" +require "elastic_graph/schema_definition/indexing/json_schema_with_metadata" + +module ElasticGraph + module SchemaDefinition + module Indexing + ::RSpec.describe JSONSchemaWithMetadata do + include_context "SchemaDefinitionHelpers" + + it "ignores derived indexed types that do not show up in the JSON schema" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "amount", "Float" + t.field "cost_currency", "String" + t.field "cost_currency_name", "String" + t.derive_indexed_type_fields "WidgetCurrency", from_id: "cost_currency" do |derive| + derive.immutable_value "name", from: "cost_currency_name" + end + end + + schema.object_type "WidgetCurrency" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index "widget_currencies" + end + end + + expect(v1_json_schema.fetch("$defs").keys).to include("Widget").and exclude("WidgetCurrency") + end + + context "when merged into an old versioned JSON schema" do + it "maintains the same metadata when a field has not changed" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "amount", "Float" + end + end + + updated_v1_json_schema = dump_versioned_json_schema(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget" do |t| + t.field "amount", "Float" + end + end + + expect( + metadata_for(v1_json_schema, "Widget", "amount") + ).to eq(metadata_for(updated_v1_json_schema, "Widget", "amount")).and have_dumped_metadata("amount", "Float") + end + + it "does not record metadata on the `__typename` field since it has special handling in our indexing logic" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "amount", "Float" + end + end + + updated_v1_json_schema = dump_versioned_json_schema(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget" do |t| + t.field "amount", "Float" + end + end + + expect( + v1_json_schema.dig("$defs", "Widget", "properties", "__typename").keys + ).to eq(updated_v1_json_schema.dig("$defs", "Widget", "properties", "__typename").keys).and exclude("ElasticGraph") + end + + it "records a changed field `type` so that the correct indexing preparer gets used when events at the old version are ingested" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "amount", "Float" + end + end + + updated_v1_json_schema = dump_versioned_json_schema(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget" do |t| + t.field "amount", "Int" + end + end + + expect(metadata_for(v1_json_schema, "Widget", "amount")).to have_dumped_metadata("amount", "Float") + expect(metadata_for(updated_v1_json_schema, "Widget", "amount")).to have_dumped_metadata("amount", "Int") + end + + it "records a changed field `name_in_index` so that the field gets written to the correct field in the index" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "description", "String" + end + end + + updated_v1_json_schema = dump_versioned_json_schema(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget" do |t| + t.field "description", "String", name_in_index: "description_text" do |f| + f.mapping type: "text" + end + end + end + + expect(metadata_for(v1_json_schema, "Widget", "description")).to have_dumped_metadata("description", "String") + expect(metadata_for(updated_v1_json_schema, "Widget", "description")).to have_dumped_metadata("description_text", "String") + end + + it "notifies of an issue when a field has been deleted or renamed without recording what happened" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "description", "String" + end + end + + missing_fields = dump_versioned_json_schema_missing_fields(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget" do |t| + t.field "full_description", "String", name_in_index: "description" + end + end + + expect(missing_fields).to contain_exactly("Widget.description", "Widget.id") + end + + it "supports renamed fields when `renamed_from` is used" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "description", "String" + end + end + + updated_v1_json_schema = dump_versioned_json_schema(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget" do |t| + t.field "full_description", "String!", name_in_index: "description" do |f| + f.renamed_from "description" + end + end + end + + expect(metadata_for(v1_json_schema, "Widget", "description")).to have_dumped_metadata("description", "String") + expect(metadata_for(updated_v1_json_schema, "Widget", "description")).to have_dumped_metadata("description", "String!") + end + + it "supports deleted fields when `deleted_field` is used" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "description", "String" + end + end + + updated_v1_json_schema = dump_versioned_json_schema(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.deleted_field "description" + end + end + + expect(metadata_for(v1_json_schema, "Widget", "description")).to have_dumped_metadata("description", "String") + expect(metadata_for(updated_v1_json_schema, "Widget", "description")).to eq nil + end + + it "notifies of an issue when a type has been deleted or renamed without recording what happened" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Options" do |t| + t.field "size", "Int" + end + + schema.object_type "Widget" do |t| + t.field "amount", "Float" + end + end + + missing_types = dump_versioned_json_schema_missing_types(v1_json_schema) do |schema| + schema.json_schema_version 2 + + # Widget has been renamed to `Component`. + schema.object_type "Component" do |t| + t.field "amount", "Float" + end + end + + expect(missing_types).to contain_exactly("Options", "Widget") + end + + it "supports renamed types when `renamed_from` is used" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "amount", "Float" + end + end + + updated_v1_json_schema = dump_versioned_json_schema(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Component" do |t| + t.field "amount", "Int", name_in_index: "amount_int" + t.renamed_from "Widget" + end + end + + expect(metadata_for(v1_json_schema, "Widget", "amount")).to have_dumped_metadata("amount", "Float") + expect(metadata_for(updated_v1_json_schema, "Widget", "amount")).to have_dumped_metadata("amount_int", "Int") + end + + it "supports deleted types when `deleted_type` is used" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "amount", "Float" + end + end + + updated_v1_json_schema = dump_versioned_json_schema(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Component" do |t| + t.field "id", "ID" + end + + schema.deleted_type "Widget" + end + + expect(metadata_for(v1_json_schema, "Widget", "amount")).to have_dumped_metadata("amount", "Float") + expect(metadata_for(updated_v1_json_schema, "Widget", "amount")).to eq(nil) + end + + it "supports deleted and renamed fields on a renamed type so long as these are indicated through `deleted_` and `renamed_` API calls" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "token", "String" + t.field "amount", "Float" + end + end + + updated_v1_json_schema = dump_versioned_json_schema(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Component" do |t| + t.renamed_from "Widget" + + t.field "id", "ID" do |f| + f.renamed_from "token" + end + + t.deleted_field "amount" + end + end + + expect(metadata_for(updated_v1_json_schema, "Widget", "token")).to have_dumped_metadata("id", "ID") + expect(metadata_for(updated_v1_json_schema, "Widget", "amount")).to eq(nil) + end + + it "keeps track of unused `deleted_field` calls" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "token", "ID" + end + end + + unused_deprecated_elements = dump_versioned_json_schema_unused_deprecated_elements(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.deleted_field "token" # used + t.deleted_field "other" # unused + end + end + + expect(unused_deprecated_elements.map(&:description)).to eq [ + %(`type.deleted_field "other"` at #{__FILE__}:#{__LINE__ - 5}) + ] + end + + it "keeps track of unused `renamed_field` calls" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "token", "ID" + end + end + + unused_deprecated_elements = dump_versioned_json_schema_unused_deprecated_elements(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget" do |t| + t.field "id", "ID" do |f| + f.renamed_from "token" # used + f.renamed_from "other" # unused + end + end + end + + expect(unused_deprecated_elements.map(&:description)).to eq [ + %(`field.renamed_from "other"` at #{__FILE__}:#{__LINE__ - 6}) + ] + end + + it "keeps track of unused `deleted_type` calls" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "token", "ID" + end + end + + unused_deprecated_elements = dump_versioned_json_schema_unused_deprecated_elements(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.deleted_type "Widget" # used + schema.deleted_type "Other" # unused + end + + expect(unused_deprecated_elements.map(&:description)).to eq [ + %(`schema.deleted_type "Other"` at #{__FILE__}:#{__LINE__ - 4}) + ] + end + + it "keeps track of unused `renamed_type` calls" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "token", "ID" + end + end + + unused_deprecated_elements = dump_versioned_json_schema_unused_deprecated_elements(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Component" do |t| + t.field "token", "ID" + t.renamed_from "Widget" # used + t.renamed_from "Other" # unused + end + end + + expect(unused_deprecated_elements.map(&:description)).to eq [ + %(`type.renamed_from "Other"` at #{__FILE__}:#{__LINE__ - 5}) + ] + end + + context "on a type that is using `route_with`" do + it "does not allow a `route_with` field to be entirely missing from an old version of the schema" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "workspace_id", "ID" + + t.index "widgets" do |f| + f.route_with "workspace_id" + end + end + end + + missing_necessary_fields = dump_versioned_json_schema_missing_necessary_fields(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "workspace_id2", "ID" + t.deleted_field "workspace_id" + + t.index "widgets" do |f| + f.route_with "workspace_id2" + end + end + end + + expect(missing_necessary_fields).to eq [missing_necessary_field_of("routing", "Widget.workspace_id2")] + end + + it "uses the `name_in_index` when determining if a `route_with` field is missing from an old version of the schema" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "workspace_id", "ID" + + t.index "widgets" do |f| + f.route_with "workspace_id" + end + end + end + + missing_necessary_fields = dump_versioned_json_schema_missing_necessary_fields(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "workspace_id2", "ID", name_in_index: "workspace_id3" + t.deleted_field "workspace_id" + + t.index "widgets" do |f| + f.route_with "workspace_id2" + end + end + end + + expect(missing_necessary_fields).to eq [missing_necessary_field_of("routing", "Widget.workspace_id3")] + + updated_v1_json_schema = dump_versioned_json_schema(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "workspace_id2", "ID", name_in_index: "workspace_id" do |f| + f.renamed_from "workspace_id" + end + + t.index "widgets" do |f| + f.route_with "workspace_id2" + end + end + end + + expect(metadata_for(updated_v1_json_schema, "Widget", "workspace_id")).to include("nameInIndex" => "workspace_id") + end + + it "handles embedded fields when determining if a `route_with` field is missing from an old schema version" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Embedded" do |t| + t.field "workspace_id", "ID" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "embedded", "Embedded" + + t.index "widgets" do |f| + f.route_with "embedded.workspace_id" + end + end + end + + missing_necessary_fields = dump_versioned_json_schema_missing_necessary_fields(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Embedded" do |t| + t.field "workspace_id", "ID" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "embedded2", "Embedded" + t.deleted_field "embedded" + + t.index "widgets" do |f| + f.route_with "embedded2.workspace_id" + end + end + end + + expect(missing_necessary_fields).to eq [missing_necessary_field_of("routing", "Widget.embedded2.workspace_id")] + + updated_v1_json_schema = dump_versioned_json_schema(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Embedded" do |t| + t.field "workspace_id", "ID" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "embedded2", "Embedded" do |f| + f.renamed_from "embedded" + end + + t.index "widgets" do |f| + f.route_with "embedded2.workspace_id" + end + end + end + + expect(metadata_for(updated_v1_json_schema, "Widget", "embedded")).to include("nameInIndex" => "embedded2") + end + + it "handles renamed types" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "workspace_id", "ID" + + t.index "widgets" do |f| + f.route_with "workspace_id" + end + end + end + + updated_v1_json_schema = dump_versioned_json_schema(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget2" do |t| + t.field "id", "ID" + t.field "workspace_id", "ID" + t.renamed_from "Widget" + + t.index "widgets" do |f| + f.route_with "workspace_id" + end + end + end + + expect(metadata_for(updated_v1_json_schema, "Widget", "workspace_id")).to include("nameInIndex" => "workspace_id") + + missing_necessary_fields = dump_versioned_json_schema_missing_necessary_fields(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget2" do |t| + t.field "id", "ID" + t.field "workspace_id2", "ID" + t.deleted_field "workspace_id" + t.renamed_from "Widget" + + t.index "widgets" do |f| + f.route_with "workspace_id2" + end + end + end + + expect(missing_necessary_fields).to eq [missing_necessary_field_of("routing", "Widget2.workspace_id2")] + end + + it "handles deleted types" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "workspace_id", "ID" + + t.index "widgets" do |f| + f.route_with "workspace_id" + end + end + end + + updated_v1_json_schema = dump_versioned_json_schema(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.deleted_type "Widget" + + schema.object_type "Widget2" do |t| + t.field "id", "ID" + t.field "workspace_id", "ID" + + t.index "widgets" do |f| + f.route_with "workspace_id" + end + end + end + + expect(metadata_for(updated_v1_json_schema, "Widget2", "workspace_id")).to eq nil + expect(metadata_for(updated_v1_json_schema, "Widget", "workspace_id")).to eq nil + end + end + + context "on a type using `rollover`" do + it "does not allow a `rollover` field to be entirely missing from an old version of the schema" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_at", "DateTime" + + t.index "widgets" do |f| + f.rollover :yearly, "created_at" + end + end + end + + missing_necessary_fields = dump_versioned_json_schema_missing_necessary_fields(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_at2", "DateTime", name_in_index: "created_at3" + t.deleted_field "created_at" + + t.index "widgets" do |f| + f.rollover :yearly, "created_at2" + end + end + end + + expect(missing_necessary_fields).to eq [missing_necessary_field_of("rollover", "Widget.created_at3")] + + updated_v1_json_schema = dump_versioned_json_schema(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_at2", "DateTime", name_in_index: "created_at" do |f| + f.renamed_from "created_at" + end + + t.index "widgets" do |f| + f.rollover :yearly, "created_at2" + end + end + end + + expect(metadata_for(updated_v1_json_schema, "Widget", "created_at")).to include("nameInIndex" => "created_at") + end + + it "uses the `name_in_index` when determining if a `rollover` field is missing from an old version of the schema" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_at", "DateTime" + + t.index "widgets" do |f| + f.rollover :yearly, "created_at" + end + end + end + + missing_necessary_fields = dump_versioned_json_schema_missing_necessary_fields(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_at2", "DateTime" + t.deleted_field "created_at" + + t.index "widgets" do |f| + f.rollover :yearly, "created_at2" + end + end + end + + expect(missing_necessary_fields).to eq [missing_necessary_field_of("rollover", "Widget.created_at2")] + end + + it "handles embedded fields when determining if a `rollover` field is missing from an old schema version" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Embedded" do |t| + t.field "created_at", "DateTime" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "embedded", "Embedded" + + t.index "widgets" do |f| + f.rollover :yearly, "embedded.created_at" + end + end + end + + missing_necessary_fields = dump_versioned_json_schema_missing_necessary_fields(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Embedded" do |t| + t.field "created_at", "DateTime" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "embedded2", "Embedded" + t.deleted_field "embedded" + + t.index "widgets" do |f| + f.rollover :yearly, "embedded2.created_at" + end + end + end + + expect(missing_necessary_fields).to eq [missing_necessary_field_of("rollover", "Widget.embedded2.created_at")] + + updated_v1_json_schema = dump_versioned_json_schema(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Embedded" do |t| + t.field "created_at", "DateTime" + end + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "embedded2", "Embedded" do |f| + f.renamed_from "embedded" + end + + t.index "widgets" do |f| + f.rollover :yearly, "embedded2.created_at" + end + end + end + + expect(metadata_for(updated_v1_json_schema, "Widget", "embedded")).to include("nameInIndex" => "embedded2") + end + + it "handles renamed types" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_at", "DateTime" + + t.index "widgets" do |f| + f.rollover :yearly, "created_at" + end + end + end + + updated_v1_json_schema = dump_versioned_json_schema(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget2" do |t| + t.field "id", "ID" + t.field "created_at", "DateTime" + t.renamed_from "Widget" + + t.index "widgets" do |f| + f.rollover :yearly, "created_at" + end + end + end + + expect(metadata_for(updated_v1_json_schema, "Widget", "created_at")).to include("nameInIndex" => "created_at") + + missing_necessary_fields = dump_versioned_json_schema_missing_necessary_fields(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget2" do |t| + t.field "id", "ID" + t.field "created_at2", "DateTime" + t.deleted_field "created_at" + t.renamed_from "Widget" + + t.index "widgets" do |f| + f.rollover :yearly, "created_at2" + end + end + end + + expect(missing_necessary_fields).to eq [missing_necessary_field_of("rollover", "Widget2.created_at2")] + end + + it "handles deleted types" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_at", "DateTime" + + t.index "widgets" do |f| + f.rollover :yearly, "created_at" + end + end + end + + updated_v1_json_schema = dump_versioned_json_schema(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.deleted_type "Widget" + + schema.object_type "Widget2" do |t| + t.field "id", "ID" + t.field "created_at", "DateTime" + + t.index "widgets" do |f| + f.rollover :yearly, "created_at" + end + end + end + + expect(metadata_for(updated_v1_json_schema, "Widget2", "created_at")).to eq nil + expect(metadata_for(updated_v1_json_schema, "Widget", "created_at")).to eq nil + end + end + + describe "conflicting definition tracking" do + it "includes a type that exists and is referenced from `deleted_type`" do + elements = dump_versioned_json_schema_definition_conflicts do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + end + + schema.deleted_type "Widget" + end + + expect(elements.map(&:description)).to contain_exactly( + %(`schema.deleted_type "Widget"` at #{__FILE__}:#{__LINE__ - 4}) + ) + end + + it "includes a type that exists and is referenced from `renamed_from`" do + elements = dump_versioned_json_schema_definition_conflicts do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + end + + schema.object_type "Component" do |t| + t.field "id", "ID" + t.renamed_from "Widget" + end + end + + expect(elements.map(&:description)).to contain_exactly( + %(`type.renamed_from "Widget"` at #{__FILE__}:#{__LINE__ - 5}) + ) + end + + it "includes a type that exists and is referenced from `deleted_type` and `renamed_from`" do + elements = dump_versioned_json_schema_definition_conflicts do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + end + + schema.object_type "Component" do |t| + t.field "id", "ID" + t.renamed_from "Widget" + end + + schema.deleted_type "Widget" + end + + expect(elements.map(&:description)).to contain_exactly( + %(`type.renamed_from "Widget"` at #{__FILE__}:#{__LINE__ - 7}), + %(`schema.deleted_type "Widget"` at #{__FILE__}:#{__LINE__ - 5}) + ) + end + + it "includes a type that is referenced from `deleted_type` and `renamed_from` but does not exist" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "token", "ID" + end + end + + elements = dump_versioned_json_schema_definition_conflicts(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Component" do |t| + t.field "id", "ID" + t.renamed_from "Widget" + end + + schema.deleted_type "Widget" + end + + expect(elements.map(&:description)).to contain_exactly( + %(`type.renamed_from "Widget"` at #{__FILE__}:#{__LINE__ - 7}), + %(`schema.deleted_type "Widget"` at #{__FILE__}:#{__LINE__ - 5}) + ) + end + + it "includes a field that exists and is referenced from `deleted_field`" do + elements = dump_versioned_json_schema_definition_conflicts do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.deleted_field "id" + end + end + + expect(elements.map(&:description)).to contain_exactly( + %(`type.deleted_field "id"` at #{__FILE__}:#{__LINE__ - 5}) + ) + end + + it "includes a field that exists and is referenced from `renamed_from`" do + elements = dump_versioned_json_schema_definition_conflicts do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "token", "ID" do |f| + f.renamed_from "id" + end + end + end + + expect(elements.map(&:description)).to contain_exactly( + %(`field.renamed_from "id"` at #{__FILE__}:#{__LINE__ - 6}) + ) + end + + it "includes a field that exists and is referenced from `deleted_field` and `renamed_from`" do + elements = dump_versioned_json_schema_definition_conflicts do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID" + t.field "token", "ID" do |f| + f.renamed_from "id" + end + t.deleted_field "id" + end + end + + expect(elements.map(&:description)).to contain_exactly( + %(`type.deleted_field "id"` at #{__FILE__}:#{__LINE__ - 5}), + %(`field.renamed_from "id"` at #{__FILE__}:#{__LINE__ - 8}) + ) + end + + it "includes a field that is referenced from `deleted_field` and `renamed_from` but does not exist" do + v1_json_schema = dump_versioned_json_schema do |schema| + schema.json_schema_version 1 + + schema.object_type "Widget" do |t| + t.field "id", "ID" + end + end + + elements = dump_versioned_json_schema_definition_conflicts(v1_json_schema) do |schema| + schema.json_schema_version 2 + + schema.object_type "Widget" do |t| + t.field "token", "ID" do |f| + f.renamed_from "id" + end + t.deleted_field "id" + end + end + + expect(elements.map(&:description)).to contain_exactly( + %(`type.deleted_field "id"` at #{__FILE__}:#{__LINE__ - 5}), + %(`field.renamed_from "id"` at #{__FILE__}:#{__LINE__ - 8}) + ) + end + end + end + + def dump_versioned_json_schema(old_versioned_json_schema = nil, &schema_definition) + merge_result = perform_merge(old_versioned_json_schema, &schema_definition) + + expect(merge_result.missing_fields).to be_empty + expect(merge_result.missing_types).to be_empty + expect(merge_result.definition_conflicts).to be_empty + expect(merge_result.missing_necessary_fields).to be_empty + + merge_result.json_schema + end + + def dump_versioned_json_schema_missing_fields(old_versioned_json_schema = nil, &schema_definition) + merge_result = perform_merge(old_versioned_json_schema, &schema_definition) + + expect(merge_result.missing_fields).not_to be_empty + expect(merge_result.missing_types).to be_empty + expect(merge_result.definition_conflicts).to be_empty + expect(merge_result.missing_necessary_fields).to be_empty + + merge_result.missing_fields + end + + def dump_versioned_json_schema_definition_conflicts(old_versioned_json_schema = nil, &schema_definition) + merge_result = perform_merge(old_versioned_json_schema, &schema_definition) + + expect(merge_result.missing_fields).to be_empty + expect(merge_result.missing_types).to be_empty + expect(merge_result.definition_conflicts).not_to be_empty + expect(merge_result.missing_necessary_fields).to be_empty + + merge_result.definition_conflicts + end + + def dump_versioned_json_schema_missing_types(old_versioned_json_schema = nil, &schema_definition) + merge_result = perform_merge(old_versioned_json_schema, &schema_definition) + + expect(merge_result.missing_fields).to be_empty + expect(merge_result.missing_types).not_to be_empty + expect(merge_result.definition_conflicts).to be_empty + expect(merge_result.missing_necessary_fields).to be_empty + + merge_result.missing_types + end + + def dump_versioned_json_schema_missing_necessary_fields(old_versioned_json_schema = nil, &schema_definition) + merge_result = perform_merge(old_versioned_json_schema, &schema_definition) + + expect(merge_result.missing_fields).to be_empty + expect(merge_result.missing_types).to be_empty + expect(merge_result.definition_conflicts).to be_empty + expect(merge_result.missing_necessary_fields).not_to be_empty + + merge_result.missing_necessary_fields + end + + def dump_versioned_json_schema_unused_deprecated_elements(old_versioned_json_schema = nil, &schema_definition) + results = define_schema(&schema_definition) + results.merge_field_metadata_into_json_schema(old_versioned_json_schema || results.current_public_json_schema) + results.unused_deprecated_elements + end + + def perform_merge(old_versioned_json_schema = nil, &schema_definition) + results = define_schema(&schema_definition) + results.merge_field_metadata_into_json_schema(old_versioned_json_schema || results.current_public_json_schema).tap do + expect(results.unused_deprecated_elements).to be_empty + end + end + + def metadata_for(json_schema, type, field) + json_schema.dig("$defs", type, "properties", field, "ElasticGraph") + end + + def define_schema(&schema_definition) + super(schema_element_name_form: "snake_case", &schema_definition) + end + + def have_dumped_metadata(name_in_index, type) + eq({"nameInIndex" => name_in_index, "type" => type}) + end + + def missing_necessary_field_of(field_type, fully_qualified_path) + JSONSchemaWithMetadata::MissingNecessaryField.new(field_type, fully_qualified_path) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_field_metadata_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_field_metadata_spec.rb new file mode 100644 index 00000000..c28ecaf1 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_field_metadata_spec.rb @@ -0,0 +1,151 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/schema_definition_helpers" + +module ElasticGraph + module SchemaDefinition + ::RSpec.describe "JSON schema field metadata generation" do + include_context "SchemaDefinitionHelpers" + + it "generates no field metadata for built-in scalar and enum types" do + metadata_by_type_and_field_name = dump_metadata + + json_schema_field_metadata = %w[ + Boolean Float ID Int String + Cursor Date DateTime DistanceUnit JsonSafeLong LocalTime LongString TimeZone Untyped + ].map do |type_name| + metadata_by_type_and_field_name.fetch(type_name) + end + + expect(json_schema_field_metadata).to all eq({}) + end + + it "generates field metadata for built-in object types" do + metadata_by_field_name = dump_metadata.fetch("GeoLocation") + + expect(metadata_by_field_name).to eq({ + "latitude" => field_meta_of("Float!", "lat"), + "longitude" => field_meta_of("Float!", "lon") + }) + end + + it "generates field metadata for user-defined object types" do + metadata_by_field_name = dump_metadata do |schema| + schema.object_type "Money" do |t| + t.field "amount", "Int" + t.field "currency", "String" + end + end.fetch("Money") + + expect(metadata_by_field_name).to eq({ + "amount" => field_meta_of("Int", "amount"), + "currency" => field_meta_of("String", "currency") + }) + end + + it "respects the type and `name_in_index` on user-defined fields" do + metadata_by_field_name = dump_metadata do |schema| + schema.object_type "Money" do |t| + t.field "amount", "Int!", name_in_index: "amount2" + t.field "currency", "[String]!", name_in_index: "currency2" + end + end.fetch("Money") + + expect(metadata_by_field_name).to eq({ + "amount" => field_meta_of("Int!", "amount2"), + "currency" => field_meta_of("[String]!", "currency2") + }) + end + + it "generates no field metadata for user-defined scalar or enum types since they have no subfields" do + metadata_by_type_and_field_name = dump_metadata do |schema| + schema.scalar_type "Url" do |t| + t.json_schema type: "string" + t.mapping type: "keyword" + end + + schema.enum_type "Color" do |t| + t.value "RED" + t.value "GREEN" + t.value "BLUE" + end + end + + json_schema_field_metadata = %w[Url Color].map do |type_name| + metadata_by_type_and_field_name.fetch(type_name) + end + + expect(json_schema_field_metadata).to all eq({}) + end + + it "generates no field metadata for user-defined union or interface types since the JSON schema" do + metadata_by_type_and_field_name = dump_metadata do |schema| + schema.interface_type "Named" do |t| + t.field "name", "String" + end + + schema.union_type "Character" do |t| + t.subtype "Droid" + t.subtype "Human" + end + + schema.object_type "Droid" do |t| + t.implements "Named" + t.field "name", "String" + t.field "model", "String" + end + + schema.object_type "Human" do |t| + t.implements "Named" + t.field "name", "String" + t.field "home_planet", "String" + end + end + + json_schema_field_metadata = %w[Named Character].map do |type_name| + metadata_by_type_and_field_name.fetch(type_name) + end + + expect(json_schema_field_metadata).to all eq({}) + end + + it "includes the JSON schema field metadata in the versioned JSON schemas but not in the current public JSON schema" do + results = define_schema do |schema| + schema.object_type "Money" do |t| + t.field "amount", "Int" + t.field "currency", "String" + end + end + + amount_path = ["$defs", "Money", "properties", "amount"] + + expect(results.json_schemas_for(1).dig(*amount_path)).to eq({ + "anyOf" => [{"$ref" => "#/$defs/Int"}, {"type" => "null"}], + "ElasticGraph" => {"nameInIndex" => "amount", "type" => "Int"} + }) + + expect(results.current_public_json_schema.dig(*amount_path)).to eq({ + "anyOf" => [{"$ref" => "#/$defs/Int"}, {"type" => "null"}] + }) + end + + def dump_metadata(&schema_definition) + define_schema(&schema_definition).json_schema_field_metadata_by_type_and_field_name + end + + def define_schema(&schema_definition) + super(schema_element_name_form: "snake_case", &schema_definition) + end + + def field_meta_of(type, name_in_index) + Indexing::JSONSchemaFieldMetadata.new(type: type, name_in_index: name_in_index) + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_pruner_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_pruner_spec.rb new file mode 100644 index 00000000..68bcf71d --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_pruner_spec.rb @@ -0,0 +1,130 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/spec_support/schema_definition_helpers" +require "elastic_graph/schema_definition/json_schema_pruner" + +module ElasticGraph + module SchemaDefinition + RSpec.describe JSONSchemaPruner do + include_context "SchemaDefinitionHelpers" + + describe ".prune" do + subject { described_class.prune(schema) } + + shared_examples "prunes types not referenced by indexed types" do |expected_type_names| + it do + expect(subject["$defs"].keys).to match_array(expected_type_names) + end + end + + context "when there are indexable types" do + let(:schema) do + dump_schema do |s| + # Widget and Boolean should be present + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "inStock", "Boolean" + t.index "widgets" + end + + # UnindexedWidget and Float should get pruned + s.object_type "UnindexedWidget" do |t| + t.field "id", "ID!" + t.field "cost", "Float" + end + end + end + + it_behaves_like "prunes types not referenced by indexed types", + [EVENT_ENVELOPE_JSON_SCHEMA_NAME, "Boolean", "ID", "Widget"] + end + + context "when there are no types defined" do + let(:schema) { dump_schema } + + it_behaves_like "prunes types not referenced by indexed types", [EVENT_ENVELOPE_JSON_SCHEMA_NAME] + end + + context "when there are no indexable types defined" do + let(:schema) do + dump_schema do |s| + # UnindexedWidget and Float should get pruned + s.object_type "UnindexedWidget" do |t| + t.field "id", "ID!" + t.field "cost", "Float" + end + end + end + + it_behaves_like "prunes types not referenced by indexed types", [EVENT_ENVELOPE_JSON_SCHEMA_NAME] + end + + context "when there are nested types referenced from an indexed type" do + let(:schema) do + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "options", "WidgetOptions" + t.index "widgets" + end + + s.object_type "WidgetOptions" do |t| + t.field "size", "Size" + t.field "color", "Color" + t.field "cost", "Money" + end + + s.enum_type "Size" do |t| + t.value "SMALL" + t.value "MEDIUM" + t.value "LARGE" + end + + s.enum_type "Color" do |t| + t.value "RED" + t.value "YELLOW" + t.value "BLUE" + end + + s.object_type "Money" do |t| + t.field "currency", "Currency" + t.field "amount_cents", "Int" + end + + s.enum_type "Currency" do |t| + t.value "USD" + t.value "CAD" + end + end + end + + it_behaves_like "prunes types not referenced by indexed types", [ + EVENT_ENVELOPE_JSON_SCHEMA_NAME, + "Color", + "Currency", + "ID", + "Int", + "Money", + "Size", + "Widget", + "WidgetOptions" + ] + end + end + + def dump_schema(&schema_definition) + schema_definition_results = define_schema(schema_element_name_form: "snake_case", &schema_definition) + latest_json_schema_version = schema_definition_results.latest_json_schema_version + + schema_definition_results.json_schemas_for(latest_json_schema_version) + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_spec.rb new file mode 100644 index 00000000..b2820ca2 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_spec.rb @@ -0,0 +1,2955 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/errors" +require "elastic_graph/spec_support/schema_definition_helpers" +require "support/json_schema_matcher" + +module ElasticGraph + module SchemaDefinition + ::RSpec.describe "JSON schema generation" do + include_context "SchemaDefinitionHelpers" + json_schema_id = {"allOf" => [{"$ref" => "#/$defs/ID"}, {"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH}]} + json_schema_float = {"$ref" => "#/$defs/Float"} + json_schema_integer = {"$ref" => "#/$defs/Int"} + json_schema_string = {"allOf" => [{"$ref" => "#/$defs/String"}, {"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH}]} + json_schema_null = {"type" => "null"} + + context "on ElasticGraph built-in types, it generates the expected JSON schema" do + attr_reader :json_schema + + before(:context) do + @json_schema = dump_schema do |s| + # Include a random version number to ensure it's getting used correctly + s.json_schema_version 42 + + # Include a basic indexed type here to validate that the envelope is getting + # generated correctly (we'll ignore it below) + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.index "widgets" + end + end + @tested_types = ::Set.new + end + + after(:context) do + built_in_types = @json_schema.fetch("$defs").keys - ["Widget"] + input_enum_types = %w[DateGroupingGranularityInput DateTimeGroupingGranularityInput DateTimeUnitInput DistanceUnitInput MatchesQueryAllowedEditsPerTerm] + + # Input enum types are named with an `Input` suffix. The JSON schema only contains the types we index, which are output types, + # and therefore it does not have the input enum types. + untested_types = built_in_types - @tested_types.to_a - input_enum_types + + expect(untested_types).to be_empty, + "It appears that #{untested_types.size} built-in type(s) lack test coverage in `json_schema_spec.rb`. " \ + "Cover them with a test to fix this failure, or ignore this if not running the entire set of built-in type tests:\n\n" \ + "- #{untested_types.sort.join("\n- ")}" + end + + example "for `#{EVENT_ENVELOPE_JSON_SCHEMA_NAME}`" do + expect(json_schema).to have_json_schema_like(EVENT_ENVELOPE_JSON_SCHEMA_NAME, { + "type" => "object", + "properties" => { + "op" => {"type" => "string", "enum" => %w[upsert]}, + "type" => {"type" => "string", "enum" => ["Widget"]}, + "id" => {"type" => "string", "maxLength" => DEFAULT_MAX_KEYWORD_LENGTH}, + "version" => {"type" => "integer", "minimum" => 0, "maximum" => (2**63) - 1}, + "record" => {"type" => "object"}, + "latency_timestamps" => { + "type" => "object", + "additionalProperties" => false, + "patternProperties" => {"^\\w+_at$" => {"type" => "string", "format" => "date-time"}} + }, + JSON_SCHEMA_VERSION_KEY => {"const" => 42}, + "message_id" => { + "type" => "string", + "description" => "The optional ID of the message containing this event from whatever messaging system is being used between the publisher and the ElasticGraph indexer." + } + }, + "additionalProperties" => false, + "required" => ["op", "type", "id", "version", JSON_SCHEMA_VERSION_KEY], + "if" => {"properties" => {"op" => {"const" => "upsert"}}}, + "then" => {"required" => ["record"]} + }, include_typename: false) + end + + %w[ID String].each do |type_name| + example "for `#{type_name}`" do + expect(json_schema).to have_json_schema_like(type_name, { + "type" => "string" + }).which_matches("abc", "a" * DEFAULT_MAX_KEYWORD_LENGTH, "a" * (DEFAULT_MAX_KEYWORD_LENGTH + 1)) + .and_fails_to_match(0, nil, true) + end + end + + example "for `Int`" do + expect(json_schema).to have_json_schema_like("Int", { + "type" => "integer", + "minimum" => -2147483648, + "maximum" => 2147483647 + }).which_matches(0, 1, -1, INT_MAX, INT_MIN) + .and_fails_to_match("a", 0.5, true, INT_MAX + 1, INT_MIN - 1) + end + + example "for `Boolean`" do + expect(json_schema).to have_json_schema_like("Boolean", { + "type" => "boolean" + }).which_matches(true, false) + .and_fails_to_match("true", "false", "yes", "no", 1, 0, nil) + end + + example "for `Float`" do + expect(json_schema).to have_json_schema_like("Float", { + "type" => "number" + }).which_matches(0, 1, -1, 0.1, -99.0) + .and_fails_to_match("a", true, nil) + end + + example "for `TimeZone`" do + expect(json_schema).to have_json_schema_like("TimeZone", { + "type" => "string", + "enum" => GraphQL::ScalarCoercionAdapters::VALID_TIME_ZONES.to_a + }) + .which_matches("America/Los_Angeles") + .and_fails_to_match("America/Seattle") # America/Seattle is not a valid time zone. + end + + example "for `Untyped`" do + expect(json_schema).to have_json_schema_like("Untyped", { + "type" => %w[array boolean integer number object string] + }).which_matches( + 3, + 3.75, + "string", + true, + %w[a b], + {"some" => "data"}, + {"some" => {"nested" => {"data" => [1, true, "3"]}}} + ).and_fails_to_match(nil) + end + + example "for `GeoLocation`" do + expect(json_schema).to have_json_schema_like("GeoLocation", { + "type" => "object", + "properties" => { + "latitude" => { + "allOf" => [ + json_schema_float, + {"minimum" => -90, "maximum" => 90} + ] + }, + "longitude" => { + "allOf" => [ + json_schema_float, + {"minimum" => -180, "maximum" => 180} + ] + } + }, + "required" => %w[latitude longitude] + }).which_matches( + {"latitude" => 0, "longitude" => 0}, + {"latitude" => -90, "longitude" => -180}, + {"latitude" => 90, "longitude" => 180} + ).and_fails_to_match( + nil, + {}, + {"latitude" => "0", "longitude" => "1"}, + {"latitude" => -91, "longitude" => 0}, + {"latitude" => 91, "longitude" => 0}, + {"latitude" => 0, "longitude" => -181}, + {"latitude" => 0, "longitude" => 181}, + {"latitude" => nil, "longitude" => 0}, + {"latitude" => 0, "longitude" => nil} + ) + end + + example "for `Cursor`" do + expect(json_schema).to have_json_schema_like("Cursor", {"type" => "string"}) + .which_matches("abc") + .and_fails_to_match(0, nil, true) + end + + example "for `Date`" do + expect(json_schema).to have_json_schema_like("Date", {"type" => "string", "format" => "date"}) + .which_matches("2023-01-01", "1999-12-31") # yyyy-MM-dd + .and_fails_to_match(0, nil, true, "01-01-2023", "0000-00-00", "2023-13-40") + end + + example "for `DateUnit`" do + expect(json_schema).to have_json_schema_like("DateUnit", { + "enum" => %w[DAY], "type" => "string" + }).which_matches(*%w[DAY]) + .and_fails_to_match(0, nil, true, "literally any other string") + end + + example "for `DateGroupingGranularity`" do + expect(json_schema).to have_json_schema_like("DateGroupingGranularity", { + "enum" => %w[YEAR QUARTER MONTH WEEK DAY], "type" => "string" + }).which_matches(*%w[YEAR QUARTER MONTH WEEK DAY]) + .and_fails_to_match(0, nil, true, "literally any other string") + end + + example "for `DateGroupingTruncationUnit`" do + expect(json_schema).to have_json_schema_like("DateGroupingTruncationUnit", { + "enum" => %w[YEAR QUARTER MONTH WEEK DAY], "type" => "string" + }).which_matches(*%w[YEAR QUARTER MONTH WEEK DAY]) + .and_fails_to_match(0, nil, true, "literally any other string") + end + + example "for `DateTime`" do + expect(json_schema).to have_json_schema_like("DateTime", { + "type" => "string", "format" => "date-time" + }).which_matches("2023-01-01T00:00:00.000Z", "1999-12-31T23:59:59.999Z") # T: yyyy-MM-dd'T'HH:mm:ss.SSSZ + .and_fails_to_match(0, nil, true, "01-01-2023", "0000-00-00 00:00", "2023-13-40 45:33") + end + + example "for `DateTimeUnit`" do + expect(json_schema).to have_json_schema_like("DateTimeUnit", { + "enum" => %w[DAY HOUR MINUTE SECOND MILLISECOND], "type" => "string" + }).which_matches(*%w[DAY HOUR MINUTE SECOND MILLISECOND]) + .and_fails_to_match(0, nil, true, "literally any other string") + end + + example "for `DateTimeGroupingGranularity`" do + expect(json_schema).to have_json_schema_like("DateTimeGroupingGranularity", { + "enum" => %w[YEAR QUARTER MONTH WEEK DAY HOUR MINUTE SECOND], "type" => "string" + }).which_matches(*%w[YEAR QUARTER MONTH WEEK DAY HOUR MINUTE SECOND]) + .and_fails_to_match(0, nil, true, "literally any other string") + end + + example "for `DateTimeGroupingTruncationUnit`" do + expect(json_schema).to have_json_schema_like("DateTimeGroupingTruncationUnit", { + "enum" => %w[YEAR QUARTER MONTH WEEK DAY HOUR MINUTE SECOND], "type" => "string" + }).which_matches(*%w[YEAR QUARTER MONTH WEEK DAY HOUR MINUTE SECOND]) + .and_fails_to_match(0, nil, true, "literally any other string") + end + + example "for `DayOfWeek`" do + expect(json_schema).to have_json_schema_like("DayOfWeek", { + "enum" => %w[MONDAY TUESDAY WEDNESDAY THURSDAY FRIDAY SATURDAY SUNDAY], "type" => "string" + }).which_matches(*%w[MONDAY TUESDAY WEDNESDAY THURSDAY FRIDAY SATURDAY SUNDAY]) + .and_fails_to_match(0, nil, true, "literally any other string") + end + + example "for `DistanceUnit`" do + expect(json_schema).to have_json_schema_like("DistanceUnit", { + "enum" => %w[MILE YARD FOOT INCH KILOMETER METER CENTIMETER MILLIMETER NAUTICAL_MILE], "type" => "string" + }).which_matches(*%w[MILE YARD FOOT INCH KILOMETER METER CENTIMETER MILLIMETER NAUTICAL_MILE]) + .and_fails_to_match(0, nil, true, "literally any other string") + end + + example "for `JsonSafeLong`" do + expect(json_schema).to have_json_schema_like("JsonSafeLong", { + "maximum" => JSON_SAFE_LONG_MAX, + "minimum" => JSON_SAFE_LONG_MIN, + "type" => "integer" + }).which_matches(0, JSON_SAFE_LONG_MIN, JSON_SAFE_LONG_MAX) + .and_fails_to_match(0.5, nil, true, JSON_SAFE_LONG_MAX + 1, JSON_SAFE_LONG_MIN - 1) + end + + example "for `LocalTime`" do + expect(json_schema).to have_json_schema_like("LocalTime", { + "type" => "string", + "pattern" => VALID_LOCAL_TIME_JSON_SCHEMA_PATTERN + }) + .which_matches("01:23:45", "14:56:39.000", "23:59:01.1", "23:59:01.12", "23:59:01.13") # HH:mm:ss, HH:mm:ss.S, HH:mm:ss.SS, HH:mm:ss.SSS + .and_fails_to_match(0, nil, true, "abc", "99:00:00", "59:59.999Z", "01:23:45.1234", "14:56:39a000") + end + + example "for `LocalTimeUnit`" do + expect(json_schema).to have_json_schema_like("LocalTimeUnit", { + "enum" => %w[HOUR MINUTE SECOND MILLISECOND], "type" => "string" + }).which_matches(*%w[HOUR MINUTE SECOND MILLISECOND]) + .and_fails_to_match(0, nil, true, "literally any other string") + end + + example "for `LocalTimeGroupingTruncationUnit`" do + expect(json_schema).to have_json_schema_like("LocalTimeGroupingTruncationUnit", { + "enum" => %w[HOUR MINUTE SECOND], "type" => "string" + }).which_matches(*%w[HOUR MINUTE SECOND]) + .and_fails_to_match(0, nil, true, "literally any other string") + end + + example "for `LongString`" do + expect(json_schema).to have_json_schema_like("LongString", { + "maximum" => LONG_STRING_MAX, + "minimum" => LONG_STRING_MIN, + "type" => "integer" + }) + .which_matches(0, LONG_STRING_MAX, LONG_STRING_MIN) + .and_fails_to_match(0.5, nil, true, LONG_STRING_MIN - 1, LONG_STRING_MAX + 1) + end + + def have_json_schema_like(type_name, *args, **kwargs) + @tested_types << type_name + super(type_name, *args, **kwargs) + end + end + + it "allows any valid JSON type for a nullable `Untyped` field" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "j1", "Untyped" + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "j1" => json_schema_ref("Untyped", is_keyword_type: true) + }, + "required" => %w[j1] + }).which_matches( + {"j1" => 3}, + {"j1" => 3.75}, + {"j1" => "string"}, + {"j1" => "a" * DEFAULT_MAX_KEYWORD_LENGTH}, + {"j1" => nil}, + {"j1" => true}, + {"j1" => %w[a b]}, + {"j1" => {"some" => "data"}}, + {"j1" => {"some" => {"nested" => {"data" => [1, true, "3"]}}}} + ).and_fails_to_match( + {"j1" => "a" * (DEFAULT_MAX_KEYWORD_LENGTH + 1)} + ) + end + + it "does not duplicate `required` fields when 2 GraphQL fields are both backed by the same indexing field" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "name", "String!" + t.field "name2", "String!", name_in_index: "name", graphql_only: true + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "name" => json_schema_ref("String!") + }, + "required" => %w[name] + }) + end + + it "does not allow multiple indexing fields with the same name because that would result in multiple JSON schema fields flowing into the same index field but with conflicting values" do + expect { + dump_schema do |s| + s.object_type "MyType" do |t| + t.field "name", "String!" + t.field "name2", "String!", name_in_index: "name" + end + end + }.to raise_error Errors::SchemaError, a_string_including("Duplicate indexing field", "MyType: name", "set `graphql_only: true`") + end + + it "raises an exception when `json_schema` on a field definition has invalid json schema option values" do + dump_schema do |s| + s.object_type "MyType" do |t| + t.field "foo", "String" do |f| + expect { + f.json_schema maxLength: "twelve" + }.to raise_error Errors::SchemaError, a_string_including("Invalid JSON schema options", "foo: String", "maxLength") + + expect(f.json_schema_options).to be_empty + + # Demonstrate that `maxLength` with an int value is allowed + f.json_schema maxLength: 12 + end + end + end + end + + it "does not allow the extra `ElasticGraph` metadata that ElasticGraph adds itself" do + dump_schema do |s| + s.object_type "MyType" do |t| + t.field "foo", "String" do |f| + expect { + f.json_schema ElasticGraph: {type: "String"} + }.to raise_error Errors::SchemaError, a_string_including("Invalid JSON schema options", "foo: String", '"data_pointer": "/ElasticGraph"') + + expect(f.json_schema_options).to be_empty + + # Demonstrate that `maxLength` with an int value is allowed + f.json_schema maxLength: 12 + end + end + end + end + + it "raises an exception when `json_schema` on a field definition has invalid json schema option names" do + dump_schema do |s| + s.object_type "MyType" do |t| + t.field "foo", "String" do |f| + expect { + f.json_schema longestLength: 14 # maxLength is correct, not longestLength + }.to raise_error Errors::SchemaError, a_string_including("Invalid JSON schema options", "foo: String", "longestLength") + end + end + end + end + + it "raises an exception when `json_schema` on a scalar type has invalid json schema option values" do + dump_schema do |s| + s.scalar_type "MyType" do |t| + t.mapping type: "keyword" + + expect { + t.json_schema type: "string", maxLength: "twelve" + }.to raise_error Errors::SchemaError, a_string_including("Invalid JSON schema options", "MyType", "twelve") + + # Demonstrate that `maxLength` with an int value is allowed + t.json_schema type: "string", maxLength: 12 + end + end + end + + it "raises an exception when `json_schema` on a scalar type has invalid json schema option values" do + dump_schema do |s| + s.scalar_type "MyType" do |t| + t.mapping type: "keyword" + + expect { + t.json_schema type: "string", longestLength: 14 # maxLength is correct, not longestLength + }.to raise_error Errors::SchemaError, a_string_including("Invalid JSON schema options", "MyType", "longestLength") + + t.json_schema type: "string" + end + end + end + + it "raises an exception when `json_schema` on an object type has invalid json schema option values" do + dump_schema do |s| + s.object_type "MyType" do |t| + expect { + t.json_schema type: "string", maxLength: "twelve" + }.to raise_error Errors::SchemaError, a_string_including("Invalid JSON schema options", "MyType", "twelve") + + # Demonstrate that `maxLength` with an int value is allowed + t.json_schema type: "string", maxLength: 12 + end + end + end + + it "raises an exception when `json_schema` on a scalar type has invalid json schema option values" do + dump_schema do |s| + s.object_type "MyType" do |t| + t.mapping type: "keyword" + + expect { + t.json_schema type: "string", longestLength: 14 # maxLength is correct, not longestLength + }.to raise_error Errors::SchemaError, a_string_including("Invalid JSON schema options", "MyType", "longestLength") + end + end + end + + context "for a field that is `sourced_from` a related type" do + it "excludes the `source_from` field because it comes from another source type and will be represented in the JSON schema of that type" do + json_schema = dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String!" + + t.index "widgets" + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.relates_to_one "widget", "Widget", via: "component_ids", dir: :in + + t.field "widget_name", "String!" do |f| + f.sourced_from "widget", "name" + end + + t.index "components" + end + end + + expect(json_schema).to have_json_schema_like("Component", { + "type" => "object", + "properties" => { + "id" => json_schema_ref("ID!") + }, + "required" => %w[id] + }) + end + + it "does not allow any JSON schema customizations of the field because they should be configured on the source type itself" do + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "size", "Int" + + t.index "widgets" + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.relates_to_one "widget", "Widget", via: "component_ids", dir: :in + + # Here we call `json_schema` after `sourced_from`... + t.field "widget_name", "String!" do |f| + f.sourced_from "widget", "name" + f.json_schema minLength: 4 + end + + # ...vs here we call it before. We do this to demonstrate the order doesn't matter. + t.field "widget_size", "Int" do |f| + f.json_schema minimum: 0 + f.sourced_from "widget", "size" + end + + t.index "components" + end + end + }.to raise_error a_string_including( + "Component` has 2 field(s) (`widget_name`, `widget_size`)", + "also have JSON schema customizations" + ) + end + end + + %w[ID String].first(1).each do |graphql_type| + it "limits the length of `#{graphql_type}!` fields based on datastore limits" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "foo", "#{graphql_type}!" + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "foo" => { + "allOf" => [ + {"$ref" => "#/$defs/#{graphql_type}"}, + {"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH} + ] + } + }, + "required" => %w[foo] + }).which_matches( + {"foo" => "abc"}, + {"foo" => "a" * DEFAULT_MAX_KEYWORD_LENGTH} + ).and_fails_to_match( + {"foo" => "a" * (DEFAULT_MAX_KEYWORD_LENGTH + 1)}, + {"foo" => nil}, + {"foo" => -129}, + {"foo" => 128} + ) + end + + it "limits the length of `#{graphql_type}` fields based on datastore limits" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "foo", graphql_type + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "foo" => { + "anyOf" => [ + { + "allOf" => [ + {"$ref" => "#/$defs/#{graphql_type}"}, + {"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH} + ] + }, + {"type" => "null"} + ] + } + }, + "required" => %w[foo] + }).which_matches( + {"foo" => "abc"}, + {"foo" => nil}, + {"foo" => "a" * DEFAULT_MAX_KEYWORD_LENGTH} + ).and_fails_to_match( + {"foo" => "a" * (DEFAULT_MAX_KEYWORD_LENGTH + 1)}, + {"foo" => -129}, + {"foo" => 128} + ) + end + + it "uses a larger `maxLength` for a #{graphql_type} if the mapping type is set to `text`" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "foo", "#{graphql_type}!" do |f| + f.mapping type: "text" + end + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "foo" => { + "allOf" => [ + {"$ref" => "#/$defs/#{graphql_type}"}, + {"maxLength" => DEFAULT_MAX_TEXT_LENGTH} + ] + } + }, + "required" => %w[foo] + }).which_matches( + {"foo" => "abc"}, + {"foo" => "a" * DEFAULT_MAX_TEXT_LENGTH} + ).and_fails_to_match( + {"foo" => "a" * (DEFAULT_MAX_TEXT_LENGTH + 1)}, + {"foo" => nil}, + {"foo" => -129}, + {"foo" => 128} + ) + end + end + + it "limits the size of custom `keyword` types based on datastore limits" do + json_schema = dump_schema do |s| + s.scalar_type "MyString" do |t| + t.json_schema type: "string" + t.mapping type: "keyword" + end + + s.object_type "MyType" do |t| + t.field "foo", "MyString" + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "foo" => { + "anyOf" => [ + { + "allOf" => [ + {"$ref" => "#/$defs/MyString"}, + {"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH} + ] + }, + {"type" => "null"} + ] + } + }, + "required" => %w[foo] + }).which_matches( + {"foo" => "abc"}, + {"foo" => nil}, + {"foo" => "a" * DEFAULT_MAX_KEYWORD_LENGTH} + ).and_fails_to_match( + {"foo" => "a" * (DEFAULT_MAX_KEYWORD_LENGTH + 1)}, + {"foo" => -129}, + {"foo" => 128} + ) + end + + it "allows the `maxLength` to be overridden on keyword and text fields" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "id", "ID!" do |f| + f.json_schema maxLength: 50 + end + + t.field "string", "String!" do |f| + f.json_schema maxLength: 100 + end + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "id" => { + "allOf" => [ + {"$ref" => "#/$defs/ID"}, + {"maxLength" => 50} + ] + }, + "string" => { + "allOf" => [ + {"$ref" => "#/$defs/String"}, + {"maxLength" => 100} + ] + } + }, + "required" => %w[id string] + }) + end + + it "does not include `maxLength` on enum fields since we already limit the values" do + json_schema = dump_schema do |s| + s.enum_type "Color" do |t| + t.value "RED" + t.value "GREEN" + t.value "BLUE" + end + + s.object_type "MyType" do |t| + t.field "color1", "Color!" + + t.field "color2", "Color!" do |f| + f.json_schema maxLength: 50 + end + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "color1" => {"$ref" => "#/$defs/Color"}, + "color2" => {"$ref" => "#/$defs/Color"} + }, + "required" => %w[color1 color2] + }).which_matches( + {"color1" => "RED", "color2" => "GREEN"}, + {"color1" => "BLUE", "color2" => "RED"} + ).and_fails_to_match( + {"color1" => "YELLOW", "color2" => "GREEN"}, + {"color1" => "BLUE", "color2" => "BROWN"} + ) + end + + it "limits byte types based on the datastore mapping type range" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "byte", "Int!" do |f| + f.mapping type: "byte" + end + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "byte" => { + "allOf" => [ + json_schema_integer, + {"minimum" => -128, "maximum" => 127} + ] + } + }, + "required" => %w[byte] + }).which_matches( + {"byte" => 0}, + {"byte" => -128}, + {"byte" => 127} + ).and_fails_to_match( + {"byte" => "a"}, + {"byte" => nil}, + {"byte" => -129}, + {"byte" => 128} + ) + end + + it "limits short types based on the datastore mapping type range" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "short", "Int!" do |f| + f.mapping type: "short" + end + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "short" => { + "allOf" => [ + json_schema_integer, + {"minimum" => -32_768, "maximum" => 32_767} + ] + } + }, + "required" => %w[short] + }).which_matches( + {"short" => 0}, + {"short" => -32_768}, + {"short" => 32_767} + ).and_fails_to_match( + {"short" => "a"}, + {"short" => nil}, + {"short" => -32_769}, + {"short" => 32_768} + ) + end + + it "limits integer types based on the datastore mapping type range" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "integer", "Int!" do |f| + f.mapping type: "integer" + end + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "integer" => json_schema_ref("Int!") + }, + "required" => %w[integer] + }).which_matches( + {"integer" => 0}, + {"integer" => INT_MAX}, + {"integer" => INT_MIN} + ).and_fails_to_match( + {"integer" => "a"}, + {"integer" => nil}, + {"integer" => INT_MAX + 1}, + {"integer" => INT_MIN - 1} + ) + end + + it "supports nullable fields by wrapping the schema in 'anyOf' with a 'null' type" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "is_happy", "Boolean" + t.field "size", "Float" + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "is_happy" => json_schema_ref("Boolean"), + "size" => json_schema_ref("Float") + }, + "required" => %w[is_happy size] + }) + end + + it "returns a JSON schema for a type with arrays" do + json_schema = dump_schema do |s| + s.object_type "Widget" do |t| + t.field "color", "[String!]" + t.field "amount_cents", "[Int!]!" + end + end + + expect(json_schema).to have_json_schema_like("Widget", { + "type" => "object", + "properties" => { + "color" => { + "anyOf" => [ + { + "type" => "array", + "items" => json_schema_string + }, + json_schema_null + ] + }, + "amount_cents" => { + "type" => "array", + "items" => json_schema_integer + } + }, + "required" => %w[color amount_cents] + }) + end + + it "returns a JSON schema for a type with enums" do + json_schema = dump_schema do |s| + s.enum_type "Color" do |t| + t.values "RED", "BLUE", "GREEN" + end + + s.enum_type "Size" do |t| + t.values "SMALL", "MEDIUM", "LARGE" + end + + s.object_type "Widget" do |t| + t.field "size", "Size!" + t.field "color", "Color" + end + end + + expect(json_schema).to have_json_schema_like("Size", { + "type" => "string", + "enum" => %w[SMALL MEDIUM LARGE] + }) + + expect(json_schema).to have_json_schema_like("Color", { + "type" => "string", + "enum" => %w[RED BLUE GREEN] + }) + + expect(json_schema).to have_json_schema_like("Widget", { + "type" => "object", + "properties" => { + "size" => json_schema_ref("Size!"), + "color" => json_schema_ref("Color") + }, + "required" => %w[size color] + }) + end + + it "respects enum value overrides" do + json_schema = dump_schema(enum_value_overrides_by_type: { + Color: {RED: "REDISH", BLUE: "BLUEISH"} + }) do |s| + s.enum_type "Color" do |t| + t.values "RED", "BLUE", "GREEN" + end + end + + expect(json_schema).to have_json_schema_like("Color", { + "type" => "string", + "enum" => %w[REDISH BLUEISH GREEN] + }) + end + + it "uses `enum` for an Enum with a single value" do + json_schema = dump_schema do |s| + s.enum_type "Color" do |t| + t.values "RED" + end + end + + expect(json_schema).to have_json_schema_like("Color", { + "type" => "string", + "enum" => ["RED"] + }) + end + + it "returns a JSON schema for a type with objects" do + json_schema = dump_schema do |s| + s.object_type "Color" do |t| + t.field "red", "Int!" + t.field "green", "Int!" + t.field "blue", "Int!" + end + + s.object_type "WidgetOptions" do |t| + t.field "color", "String!" + t.field "color_breakdown", "Color!" + end + + s.object_type "Widget" do |t| + t.field "options", "WidgetOptions" + end + end + + expect(json_schema).to have_json_schema_like("Color", { + "type" => "object", + "properties" => { + "red" => json_schema_ref("Int!"), + "green" => json_schema_ref("Int!"), + "blue" => json_schema_ref("Int!") + }, + "required" => %w[red green blue] + }) + + expect(json_schema).to have_json_schema_like("WidgetOptions", { + "type" => "object", + "properties" => { + "color" => json_schema_ref("String!"), + "color_breakdown" => json_schema_ref("Color!") + }, + "required" => %w[color color_breakdown] + }) + + expect(json_schema).to have_json_schema_like("Widget", { + "type" => "object", + "properties" => { + "options" => json_schema_ref("WidgetOptions") + }, + "required" => %w[options] + }) + end + + it "returns a JSON schema with definitions for custom scalar types" do + json_schema = dump_schema do |s| + s.scalar_type "PhoneNumber" do |t| + t.mapping type: "keyword" + t.json_schema type: "string", format: "^\\+[1-9][0-9]{1,14}$" + end + end + + expect(json_schema).to have_json_schema_like("PhoneNumber", { + "type" => "string", + "format" => "^\\+[1-9][0-9]{1,14}$" + }) + end + + it "returns a JSON schema for a type with wrapped enums" do + json_schema = dump_schema do |s| + s.enum_type "Size" do |t| + t.values "SMALL", "MEDIUM", "LARGE" + end + + s.object_type "Widget" do |t| + t.field "null_array_null", "[Size]" + t.field "non_null_array_null", "[Size]!" + t.field "null_array_non_null", "[Size!]" + t.field "non_null_array_non_null", "[Size!]!" + t.field "null_null_array_null", "[[Size]]" + end + end + + expect(json_schema).to have_json_schema_like("Widget", { + "type" => "object", + "properties" => { + "null_array_null" => { + "anyOf" => [ + { + "type" => "array", + "items" => json_schema_ref("Size") + }, + json_schema_null + ] + }, + "non_null_array_null" => { + "type" => "array", + "items" => json_schema_ref("Size") + }, + "null_array_non_null" => { + "anyOf" => [ + { + "type" => "array", + "items" => json_schema_ref("Size!") + }, + json_schema_null + ] + }, + "non_null_array_non_null" => { + "type" => "array", + "items" => json_schema_ref("Size!") + }, + "null_null_array_null" => { + "anyOf" => [ + { + "type" => "array", + "items" => { + "anyOf" => [ + { + "type" => "array", + "items" => json_schema_ref("Size") + }, + json_schema_null + ] + } + }, + json_schema_null + ] + } + }, + "required" => %w[null_array_null non_null_array_null null_array_non_null non_null_array_non_null null_null_array_null] + }) + end + + it "returns a JSON schema for a type with wrapped objects" do + json_schema = dump_schema do |s| + s.object_type "Color" do |t| + t.field "red", "Int!" + t.field "green", "Int!" + t.field "blue", "Int!" + end + + s.object_type "WidgetOptions" do |t| + t.field "color_breakdown", "Color!" + end + + s.object_type "Widget" do |t| + t.field "nullable", "WidgetOptions" + t.field "non_null", "WidgetOptions!" + t.field "null_array_null", "[WidgetOptions]" do |f| + f.mapping type: "object" + end + t.field "non_null_array_null", "[WidgetOptions]!" do |f| + f.mapping type: "object" + end + t.field "null_array_non_null", "[WidgetOptions!]" do |f| + f.mapping type: "object" + end + t.field "non_null_array_non_null", "[WidgetOptions!]!" do |f| + f.mapping type: "object" + end + t.field "null_null_array_null", "[[WidgetOptions]]" do |f| + f.mapping type: "object" + end + end + end + + expect(json_schema).to have_json_schema_like("Widget", { + "type" => "object", + "properties" => { + "nullable" => json_schema_ref("WidgetOptions"), + "non_null" => json_schema_ref("WidgetOptions!"), + "null_array_null" => { + "anyOf" => [ + { + "type" => "array", + "items" => { + "anyOf" => [ + {"$ref" => "#/$defs/WidgetOptions"}, + json_schema_null + ] + } + }, + json_schema_null + ] + }, + "non_null_array_null" => { + "type" => "array", + "items" => { + "anyOf" => [ + {"$ref" => "#/$defs/WidgetOptions"}, + json_schema_null + ] + } + }, + "null_array_non_null" => { + "anyOf" => [ + { + "type" => "array", + "items" => {"$ref" => "#/$defs/WidgetOptions"} + }, + json_schema_null + ] + }, + "non_null_array_non_null" => { + "type" => "array", + "items" => { + "$ref" => "#/$defs/WidgetOptions" + } + }, + "null_null_array_null" => { + "anyOf" => [ + { + "type" => "array", + "items" => { + "anyOf" => [ + { + "type" => "array", + "items" => { + "anyOf" => [ + {"$ref" => "#/$defs/WidgetOptions"}, + json_schema_null + ] + } + }, + json_schema_null + ] + } + }, + json_schema_null + ] + } + }, + "required" => %w[nullable non_null null_array_null non_null_array_null null_array_non_null non_null_array_non_null null_null_array_null] + }) + end + + context "on an indexed type with a rollover index" do + it "makes the JSON schema for a rollover index timestamp field on the indexed type non-nullable since a target index cannot be chosen without it" do + json_schema = dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime" + t.index "widgets" do |i| + i.rollover :monthly, "created_at" + end + end + end + + expect(json_schema).to have_json_schema_like("Widget", { + "type" => "object", + "properties" => { + "id" => json_schema_ref("ID!"), + "created_at" => json_schema_ref("DateTime!") + }, + "required" => %w[id created_at] + }) + + expect(json_schema).to have_json_schema_like("DateTime", { + "type" => "string", + "format" => "date-time" + }) + end + + it "does not break other configured JSON schema customizations when forcing the non-nullability on a rollover timestamp field" do + json_schema = dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime" do |f| + f.json_schema pattern: "\w+" + end + t.index "widgets" do |i| + i.rollover :monthly, "created_at" + end + end + end + + expect(json_schema).to have_json_schema_like("Widget", { + "type" => "object", + "properties" => { + "id" => json_schema_ref("ID!"), + "created_at" => { + "allOf" => [ + {"$ref" => "#/$defs/DateTime"}, + {"pattern" => "\w+"} + ] + } + }, + "required" => %w[id created_at] + }) + + expect(json_schema).to have_json_schema_like("DateTime", { + "type" => "string", + "format" => "date-time" + }) + end + + it "supports nested timestamp fields, applying non-nullability to every field in the path" do + json_schema = dump_schema do |s| + s.object_type "WidgetTimestamps" do |t| + t.field "created_at", "DateTime" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "timestamps", "WidgetTimestamps" + t.index "widgets" do |i| + i.rollover :monthly, "timestamps.created_at" + end + end + end + + expect(json_schema).to have_json_schema_like("Widget", { + "type" => "object", + "properties" => { + "id" => json_schema_ref("ID!"), + "timestamps" => json_schema_ref("WidgetTimestamps!") + }, + "required" => %w[id timestamps] + }) + + expect(json_schema).to have_json_schema_like("WidgetTimestamps", { + "type" => "object", + "properties" => { + "created_at" => json_schema_ref("DateTime!") + }, + "required" => %w[created_at] + }) + + expect(json_schema).to have_json_schema_like("DateTime", { + "type" => "string", + "format" => "date-time" + }) + end + + it "raises an error if the timestamp field specified in `rollover` is absent from the index" do + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.index "widgets" do |i| + i.rollover :monthly, "created_at" + end + end + end + }.to raise_error(Errors::SchemaError, a_string_including("Field `Widget.created_at` cannot be resolved, but it is referenced as an index `rollover` field.")) + end + + it "allows the timestamp field to be an indexing-only field since it need not be exposed to GraphQL clients" do + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_at", "DateTime", indexing_only: true + t.index "widgets" do |i| + i.rollover :monthly, "created_at" + end + end + end + }.not_to raise_error + end + + it "allows the timestamp field to be a `DateTime` or `Date` field" do + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_on", "Date" + t.index "widgets" do |i| + i.rollover :monthly, "created_on" + end + end + end + }.not_to raise_error + + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_at", "DateTime" + t.index "widgets" do |i| + i.rollover :monthly, "created_at" + end + end + end + }.not_to raise_error + end + + it "allows the timestamp field to be a non-nullable field" do + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_on", "Date!" + t.index "widgets" do |i| + i.rollover :monthly, "created_on" + end + end + end + }.not_to raise_error + + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_at", "DateTime!" + t.index "widgets" do |i| + i.rollover :monthly, "created_at" + end + end + end + }.not_to raise_error + end + + it "raises an error if a nested rollover timestamp field references an undefined type" do + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "options", "WidgetOptions" + t.index "widgets" do |i| + i.rollover :monthly, "options.created_at" + end + end + end + }.to raise_error(Errors::SchemaError, a_string_including( + "Field `Widget.options.created_at` cannot be resolved", + "Verify that all fields and types referenced by `options.created_at` are defined." + )) + end + + it "raises an error if a rollover timestamp field references an object type" do + expect { + dump_schema do |s| + s.object_type "WidgetOpts" do |t| + t.field "size", "Int" + end + + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "opts", "WidgetOpts" + t.index "widgets" do |i| + i.rollover :monthly, "opts" + end + end + end + }.to raise_error(Errors::SchemaError, a_string_including("rollover field `Widget.opts: WidgetOpts` cannot be used for rollover since it is not a `Date` or `DateTime` field")) + end + + it "raises an error if a rollover timestamp field references an enum type" do + expect { + dump_schema do |s| + s.enum_type "Color" do |t| + t.values "RED", "GREEN", "BLUE" + end + + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "color", "Color" + t.index "widgets" do |i| + i.rollover :monthly, "color" + end + end + end + }.to raise_error(Errors::SchemaError, a_string_including("rollover field `Widget.color: Color` cannot be used for rollover since it is not a `Date` or `DateTime` field")) + end + + it "raises an error if a rollover timestamp field references an scalar type that can't be used for rollover" do + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_at", "String" # not a DateTime! + t.index "widgets" do |i| + i.rollover :monthly, "created_at" + end + end + end + }.to raise_error(Errors::SchemaError, a_string_including("rollover field `Widget.created_at: String` cannot be used for rollover since it is not a `Date` or `DateTime` field")) + end + + it "respects configured type name overrides when determining if a rollover field is a valid type" do + json_schema = dump_schema(type_name_overrides: {"Date" => "Etad", "DateTime" => "EmitEtad"}) do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_at", "EmitEtad" + t.index "widgets" do |i| + i.rollover :monthly, "created_at" + end + end + + s.object_type "Component" do |t| + t.field "id", "ID" + t.field "created_on", "Etad" + t.index "widgets" do |i| + i.rollover :monthly, "created_on" + end + end + + expect { + s.object_type "Part" do |t| + t.field "id", "ID" + t.field "created_at", "String" + t.index "widgets" do |i| + i.rollover :monthly, "created_at" + end + end + }.to raise_error(Errors::SchemaError, a_string_including( + "rollover field `Part.created_at: String` cannot be used for rollover since it is not a `Etad` or `EmitEtad` field" + )) + end + + expect(json_schema.fetch("$defs").keys).to include("Widget", "Component") + end + + it "raises an error if a rollover timestamp field references a list field" do + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_ats", "[DateTime]" + t.index "widgets" do |i| + i.rollover :monthly, "created_ats" + end + end + end + }.to raise_error(Errors::SchemaError, a_string_including("rollover field `Widget.created_ats: [DateTime]` cannot be used for rollover since it is a list field.")) + + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_ons", "[Date]" + t.index "widgets" do |i| + i.rollover :monthly, "created_ons" + end + end + end + }.to raise_error(Errors::SchemaError, a_string_including("rollover field `Widget.created_ons: [Date]` cannot be used for rollover since it is a list field.")) + end + + it "raises an error if the timestamp field specified in `rollover` is defined after the `index` call" do + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.index "widgets" do |i| + i.rollover :monthly, "created_at" + end + # :nocov: -- the error is raised before we get here + t.field "created_at", "DateTime" + # :nocov: + end + end + }.to raise_error(Errors::SchemaError, a_string_including("the `Widget.created_at` definition must come before the `index` call")) + end + end + + context "on an indexed type with custom shard routing" do + it "makes the custom routing field non-nullable in the JSON schema since we cannot target a shard without it" do + json_schema = dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "user_id", "ID" + + t.index "widgets" do |i| + i.route_with "user_id" + end + end + end + + expect(json_schema).to have_json_schema_like("ID", { + "type" => "string" + }) + + expect(json_schema).to have_json_schema_like("Widget", { + "type" => "object", + "properties" => { + "id" => json_schema_ref("ID!"), + "user_id" => shard_routing_string_field + }, + "required" => %w[id user_id] + }).which_matches( + {"id" => "abc", "user_id" => "def"}, + {"id" => "abc", "user_id" => " d"}, + {"id" => "abc", "user_id" => "\td"}, + {"id" => "abc", "user_id" => "d\n"} + ).and_fails_to_match( + {"id" => "abc", "user_id" => nil}, + {"id" => "abc", "user_id" => ""}, + {"id" => "abc", "user_id" => " "}, + {"id" => "abc", "user_id" => " \t"}, + {"id" => "abc", "user_id" => " \n"} + ) + end + + it "supports nested routing fields, applying non-nullability to every field in the path" do + json_schema = dump_schema do |s| + s.object_type "WidgetIDs" do |t| + t.field "user_id", "ID" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "widget_ids", "WidgetIDs" + t.index "widgets" do |i| + i.route_with "widget_ids.user_id" + end + end + end + + expect(json_schema).to have_json_schema_like("WidgetIDs", { + "type" => "object", + "properties" => { + "user_id" => shard_routing_string_field + }, + "required" => ["user_id"] + }) + + expect(json_schema).to have_json_schema_like("Widget", { + "type" => "object", + "properties" => { + "id" => json_schema_ref("ID!"), + "widget_ids" => json_schema_ref("WidgetIDs!") + }, + "required" => %w[id widget_ids] + }) + end + + it "raises an error if the specified custom shard routing field is absent from the index" do + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.index "widgets" do |i| + i.route_with "user_id" + end + end + end + }.to raise_error(Errors::SchemaError, a_string_including("Field `Widget.user_id` cannot be resolved, but it is referenced as an index `route_with` field.")) + end + + it "raises an error if a shard routing field references an object type" do + expect { + dump_schema do |s| + s.object_type "WidgetOpts" do |t| + t.field "size", "Int" + end + + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "opts", "WidgetOpts" + t.index "widgets" do |i| + i.route_with "opts" + end + end + end + }.to raise_error(Errors::SchemaError, a_string_including("shard routing field `Widget.opts: WidgetOpts` cannot be used for routing since it is not a leaf field.")) + end + + it "raises an error if a shard routing field references a list field" do + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "tags", "[String]" + t.index "widgets" do |i| + i.route_with "tags" + end + end + end + }.to raise_error(Errors::SchemaError, a_string_including("shard routing field `Widget.tags: [String]` cannot be used for routing since it is not a leaf field.")) + end + + it "allows the custom shard routing field to be an indexing-only field since it need not be exposed to GraphQL clients" do + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "user_id", "ID", indexing_only: true + t.index "widgets" do |i| + i.route_with "user_id" + end + end + end + }.not_to raise_error + end + + it "raises an error if a nested custom shard routing field references an undefined type" do + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "options", "WidgetOptions" + t.index "widgets" do |i| + i.route_with "options.user_id" + end + end + end + }.to raise_error(Errors::SchemaError, a_string_including( + "Field `Widget.options.user_id` cannot be resolved", + "Verify that all fields and types referenced by `options.user_id` are defined" + )) + end + + it "allows the custom shard routing field to be nullable or non-null" do + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "user_id", "ID" + t.index "widgets" do |i| + i.route_with "user_id" + end + end + end + }.not_to raise_error + + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "user_id", "ID!" + t.index "widgets" do |i| + i.route_with "user_id" + end + end + end + }.not_to raise_error + end + + it "raises an error if the specified custom shard routing field is defined after `index`" do + expect { + dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.index "widgets" do |i| + i.route_with "user_id" + end + # :nocov: -- the error is raised before we get here + t.field "user_id", "ID" + # :nocov: + end + end + }.to raise_error(Errors::SchemaError, a_string_including("the `Widget.user_id` definition must come before the `index` call")) + end + + it "mentions the expected field in the error message when dealing with nested fields" do + expect { + dump_schema do |s| + s.object_type "Nested" do |t| + t.field "user_id", "ID" + end + + s.object_type "Widget" do |t| + t.field "id", "ID" + t.index "widgets" do |i| + i.route_with "nested.user_id" + end + end + end + }.to raise_error(Errors::SchemaError, a_string_including("Field `Widget.nested.user_id` cannot be resolved, but it is referenced as an index `route_with` field.")) + end + + it "does not include a confusing 'must come after' message..." do + expect { + dump_schema do |s| + s.object_type "Nested" do |t| + t.field "user_id", "ID" + end + + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "nested", "Nested" + t.index "widgets" do |i| + i.route_with "nested.some_id" + end + end + end + }.to raise_error(Errors::SchemaError, a_string_including("Field `Widget.nested.some_id` cannot be resolved, but it is referenced as an index `route_with` field").and(excluding("must come before"))) + end + end + + it "correctly overwrites built-in type customizations" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "month", "Int" do |f| + f.mapping type: "byte" + f.json_schema minimum: 0, maximum: 99 + end + + t.field "year", "Int" do |f| + f.mapping type: "short" + f.json_schema minimum: 2000, maximum: 2099 + end + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "month" => { + "anyOf" => [ + { + "allOf" => [ + json_schema_integer, + {"minimum" => 0, "maximum" => 99} + ] + }, + json_schema_null + ] + }, + "year" => { + "anyOf" => [ + { + "allOf" => [ + json_schema_integer, + {"minimum" => 2000, "maximum" => 2099} + ] + }, + json_schema_null + ] + } + }, + "required" => %w[month year] + }) + end + + it "allows JSON schema options to be built up over multiple `json_schema` calls" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "month", "Int" do |f| + f.json_schema minimum: 0 + f.json_schema maximum: 99 + f.json_schema minimum: 20 # demonstrate that the last call wins + end + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "month" => { + "anyOf" => [ + { + "allOf" => [ + json_schema_integer, + {"minimum" => 20, "maximum" => 99} + ] + }, + json_schema_null + ] + } + }, + "required" => %w[month] + }) + end + + it "correctly restricts enum types with customizations" do + json_schema = dump_schema do |s| + s.enum_type "Color" do |t| + t.values "RED", "ORANGE", "YELLOW", "GREEN", "BLUE", "INDIGO", "VIOLET" + end + + s.object_type "MyType" do |t| + t.field "primaryColor", "Color!" do |f| + f.json_schema enum: %w[RED YELLOW BLUE] + end + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "primaryColor" => { + "allOf" => [ + {"$ref" => "#/$defs/Color"}, + {"enum" => %w[RED YELLOW BLUE]} + ] + } + }, + "required" => %w[primaryColor] + }) + end + + it "applies customizations defined on a list field to the JSON schema array instead of applying them to the items" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "tags", "[String!]!" do |f| + f.json_schema uniqueItems: true, maxItems: 1000 + end + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "tags" => { + "type" => "array", + "items" => json_schema_string, + "uniqueItems" => true, + "maxItems" => 1000 + } + }, + "required" => %w[tags] + }) + end + + it "still applies customizations from the mapping type to array items" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "values", "[Int!]!" do |f| + f.json_schema minItems: 1 + f.mapping type: "short" + end + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "values" => { + "type" => "array", + "items" => { + "allOf" => [ + json_schema_integer, + {"minimum" => -32768, "maximum" => 32767} + ] + }, + "minItems" => 1 + } + }, + "required" => %w[values] + }) + end + + it "raises a Errors::SchemaError when a conflicting type is specified" do + dump_schema do |s| + s.object_type "MyType" do |t| + t.field "built_in_scalar_replaced", "String!" do |f| + expect { + f.json_schema type: "boolean" + }.to raise_error(Errors::SchemaError, a_string_including( + "Cannot override JSON schema type of field `built_in_scalar_replaced` with `boolean`" + )) + end + end + end + end + + it "respects `json_schema` replacements set on a field definition, except when conflicting" do + json_schema = dump_schema do |s| + s.scalar_type "MyText" do |t| + t.json_schema type: "string" + t.mapping type: "keyword" + end + + s.object_type "MyType" do |t| + t.field "built_in_scalar_augmented", "String!" do |f| + f.json_schema minLength: 4 + end + t.field "custom_scalar", "MyText!" + t.field "custom_scalar_augmented", "MyText!" do |f| + f.json_schema minLength: 4 + end + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "built_in_scalar_augmented" => { + "allOf" => [ + {"$ref" => "#/$defs/String"}, + {"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH, "minLength" => 4} + ] + }, + "custom_scalar" => json_schema_ref("MyText!", is_keyword_type: true), + "custom_scalar_augmented" => { + "allOf" => [ + {"$ref" => "#/$defs/MyText"}, + {"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH, "minLength" => 4} + ] + } + }, + "required" => %w[built_in_scalar_augmented custom_scalar custom_scalar_augmented] + }) + end + + it "respects `json_schema` customizations set on an object type definition" do + define_point = lambda do |s| + s.object_type "Point" do |t| + t.field "x", "Float" + t.field "y", "Float" + t.json_schema type: "array", items: [{type: "number"}, {type: "number"}] + end + end + + define_my_type = lambda do |s| + s.object_type "MyType" do |t| + t.field "location", "Point" + end + end + + # We should get the same json schema regardless of which type is defined first. + type_before_reference_json_schema = dump_schema do |s| + define_point.call(s) + define_my_type.call(s) + end + + type_after_reference_json_schema = dump_schema do |s| + define_my_type.call(s) + define_point.call(s) + end + + expect(type_before_reference_json_schema).to eq(type_after_reference_json_schema) + .and have_json_schema_like("Point", { + "type" => "array", + "items" => [ + {"type" => "number"}, + {"type" => "number"} + ] + }).which_matches( + [0, 0], + [1, 2], + [1234567890, 1234567890] + ).and_fails_to_match( + [nil, nil], + %w[a b], + nil + ) + end + + describe "indexing-only fields" do + it "allows the indexing-only fields to specify their customized json schema" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "date", "String", indexing_only: true do |f| + f.mapping type: "date" + f.json_schema format: "date-time" + end + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "date" => { + "anyOf" => [ + { + "allOf" => [ + {"$ref": "#/$defs/String"}, + {"format" => "date-time"} + ] + }, + json_schema_null + ] + } + }, + "required" => %w[date] + }) + end + + it "allows the indexing-only fields to be objects with nested fields" do + json_schema = dump_schema do |s| + s.object_type "NestedType" do |t| + t.field "name", "String!" + end + + s.object_type "MyType" do |t| + t.field "nested", "NestedType!", indexing_only: true + end + end + + expect(json_schema).to have_json_schema_like("NestedType", { + "type" => "object", + "properties" => { + "name" => json_schema_ref("String!") + }, + "required" => ["name"] + }) + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "nested" => json_schema_ref("NestedType!") + }, + "required" => %w[nested] + }) + end + + it "raises an error when same mapping field is defined twice with different JSON schemas" do + expect { + dump_schema do |s| + s.object_type "Card" do |t| + t.field "meta", "Int" do |f| + f.mapping type: "integer" + f.json_schema minimum: 10 + end + + t.field "meta", "Int", indexing_only: true do |f| + f.mapping type: "integer" + f.json_schema minimum: 20 + end + end + end + }.to raise_error Errors::SchemaError, a_string_including("Duplicate indexing field", "Card", "meta", "graphql_only: true") + end + end + + it "generates the JSON schema of an array for a `paginated_collection_field`" do + json_schema = dump_schema do |s| + s.object_type "Widget" do |t| + t.paginated_collection_field "names", "String" + end + end + + expect(json_schema).to have_json_schema_like("Widget", { + "type" => "object", + "properties" => { + "names" => { + "type" => "array", + "items" => json_schema_string + } + }, + "required" => %w[names] + }) + end + + it "honors JSON schema customizations of a `paginated_collection_field`" do + json_schema = dump_schema do |s| + s.object_type "Widget" do |t| + t.paginated_collection_field "names", "String" do |f| + f.json_schema uniqueItems: true, maxItems: 1000 + end + end + end + + expect(json_schema).to have_json_schema_like("Widget", { + "type" => "object", + "properties" => { + "names" => { + "type" => "array", + "items" => json_schema_string, + "uniqueItems" => true, + "maxItems" => 1000 + } + }, + "required" => %w[names] + }) + end + + describe "relation fields" do + context "on a relation with an outbound foreign key" do + it "includes a non-null foreign key field if the GraphQL relation field is non-null" do + json_schema = dump_schema do |s| + s.object_type "OtherType" do |t| + t.field "id", "ID!" + end + + s.object_type "MyType" do |t| + t.relates_to_one "other", "OtherType!", via: "other_id", dir: :out + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "other_id" => json_schema_ref("ID!") + }, + "required" => %w[other_id] + }) + end + + it "includes a nullable foreign key field if the GraphQL relation field is nullable" do + json_schema = dump_schema do |s| + s.object_type "OtherType" do |t| + t.field "id", "ID!" + end + + s.object_type "MyType" do |t| + t.relates_to_one "other", "OtherType", via: "other_id", dir: :out + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "other_id" => json_schema_ref("ID") + }, + "required" => %w[other_id] + }) + end + + it "includes an array foreign key field if its a `relates_to_many` field" do + json_schema = dump_schema do |s| + s.object_type "OtherType" do |t| + t.field "id", "ID!" + t.index "other_type" + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.relates_to_many "others", "OtherType", via: "other_ids", dir: :out, singular: "other" + t.index "my_type" + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "id" => json_schema_ref("ID!"), + "other_ids" => { + "type" => "array", + "items" => json_schema_id + } + }, + "required" => %w[id other_ids] + }) + end + + it "includes a non-null `id` field if the relation is self-referential, even if there is no `id` GraphQL field (for a `relates_to_one` case)" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.relates_to_one "parent", "MyType!", via: "parent_id", dir: :out + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "parent_id" => json_schema_ref("ID!"), + "id" => json_schema_ref("ID!") + }, + "required" => %w[parent_id id] + }) + end + end + + context "on a relation with an inbound foreign key" do + it "includes the foreign key field when the relation is self-referential, regardless of the details of the relation (nullable or not, one or many)" do + json_schema = dump_schema do |s| + s.object_type "MyTypeOneNullable" do |t| + t.field "id", "ID!" + t.relates_to_one "parent", "MyTypeOneNullable", via: "children_ids", dir: :in + t.index "my_type1" + end + + s.object_type "MyTypeOneNonNull" do |t| + t.field "id", "ID!" + t.relates_to_one "parent", "MyTypeOneNonNull!", via: "children_ids", dir: :in + t.index "my_type2" + end + + s.object_type "MyTypeBothDirections" do |t| + t.field "id", "ID!" + t.relates_to_one "parent", "MyTypeBothDirections!", via: "children_ids", dir: :in + t.relates_to_many "children", "MyTypeBothDirections", via: "children_ids", dir: :out, singular: "child" + t.index "my_type2" + end + + s.object_type "MyTypeMany" do |t| + t.field "id", "ID!" + t.relates_to_many "children", "MyTypeMany", via: "parent_id", dir: :in, singular: "child" + t.index "my_type3" + end + end + + expect(json_schema).to have_json_schema_like("MyTypeOneNullable", { + "type" => "object", + "properties" => { + "id" => json_schema_ref("ID!"), + # technically this would probably be an array field, but there's not enough info on this side of the relation to know. + # When the other side is also defined (as in `both_dirs`) it is more accurate. + "children_ids" => json_schema_ref("ID") + }, + "required" => %w[id children_ids] + }) + + expect(json_schema).to have_json_schema_like("MyTypeOneNonNull", { + "type" => "object", + "properties" => { + "id" => json_schema_ref("ID!"), + # technically this would probably be an array field, but there's not enough info on this side of the relation to know. + # When the other side is also defined (see another test) it is more accurate. + "children_ids" => json_schema_ref("ID!") + }, + "required" => %w[id children_ids] + }) + + expect(json_schema).to have_json_schema_like("MyTypeBothDirections", { + "type" => "object", + "properties" => { + "id" => json_schema_ref("ID!"), + "children_ids" => { + "type" => "array", + "items" => json_schema_id + } + }, + "required" => %w[id children_ids] + }) + + expect(json_schema).to have_json_schema_like("MyTypeMany", { + "type" => "object", + "properties" => { + "id" => json_schema_ref("ID!"), + "parent_id" => json_schema_ref("ID") + }, + "required" => %w[id parent_id] + }) + end + end + + it "prefers defined fields to fields inferred by relations when the same field is created by both, as defined fields are more accurate" do + json_schema = dump_schema do |s| + s.object_type "CardInferred" do |t| + t.relates_to_one "cloned_from_card", "CardInferred", via: "cloned_from_card_id", dir: :out + end + + s.object_type "CardExplicit" do |t| + t.relates_to_one "cloned_from_card", "CardInferred", via: "cloned_from_card_id", dir: :out + t.field "cloned_from_card_id", "ID!" + end + end + + expect(json_schema).to have_json_schema_like("CardInferred", { + "type" => "object", + "properties" => { + "cloned_from_card_id" => json_schema_ref("ID"), + "id" => json_schema_ref("ID!") + }, + "required" => %w[cloned_from_card_id id] + }) + + expect(json_schema).to have_json_schema_like("CardExplicit", { + "type" => "object", + "properties" => { + "cloned_from_card_id" => json_schema_ref("ID!") + }, + "required" => %w[cloned_from_card_id] + }) + end + end + + context "`nullable:` option inside `json_schema`" do + it "forces field that is nullable in GraphQL to be non-nullable in the generated JSON schema" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "size", "Float" do |f| + f.json_schema nullable: false + end + t.field "cost", "Float" + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "size" => json_schema_ref("Float!"), + "cost" => json_schema_ref("Float") + }, + "required" => %w[size cost] + }) + end + + it "has no effect on an already-non-nullable field" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "size", "Float!" do |f| + f.json_schema nullable: false + end + t.field "cost", "Float!" + t.index "my_type" + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "id" => json_schema_ref("ID!"), + "size" => json_schema_ref("Float!"), + "cost" => json_schema_ref("Float!") + }, + "required" => %w[id size cost] + }) + end + + it "forces wrapped field that is nullable in GraphQL to be non-nullable in the generated JSON schema" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "size", "[[Float!]]" do |f| + f.json_schema nullable: false + end + t.field "cost", "[[Float!]]" + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "size" => { + "type" => "array", + "items" => { + "anyOf" => [ + { + "type" => "array", + "items" => json_schema_float + }, + json_schema_null + ] + } + }, + "cost" => { + "anyOf" => [ + { + "type" => "array", + "items" => { + "anyOf" => [ + { + "type" => "array", + "items" => json_schema_float + }, + json_schema_null + ] + } + }, + json_schema_null + ] + } + }, + "required" => %w[size cost] + }) + end + + it "has no effect on an already-non-nullable wrapped field" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "size", "[[Float!]]!" do |f| + f.json_schema nullable: false + end + t.field "cost", "[[Float!]]!" + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "size" => { + "type" => "array", + "items" => { + "anyOf" => [ + { + "type" => "array", + "items" => json_schema_float + }, + json_schema_null + ] + } + }, + "cost" => { + "type" => "array", + "items" => { + "anyOf" => [ + { + "type" => "array", + "items" => json_schema_float + }, + json_schema_null + ] + } + } + }, + "required" => %w[size cost] + }) + end + + it "raises an exception on `nullable: true` because we cannot allow that for non-null GraphQL fields and `nullable: true` does nothing on an already nullable GraphQL field`" do + dump_schema do |s| + s.object_type "MyType" do |t| + t.field "size", "[[Float!]]" do |f| + expect { + f.json_schema nullable: true + }.to raise_error(Errors::SchemaError, a_string_including("`nullable: true` is not allowed on a field--just declare the GraphQL field as being nullable (no `!` suffix) instead.")) + end + end + end + end + + it "is not allowed on an object or scalar type (it is only intended for use on fields)" do + dump_schema do |s| + s.object_type "MyType" do |t| + expect { + t.json_schema nullable: false + }.to raise_error(Errors::SchemaError, a_string_including("Invalid JSON schema options", "nullable")) + end + + s.scalar_type "ScalarType" do |t| + t.mapping type: "boolean" + t.json_schema type: "boolean" + + expect { + t.json_schema nullable: false + }.to raise_error(Errors::SchemaError, a_string_including("Invalid JSON schema options", "nullable")) + end + end + end + end + + it "dumps object schemas with a __typename property" do + json_schema = dump_schema do |s| + s.object_type "MyType" do |t| + t.field "id", "ID!" + end + end + + expect(json_schema.dig("$defs", "MyType", "properties", "__typename")).to eq({ + "const" => "MyType", + "default" => "MyType", + "type" => "string" + }) + end + + shared_examples_for "a type with subtypes" do |type_def_method| + context "composed of 2 indexed types" do + it "generates separate json schemas for the two subtypes and the supertype" do + schemas = dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "amount_cents", "Int!" + link_subtype_to_supertype(t, "Thing") + t.index "widgets" + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "weight", "Int!" + link_subtype_to_supertype(t, "Thing") + t.index "components" + end + + s.public_send type_def_method, "Thing" do |t| + link_supertype_to_subtypes(t, "Widget", "Component") + end + end + + expect(schemas).to have_json_schema_like("Widget", { + "type" => "object", + "properties" => { + "id" => json_schema_ref("ID!"), + "name" => json_schema_ref("String!"), + "amount_cents" => json_schema_ref("Int!") + }, + "required" => %w[id name amount_cents] + }) + + expect(schemas).to have_json_schema_like("Component", { + "type" => "object", + "properties" => { + "id" => json_schema_ref("ID!"), + "name" => json_schema_ref("String!"), + "weight" => json_schema_ref("Int!") + }, + "required" => %w[id name weight] + }) + + expect(schemas).to have_json_schema_like("Thing", { + "required" => [ + "__typename" + ], + "oneOf" => [ + { + "$ref" => "#/$defs/Widget" + }, + { + "$ref" => "#/$defs/Component" + } + ] + }) + + type_definitions = schemas.fetch("$defs") + expect(type_definitions.keys).to include("Thing") + expect(envelope_type_enum_values(type_definitions)).to contain_exactly("Widget", "Component") + end + end + + context "that is itself indexed" do + it "uses `oneOf` to produce a JSON schema that exclusively validates one or the other type" do + json_schema = dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "amount_cents", "Int!" + link_subtype_to_supertype(t, "Thing") + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "weight", "Int!" + link_subtype_to_supertype(t, "Thing") + end + + s.public_send type_def_method, "Thing" do |t| + link_supertype_to_subtypes(t, "Widget", "Component") + t.index "things" + end + end + + expect(json_schema).to have_json_schema_like("Thing", { + "required" => ["__typename"], + "oneOf" => [ + {"$ref" => "#/$defs/Widget"}, + {"$ref" => "#/$defs/Component"} + ] + }).which_matches( + {"id" => "1", "name" => "foo", "amount_cents" => 12, "__typename" => "Widget"}, + {"id" => "1", "name" => "foo", "weight" => 12, "__typename" => "Component"} + ).and_fails_to_match( + {"id" => "1", "name" => "foo", "__typename" => "Widget"}, + nil + ) + end + end + + context "that is an embedded type" do + it "uses `oneOf` to produce a JSON schema that exclusively validates one or the other type" do + json_schema = dump_schema do |s| + s.object_type "Widget" do |t| + t.field "name", "String!" + t.field "amount_cents", "Int!" + link_subtype_to_supertype(t, "Thing") + end + + s.object_type "Component" do |t| + t.field "name", "String!" + t.field "weight", "Int!" + link_subtype_to_supertype(t, "Thing") + end + + s.public_send type_def_method, "Thing" do |t| + link_supertype_to_subtypes(t, "Widget", "Component") + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "thing", "Thing!" + t.index "my_type" + end + end + + expect(json_schema).to have_json_schema_like("Thing", { + "required" => ["__typename"], + "oneOf" => [ + {"$ref" => "#/$defs/Widget"}, + {"$ref" => "#/$defs/Component"} + ] + }) + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "id" => json_schema_ref("ID!"), + "thing" => json_schema_ref("Thing!"), + "__typename" => { + "type" => "string", + "const" => "MyType", + "default" => "MyType" + } + }, + "required" => %w[id thing] + }).which_matches( + {"id" => "a", "thing" => {"id" => "a", "name" => "foo", "amount_cents" => 12, "__typename" => "Widget"}}, + {"id" => "a", "thing" => {"id" => "a", "name" => "foo", "weight" => 12, "__typename" => "Component"}} + ).and_fails_to_match( + {"id" => "a", "name" => "foo", "__typename" => "Widget"}, + {"id" => "a", "thing" => nil}, + nil + ) + end + + it "generates a JSON schema that correctly allows null values when the supertype field is nullable" do + json_schema = dump_schema do |s| + s.object_type "Widget" do |t| + t.field "name", "String!" + t.field "amount_cents", "Int!" + link_subtype_to_supertype(t, "Thing") + end + + s.object_type "Component" do |t| + t.field "name", "String!" + t.field "weight", "Int!" + link_subtype_to_supertype(t, "Thing") + end + + s.public_send type_def_method, "Thing" do |t| + link_supertype_to_subtypes(t, "Widget", "Component") + end + + s.object_type "MyType" do |t| + t.field "thing", "Thing" + end + end + + expect(json_schema).to have_json_schema_like("MyType", { + "type" => "object", + "properties" => { + "thing" => json_schema_ref("Thing"), + "__typename" => { + "type" => "string", + "const" => "MyType", + "default" => "MyType" + } + }, + "required" => %w[thing] + }).which_matches( + {"thing" => {"id" => "a", "name" => "foo", "amount_cents" => 12, "__typename" => "Widget"}}, + {"thing" => {"id" => "a", "name" => "foo", "weight" => 12, "__typename" => "Component"}}, + {"thing" => nil} + ).and_fails_to_match( + {"name" => "foo", "__typename" => "Widget"}, + nil + ) + end + + it "allows the same field on two subtypes to have different json_schema" do + json_schema = dump_schema do |s| + s.object_type "Person" do |t| + t.field "name", "String" do |f| + f.json_schema nullable: false + end + t.field "nationality", "String!" + link_subtype_to_supertype(t, "Inventor") + end + + s.object_type "Company" do |t| + t.field "name", "String" do |f| + f.json_schema maxLength: 20 + end + t.field "stock_ticker", "String!" + link_subtype_to_supertype(t, "Inventor") + end + + s.public_send type_def_method, "Inventor" do |t| + link_supertype_to_subtypes(t, "Person", "Company") + end + end + + expect(json_schema).to have_json_schema_like("Person", { + "type" => "object", + "properties" => { + "name" => json_schema_ref("String!"), + "nationality" => json_schema_ref("String!") + }, + "required" => %w[name nationality] + }) + + expect(json_schema).to have_json_schema_like("Company", { + "type" => "object", + "properties" => { + "name" => { + "anyOf" => [ + { + "allOf" => [ + {"$ref" => "#/$defs/String"}, + {"maxLength" => 20} + ] + }, + {"type" => "null"} + ] + }, + "stock_ticker" => json_schema_ref("String!") + }, + "required" => %w[name stock_ticker] + }) + end + end + end + + context "on a type union" do + include_examples "a type with subtypes", :union_type do + def link_subtype_to_supertype(object_type, supertype_name) + # nothing to do; the linkage happens via a `subtypes` call on the supertype + end + + def link_supertype_to_subtypes(union_type, *subtype_names) + union_type.subtypes(*subtype_names) + end + end + end + + context "on an interface type" do + include_examples "a type with subtypes", :interface_type do + def link_subtype_to_supertype(object_type, interface_name) + object_type.implements interface_name + end + + def link_supertype_to_subtypes(interface_type, *subtype_names) + # nothing to do; the linkage happens via an `implements` call on the subtype + end + end + + it "supports interface recursion (e.g. an interface that implements an interface)" do + json_schema = dump_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "amount_cents", "Int!" + t.implements "WidgetOrComponent" + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.field "name", "String!" + t.field "weight", "Int!" + t.implements "WidgetOrComponent" + end + + s.interface_type "WidgetOrComponent" do |t| + t.implements "Thing" + end + + s.object_type "Object" do |t| + t.field "id", "ID!" + t.field "description", "String!" + t.implements "Thing" + end + + s.interface_type "Thing" do |t| + t.field "id", "ID!" + t.index "things" + end + end + + expect(json_schema).to have_json_schema_like("Thing", { + "required" => ["__typename"], + "oneOf" => [ + {"$ref" => "#/$defs/Widget"}, + {"$ref" => "#/$defs/Component"}, + {"$ref" => "#/$defs/Object"} + ] + }).which_matches( + {"id" => "1", "name" => "foo", "amount_cents" => 12, "__typename" => "Widget"}, + {"id" => "1", "name" => "foo", "weight" => 12, "__typename" => "Component"}, + {"id" => "1", "description" => "foo", "__typename" => "Object"} + ).and_fails_to_match( + {"id" => "1", "name" => "foo", "__typename" => "Widget"}, + nil + ) + end + end + + it "dumps the types by name in alphabetical order (minus the envelope type at the start) for consistent dump output" do + schemas1 = all_type_definitions_for do |s| + s.object_type "AType" do |t| + t.field "id", "ID!" + t.index "a_type" + end + + s.object_type "BType" do |t| + t.field "id", "ID!" + t.index "b_type" + end + end + + schemas2 = all_type_definitions_for do |s| + s.object_type "BType" do |t| + t.field "id", "ID!" + t.index "b_type" + end + + s.object_type "AType" do |t| + t.field "id", "ID!" + t.index "a_type" + end + end + + # The types should have alphabetical keys (except the envelope always goes first; hence the `drop(1)`) + expect(schemas1.keys.drop(1)).to eq schemas1.keys.drop(1).sort + expect(schemas2.keys.drop(1)).to eq schemas2.keys.drop(1).sort + + # ...and the types should be alphabetically listed within the envelope, too. + expect(envelope_type_enum_values(schemas1)).to eq %w[AType BType] + expect(envelope_type_enum_values(schemas2)).to eq %w[AType BType] + end + + it "does not dump a schema for a derived indexed type because it cannot be directly ingested by the indexer" do + schemas = all_type_definitions_for do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "workspace_id", "ID" + t.index "widgets" + t.derive_indexed_type_fields "WidgetWorkspace", from_id: "workspace_id" do |derive| + derive.append_only_set "widget_ids", from: "id" + end + end + + s.object_type "WidgetWorkspace" do |t| + t.field "id", "ID!" + t.field "widget_ids", "[ID!]!" + t.index "widget_workspaces" + end + end + + expect(schemas.keys).to include(EVENT_ENVELOPE_JSON_SCHEMA_NAME, "Widget") + expect(schemas.keys).to exclude("WidgetWorkspace") + expect(envelope_type_enum_values(schemas)).to eq ["Widget"] + end + + it "raises a clear error if the schema defines a type with a reserved name" do + dump_schema do |s| + expect { + s.object_type EVENT_ENVELOPE_JSON_SCHEMA_NAME + }.to raise_error Errors::SchemaError, a_string_including(EVENT_ENVELOPE_JSON_SCHEMA_NAME, "reserved name") + end + end + + it "sets json_schema_version to the specified (valid) value" do + result = define_schema(schema_element_name_form: "snake_case") do |s| + s.json_schema_version 1 + end.json_schemas_for(1) + + expect(result[JSON_SCHEMA_VERSION_KEY]).to eq(1) + end + + it "fails if json_schema_version is set to invalid values" do + expect { + define_schema(schema_element_name_form: "snake_case") do |s| + s.json_schema_version 0.5 + end + }.to raise_error(Errors::SchemaError, a_string_including("must be a positive integer. Specified version: 0.5")) + + expect { + define_schema(schema_element_name_form: "snake_case") do |s| + s.json_schema_version "asd" + end + }.to raise_error(Errors::SchemaError, a_string_including("must be a positive integer. Specified version: asd")) + + expect { + define_schema(schema_element_name_form: "snake_case") do |s| + s.json_schema_version 0 + end + }.to raise_error(Errors::SchemaError, a_string_including("must be a positive integer. Specified version: 0")) + + expect { + define_schema(schema_element_name_form: "snake_case") do |s| + s.json_schema_version(-1) + end + }.to raise_error(Errors::SchemaError, a_string_including("must be a positive integer. Specified version: -1")) + end + + it "fails if json_schema_version is left unset" do + expect { + define_schema(schema_element_name_form: "snake_case", json_schema_version: nil) {}.available_json_schema_versions + }.to raise_error(Errors::SchemaError, a_string_including("must be specified in the schema")) + end + + it "fails if json_schema_version is set multiple times" do + expect { + define_schema(schema_element_name_form: "snake_case") do |s| + s.json_schema_version 1 + s.json_schema_version 2 + end + }.to raise_error(Errors::SchemaError, a_string_including("can only be set once", "Previously-set version: 1")) + end + + it "is unable to return a non-existent schema version" do + expect { + define_schema(schema_element_name_form: "snake_case") do |s| + s.json_schema_version 1 + end.json_schemas_for(2) + }.to raise_error(Errors::NotFoundError, a_string_including("The requested json schema version (2) is not available", "Available versions: 1")) + end + + it "ignores runtime fields during json schema generation" do + json_schema = dump_schema do |schema| + schema.object_type "Widget" do |t| + t.field "test_runtime_field", "String" do |f| + f.runtime_script "example test script" + end + end + end + + widget_def = json_schema.fetch("$defs").fetch("Widget") + expect(widget_def["properties"].keys).not_to include("test_runtime_field") + end + + def all_type_definitions_for(&schema_definition) + dump_schema(&schema_definition).fetch("$defs") + end + + def dump_schema(type_name_overrides: {}, enum_value_overrides_by_type: {}, &schema_definition) + define_schema( + schema_element_name_form: "snake_case", + type_name_overrides: type_name_overrides, + enum_value_overrides_by_type: enum_value_overrides_by_type, + &schema_definition + ).current_public_json_schema + end + + def envelope_type_enum_values(schemas) + schemas.dig(EVENT_ENVELOPE_JSON_SCHEMA_NAME, "properties", "type", "enum") + end + + def json_schema_ref(type, is_keyword_type: %w[ID! ID String! String].include?(type)) + if type.end_with?("!") + basic_json_schema_ref = {"$ref" => "#/$defs/#{type.delete_suffix("!")}"} + + if is_keyword_type + { + "allOf" => [ + basic_json_schema_ref, + {"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH} + ] + } + else + basic_json_schema_ref + end + else + { + "anyOf" => [ + json_schema_ref("#{type}!", is_keyword_type: is_keyword_type), + {"type" => "null"} + ] + } + end + end + + def shard_routing_string_field + { + "allOf" => [ + {"$ref" => "#/$defs/ID"}, + {"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH, "pattern" => Indexing::Index::HAS_NON_WHITE_SPACE_REGEX} + ] + } + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/enum_types_by_name_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/enum_types_by_name_spec.rb new file mode 100644 index 00000000..f17a3665 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/enum_types_by_name_spec.rb @@ -0,0 +1,242 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "runtime_metadata_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "RuntimeMetadata #enum_types_by_name" do + include_context "RuntimeMetadata support" + + it "dumps sort info for sort enum types" do + metadata = enum_type_metadata_for "WidgetSortOrderInput" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.index "widgets" + end + end + + expect(metadata.values_by_name.transform_values(&:sort_field)).to eq( + "id_ASC" => sort_field_with(field_path: "id", direction: :asc), + "id_DESC" => sort_field_with(field_path: "id", direction: :desc), + "name_ASC" => sort_field_with(field_path: "name", direction: :asc), + "name_DESC" => sort_field_with(field_path: "name", direction: :desc) + ) + end + + it "dumps `DateGroupingGranularity` metadata" do + metadata = enum_type_metadata_for "DateGroupingGranularity", expect_matching_input: true + + expect(metadata.values_by_name.transform_values(&:datastore_value)).to eq( + "YEAR" => "year", + "QUARTER" => "quarter", + "MONTH" => "month", + "WEEK" => "week", + "DAY" => "day" + ) + end + + it "dumps `DateTimeGroupingGranularity` metadata" do + metadata = enum_type_metadata_for "DateTimeGroupingGranularity", expect_matching_input: true + + expect(metadata.values_by_name.transform_values(&:datastore_value)).to eq( + "YEAR" => "year", + "QUARTER" => "quarter", + "MONTH" => "month", + "WEEK" => "week", + "DAY" => "day", + "HOUR" => "hour", + "MINUTE" => "minute", + "SECOND" => "second" + ) + end + + it "dumps `DistanceUnit` metadata" do + metadata = enum_type_metadata_for "DistanceUnit", expect_matching_input: true + + expect(metadata.values_by_name.transform_values(&:datastore_abbreviation)).to eq( + "MILE" => :mi, + "YARD" => :yd, + "FOOT" => :ft, + "INCH" => :in, + "KILOMETER" => :km, + "METER" => :m, + "CENTIMETER" => :cm, + "MILLIMETER" => :mm, + "NAUTICAL_MILE" => :nmi + ) + end + + it "dumps `DateTimeUnit` metadata" do + metadata = enum_type_metadata_for "DateTimeUnit", expect_matching_input: true + + expect(metadata.values_by_name.transform_values(&:datastore_abbreviation)).to eq( + "DAY" => :d, + "HOUR" => :h, + "MILLISECOND" => :ms, + "MINUTE" => :m, + "SECOND" => :s + ) + + expect(metadata.values_by_name.transform_values(&:datastore_value)).to eq( + "DAY" => 86_400_000, + "HOUR" => 3_600_000, + "MILLISECOND" => 1, + "MINUTE" => 60_000, + "SECOND" => 1_000 + ) + end + + it "dumps `DateUnit` metadata" do + metadata = enum_type_metadata_for "DateUnit", expect_matching_input: true + + expect(metadata.values_by_name.transform_values(&:datastore_abbreviation)).to eq( + "DAY" => :d + ) + + expect(metadata.values_by_name.transform_values(&:datastore_value)).to eq( + "DAY" => 86_400_000 + ) + end + + it "dumps `LocalTimeUnit` metadata" do + metadata = enum_type_metadata_for "LocalTimeUnit", expect_matching_input: true + + expect(metadata.values_by_name.transform_values(&:datastore_abbreviation)).to eq( + "HOUR" => :h, + "MILLISECOND" => :ms, + "MINUTE" => :m, + "SECOND" => :s + ) + + expect(metadata.values_by_name.transform_values(&:datastore_value)).to eq( + "HOUR" => 3_600_000, + "MILLISECOND" => 1, + "MINUTE" => 60_000, + "SECOND" => 1_000 + ) + end + + it "respects enum value name overrides" do + metadata = enum_type_metadata_for "DistanceUnit", expect_matching_input: true, enum_value_overrides_by_type: { + DistanceUnit: {MILE: "MI", YARD: "YD"} + } + + expect(metadata.values_by_name.transform_values(&:datastore_abbreviation)).to include( + "MI" => :mi, + "YD" => :yd, + "FOOT" => :ft # demonstrate one that's not overridden + ) + + expect(metadata.values_by_name.transform_values(&:alternate_original_name)).to include( + "MI" => "MILE", + "YD" => "YARD", + "FOOT" => nil # demonstrate one that's not overridden + ) + end + + it "is not dumped for any other types of enums (including user-defined ones) since there is no runtime metadata to store for them" do + results = define_schema do |s| + s.enum_type "Color" do |t| + t.values "RED", "BLUE", "GREEN" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "color", "Color" + t.index "widgets" + end + end + + runtime_metadata = SchemaArtifacts::RuntimeMetadata::Schema.from_hash(results.runtime_metadata.to_dumpable_hash, for_context: :admin) + expect(runtime_metadata.enum_types_by_name.keys).to contain_exactly( + "DateGroupingGranularity", "DateGroupingGranularityInput", + "DateGroupingTruncationUnit", "DateGroupingTruncationUnitInput", + "DateTimeGroupingGranularity", "DateTimeGroupingGranularityInput", + "DateTimeGroupingTruncationUnit", "DateTimeGroupingTruncationUnitInput", + "DateTimeUnit", "DateTimeUnitInput", + "DateUnit", "DateUnitInput", + "DistanceUnit", "DistanceUnitInput", + "LocalTimeGroupingTruncationUnit", "LocalTimeGroupingTruncationUnitInput", + "LocalTimeUnit", "LocalTimeUnitInput", + "MatchesQueryAllowedEditsPerTerm", "MatchesQueryAllowedEditsPerTermInput", + "WidgetSortOrderInput" + ) + end + + it "uses dot-separated paths for nested sort fields" do + metadata = enum_type_metadata_for "WidgetSortOrderInput" do |s| + s.object_type "WidgetOptions" do |t| + t.field "size", "Int" + t.field "color", "String" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "options", "WidgetOptions" + t.index "widgets" + end + end + + expect(metadata.values_by_name.transform_values(&:sort_field)).to include( + "options_size_ASC" => sort_field_with(field_path: "options.size", direction: :asc), + "options_size_DESC" => sort_field_with(field_path: "options.size", direction: :desc), + "options_color_ASC" => sort_field_with(field_path: "options.color", direction: :asc), + "options_color_DESC" => sort_field_with(field_path: "options.color", direction: :desc) + ) + end + + it "uses a field's `name_in_index` in the sort field path" do + metadata = enum_type_metadata_for "WidgetSortOrderInput" do |s| + s.object_type "WidgetOptions" do |t| + t.field "size", "Int", name_in_index: "size2" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "options", "WidgetOptions", name_in_index: "options2" + t.index "widgets" + end + end + + expect(metadata.values_by_name.transform_values(&:sort_field)).to include( + "options_size_ASC" => sort_field_with(field_path: "options2.size2", direction: :asc), + "options_size_DESC" => sort_field_with(field_path: "options2.size2", direction: :desc) + ) + end + + it "omits unsortable fields" do + metadata = enum_type_metadata_for "WidgetSortOrderInput" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String", sortable: false + t.index "widgets" + end + end + + expect(metadata.values_by_name.transform_values(&:sort_field)).to eq( + "id_ASC" => sort_field_with(field_path: "id", direction: :asc), + "id_DESC" => sort_field_with(field_path: "id", direction: :desc) + ) + end + + def enum_type_metadata_for(name, expect_matching_input: false, **schema_options, &block) + enum_types_by_name = define_schema(**schema_options, &block).runtime_metadata.enum_types_by_name + + enum_types_by_name[name].tap do |metadata| + if expect_matching_input + input_metadata = enum_types_by_name["#{name}Input"] + expect(input_metadata).to eq(metadata) + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/graphql_extension_modules_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/graphql_extension_modules_spec.rb new file mode 100644 index 00000000..01e4c9e1 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/graphql_extension_modules_spec.rb @@ -0,0 +1,37 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "runtime_metadata_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "RuntimeMetadata #graphql_extension_modules" do + include_context "RuntimeMetadata support" + + it "includes any modules registered during schema definition" do + extension_module1 = Module.new + extension_module2 = Module.new + + metadata = define_schema do |s| + s.register_graphql_extension extension_module1, defined_at: __FILE__ + s.register_graphql_extension extension_module2, defined_at: __FILE__ + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.index "widgets" + end + end.runtime_metadata + + expect(metadata.graphql_extension_modules).to eq [ + SchemaArtifacts::RuntimeMetadata::Extension.new(extension_module1, __FILE__, {}), + SchemaArtifacts::RuntimeMetadata::Extension.new(extension_module2, __FILE__, {}) + ] + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/index_definitions_by_name_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/index_definitions_by_name_spec.rb new file mode 100644 index 00000000..d5295d9c --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/index_definitions_by_name_spec.rb @@ -0,0 +1,903 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "runtime_metadata_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "RuntimeMetadata #index_definitions_by_name" do + include_context "RuntimeMetadata support" + + it "dumps the `route_with` value" do + widgets = index_definition_metadata_for("widgets") do |i| + i.route_with "group_id" + end + expect(widgets.route_with).to eq "group_id" + end + + it "dumps the `route_with` field's `name_in_index`" do + widgets = index_definition_metadata_for("widgets") do |i| + i.route_with "group_id_gql" + end + expect(widgets.route_with).to eq "group_id_index" + end + + it "supports nested `route_with` fields, using the `name_in_index` at each layer" do + widgets = index_definition_metadata_for("widgets") do |i| + i.route_with "nested_fields_gql.some_id_gql" + end + expect(widgets.route_with).to eq "nested_fields_index.some_id_index" + end + + it "defaults `route_with` to `id` because that's the default routing the datastore uses" do + components = index_definition_metadata_for("components") + expect(components.route_with).to eq "id" + end + + it "dumps the `rollover` options, if set" do + widgets = index_definition_metadata_for("widgets") do |i| + i.rollover :monthly, "created_at" + end + expect(widgets.rollover).to eq SchemaArtifacts::RuntimeMetadata::IndexDefinition::Rollover.new( + frequency: :monthly, + timestamp_field_path: "created_at" + ) + + components = index_definition_metadata_for("components") + expect(components.rollover).to eq nil + end + + it "dumps the `rollover` timestamp field's `name_in_index`" do + widgets = index_definition_metadata_for("widgets") do |i| + i.rollover :monthly, "created_at_gql" + end + expect(widgets.rollover).to eq SchemaArtifacts::RuntimeMetadata::IndexDefinition::Rollover.new( + frequency: :monthly, + timestamp_field_path: "created_at_index" + ) + end + + it "supports nested `rollover` timestamp fields, using the `name_in_index` at each layer" do + widgets = index_definition_metadata_for("widgets") do |i| + i.rollover :monthly, "nested_fields_gql.some_timestamp_gql" + end + expect(widgets.rollover).to eq SchemaArtifacts::RuntimeMetadata::IndexDefinition::Rollover.new( + frequency: :monthly, + timestamp_field_path: "nested_fields_index.some_timestamp_index" + ) + end + + it "dumps the `default_sort_fields`" do + widgets = index_definition_metadata_for("widgets") do |i| + i.default_sort "created_at", :asc, "group_id", :desc + end + + expect(widgets.default_sort_fields).to eq [ + sort_field_with(field_path: "created_at", direction: :asc), + sort_field_with(field_path: "group_id", direction: :desc) + ] + end + + it "defaults `default_sort_fields` to `[]`" do + widgets = index_definition_metadata_for("widgets") + + expect(widgets.default_sort_fields).to eq [] + end + + it "dumps the `default_sort_fields` `name_in_index`" do + widgets = index_definition_metadata_for("widgets") do |i| + i.default_sort "created_at_gql", :asc, "group_id_gql", :desc + end + + expect(widgets.default_sort_fields).to eq [ + sort_field_with(field_path: "created_at_index", direction: :asc), + sort_field_with(field_path: "group_id_index", direction: :desc) + ] + end + + it "supports nested `default_sort_fields`, correctly using `name_in_index` at each layer" do + widgets = index_definition_metadata_for("widgets") do |i| + i.default_sort "nested_fields_gql.some_timestamp_gql", :asc, "nested_fields_gql.some_id_gql", :desc + end + expect(widgets.default_sort_fields).to eq [ + sort_field_with(field_path: "nested_fields_index.some_timestamp_index", direction: :asc), + sort_field_with(field_path: "nested_fields_index.some_id_index", direction: :desc) + ] + end + + it "raises a clear error when a sort field has not been defined" do + expect { + index_definition_metadata_for("widgets") do |i| + i.default_sort "unknown_field", :asc + end + }.to raise_error(Errors::SchemaError, a_string_including("Field `MyType.unknown_field` cannot be resolved, but it is referenced as an index `default_sort` field")) + end + + it "allows a referenced sort field to be indexing-only since it need not be exposed to GraphQL clients" do + expect { + index_definition_metadata_for("widgets", on_my_type: ->(t) { t.field "index_value", "String", indexing_only: true }) do |i| + i.default_sort "index_value", :asc + end + }.not_to raise_error + end + + it "raises a clear error when a nested sort field references a type that does not exist", :dont_validate_graphql_schema do + expect { + index_definition_metadata_for("widgets", on_my_type: ->(t) { t.field "options", "Opts" }) do |i| + i.default_sort "options.unknown_field", :asc + end + }.to raise_error(Errors::SchemaError, a_string_including("Type `Opts` cannot be resolved")) + end + + describe "#current_sources" do + it "only contains `#{SELF_RELATIONSHIP_NAME}` on a index that has no `sourced_from` fields (but has fields)" do + current_sources = current_sources_for "widgets" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.index "widgets" + end + end + + expect(current_sources).to contain_exactly SELF_RELATIONSHIP_NAME + end + + it "includes the sources of all `sourced_from` fields defined on the index" do + current_sources = current_sources_for "components" do |s| + s.object_type "Component" do |t| + t.field "id", "ID!" + t.relates_to_one "widget", "Widget", via: "component_ids", dir: :in + + t.field "widget_name", "String" do |f| + f.sourced_from "widget", "name" + end + + t.field "widget_type", "String" do |f| + f.sourced_from "widget", "type" + end + + t.index "components" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "type", "String" + t.field "component_ids", "[ID!]!" + t.index "widgets" + end + end + + expect(current_sources).to contain_exactly SELF_RELATIONSHIP_NAME, "widget" + end + + it "considers `indexing_only` fields" do + current_sources = current_sources_for "components" do |s| + s.object_type "Component" do |t| + t.field "id", "ID!" + t.relates_to_one "widget", "Widget", via: "component_ids", dir: :in + + t.field "widget_name", "String", indexing_only: true do |f| + f.sourced_from "widget", "name" + end + + t.index "components" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "type", "String" + t.field "component_ids", "[ID!]!" + t.index "widgets" + end + end + + expect(current_sources).to contain_exactly SELF_RELATIONSHIP_NAME, "widget" + end + + it "omits `#{SELF_RELATIONSHIP_NAME}` when _every_ field is a `sourced_from` field" do + current_sources = current_sources_for "components" do |s| + s.object_type "Component" do |t| + t.field "id", "ID" do |f| + f.sourced_from "widget", "component_id" + end + + t.relates_to_one "widget", "Widget", via: "component_id", dir: :in + + t.field "widget_name", "String" do |f| + f.sourced_from "widget", "name" + end + + t.field "widget_type", "String" do |f| + f.sourced_from "widget", "type" + end + + t.index "components" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "type", "String" + t.field "component_id", "ID" + t.index "widgets" + end + end + + expect(current_sources).to contain_exactly "widget" + end + + it "considers `sourced_from` fields on embedded types" do + current_sources = current_sources_for "components" do |s| + s.object_type "ComponentWidgetFields" do |t| + t.relates_to_one "widget", "Widget", via: "component_ids", dir: :in + + t.field "widget_name", "String" do |f| + f.sourced_from "widget", "name" + end + + t.field "widget_type", "String" do |f| + f.sourced_from "widget", "type" + end + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.field "widget_fields", "ComponentWidgetFields" + t.index "components" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "type", "String" + t.field "component_ids", "[ID!]!" + t.index "widgets" + end + end + + expect(current_sources).to contain_exactly SELF_RELATIONSHIP_NAME, "widget" + end + + it "considers the sources of all subtypes of a type union" do + current_sources = current_sources_for "widgets_or_components" do |s| + s.object_type "Component" do |t| + t.field "id", "ID!" + t.relates_to_one "widget", "Widget", via: "component_ids", dir: :in + + t.field "widget_name", "String", indexing_only: true do |f| + f.sourced_from "widget", "name" + end + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "type", "String" + t.field "component_ids", "[ID!]!" + t.index "widgets" + end + + s.union_type "WidgetOrComponent" do |t| + t.subtypes "Widget", "Component" + t.index "widgets_or_components" + end + end + + expect(current_sources).to contain_exactly SELF_RELATIONSHIP_NAME, "widget" + end + + it "considers the sources of all subtypes of an interface" do + current_sources = current_sources_for "types_with_ids" do |s| + s.object_type "Component" do |t| + t.implements "TypeWithID" + t.field "id", "ID!" + t.relates_to_one "widget", "Widget", via: "component_ids", dir: :in + + t.field "widget_name", "String", indexing_only: true do |f| + f.sourced_from "widget", "name" + end + end + + s.object_type "Widget" do |t| + t.implements "TypeWithID" + t.field "id", "ID!" + t.field "name", "String" + t.field "type", "String" + t.field "component_ids", "[ID!]!" + t.index "widgets" + end + + s.interface_type "TypeWithID" do |t| + t.field "id", "ID!" + t.index "types_with_ids" + end + end + + expect(current_sources).to contain_exactly SELF_RELATIONSHIP_NAME, "widget" + end + + def current_sources_for(index_name, &block) + define_schema(&block) + .runtime_metadata + .index_definitions_by_name + .fetch(index_name) + .current_sources + end + end + + describe "#fields_by_path" do + it "records the source of each field, defaulting to `#{SELF_RELATIONSHIP_NAME}` for fields that do not use `sourced_from`" do + fields_by_path = fields_by_path_for "components" do |s| + s.object_type "Component" do |t| + t.field "id", "ID!" + t.relates_to_one "widget", "Widget", via: "component_ids", dir: :in + + t.field "widget_name", "String" do |f| + f.sourced_from "widget", "name" + end + + t.field "widget_type", "String" do |f| + f.sourced_from "widget", "type" + end + + t.index "components" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "type", "String" + t.field "component_ids", "[ID!]!" + t.index "widgets" + end + end + + expect(fields_by_path).to eq({ + "id" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "widget_name" => index_field_with(source: "widget"), + "widget_type" => index_field_with(source: "widget") + }) + end + + it "uses dot-separated paths for keys" do + fields_by_path = fields_by_path_for "components" do |s| + s.object_type "ComponentWidgetFields" do |t| + t.relates_to_one "widget", "Widget", via: "component_ids", dir: :in + t.field "id", "ID" + + t.field "name", "String" do |f| + f.sourced_from "widget", "name" + end + + t.field "type", "String" do |f| + f.sourced_from "widget", "type" + end + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.field "widget_fields", "ComponentWidgetFields" + t.index "components" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "type", "String" + t.field "component_ids", "[ID!]!" + t.index "widgets" + end + end + + expect(fields_by_path).to eq({ + "id" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "widget_fields.id" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "widget_fields.name" => index_field_with(source: "widget"), + "widget_fields.type" => index_field_with(source: "widget") + }) + end + + it "includes `indexing_only` fields and excludes `graphql_only` fields" do + fields_by_path = fields_by_path_for "components" do |s| + s.object_type "Component" do |t| + t.field "id", "ID!" + t.field "name", "String", graphql_only: true + t.field "type", "String", indexing_only: true + t.index "components" + end + end + + expect(fields_by_path).to eq({ + "id" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "type" => index_field_with(source: SELF_RELATIONSHIP_NAME) + }) + end + + it "uses the `name_in_index` instead of the GraphQL name of a field" do + fields_by_path = fields_by_path_for "components" do |s| + s.object_type "ComponentWidgetFields" do |t| + t.relates_to_one "widget", "Widget", via: "component_ids", dir: :in + t.field "id", "ID" + + t.field "name", "String", name_in_index: "name2" do |f| + f.sourced_from "widget", "name" + end + + t.field "type", "String" do |f| + f.sourced_from "widget", "type" + end + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.field "widget_fields", "ComponentWidgetFields", name_in_index: "nested" + t.index "components" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "type", "String" + t.field "component_ids", "[ID!]!" + t.index "widgets" + end + end + + expect(fields_by_path).to eq({ + "id" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "nested.id" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "nested.name2" => index_field_with(source: "widget"), + "nested.type" => index_field_with(source: "widget") + }) + end + + it "includes fields from all subtypes of a type union" do + fields_by_path = fields_by_path_for "widgets_or_components" do |s| + s.object_type "Component" do |t| + t.field "id", "ID!" + t.relates_to_one "widget", "Widget", via: "component_id", dir: :in + + t.field "widget_name", "String", indexing_only: true do |f| + f.sourced_from "widget", "name" + end + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "type", "String" + t.field "component_id", "ID" + t.index "widgets" + end + + s.union_type "WidgetOrComponent" do |t| + t.subtypes "Widget", "Component" + t.index "widgets_or_components" + end + end + + expect(fields_by_path).to eq({ + "component_id" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "id" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "name" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "type" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "widget_name" => index_field_with(source: "widget") + }) + end + + it "considers the sources of all subtypes of an interface" do + fields_by_path = fields_by_path_for "types_with_ids" do |s| + s.object_type "Component" do |t| + t.implements "TypeWithID" + t.field "id", "ID!" + t.relates_to_one "widget", "Widget", via: "component_id", dir: :in + + t.field "widget_name", "String", indexing_only: true do |f| + f.sourced_from "widget", "name" + end + end + + s.object_type "Widget" do |t| + t.implements "TypeWithID" + t.field "id", "ID!" + t.field "name", "String" + t.field "type", "String" + t.field "component_id", "ID" + t.index "widgets" + end + + s.interface_type "TypeWithID" do |t| + t.field "id", "ID!" + t.index "types_with_ids" + end + end + + expect(fields_by_path).to eq({ + "component_id" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "id" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "name" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "type" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "widget_name" => index_field_with(source: "widget") + }) + end + + it "propagates an alternative source from a parent field to a child field" do + fields_by_path = fields_by_path_for "components" do |s| + s.object_type "NameAndType" do |t| + t.field "name", "String" + t.field "type", "String" + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.relates_to_one "widget", "Widget", via: "component_id", dir: :in + + t.field "widget_name_and_type", "NameAndType" do |f| + f.sourced_from "widget", "name_and_type" + end + + t.index "components" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name_and_type", "NameAndType" + t.field "component_id", "ID" + t.index "widgets" + end + end + + expect(fields_by_path).to eq({ + "id" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "widget_name_and_type.name" => index_field_with(source: "widget"), + "widget_name_and_type.type" => index_field_with(source: "widget") + }) + end + + it "indicates where the subfields of `#{LIST_COUNTS_FIELD}` are sourced from" do + fields_by_path = fields_by_path_for "components" do |s| + s.object_type "Component" do |t| + t.field "id", "ID!" + t.field "tags", "[String!]!" + t.relates_to_one "widget", "Widget", via: "component_ids", dir: :in + + t.field "widget_name", "String" do |f| + f.sourced_from "widget", "name" + end + + t.field "widget_tags", "[String!]" do |f| + f.sourced_from "widget", "tags" + end + + t.index "components" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "tags", "[String!]!" + t.field "component_ids", "[ID!]!" + t.index "widgets" + end + end + + expect(fields_by_path.select { |k, v| k.start_with?(LIST_COUNTS_FIELD) }).to eq({ + "#{LIST_COUNTS_FIELD}.tags" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "#{LIST_COUNTS_FIELD}.widget_tags" => index_field_with(source: "widget") + }) + end + + it "correctly uses `#{LIST_COUNTS_FIELD_PATH_KEY_SEPARATOR}` to separate path parts under `__counts`" do + fields_by_path = fields_by_path_for "components" do |s| + s.object_type "Tags" do |t| + t.field "tags", "[String!]!" + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.field "subfield", "Tags" + t.relates_to_one "widget", "Widget", via: "component_ids", dir: :in + + t.field "widget_name", "String" do |f| + f.sourced_from "widget", "name" + end + + t.field "widget_subfield", "Tags" do |f| + f.sourced_from "widget", "subfield" + end + + t.index "components" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "subfield", "Tags" + t.field "component_ids", "[ID!]!" + t.index "widgets" + end + end + + expect(fields_by_path.select { |k, v| k.start_with?(LIST_COUNTS_FIELD) }).to eq({ + "#{LIST_COUNTS_FIELD}.subfield|tags" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "#{LIST_COUNTS_FIELD}.widget_subfield|tags" => index_field_with(source: "widget") + }) + end + + it "deals with `nested` list fields which have no `object` list fields correctly" do + fields_by_path = fields_by_path_for "components" do |s| + s.object_type "NameAndType" do |t| + t.field "name", "String" + t.field "types", "[String]" + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.relates_to_one "widget", "Widget", via: "component_id", dir: :in + + t.field "widget_name_and_type", "[NameAndType!]" do |f| + f.mapping type: "nested" + f.sourced_from "widget", "name_and_type" + end + + t.index "components" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name_and_type", "[NameAndType!]!" do |f| + f.mapping type: "nested" + end + t.field "component_id", "ID" + t.index "widgets" + end + end + + expect(fields_by_path).to eq({ + "id" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "widget_name_and_type.name" => index_field_with(source: "widget"), + "widget_name_and_type.types" => index_field_with(source: "widget"), + "#{LIST_COUNTS_FIELD}.widget_name_and_type" => index_field_with(source: "widget"), + "widget_name_and_type.#{LIST_COUNTS_FIELD}.types" => index_field_with(source: "widget") + }) + end + + it "deals with `nested` list fields which have `object` list fields correctly" do + fields_by_path = fields_by_path_for "components" do |s| + s.object_type "NameAndType" do |t| + t.field "name", "String" + t.field "types", "[String]" + end + + s.object_type "Details" do |t| + t.field "size", "Int" + t.field "name_and_type", "[NameAndType!]!" do |f| + f.mapping type: "object" + end + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.relates_to_one "widget", "Widget", via: "component_id", dir: :in + + t.field "widget_details", "[Details!]" do |f| + f.mapping type: "nested" + f.sourced_from "widget", "details" + end + + t.index "components" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "details", "[Details!]!" do |f| + f.mapping type: "nested" + end + t.field "component_id", "ID" + t.index "widgets" + end + end + + expect(fields_by_path).to eq({ + "id" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "#{LIST_COUNTS_FIELD}.widget_details" => index_field_with(source: "widget"), + "widget_details.#{LIST_COUNTS_FIELD}.name_and_type" => index_field_with(source: "widget"), + "widget_details.#{LIST_COUNTS_FIELD}.name_and_type|name" => index_field_with(source: "widget"), + "widget_details.#{LIST_COUNTS_FIELD}.name_and_type|types" => index_field_with(source: "widget"), + "widget_details.name_and_type.name" => index_field_with(source: "widget"), + "widget_details.name_and_type.types" => index_field_with(source: "widget"), + "widget_details.size" => index_field_with(source: "widget") + }) + end + + it "deals with `object` list fields which have `nested` list fields correctly" do + fields_by_path = fields_by_path_for "components" do |s| + s.object_type "NameAndType" do |t| + t.field "name", "String" + t.field "types", "[String]" + end + + s.object_type "Details" do |t| + t.field "size", "Int" + t.field "name_and_type", "[NameAndType!]!" do |f| + f.mapping type: "nested" + end + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.relates_to_one "widget", "Widget", via: "component_id", dir: :in + + t.field "widget_details", "[Details!]" do |f| + f.mapping type: "object" + f.sourced_from "widget", "details" + end + + t.index "components" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "details", "[Details!]!" do |f| + f.mapping type: "object" + end + t.field "component_id", "ID" + t.index "widgets" + end + end + + expect(fields_by_path).to eq({ + "id" => index_field_with(source: SELF_RELATIONSHIP_NAME), + "#{LIST_COUNTS_FIELD}.widget_details" => index_field_with(source: "widget"), + "#{LIST_COUNTS_FIELD}.widget_details|name_and_type" => index_field_with(source: "widget"), + "#{LIST_COUNTS_FIELD}.widget_details|size" => index_field_with(source: "widget"), + "widget_details.name_and_type.name" => index_field_with(source: "widget"), + "widget_details.name_and_type.types" => index_field_with(source: "widget"), + "widget_details.name_and_type.#{LIST_COUNTS_FIELD}.types" => index_field_with(source: "widget"), + "widget_details.size" => index_field_with(source: "widget") + }) + end + + it "aligns the `#{LIST_COUNTS_FIELD}` subfields with mapping generated for `#{LIST_COUNTS_FIELD}`" do + results = define_schema do |schema| + schema.object_type "TeamDetails" do |t| + t.field "uniform_colors", "[String!]!" + + # `details.count` isn't really meaningful on our team model here, but we need this field + # to test that ElasticGraph handles a domain field named `count` even while it offers a + # `count` operator on list fields. + t.field schema.state.schema_elements.count, "Int" + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "current_name", "String" + t.field "past_names", "[String!]!" + t.field "won_championships_at", "[DateTime!]!" + t.field "details", "TeamDetails" + t.field "stadium_location", "GeoLocation" + t.field "forbes_valuations", "[JsonSafeLong!]!" + + t.field "current_players_nested", "[Player!]!" do |f| + f.mapping type: "nested" + end + + t.field "current_players_object", "[Player!]!" do |f| + f.mapping type: "object" + end + + t.field "seasons_nested", "[TeamSeason!]!" do |f| + f.mapping type: "nested" + end + + t.field "seasons_object", "[TeamSeason!]!" do |f| + f.mapping type: "object" + end + + t.index "teams" + end + + schema.object_type "Player" do |t| + t.field "name", "String" + t.field "nicknames", "[String!]!" + + t.field "seasons_nested", "[PlayerSeason!]!" do |f| + f.mapping type: "nested" + end + + t.field "seasons_object", "[PlayerSeason!]!" do |f| + f.mapping type: "object" + end + end + + schema.object_type "TeamSeason" do |t| + t.field "year", "Int" + t.field "notes", "[String!]!" + t.field "started_at", "DateTime" + t.field "won_games_at", "[DateTime!]!" + + t.field "players_nested", "[Player!]!" do |f| + f.mapping type: "nested" + end + + t.field "players_object", "[Player!]!" do |f| + f.mapping type: "object" + end + end + + schema.object_type "PlayerSeason" do |t| + t.field "year", "Int" + t.field "games_played", "Int" + t.paginated_collection_field "awards", "String" + end + end + + from_runtime_metadata = results + .runtime_metadata + .index_definitions_by_name.fetch("teams") + .fields_by_path.keys + .select { |path| path.include?(LIST_COUNTS_FIELD) } + + from_mapping = build_count_paths_from_mapping( + results.index_mappings_by_index_def_name.fetch("teams") + ) + + expect(from_runtime_metadata.sort.join("\n")).to eq from_mapping.sort.join("\n") + expect(logged_output).to include "a `TeamDetails.count` field exists" + end + + def fields_by_path_for(index_name, &block) + define_schema(&block) + .runtime_metadata + .index_definitions_by_name + .fetch(index_name) + .fields_by_path + end + + def build_count_paths_from_mapping(mapping) + mapping.fetch("properties").flat_map do |field_name, value| + if field_name == LIST_COUNTS_FIELD + value.fetch("properties").keys.map { |subfield| "#{LIST_COUNTS_FIELD}.#{subfield}" } + elsif value.key?("properties") + build_count_paths_from_mapping(value).map do |subfield| + "#{field_name}.#{subfield}" + end + else + [] + end + end + end + end + + def index_definition_metadata_for(name, on_my_type: nil, **options, &block) + runtime_metadata = define_schema do |s| + s.object_type "NestedFields" do |t| + t.field "some_id_gql", "ID", name_in_index: "some_id_index" + t.field "some_timestamp_gql", "DateTime", name_in_index: "some_timestamp_index" + end + + s.object_type "MyType" do |t| + t.field "id", "ID!" + t.field "group_id", "ID!" + t.field "group_id_gql", "ID", name_in_index: "group_id_index" + t.field "created_at", "DateTime!" + t.field "created_at_gql", "DateTime!", name_in_index: "created_at_index" + t.field "nested_fields_gql", "NestedFields", name_in_index: "nested_fields_index" + on_my_type&.call(t) + t.index(name, **options, &block) + end + end.runtime_metadata + + runtime_metadata + .index_definitions_by_name + .fetch(name) + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/derived_aggregation_types_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/derived_aggregation_types_spec.rb new file mode 100644 index 00000000..c7b354b5 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/derived_aggregation_types_spec.rb @@ -0,0 +1,118 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "object_type_metadata_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "RuntimeMetadata #object_types_by_name on aggregation types derived from indexed types" do + include_context "object type metadata support" + + it "records the necessary metadata on indexed aggregation types" do + metadata = object_type_metadata_for "WidgetAggregation" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "size", "Int" + t.index "widgets" + end + end + + expect(metadata.elasticgraph_category).to eq :indexed_aggregation + expect(metadata.source_type).to eq "Widget" + expect(metadata.index_definition_names).to eq [] + end + + it "records the necessary metadata on nested sub aggregation types" do + metadata = object_type_metadata_for "TeamPlayerSubAggregationConnection" do |s| + s.object_type "Player" do |t| + t.field "id", "ID!" + t.field "name", "String" + end + + s.object_type "Team" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "players", "[Player!]!" do |f| + f.mapping type: "nested" + end + t.index "teams" + end + end + + expect(metadata.elasticgraph_category).to eq :nested_sub_aggregation_connection + expect(metadata.index_definition_names).to eq [] + end + + it "dumps the customized `name_in_index` on aggregation grouped by types, too, so that the query engine is made aware of the alternate name" do + metadata = object_type_metadata_for "WidgetGroupedBy" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "description", "String", name_in_index: "description_index" + t.index "widgets" + end + end + + expect(metadata.graphql_fields_by_name).to eq({ + "description" => graphql_field_with( + name_in_index: "description_index", + relation: nil + ) + }) + end + + it "dumps the customized `name_in_index` on aggregated values types, too, so that the query engine is made aware of the alternate name" do + metadata = object_type_metadata_for "WidgetAggregatedValues" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "cost", "Int", name_in_index: "cost_index" + t.index "widgets" + end + end + + expect(metadata.graphql_fields_by_name).to eq({ + "cost" => graphql_field_with( + name_in_index: "cost_index", + relation: nil + ) + }) + end + + it "dumps the customized `name_in_index` on sub-aggregation types, too so that the query engine is made aware of the alternate name" do + team_sub_aggs, team_collections_sub_aggs = object_type_metadata_for( + "TeamAggregationSubAggregations", + "TeamAggregationCollectionsSubAggregations" + ) do |schema| + schema.object_type "Player" do |t| + t.field "name", "String" + end + + schema.object_type "TeamCollections" do |t| + t.field "players", "[Player!]!", name_in_index: "the_players" do |f| + f.mapping type: "nested" + end + end + + schema.object_type "Team" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "collections", "TeamCollections", name_in_index: "collections_in_index" + t.index "teams" + end + end + + expect(team_sub_aggs.graphql_fields_by_name).to eq({ + "collections" => graphql_field_with(name_in_index: "collections_in_index") + }) + + expect(team_collections_sub_aggs.graphql_fields_by_name).to eq({ + "players" => graphql_field_with(name_in_index: "the_players") + }) + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/for_built_in_types_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/for_built_in_types_spec.rb new file mode 100644 index 00000000..e096e2dc --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/for_built_in_types_spec.rb @@ -0,0 +1,142 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "object_type_metadata_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "RuntimeMetadata #object_types_by_name for built-in types" do + include_context "object type metadata support" + + context "`AggregatedValues` types" do + it "includes aggregation functions on `IntAggregatedValues` fields" do + metadata = object_type_metadata_for "IntAggregatedValues" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "size", "Int" + t.index "widgets" + end + end + + expect(metadata.elasticgraph_category).to eq :scalar_aggregated_values + expect(metadata.graphql_fields_by_name.transform_values(&:computation_detail)).to eq( + "approximate_avg" => agg_detail_of(:avg, nil), + "approximate_sum" => agg_detail_of(:sum, 0), + "exact_sum" => agg_detail_of(:sum, 0), + "exact_max" => agg_detail_of(:max, nil), + "exact_min" => agg_detail_of(:min, nil), + "approximate_distinct_value_count" => agg_detail_of(:cardinality, 0) + ) + end + + it "includes aggregation functions on `FloatAggregatedValues` fields" do + metadata = object_type_metadata_for "FloatAggregatedValues" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "amount", "Float" + t.index "widgets" + end + end + + expect(metadata.elasticgraph_category).to eq :scalar_aggregated_values + expect(metadata.graphql_fields_by_name.transform_values(&:computation_detail)).to eq( + "approximate_avg" => agg_detail_of(:avg, nil), + "approximate_sum" => agg_detail_of(:sum, 0), + "exact_max" => agg_detail_of(:max, nil), + "exact_min" => agg_detail_of(:min, nil), + "approximate_distinct_value_count" => agg_detail_of(:cardinality, 0) + ) + end + + it "includes aggregation functions on `JsonSafeLongAggregatedValues` fields" do + metadata = object_type_metadata_for "JsonSafeLongAggregatedValues" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "size", "JsonSafeLong" + t.index "widgets" + end + end + + expect(metadata.elasticgraph_category).to eq :scalar_aggregated_values + expect(metadata.graphql_fields_by_name.transform_values(&:computation_detail)).to eq( + "approximate_avg" => agg_detail_of(:avg, nil), + "approximate_sum" => agg_detail_of(:sum, 0), + "exact_sum" => agg_detail_of(:sum, 0), + "exact_max" => agg_detail_of(:max, nil), + "exact_min" => agg_detail_of(:min, nil), + "approximate_distinct_value_count" => agg_detail_of(:cardinality, 0) + ) + end + + it "includes aggregation functions on `LongStringAggregatedValues` fields" do + metadata = object_type_metadata_for "LongStringAggregatedValues" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "size", "LongString" + t.index "widgets" + end + end + + expect(metadata.elasticgraph_category).to eq :scalar_aggregated_values + expect(metadata.graphql_fields_by_name.transform_values(&:computation_detail)).to eq( + "approximate_avg" => agg_detail_of(:avg, nil), + "approximate_sum" => agg_detail_of(:sum, 0), + "exact_sum" => agg_detail_of(:sum, 0), + "approximate_max" => agg_detail_of(:max, nil), + "exact_max" => agg_detail_of(:max, nil), + "approximate_min" => agg_detail_of(:min, nil), + "exact_min" => agg_detail_of(:min, nil), + "approximate_distinct_value_count" => agg_detail_of(:cardinality, 0) + ) + end + end + + context "date grouped by objects" do + it "sets `elasticgraph_category = :date_grouped_by_object` on the `DateGroupedBy` type" do + metadata = object_type_metadata_for "DateGroupedBy" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_at", "DateTime" + t.index "widgets" + end + end + + expect(metadata.elasticgraph_category).to eq :date_grouped_by_object + end + + it "sets `elasticgraph_category = :date_grouped_by_object` on the `DateTimeGroupedBy` type" do + metadata = object_type_metadata_for "DateTimeGroupedBy" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "created_at", "DateTime" + t.index "widgets" + end + end + + expect(metadata.elasticgraph_category).to eq :date_grouped_by_object + end + end + + def agg_detail_of(function, empty_bucket_value) + SchemaArtifacts::RuntimeMetadata::ComputationDetail.new( + function: function, + empty_bucket_value: empty_bucket_value + ) + end + + prepend Module.new { + def object_type_metadata_for(...) + super(...).tap do |metadata| + # All built-in return types should be graphql-only. + expect(metadata.graphql_only_return_type).to eq true + end + end + } + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/for_relay_types_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/for_relay_types_spec.rb new file mode 100644 index 00000000..6eab4910 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/for_relay_types_spec.rb @@ -0,0 +1,39 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "object_type_metadata_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "RuntimeMetadata #object_types_by_name for generated relay types" do + include_context "object type metadata support" + + it "tags relay connection types with `elasticgraph_category: :relay_connection`" do + metadata = object_type_metadata_for "WidgetConnection" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + end + end + + expect(metadata.elasticgraph_category).to eq :relay_connection + end + + it "tags relay connection types with `elasticgraph_category: :relay_edge`" do + metadata = object_type_metadata_for "WidgetEdge" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + end + end + + expect(metadata.elasticgraph_category).to eq :relay_edge + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/graphql_fields_by_name_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/graphql_fields_by_name_spec.rb new file mode 100644 index 00000000..596c38ce --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/graphql_fields_by_name_spec.rb @@ -0,0 +1,173 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "object_type_metadata_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "RuntimeMetadata #object_types_by_name #graphql_fields_by_name" do + include_context "object type metadata support" + + context "on a normal indexed type" do + it "dumps the `name_in_index` of any fields" do + metadata = object_type_metadata_for "Widget" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "description", "String", name_in_index: "description_index" + t.index "widgets" + end + end + + expect(metadata.graphql_fields_by_name).to eq({ + "description" => graphql_field_with( + name_in_index: "description_index", + relation: nil + ) + }) + end + + it "dumps the customized `name_in_index` on filter types, too, so that the query engine is made aware of the alternate name" do + metadata = object_type_metadata_for "WidgetFilterInput" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "description", "String", name_in_index: "description_index" + t.index "widgets" + end + end + + expect(metadata.graphql_fields_by_name).to eq({ + "description" => graphql_field_with( + name_in_index: "description_index", + relation: nil + ) + }) + end + + it "honors `name_in_index` passed to `paginated_collection_field`" do + metadata = object_types_by_name do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.paginated_collection_field "names", "String", name_in_index: "names2" + t.index "widgets" + end + end + + expected_graphql_fields_by_name = { + "names" => graphql_field_with( + name_in_index: "names2", + relation: nil + ) + } + + expect(metadata.fetch("Widget").graphql_fields_by_name).to eq(expected_graphql_fields_by_name) + expect(metadata.fetch("WidgetFilterInput").graphql_fields_by_name).to eq(expected_graphql_fields_by_name) + end + end + + context "on an embedded object type" do + it "dumps the `name_in_index` of any fields" do + metadata = object_type_metadata_for "Widget" do |s| + s.object_type "Widget" do |t| + t.field "description", "String", name_in_index: "description_index" + end + end + + expect(metadata.graphql_fields_by_name).to eq({ + "description" => graphql_field_with( + name_in_index: "description_index", + relation: nil + ) + }) + end + end + + it "dumps the `name_in_index: #{LIST_COUNTS_FIELD}` for all `*ListFilterInput` and `*FieldsListFilterInput` types so that our filter interpreter knows that its for the special `#{LIST_COUNTS_FIELD}` field" do + results = define_schema do |schema| + schema.enum_type "Color" do |t| + t.values "RED", "GREEN", "BLUE" + end + + schema.scalar_type "Duration" do |t| + t.mapping type: "keyword" + t.json_schema type: "string" + end + + schema.object_type "Options" do |t| + t.field "size", "Int" + end + + schema.object_type "GrabBag" do |t| + t.field "id", "ID!" + t.index "grabbags" + end + end + + list_filter_types = results.graphql_schema_string.scan(/input ((?:\w+)ListFilterInput)\b/).flatten + + # This contains the `*ListFilterInput` type for the types defined above plus all built-in types as of 2023-09-21. + # It's not expected that this be kept in sync as we add new types to ElasticGraph over time--we just want + # to verify that we're covering a wide swath of different kinds of types here. + expected_minimum_list_filter_types = %w[ + BooleanListFilterInput ColorListFilterInput CursorListFilterInput DateListFilterInput DateGroupingGranularityListFilterInput DateTimeListFilterInput + DateTimeGroupingGranularityListFilterInput DateTimeUnitListFilterInput DistanceUnitListFilterInput DurationListFilterInput FloatListFilterInput + GeoLocationListFilterInput GrabBagListFilterInput IDListFilterInput IntListFilterInput JsonSafeLongListFilterInput LocalTimeListFilterInput + LongStringListFilterInput OptionsListFilterInput StringListFilterInput TextListFilterInput TimeZoneListFilterInput UntypedListFilterInput + GrabBagFieldsListFilterInput OptionsFieldsListFilterInput + ] + + expect(list_filter_types).to include(*expected_minimum_list_filter_types) + + object_types_by_name = results.runtime_metadata.object_types_by_name + has_runtime_metadata, missing_runtime_metadata = list_filter_types.partition { |type| object_types_by_name.key?(type) } + + # All `*ListFilterInput` types should be in runtime metadata so that their `count` field can have `name_in_index = __counts`. + expect(missing_runtime_metadata).to be_empty + + missing_count_name_in_index = has_runtime_metadata.reject do |type| + # :nocov: -- some branches of the line below are only covered when the test fails. + object_types_by_name.fetch(type).graphql_fields_by_name.dig("count")&.name_in_index == LIST_COUNTS_FIELD + # :nocov: + end + + expect(missing_count_name_in_index).to be_empty + end + + it "omits `name_in_index: #{LIST_COUNTS_FIELD}` when there is a user-defined field named `count` to avoid overriding the field that gets generated for the user-defined field" do + count_type_meta, count1_type_meta = object_type_metadata_for "CountTypeFieldsListFilterInput", "Count1TypeFieldsListFilterInput" do |schema| + schema.object_type "CountType" do |t| + t.field "count", "Int" + end + + schema.object_type "Count1Type" do |t| + t.field "count1", "Int" + end + + schema.object_type "RootType" do |t| + t.field "id", "ID!" + t.field "list", "[CountType!]!" do |f| + f.mapping type: "object" + end + t.field "list1", "[Count1Type!]!" do |f| + f.mapping type: "object" + end + t.index "roots" + end + end + + # Metadata is not dumped for this case because `count` is a normal, user-defined field without a custom `name_in_index`. + expect(count_type_meta).to be nil + # ...but it's still dumped for this one because it's our special LIST_COUNTS_FIELD. + expect(count1_type_meta.graphql_fields_by_name["count"].name_in_index).to eq LIST_COUNTS_FIELD + expect(logged_output).to include( + "WARNING: Since a `CountType.count` field exists, ElasticGraph is not able to\n" \ + "define its typical `CountTypeFieldsListFilterInput.count` field" + ) + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/graphql_only_return_type_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/graphql_only_return_type_spec.rb new file mode 100644 index 00000000..205590ab --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/graphql_only_return_type_spec.rb @@ -0,0 +1,61 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "object_type_metadata_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "RuntimeMetadata #object_types_by_name #graphql_only_return_type" do + include_context "object type metadata support" + + it "is set to `true` on a return type that has `t.graphql_only true`" do + metadata = object_type_metadata_for "Widget" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID", name_in_index: "id_index" + t.graphql_only true + end + end + + expect(metadata.graphql_only_return_type).to eq true + end + + it "is set to `false` on a return type that has `t.graphql_only = false`" do + metadata = object_type_metadata_for "Widget" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID", name_in_index: "id_index" + t.graphql_only false + end + end + + expect(metadata.graphql_only_return_type).to eq false + end + + it "is set to `false` on a return type that has `graphql_only` unset" do + metadata = object_type_metadata_for "Widget" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID", name_in_index: "id_index" + end + end + + expect(metadata.graphql_only_return_type).to eq false + end + + it "is set to `false` on an input type regardless of `graphql_only`" do + metadata = object_type_metadata_for "Widget" do |s| + s.factory.new_input_type "Widget" do |t| + t.field "id", "ID", name_in_index: "id_index" + t.graphql_only true + s.state.register_input_type(t) + end + end + + expect(metadata.graphql_only_return_type).to eq false + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/index_definition_names_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/index_definition_names_spec.rb new file mode 100644 index 00000000..f67896a1 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/index_definition_names_spec.rb @@ -0,0 +1,147 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "object_type_metadata_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "RuntimeMetadata #object_types_by_name #index_definition_names" do + include_context "object type metadata support" + + context "on a normal indexed type" do + it "dumps them based on the defined indices" do + metadata = object_type_metadata_for "Widget" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.index "widgets" + end + end + + expect(metadata.index_definition_names).to eq ["widgets"] + end + + it "does not allow it to change its indexable status after the `object_type` call" do + expect { + object_type_metadata_for "WidgetAggregation" do |s| + the_type = nil + + s.object_type "Widget" do |t| + the_type = t + t.field "id", "ID" + t.field "description", "String", name_in_index: "description_index" + t.index "widgets" + end + + the_type.indices.clear + end + }.to raise_error(a_string_including("can't modify frozen Array")) + end + end + + context "on an embedded object type" do + it "does not dump any" do + metadata = object_type_metadata_for "WidgetOptions" do |s| + s.object_type "WidgetOptions" do |t| + t.field "size", "Int", name_in_index: "size_index" + end + end + + expect(metadata.index_definition_names).to be_empty + end + + it "does not allow it to change its indexable status after the `object_type` call" do + expect { + object_type_metadata_for "WidgetAggregation" do |s| + the_type = nil + + s.object_type "Widget" do |t| + the_type = t + t.field "id", "ID" + t.field "description", "String", name_in_index: "description_index" + end + + the_type.index "widgets" + end + }.to raise_error(a_string_including("can't modify frozen Array")) + end + end + + on_a_type_union_or_interface_type do |type_def_method| + it "dumps them based on indices defined directly on the supertype" do + metadata = object_type_metadata_for "Thing" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + link_subtype_to_supertype(t, "Thing") + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + link_subtype_to_supertype(t, "Thing") + end + + s.public_send type_def_method, "Thing" do |t| + link_supertype_to_subtypes(t, "Widget", "Component") + t.index "things" + end + end + + expect(metadata.index_definition_names).to eq ["things"] + end + + it "does not dump any when no direct index is defined on it (even if the subtypes have indices)" do + metadata = object_type_metadata_for "Thing" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + # Use an alternate `name_in_index` to force `metadata` not to be `nil`. + t.field "workspace_id", "ID!", name_in_index: "wid" + link_subtype_to_supertype(t, "Thing") + t.index "widgets" + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + link_subtype_to_supertype(t, "Thing") + t.index "components" + end + + s.public_send type_def_method, "Thing" do |t| + link_supertype_to_subtypes(t, "Widget", "Component") + end + end + + expect(metadata.index_definition_names).to eq([]) + end + + it "does not allow it to change its indexable status after the `#{type_def_method}` call" do + expect { + object_type_metadata_for "Thing" do |s| + the_type = nil + + s.object_type "Widget" do |t| + t.field "id", "ID!" + link_subtype_to_supertype(t, "Thing") + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + link_subtype_to_supertype(t, "Thing") + end + + s.public_send type_def_method, "Thing" do |t| + the_type = t + link_supertype_to_subtypes(t, "Widget", "Component") + end + + the_type.index "widgets" + end + }.to raise_error(a_string_including("can't modify frozen Array")) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/object_type_metadata_support.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/object_type_metadata_support.rb new file mode 100644 index 00000000..92cc5918 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/object_type_metadata_support.rb @@ -0,0 +1,64 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../runtime_metadata_support" + +module ElasticGraph + module SchemaDefinition + RSpec.shared_context "object type metadata support" do + include_context "RuntimeMetadata support" + + def object_types_by_name(**options, &block) + runtime_metadata = define_schema(**options, &block).runtime_metadata + runtime_metadata = SchemaArtifacts::RuntimeMetadata::Schema.from_hash(runtime_metadata.to_dumpable_hash, for_context: :admin) + runtime_metadata.object_types_by_name + end + + def object_type_metadata_for(*names, **options, &block) + if names.one? + object_types_by_name(**options, &block)[names.first] + else + metadata = object_types_by_name(**options, &block) + names.map { |name| metadata[name] } + end + end + + def self.on_a_type_union_or_interface_type(&block) + context "on a type union" do + include ObjectTypeMetadataUnionTypeImplementation + module_exec(:union_type, &block) + end + + context "on an interface type" do + include ObjectTypeMetadataInterfaceTypeImplementation + module_exec(:interface_type, &block) + end + end + end + + module ObjectTypeMetadataUnionTypeImplementation + def link_subtype_to_supertype(object_type, supertype_name) + # nothing to do; the linkage happens via a `subtypes` call on the supertype + end + + def link_supertype_to_subtypes(union_type, *subtype_names) + union_type.subtypes(*subtype_names) + end + end + + module ObjectTypeMetadataInterfaceTypeImplementation + def link_subtype_to_supertype(object_type, interface_name) + object_type.implements interface_name + end + + def link_supertype_to_subtypes(interface_type, *subtype_names) + # nothing to do; the linkage happens via an `implements` call on the subtype + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/pruning_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/pruning_spec.rb new file mode 100644 index 00000000..adaf9ab9 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/pruning_spec.rb @@ -0,0 +1,127 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "object_type_metadata_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "RuntimeMetadata #object_types_by_name--pruning" do + include_context "object type metadata support" + + it "prunes down the dumped types to ones that have runtime metadata, avoiding dumping a ton of ElasticGraph built-ins (`PageInfo`, etc), but dumping derived graphql types (aggregation, filters) and aggregated value types" do + results = define_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "options", "WidgetOptions" + t.field "description", "String", name_in_index: "description_in_index" + t.index "widgets" + end + + s.object_type "WidgetOptions" do |t| + t.field "size", "Int", name_in_index: "size_in_index" + end + + # NoCustomizations should not be dumped because it needs no runtime metadata + s.object_type "NoCustomizations" do |t| + t.field "size", "Int", graphql_only: true + end + end + + runtime_metadata = SchemaArtifacts::RuntimeMetadata::Schema.from_hash(results.runtime_metadata.to_dumpable_hash, for_context: :admin) + + # Note: this list has greatly grown over time. When you make a change causing a new type to be dumped, this'll fail + # and force you to consider if that new type should be dumped or not. Add types to this as needed for ones that we + # do want dumped. + list = %w[ + AggregationCountDetail + BooleanListFilterInput + CursorListFilterInput + DateAggregatedValues + DateGroupedBy + DateGroupingGranularityListFilterInput + DateGroupingTruncationUnitListFilterInput + DateListFilterInput + DateTimeAggregatedValues + DateTimeGroupedBy + DateTimeGroupingGranularityListFilterInput + DateTimeGroupingTruncationUnitListFilterInput + DateTimeListFilterInput + DateTimeUnitListFilterInput + DateUnitListFilterInput + DayOfWeekListFilterInput + DistanceUnitListFilterInput + FloatAggregatedValues + FloatListFilterInput + GeoLocation + GeoLocationListFilterInput + IDListFilterInput + IntAggregatedValues + IntListFilterInput + JsonSafeLongAggregatedValues + JsonSafeLongListFilterInput + LocalTimeAggregatedValues + LocalTimeGroupingTruncationUnitListFilterInput + LocalTimeListFilterInput + LocalTimeUnitListFilterInput + LongStringAggregatedValues + LongStringListFilterInput + MatchesQueryAllowedEditsPerTermListFilterInput + NoCustomizationsFieldsListFilterInput + NoCustomizationsListFilterInput + NonNumericAggregatedValues + PageInfo + StringListFilterInput + TextListFilterInput + TimeZoneListFilterInput + UntypedListFilterInput + Widget + WidgetAggregatedValues + WidgetAggregation + WidgetAggregationConnection + WidgetAggregationEdge + WidgetConnection + WidgetEdge + WidgetFieldsListFilterInput + WidgetFilterInput + WidgetGroupedBy + WidgetListFilterInput + WidgetOptions + WidgetOptionsAggregatedValues + WidgetOptionsFieldsListFilterInput + WidgetOptionsFilterInput + WidgetOptionsGroupedBy + WidgetOptionsListFilterInput + ].sort + + expect(runtime_metadata.object_types_by_name.keys).to match_array(list) + end + + it "excludes derived types that come from a `graphql_only` type so that extensions (like `elasticgraph-apollo`) can define GraphQL-only types" do + object_types_by_name = define_schema do |s| + s.object_type "CustomFrameworkObject" do |t| + t.field "foo", "String" + t.graphql_only true + end + + s.scalar_type "CustomFrameworkScalar" do |t| + t.mapping type: "keyword" + t.json_schema type: "string" + t.graphql_only true + end + + s.enum_type "CustomFrameworkEnum" do |t| + t.value "FOO" + t.graphql_only true + end + end.runtime_metadata.object_types_by_name + + expect(object_types_by_name.keys.grep(/CustomFramework/)).to contain_exactly("CustomFrameworkObject") + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/relation_metadata_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/relation_metadata_spec.rb new file mode 100644 index 00000000..79a8f1a1 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/relation_metadata_spec.rb @@ -0,0 +1,261 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "object_type_metadata_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "RuntimeMetadata #object_types_by_name--relation metadata" do + include_context "object type metadata support" + + it "dumps relation metadata from a `relates_to_one` field" do + metadata = object_type_metadata_for "Widget" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.relates_to_one "parent", "Widget", via: "parent_id", dir: :out + t.index "widgets" + end + end + + expect(metadata.graphql_fields_by_name).to eq({ + "parent" => graphql_field_with( + name_in_index: nil, + relation: SchemaArtifacts::RuntimeMetadata::Relation.new( + foreign_key: "parent_id", + direction: :out, + additional_filter: {}, + foreign_key_nested_paths: [] + ) + ) + }) + end + + it "dumps relation metadata from a `relates_to_many` field" do + metadata = object_type_metadata_for "Widget" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.relates_to_many "children", "Widget", via: "parent_id", dir: :in, singular: "child" + t.index "widgets" + end + end + + expected_relation_field = graphql_field_with( + name_in_index: nil, + relation: SchemaArtifacts::RuntimeMetadata::Relation.new( + foreign_key: "parent_id", + direction: :in, + additional_filter: {}, + foreign_key_nested_paths: [] + ) + ) + + expect(metadata.graphql_fields_by_name).to eq({ + "children" => expected_relation_field, + "child_aggregations" => expected_relation_field + }) + end + + it "stores an additional filter on the relation" do + filter = {"is_enabled" => {"equal_to_any_of" => [true]}} + + metadata = object_type_metadata_for "Widget" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + + t.relates_to_one "parent", "Widget", via: "parent_id", dir: :in do |r| + r.additional_filter filter + end + + t.relates_to_many "children", "Widget", via: "parent_id", dir: :in, singular: "child" do |r| + r.additional_filter filter + end + + t.index "widgets" + end + end + + expected_relation_field = graphql_field_with( + name_in_index: nil, + relation: SchemaArtifacts::RuntimeMetadata::Relation.new( + foreign_key: "parent_id", + direction: :in, + additional_filter: filter, + foreign_key_nested_paths: [] + ) + ) + + expect(metadata.graphql_fields_by_name).to eq({ + "children" => expected_relation_field, + "child_aggregations" => expected_relation_field, + "parent" => expected_relation_field + }) + end + + it "converts an additional filter with symbol keys to string keys so it serializes properly" do + filter = {is_enabled: {equal_to_any_of: [true]}} + + metadata = object_type_metadata_for "Widget" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + + t.relates_to_one "parent", "Widget", via: "parent_id", dir: :in do |r| + r.additional_filter filter + end + + t.relates_to_many "children", "Widget", via: "parent_id", dir: :in, singular: "child" do |r| + r.additional_filter filter + end + + t.index "widgets" + end + end + + expected_relation_field = graphql_field_with( + name_in_index: nil, + relation: SchemaArtifacts::RuntimeMetadata::Relation.new( + foreign_key: "parent_id", + direction: :in, + additional_filter: {"is_enabled" => {"equal_to_any_of" => [true]}}, + foreign_key_nested_paths: [] + ) + ) + + expect(metadata.graphql_fields_by_name).to eq({ + "children" => expected_relation_field, + "child_aggregations" => expected_relation_field, + "parent" => expected_relation_field + }) + end + + it "merges the filters when `additional_filter` is called multiple times" do + filter1 = { + is_enabled: {equal_to_any_of: [true]}, + details: {foo: {lt: 3}} + } + filter2 = { + "details" => {"bar" => {"gt" => 5}}, + "other" => {"lte" => 100} + } + + metadata = object_type_metadata_for "Widget" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + + t.relates_to_one "parent", "Widget", via: "parent_id", dir: :in do |r| + r.additional_filter filter1 + r.additional_filter filter2 + end + + t.relates_to_many "children", "Widget", via: "parent_id", dir: :in, singular: "child" do |r| + r.additional_filter filter1 + r.additional_filter filter2 + end + + t.index "widgets" + end + end + + expected_relation_field = graphql_field_with( + name_in_index: nil, + relation: SchemaArtifacts::RuntimeMetadata::Relation.new( + foreign_key: "parent_id", + direction: :in, + additional_filter: { + "is_enabled" => {"equal_to_any_of" => [true]}, + "details" => {"foo" => {"lt" => 3}, "bar" => {"gt" => 5}}, + "other" => {"lte" => 100} + }, + foreign_key_nested_paths: [] + ) + ) + + expect(metadata.graphql_fields_by_name).to eq({ + "children" => expected_relation_field, + "child_aggregations" => expected_relation_field, + "parent" => expected_relation_field + }) + end + + context "when the relation foreign key involves `nested` fields" do + let(:types_in_dependency_order) do + [ + "Sponsorship", # references no types + "Affiliation", # references Sponsorship + "Player", # references Affiliation + "Team", # references Player + "Sponsor" # references Team + ] + end + + it "resolves `foreign_key_nested_paths` when types are only referenced after they have been defined" do + test_foreign_key_nested_paths(types_in_dependency_order) + end + + it "resolves `foreign_key_nested_paths` when types are referenced before they have been defined" do + test_foreign_key_nested_paths(types_in_dependency_order.reverse) + end + + def test_foreign_key_nested_paths(type_definition_order) + type_defs_by_name = { + "Affiliation" => lambda do |t| + t.field "sponsorships", "[Sponsorship!]!" do |f| + f.mapping type: "nested" + end + end, + + "Player" => lambda do |t| + t.field "name", "String" + t.field "affiliations", "Affiliation" + end, + + "Sponsor" => lambda do |t| + t.field "id", "ID" + t.field "name", "String" + t.relates_to_many "affiliated_teams", "Team", via: "players.affiliations.sponsorships.sponsor_id", dir: :in, singular: "affiliated_team" + + t.index "sponsors" + end, + + "Sponsorship" => lambda do |t| + t.field "sponsor_id", "ID" + end, + + "Team" => lambda do |t| + t.field "id", "ID" + t.field "players", "[Player!]!" do |f| + f.mapping type: "nested" + end + + t.index "teams" + end + } + + metadata = object_type_metadata_for "Sponsor" do |s| + type_definition_order.each do |type| + s.object_type(type, &type_defs_by_name.fetch(type)) + end + end + + expected_relation_field = graphql_field_with( + name_in_index: nil, + relation: SchemaArtifacts::RuntimeMetadata::Relation.new( + foreign_key: "players.affiliations.sponsorships.sponsor_id", + direction: :in, + additional_filter: {}, + foreign_key_nested_paths: ["players", "players.affiliations.sponsorships"] + ) + ) + expect(metadata.graphql_fields_by_name).to eq({ + "affiliated_teams" => expected_relation_field, + "affiliated_team_aggregations" => expected_relation_field + }) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/update_targets_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/update_targets_spec.rb new file mode 100644 index 00000000..e7d23116 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/object_types_by_name/update_targets_spec.rb @@ -0,0 +1,1526 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "object_type_metadata_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "RuntimeMetadata #object_types_by_name #update_targets" do + include_context "object type metadata support" + + context "when `sourced_from` is used" do + it "dumps an update target on the related type, regardless of the definition order of the relation compared to the `sourced_from` fields" do + update_targets_relation_before_fields = update_targets_for("WidgetWorkspace") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "DateTime" do |f| + f.sourced_from "workspace", "created_at" + end + end + + update_targets_fields_before_relation = update_targets_for("WidgetWorkspace") do |t| + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "DateTime" do |f| + f.sourced_from "workspace", "created_at" + end + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + end + + expect(update_targets_relation_before_fields).to eq(update_targets_fields_before_relation) + expect_widget_update_target_with( + update_targets_relation_before_fields, + id_source: "widget_ids", + routing_value_source: nil, + relationship: "workspace", + data_params: { + "workspace_name" => dynamic_param_with(source_path: "name", cardinality: :one), + "workspace_created_at" => dynamic_param_with(source_path: "created_at", cardinality: :one) + } + ) + end + + it "excludes the `sourced_from` fields from the `params` of the main type since we don't want updates of that type stomping a value indexed from an alternate source" do + update_targets = update_targets_for("Widget") + + expect_widget_update_target_with( + update_targets, + id_source: "id", + routing_value_source: "id", + relationship: SELF_RELATIONSHIP_NAME, + data_params: { + # Importantly, `workspace_name` and `workspace_created_at` are NOT in this map. + "name" => dynamic_param_with(source_path: "name", cardinality: :one) + } + ) + end + + it "allows an alternate `name_in_index` to be used on the referenced field" do + update_targets = update_targets_for("WidgetWorkspace", widget_workspace_name_opts: {name_in_index: "name2"}) do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + end + + expect_widget_update_target_with( + update_targets, + id_source: "widget_ids", + routing_value_source: nil, + relationship: "workspace", + data_params: { + "workspace_name" => dynamic_param_with(source_path: "name2", cardinality: :one) + } + ) + end + + it "allows an alternate `name_in_index` to be used on the local field" do + update_targets = update_targets_for("WidgetWorkspace") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String", name_in_index: "workspace_name2" do |f| + f.sourced_from "workspace", "name" + end + end + + expect_widget_update_target_with( + update_targets, + id_source: "widget_ids", + routing_value_source: nil, + relationship: "workspace", + data_params: { + "workspace_name2" => dynamic_param_with(source_path: "name", cardinality: :one) + } + ) + end + + it "allows the field to be sourced from a nested field" do + update_targets = update_targets_for("WidgetWorkspace") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "nested.further_nested.name" + end + end + + expect_widget_update_target_with( + update_targets, + id_source: "widget_ids", + routing_value_source: nil, + relationship: "workspace", + data_params: { + "workspace_name" => dynamic_param_with(source_path: "nested.further_nested_in_index.name", cardinality: :one) + } + ) + end + + it "allows non-nullability on the parent parts of a nested field" do + update_targets = update_targets_for("WidgetWorkspace", widget_workspace_nested_1_further_nested: "WidgetWorkspaceNested2!") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "nested.further_nested.name" + end + end + + expect_widget_update_target_with( + update_targets, + id_source: "widget_ids", + routing_value_source: nil, + relationship: "workspace", + data_params: { + "workspace_name" => dynamic_param_with(source_path: "nested.further_nested_in_index.name", cardinality: :one) + } + ) + end + + it "does not dump any update targets for interface types with no defined indices, even if they are implemented by other types with defined indices" do + metadata = object_type_metadata_for "WidgetWorkspace" do |s| + s.interface_type "NamedEntity" do |t| + t.field "name", "String" + end + + s.object_type "Widget" do |t| + t.implements "NamedEntity" + t.field "id", "ID!" + t.field "name", "String" + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.index "widgets" + end + + s.object_type "WidgetWorkspace" do |t| + t.implements "NamedEntity" + t.field "id", "ID!" + t.field "name", "String" + t.field "widget_ids", "[ID!]!" + t.index "widget_workspaces" + end + end + + expect(metadata.update_targets.map(&:type)).to exclude "NamedEntity" + end + + context "on a type that uses custom routing" do + it "determines the `routing_value_source` from an `equivalent_field` configured on the relation" do + source = routing_value_source_for_widget_update_target_of("WidgetWorkspace") do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + # Including multiple `equivalent_field` calls to force it to use the right one. + r.equivalent_field "id", locally_named: "id" + r.equivalent_field "workspace_owner_id", locally_named: "owner_id" + r.equivalent_field "name", locally_named: "name" + end + end + + expect(source).to eq "workspace_owner_id" + end + + it "allows `locally_named:` to be omitted when defining equivalent fields, defaulting the local name to the same as the remote name" do + source = routing_value_source_for_widget_update_target_of("WidgetWorkspace", on_widget_workspace_type: ->(t) { t.field "owner_id", "ID!" }) do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "owner_id" + end + end + + expect(source).to eq "owner_id" + end + + it "raises a clear error if an `equivalent_field` is not defined for the custom routing field" do + expect { + routing_value_source_for_widget_update_target_of("WidgetWorkspace") do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "another_id" + end + end + }.to raise_error Errors::SchemaError, a_string_including( + "Cannot update `Widget` documents", + "related `workspace` events", + "`Widget` uses custom shard routing", + "don't know what `workspace` field to use", + "`Widget` update requests", + "`Widget.workspace` relationship definition", + '`rel.equivalent_field "[WidgetWorkspace field]", locally_named: "owner_id"`' + ) + end + + it "resolves the custom routing field using the public GraphQL field name instead of the internal `name_in_index`" do + source = routing_value_source_for_widget_update_target_of("WidgetWorkspace", owner_id_field_opts: {name_in_index: "owner_id_in_index"}) do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "workspace_owner_id", locally_named: "owner_id" + end + end + + expect(source).to eq "workspace_owner_id" + end + + it "records the `name_in_index` as the `routing_value_source` since the `elasticgraph-indexer` logic that uses it expects the index field name" do + source = routing_value_source_for_widget_update_target_of("WidgetWorkspace", on_widget_workspace_type: ->(t) { t.field "woid", "ID!", name_in_index: "widget_order_id" }) do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "woid", locally_named: "owner_id" + end + end + + expect(source).to eq "widget_order_id" + end + + it "allows the `routing_value_source` to be an indexing-only field, but does not allow it to be a graphql-only field" do + source = routing_value_source_for_widget_update_target_of("WidgetWorkspace", on_widget_workspace_type: ->(t) { t.field "owner_id", "ID!", indexing_only: true }) do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "owner_id" + end + end + + expect(source).to eq "owner_id" + + expect { + routing_value_source_for_widget_update_target_of("WidgetWorkspace", on_widget_workspace_type: ->(t) { t.field "owner_id", "ID", graphql_only: true }) do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "owner_id" + end + end + }.to raise_error Errors::SchemaError, a_string_including("`WidgetWorkspace.owner_id` (referenced from an `equivalent_field` defined on `Widget.workspace`) does not exist") + end + + it "supports nested fields as a `routing_value_source`" do + source = routing_value_source_for_widget_update_target_of("WidgetWorkspace") do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "nested.owner_id", locally_named: "owner_id" + end + end + + expect(source).to eq "nested.owner_id" + end + + def routing_value_source_for_widget_update_target_of(type, owner_id_field_opts: {}, **options) + update_targets = update_targets_for(type, on_widgets_index: ->(index) { index.route_with "owner_id" }, **options) do |t| + t.field "owner_id", "ID!", **owner_id_field_opts + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + yield t + end + + expect(update_targets.count { |t| t.type == "Widget" }).to eq(1) + widget_target = update_targets.find { |t| t.type == "Widget" } + widget_target.routing_value_source + end + end + + context "on a type that uses a rollover index" do + it "determines the `rollover_timestamp_value_source` from an `equivalent_field` configured on the relation" do + source = rollover_timestamp_value_source_for_widget_update_target_of("WidgetWorkspace") do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + # Including multiple `equivalent_field` calls to force it to use the right one. + r.equivalent_field "workspace_owner_id", locally_named: "id" + r.equivalent_field "workspace_created_at", locally_named: "created_at" + r.equivalent_field "name", locally_named: "name" + end + end + + expect(source).to eq "workspace_created_at" + end + + it "allows `locally_named:` to be omitted when defining equivalent fields, defaulting the local name to the same as the remote name" do + source = rollover_timestamp_value_source_for_widget_update_target_of("WidgetWorkspace") do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "created_at" + end + end + + expect(source).to eq "created_at" + end + + it "raises a clear error if an `equivalent_field` is not defined for the custom routing field" do + expect { + rollover_timestamp_value_source_for_widget_update_target_of("WidgetWorkspace") do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "another_timestamp_field" + end + end + }.to raise_error Errors::SchemaError, a_string_including( + "Cannot update `Widget` documents", + "related `workspace` events", + "`Widget` uses a rollover index", + "don't know what `workspace` timestamp field to use", + "`Widget` update requests", + "`Widget.workspace` relationship definition", + '`rel.equivalent_field "[WidgetWorkspace field]", locally_named: "created_at"`' + ) + end + + it "resolves the custom routing field using the public GraphQL field name instead of the internal `name_in_index`" do + source = rollover_timestamp_value_source_for_widget_update_target_of("WidgetWorkspace", created_at_field_opts: {name_in_index: "created_at_in_index"}) do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "workspace_created_at", locally_named: "created_at" + end + end + + expect(source).to eq "workspace_created_at" + end + + it "records the `name_in_index` as the `routing_value_source` since the `elasticgraph-indexer` logic that uses it expects the index field name" do + source = rollover_timestamp_value_source_for_widget_update_target_of("WidgetWorkspace", on_widget_workspace_type: ->(t) { t.field "w_created_at", "DateTime", name_in_index: "widget_created_at" }) do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "w_created_at", locally_named: "created_at" + end + end + + expect(source).to eq "widget_created_at" + end + + it "allows the `routing_value_source` to be an indexing-only field, but does not allow it to be a graphql-only field" do + source = rollover_timestamp_value_source_for_widget_update_target_of("WidgetWorkspace", on_widget_workspace_type: ->(t) { t.field "alt_created_at", "DateTime", indexing_only: true }) do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "alt_created_at", locally_named: "created_at" + end + end + + expect(source).to eq "alt_created_at" + + expect { + rollover_timestamp_value_source_for_widget_update_target_of("WidgetWorkspace", on_widget_workspace_type: ->(t) { t.field "alt_created_at", "DateTime", graphql_only: true }) do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "alt_created_at", locally_named: "created_at" + end + end + }.to raise_error Errors::SchemaError, a_string_including("`WidgetWorkspace.alt_created_at` (referenced from an `equivalent_field` defined on `Widget.workspace`) does not exist") + end + + it "supports nested fields as a `routing_value_source`" do + source = rollover_timestamp_value_source_for_widget_update_target_of("WidgetWorkspace") do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "nested.timestamp", locally_named: "created_at" + end + end + + expect(source).to eq "nested.timestamp" + end + + def rollover_timestamp_value_source_for_widget_update_target_of(type, created_at_field_opts: {}, **options) + update_targets = update_targets_for(type, on_widgets_index: ->(index) { index.rollover :yearly, "created_at" }, **options) do |t| + t.field "created_at", "DateTime", **created_at_field_opts + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + yield t + end + + expect(update_targets.count { |t| t.type == "Widget" }).to eq(1) + widget_target = update_targets.find { |t| t.type == "Widget" } + widget_target.rollover_timestamp_value_source + end + end + + def expect_widget_update_target_with( + update_targets, + id_source:, + data_params:, + relationship:, routing_value_source: nil, + rollover_timestamp_value_source: nil + ) + expect(update_targets.count { |t| t.type == "Widget" }).to eq(1) + widget_target = update_targets.find { |t| t.type == "Widget" } + + expect(widget_target).not_to eq nil + expect(widget_target.type).to eq "Widget" + expect(widget_target.relationship).to eq relationship + expect(widget_target.script_id).to eq(INDEX_DATA_UPDATE_SCRIPT_ID) + expect(widget_target.id_source).to eq id_source + expect(widget_target.routing_value_source).to eq(routing_value_source) + expect(widget_target.rollover_timestamp_value_source).to eq(rollover_timestamp_value_source) + expect(widget_target.data_params).to eq(data_params) + expect(widget_target.metadata_params).to eq(standard_metadata_params(relationship: relationship)) + end + + describe "validations" do + context "on the relationship" do + it "respects a type name override on the related type" do + expect { + update_targets_for("Widget", type_name_overrides: {WidgetWorkspace: "WorkspaceForWidget"}) do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "workspace_id", dir: :out + expect(t.fields_with_sources).to be_empty + end + }.not_to raise_error + end + + it "raises an error if a relationship referenced from a `sourced_from` uses `additional_filter`" do + expect { + update_targets_for("Widget") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |rel| + rel.additional_filter is_enabled: {equal_to_any_of: [true]} + end + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "DateTime" do |f| + f.sourced_from "workspace", "created_at" + end + end + }.to raise_error_about_workspace_relationship("is a `relationship` using an `additional_filter` but `sourced_from` is not supported on relationships with `additional_filter`.") + end + + it "raises an error if the referenced relationship is not defined" do + expect { + update_targets_for("Widget") do |t| + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "DateTime" do |f| + f.sourced_from "workspace", "created_at" + end + end + }.to raise_error_about_workspace_relationship("is not defined. Is it misspelled?") + end + + it "raises an error if the referenced relationship is a normal field instead of a `relates_to_one` field", :dont_validate_graphql_schema do + expect { + update_targets_for("Widget") do |t| + t.field "workspace", "WidgetWorkspace" + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "DateTime" do |f| + f.sourced_from "workspace", "created_at" + end + end + }.to raise_error_about_workspace_relationship("is not a relationship. It must be defined using `relates_to_one` or `relates_to_many`.") + end + + it "raises an error if a relationship referenced from a `sourced_from` field is a `relates_to_many` since we don't yet support that" do + expect { + update_targets_for("Widget") do |t| + t.relates_to_many "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in, singular: "workspace" + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "DateTime" do |f| + f.sourced_from "workspace", "created_at" + end + end + }.to raise_error_about_workspace_relationship("is a `relates_to_many` relationship, but `sourced_from` is only supported on a `relates_to_one` relationship.") + end + + it "still allows other relationships to be defined as a `relates_to_many`" do + expect { + update_targets_for("Widget") do |t| + t.relates_to_many "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in, singular: "workspace" + expect(t.fields_with_sources).to be_empty + end + }.not_to raise_error + end + + it "raises an error if the referenced relationship uses an outbound foreign key instead of an inbound foreign key since we don't yet support that type of foreign key" do + expect { + update_targets_for("Widget") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "workspace_id", dir: :out + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "DateTime" do |f| + f.sourced_from "workspace", "created_at" + end + end + }.to raise_error_about_workspace_relationship("has an outbound foreign key (`dir: :out`), but `sourced_from` is only supported via inbound foreign key (`dir: :in`) relationships.") + end + + it "still allows other relationships to be defined with an outbound foreign key" do + expect { + update_targets_for("Widget") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "workspace_id", dir: :out + expect(t.fields_with_sources).to be_empty + end + }.not_to raise_error + end + + it "raises an error if the related type does not exist, regardless of whether there are any `sourced_from` fields or not", :dont_validate_graphql_schema do + expect { + update_targets_for("Widget") do |t| + t.relates_to_one "workspace", "WidgetWorkspaceTypo", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "DateTime" do |f| + f.sourced_from "workspace", "created_at" + end + end + }.to raise_error_about_workspace_relationship("references an unknown type: `WidgetWorkspaceTypo`. Is it misspelled?") + + expect { + update_targets_for("Widget") do |t| + t.relates_to_one "workspace", "WidgetWorkspaceTypo", via: "widget_ids", dir: :in + expect(t.fields_with_sources).to be_empty + end + }.to raise_error_about_workspace_relationship( + "references an unknown type: `WidgetWorkspaceTypo`. Is it misspelled?", + sourced_fields: false + ) + end + + it "raises an error if the related type exists but is a scalar type, regardless of whether there are any `sourced_from` fields or not" do + expect { + update_targets_for("Widget") do |t| + t.relates_to_one "workspace", "DateTime", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "DateTime" do |f| + f.sourced_from "workspace", "created_at" + end + end + }.to raise_error_about_workspace_relationship( + "references a type which is not an object type: `DateTime`. Only object types can be used in relations." + ) + + expect { + update_targets_for("Widget") do |t| + t.relates_to_one "workspace", "DateTime", via: "widget_ids", dir: :in + expect(t.fields_with_sources).to be_empty + end + }.to raise_error_about_workspace_relationship( + "references a type which is not an object type: `DateTime`. Only object types can be used in relations.", + sourced_fields: false + ) + end + + it "raises an error if the related type exists but is a list type, regardless of whether there are any `sourced_from` fields or not" do + expect { + update_targets_for("Widget") do |t| + t.relates_to_one "workspace", "[WidgetWorkspace]", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "DateTime" do |f| + f.sourced_from "workspace", "created_at" + end + end + }.to raise_error_about_workspace_relationship( + "references a type which is not an object type: `[WidgetWorkspace]`. Only object types can be used in relations." + ) + + expect { + update_targets_for("Widget") do |t| + t.relates_to_one "workspace", "[WidgetWorkspace]", via: "widget_ids", dir: :in + expect(t.fields_with_sources).to be_empty + end + }.to raise_error_about_workspace_relationship( + "references a type which is not an object type: `[WidgetWorkspace]`. Only object types can be used in relations.", + sourced_fields: false + ) + end + + it "raises an error if the related type exists but is an enum type, regardless of whether there are any `sourced_from` fields or not" do + expect { + update_targets_for("Widget") do |t| + t.relates_to_one "workspace", "Color", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "DateTime" do |f| + f.sourced_from "workspace", "created_at" + end + end + }.to raise_error_about_workspace_relationship( + "references a type which is not an object type: `Color`. Only object types can be used in relations." + ) + + expect { + update_targets_for("Widget") do |t| + t.relates_to_one "workspace", "Color", via: "widget_ids", dir: :in + expect(t.fields_with_sources).to be_empty + end + }.to raise_error_about_workspace_relationship( + "references a type which is not an object type: `Color`. Only object types can be used in relations.", + sourced_fields: false + ) + end + + it "raises an error if the related type exists but is a non-indexed object type, regardless of whether there are any `sourced_from` fields or not" do + expect { + update_targets_for("Widget", index_widget_workspaces: false) do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "DateTime" do |f| + f.sourced_from "workspace", "created_at" + end + end + }.to raise_error_about_workspace_relationship( + "references a type which is not indexed: `WidgetWorkspace`. Only indexed types can be used in relations." + ) + + expect { + update_targets_for("Widget", index_widget_workspaces: false) do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + expect(t.fields_with_sources).to be_empty + end + }.to raise_error_about_workspace_relationship( + "references a type which is not indexed: `WidgetWorkspace`. Only indexed types can be used in relations.", + sourced_fields: false + ) + end + + it "raises an error if an inbound foreign key field does not exist on the related type, regardless of whether there are any `sourced_from` fields or not" do + expect { + update_targets_for("Widget") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "unknown_id", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "DateTime" do |f| + f.sourced_from "workspace", "created_at" + end + end + }.to raise_error_about_workspace_relationship( + "uses `WidgetWorkspace.unknown_id` as the foreign key, but that field does not exist as an indexing field." + ) + + expect { + update_targets_for("Widget") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "unknown_id", dir: :in + expect(t.fields_with_sources).to be_empty + end + }.to raise_error_about_workspace_relationship( + "uses `WidgetWorkspace.unknown_id` as the foreign key, but that field does not exist as an indexing field.", + sourced_fields: false + ) + end + + it "raises an error if an inbound foreign key field exists as a GraphQL-only field on the related type, regardless of whether there are any `sourced_from` fields or not" do + expect { + update_targets_for("Widget", on_widget_workspace_type: ->(t) { t.field "gql_only_id", "ID", graphql_only: true }) do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "gql_only_id", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "DateTime" do |f| + f.sourced_from "workspace", "created_at" + end + end + }.to raise_error_about_workspace_relationship( + "uses `WidgetWorkspace.gql_only_id` as the foreign key, but that field does not exist as an indexing field." + ) + + expect { + update_targets_for("Widget", on_widget_workspace_type: ->(t) { t.field "gql_only_id", "ID", graphql_only: true }) do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "gql_only_id", dir: :in + expect(t.fields_with_sources).to be_empty + end + }.to raise_error_about_workspace_relationship( + "uses `WidgetWorkspace.gql_only_id` as the foreign key, but that field does not exist as an indexing field.", + sourced_fields: false + ) + end + + it "raises an error if an inbound foreign key field is not an `ID`, regardless of whether there are any `sourced_from` fields or not" do + expect { + update_targets_for("Widget", on_widget_workspace_type: ->(t) { t.field "numeric_id", "Int" }) do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "numeric_id", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "DateTime" do |f| + f.sourced_from "workspace", "created_at" + end + end + }.to raise_error_about_workspace_relationship( + "uses `WidgetWorkspace.numeric_id` as the foreign key, but that field is not an `ID` field as expected." + ) + + expect { + update_targets_for("Widget", on_widget_workspace_type: ->(t) { t.field "numeric_id", "Int" }) do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "numeric_id", dir: :in + expect(t.fields_with_sources).to be_empty + end + }.to raise_error_about_workspace_relationship( + "uses `WidgetWorkspace.numeric_id` as the foreign key, but that field is not an `ID` field as expected.", + sourced_fields: false + ) + end + + it "raises an error if an outbound foreign key field is not an `ID`" do + expect { + update_targets_for("Widget", on_widget_workspace_type: ->(t) { + t.field "numeric_id", "Int" + t.relates_to_one "widget", "Widget", via: "numeric_id", dir: :out + }) + }.to raise_error Errors::SchemaError, a_string_including( + "`WidgetWorkspace.widget` uses `WidgetWorkspace.numeric_id` as the foreign key, but that field is not an `ID` field as expected." + ) + end + + it "does not raise an error if the outbound foreign key field is inferred instead of explicitly defined" do + expect { + update_targets_for("Widget", on_widget_workspace_type: ->(t) { + t.relates_to_one "widget", "Widget", via: "numeric_id", dir: :out + }) + }.not_to raise_error + end + + it "allows a foreign key whose type is nested inside of an `object` array" do + expect { + object_type_metadata_for "WidgetWorkspace" do |s| + s.object_type "WorkspaceReference" do |t| + t.field "workspace_id", "ID!" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "workspaces", "[WorkspaceReference!]!" do |f| + f.mapping type: "object" + end + t.index "widgets" + end + + s.object_type "WidgetWorkspace" do |t| + t.field "id", "ID!" + t.relates_to_many "widgets", "Widget", via: "workspaces.workspace_id", dir: :in, singular: "widget" + t.index "widget_workspaces" + end + end + }.not_to raise_error + end + + it "allows a foreign key whose type is nested inside of a `nested` array" do + expect { + object_type_metadata_for "WidgetWorkspace" do |s| + s.object_type "WorkspaceReference" do |t| + t.field "workspace_id", "ID!" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "workspaces", "[WorkspaceReference!]!" do |f| + f.mapping type: "nested" + end + t.index "widgets" + end + + s.object_type "WidgetWorkspace" do |t| + t.field "id", "ID!" + t.relates_to_many "widgets", "Widget", via: "workspaces.workspace_id", dir: :in, singular: "widget" + t.index "widget_workspaces" + end + end + }.not_to raise_error + end + + it "validates relationships on unindexed object types" do + expect { + object_type_metadata_for "WidgetWorkspace" do |s| + s.object_type "WorkspaceReference" do |t| + t.field "workspace_id", "ID!" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "workspaces", "[WorkspaceReference!]!" do |f| + f.mapping type: "object" + end + t.index "widgets" + end + + s.object_type "WidgetWorkspace" do |t| + t.field "id", "ID!" + t.relates_to_many "widgets", "Widget", via: "workspaces.invalid_key", dir: :in, singular: "widget" + end + end + }.to raise_error Errors::SchemaError, a_string_including( + "`WidgetWorkspace.widgets` uses `Widget.workspaces.invalid_key` as the foreign key", + "but that field does not exist as an indexing field" + ) + end + end + + context "on `equivalent_field` definitions" do + it "does not allow `equivalent_field` definitions to stomp each other" do + expect { + update_targets_for("WidgetWorkspace") do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "created_at1", locally_named: "created_at" + r.equivalent_field "created_at2", locally_named: "created_at" + end + end + }.to raise_error Errors::SchemaError, a_string_including( + "`equivalent_field` has been called multiple times on `Widget.workspace", + 'same `locally_named` value ("created_at")' + ) + end + + it "raises a clear error if the configured `equivalent_field` does not exist" do + expect { + update_targets_for("WidgetWorkspace") do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "unknown_id", locally_named: "id" + end + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + end + }.to raise_error Errors::SchemaError, a_string_including( + "Field `WidgetWorkspace.unknown_id` (referenced from an `equivalent_field` defined on `Widget.workspace`) does not exist. Either define it or correct the `equivalent_field` definition." + ) + end + + it "raises a clear error if the `locally_named` field of a configured `equivalent_field` does not exist" do + expect { + update_targets_for("WidgetWorkspace") do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "workspace_owner_id", locally_named: "unknown_id" + end + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + end + }.to raise_error Errors::SchemaError, a_string_including( + "Field `Widget.unknown_id` (referenced from an `equivalent_field` defined on `Widget.workspace`) does not exist. Either define it or correct the `equivalent_field` definition." + ) + end + + it "requires both sides of an `equivalent_field` to have the same type" do + expect { + update_targets_for("WidgetWorkspace") do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "name", locally_named: "id" + end + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + end + }.to raise_error Errors::SchemaError, a_string_including( + "Field `WidgetWorkspace.name: String` is defined as an equivalent of `Widget.id: ID!` via an `equivalent_field` definition on `Widget.workspace`, but their types do not agree. To continue, change one or the other so that they agree." + ) + end + + it "allows the two sides of an `equivalent_field` to differ in terms of nullability" do + expect { + update_targets_for("WidgetWorkspace", widget_workspace_name: "String!", widget_name: "String") do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "name", locally_named: "name" + end + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + end + }.not_to raise_error + end + + it "does not allow the two sides of an `equivalent_field` to differ in terms of list wrappings" do + expect { + update_targets_for("WidgetWorkspace", widget_workspace_name: "String", widget_name: "[String]") do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in do |r| + r.equivalent_field "name", locally_named: "name" + end + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + end + }.to raise_error Errors::SchemaError, a_string_including( + "Field `WidgetWorkspace.name: String` is defined as an equivalent of `Widget.name: [String]` via an `equivalent_field` definition on `Widget.workspace`, but their types do not agree. To continue, change one or the other so that they agree." + ) + end + + it "does not interfere with the non-nullability of a `relates_to_one` field" do + expect { + update_targets_for("WidgetWorkspace") do |t| + expect(t.name).to eq "Widget" + + t.relates_to_one "workspace", "WidgetWorkspace!", via: "widget_ids", dir: :in do |r| + r.equivalent_field "name", locally_named: "name" + end + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + end + }.not_to raise_error + end + end + + context "on the referenced field" do + it "raises an error if the referenced field doesn't exist on the related type" do + expect { + update_targets_for("Widget") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name2" + end + + t.field "workspace_created_at", "DateTime" do |f| + f.sourced_from "workspace", "created_at3" + end + end + }.to raise_error a_string_including( + "1. `Widget.workspace_name` has an invalid `sourced_from` argument: `WidgetWorkspace.name2` does not exist as an indexing field.", + "2. `Widget.workspace_created_at` has an invalid `sourced_from` argument: `WidgetWorkspace.created_at3` does not exist as an indexing field." + ) + end + + it "allows the referenced field to be an `indexing_only` field since it must be ingested but need not be exposed in GraphQL" do + expect { + update_targets_for("Widget", widget_workspace_name_opts: {indexing_only: true}) do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + end + }.not_to raise_error + end + + it "does not allow the referenced field to be a `graphql_only_field` field" do + expect { + update_targets_for("Widget", widget_workspace_name_opts: {graphql_only: true}) do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + end + }.to raise_error a_string_including( + "`Widget.workspace_name` has an invalid `sourced_from` argument: `WidgetWorkspace.name` does not exist as an indexing field." + ) + end + + it "raises an error if any part of nested field path doesn't exist" do + try_sourced_from_field_path = lambda do |field_path| + update_targets_for("WidgetWorkspace") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", field_path + end + end + end + + # verify that `nested.further_nested.name` is correct. + expect { try_sourced_from_field_path.call("nested.further_nested.name") }.not_to raise_error + + expect { + try_sourced_from_field_path.call("nesteth.further_nested.name") + }.to raise_error a_string_including( + "1. `Widget.workspace_name` has an invalid `sourced_from` argument: `WidgetWorkspace.nesteth.further_nested.name` could not be resolved" + ) + + expect { + try_sourced_from_field_path.call("nested.further_nesteth.name") + }.to raise_error a_string_including( + "1. `Widget.workspace_name` has an invalid `sourced_from` argument: `WidgetWorkspace.nested.further_nesteth.name` could not be resolved" + ) + + expect { + try_sourced_from_field_path.call("nested.further_nested.missing") + }.to raise_error a_string_including( + "1. `Widget.workspace_name` has an invalid `sourced_from` argument: `WidgetWorkspace.nested.further_nested.missing` could not be resolved" + ) + end + + it "raises an error if an empty field path is provided" do + expect { + update_targets_for("WidgetWorkspace") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "" + end + end + }.to raise_error a_string_including( + "1. `Widget.workspace_name` has an invalid `sourced_from` argument: `WidgetWorkspace.` does not exist as an indexing field." + ) + end + + it "raises an error if any of the parent parts of the field path refer to non-object fields" do + expect { + update_targets_for("WidgetWorkspace") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name.nested" + end + end + }.to raise_error a_string_including( + "1. `Widget.workspace_name` has an invalid `sourced_from` argument: `WidgetWorkspace.name.nested` could not be resolved" + ) + end + + it "raises an error if any of the parts of a nested field are list fields" do + expect { + update_targets_for("WidgetWorkspace", widget_workspace_nested_1_further_nested: "[WidgetWorkspaceNested2]") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "nested.further_nested.name" + end + end + }.to raise_error a_string_including( + "`Widget.workspace_name` has an invalid `sourced_from` argument: `WidgetWorkspace.nested.further_nested.name` could not be resolved", + "some parts do not exist on their respective types as non-list fields" + ) + end + end + + context "on the referenced field's type" do + it "requires the referenced field to have the same GraphQL type as the `sourced_from` field" do + expect { + update_targets_for("Widget") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "DateTime" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "String" do |f| + f.sourced_from "workspace", "created_at" + end + end + }.to raise_error a_string_including( + "1. The type of `Widget.workspace_name` is `DateTime`, but the type of it's source (`WidgetWorkspace.name`) is `String`. These must agree to use `sourced_from`.", + "2. The type of `Widget.workspace_created_at` is `String`, but the type of it's source (`WidgetWorkspace.created_at`) is `DateTime`. These must agree to use `sourced_from`." + ) + end + + it "allows a `sourced_from` field to be nullable even if its referenced field is not" do + expect { + update_targets_for("Widget", widget_workspace_name: "String!") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + end + }.not_to raise_error + end + + it "does not allow a `sourced_from` field to be non-nullable when its referenced field is nullable" do + expect { + update_targets_for("Widget", widget_workspace_name: "String") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String!" do |f| + f.sourced_from "workspace", "name" + end + end + }.to raise_error a_string_including( + "The type of `Widget.workspace_name` (`String!`) is not nullable, but this is not allowed for `sourced_from` fields since the value will be `null` before the related type's event is ingested." + ) + end + + it "does not allow a `sourced_from` field to be non-nullable even if its referenced field is non-nullable because if the related event is ingested 2nd the value will initially be null" do + expect { + update_targets_for("Widget", widget_workspace_name: "String!") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String!" do |f| + f.sourced_from "workspace", "name" + end + end + }.to raise_error a_string_including( + "The type of `Widget.workspace_name` (`String!`) is not nullable, but this is not allowed for `sourced_from` fields since the value will be `null` before the related type's event is ingested." + ) + end + + it "does not consider a list and scalar field to be the same type" do + expect { + update_targets_for("Widget", widget_workspace_name: "[String]") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "[DateTime]" do |f| + f.sourced_from "workspace", "created_at" + end + end + }.to raise_error a_string_including( + "1. The type of `Widget.workspace_name` is `String`, but the type of it's source (`WidgetWorkspace.name`) is `[String]`. These must agree to use `sourced_from`.", + "2. The type of `Widget.workspace_created_at` is `[DateTime]`, but the type of it's source (`WidgetWorkspace.created_at`) is `DateTime`. These must agree to use `sourced_from`." + ) + end + + it "otherwise supports list fields" do + expect { + update_targets_for("Widget", widget_workspace_name: "[String]") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "[String]" do |f| + f.sourced_from "workspace", "name" + end + end + }.not_to raise_error + end + + it "allows the referenced field to have a different mapping type from the `sourced_from` field" do + expect { + update_targets_for("Widget") do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.mapping type: "text" + f.sourced_from "workspace", "name" + end + end + }.not_to raise_error + end + end + + def raise_error_about_workspace_relationship(details, sourced_fields: true) + expected_string = + if sourced_fields + "`Widget.workspace` (referenced from `sourced_from` on field(s): `workspace_name`, `workspace_created_at`) #{details}" + else + "`Widget.workspace` #{details}" + end + + raise_error Errors::SchemaError, a_string_including(expected_string) + end + end + + def update_targets_for( + type, + widget_name: "String", + widget_workspace_name: "String", + widget_workspace_name_opts: {}, + widget_workspace_created_at: "DateTime", + widget_workspace_nested_1_further_nested: "WidgetWorkspaceNested2", + on_widgets_index: nil, + on_widget_workspace_type: nil, + index_widget_workspaces: true, + type_name_overrides: {}, + &define_relation_and_sourced_from_fields + ) + define_relation_and_sourced_from_fields ||= lambda do |t| + t.relates_to_one "workspace", "WidgetWorkspace", via: "widget_ids", dir: :in + + t.field "workspace_name", "String" do |f| + f.sourced_from "workspace", "name" + end + + t.field "workspace_created_at", "DateTime" do |f| + f.sourced_from "workspace", "created_at" + end + end + + metadata = object_type_metadata_for type, type_name_overrides: type_name_overrides do |s| + s.enum_type "Color" do |t| + t.value "RED" + t.value "GREEN" + t.value "BLUE" + end + + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", widget_name + + define_relation_and_sourced_from_fields.call(t) + + t.index "widgets" do |index| + on_widgets_index&.call(index) + end + end + + s.object_type "WidgetWorkspaceNested1" do |t| + t.field "further_nested", widget_workspace_nested_1_further_nested, name_in_index: "further_nested_in_index" do |f| + f.mapping type: "object" if widget_workspace_nested_1_further_nested.include?("[") + end + t.field "owner_id", "ID!" + t.field "timestamp", "DateTime" + end + + s.object_type "WidgetWorkspaceNested2" do |t| + t.field "name", "String" + end + + s.object_type "WidgetWorkspace" do |t| + t.field "id", "ID!" + t.field "workspace_owner_id", "ID!" + t.field "name", widget_workspace_name, **widget_workspace_name_opts + t.field "created_at", widget_workspace_created_at + t.field "workspace_created_at", widget_workspace_created_at + t.field "nested", "WidgetWorkspaceNested1" + t.relates_to_many "widgets", "Widget", via: "widget_ids", dir: :out, singular: "widget" + + on_widget_workspace_type&.call(t) + + t.index "widget_workspaces" if index_widget_workspaces + end + end + + metadata.update_targets + end + end + + context "on a normal indexed type" do + it "dumps information about update targets" do + metadata = object_type_metadata_for "Widget" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "workspace_id", "ID!" + t.field "name", "String!" + t.index "widgets" + t.derive_indexed_type_fields "WidgetWorkspace", from_id: "workspace_id" do |derive| + derive.append_only_set "widget_ids", from: "id" + end + end + + s.object_type "WidgetWorkspace" do |t| + t.field "id", "ID!" + t.index "widget_workspaces" + end + end + + expect(metadata.update_targets.map(&:type)).to contain_exactly("WidgetWorkspace", "Widget") + + widget_workspace_target = metadata.update_targets.find { |t| t.type == "WidgetWorkspace" } + expect(widget_workspace_target.type).to eq "WidgetWorkspace" + expect(widget_workspace_target.relationship).to eq nil + expect(widget_workspace_target.script_id).to start_with "update_WidgetWorkspace_from_Widget_" + expect(widget_workspace_target.id_source).to eq "workspace_id" + expect(widget_workspace_target.routing_value_source).to eq(nil) + expect(widget_workspace_target.rollover_timestamp_value_source).to eq(nil) + expect(widget_workspace_target.data_params).to eq({"id" => dynamic_param_with(source_path: "id", cardinality: :many)}) + expect(widget_workspace_target.metadata_params).to eq({}) + + widget_target = metadata.update_targets.find { |t| t.type == "Widget" } + expect(widget_target.type).to eq "Widget" + expect(widget_target.relationship).to eq SELF_RELATIONSHIP_NAME + expect(widget_target.script_id).to eq(INDEX_DATA_UPDATE_SCRIPT_ID) + expect(widget_target.id_source).to eq "id" + expect(widget_target.routing_value_source).to eq("id") + expect(widget_target.rollover_timestamp_value_source).to eq(nil) + expect(widget_target.data_params).to eq({ + "name" => dynamic_param_with(source_path: "name", cardinality: :one), + "workspace_id" => dynamic_param_with(source_path: "workspace_id", cardinality: :one) + }) + expect(widget_target.metadata_params).to eq(standard_metadata_params(relationship: SELF_RELATIONSHIP_NAME)) + end + + it "respects a type name override for the destination type" do + metadata = object_type_metadata_for("Widget", type_name_overrides: {WidgetWorkspace: "WorkspaceOfWidget"}) do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "workspace_id", "ID!" + t.field "name", "String!" + t.index "widgets" + t.derive_indexed_type_fields "WidgetWorkspace", from_id: "workspace_id" do |derive| + derive.append_only_set "widget_ids", from: "id" + end + end + + s.object_type "WidgetWorkspace" do |t| + t.field "id", "ID!" + t.index "widget_workspaces" + end + end + + expect(metadata.update_targets.map(&:type)).to contain_exactly("WorkspaceOfWidget", "Widget") + end + + it "dumps only the 'self' update target when the indexed type has no derived indexing types" do + metadata = object_type_metadata_for "Widget" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "cost", "Int" + t.index "widgets" + end + end + + expect(metadata.update_targets.map(&:type)).to contain_exactly("Widget") + + widget_target = metadata.update_targets.find { |t| t.type == "Widget" } + expect(widget_target.type).to eq "Widget" + expect(widget_target.relationship).to eq SELF_RELATIONSHIP_NAME + expect(widget_target.script_id).to eq(INDEX_DATA_UPDATE_SCRIPT_ID) + expect(widget_target.id_source).to eq "id" + expect(widget_target.routing_value_source).to eq("id") + expect(widget_target.rollover_timestamp_value_source).to eq(nil) + expect(widget_target.data_params).to eq({"cost" => dynamic_param_with(source_path: "cost", cardinality: :one)}) + expect(widget_target.metadata_params).to eq(standard_metadata_params(relationship: SELF_RELATIONSHIP_NAME)) + end + + it "sets the `routing_value_source` correctly when the index uses custom shard routing" do + metadata = object_type_metadata_for "Widget" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "workspace_id", "ID" + t.index "widgets" do |i| + i.route_with "workspace_id" + end + end + end + + expect(metadata.update_targets.map(&:type)).to contain_exactly("Widget") + + widget_target = metadata.update_targets.find { |t| t.type == "Widget" } + expect(widget_target.routing_value_source).to eq("workspace_id") + end + + it "sets the `rollover_timestamp_value_source` correctly when the index uses rollover indexes" do + metadata = object_type_metadata_for "Widget" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime" + t.index "widgets" do |i| + i.rollover :monthly, "created_at" + end + end + end + + expect(metadata.update_targets.map(&:type)).to contain_exactly("Widget") + + widget_target = metadata.update_targets.find { |t| t.type == "Widget" } + expect(widget_target.rollover_timestamp_value_source).to eq("created_at") + end + end + + context "on an embedded object type" do + it "dumps no `update_targets`" do + metadata = object_type_metadata_for "WidgetOptions" do |s| + s.object_type "WidgetOptions" do |t| + t.field "size", "Int", name_in_index: "size_index" + end + end + + expect(metadata.update_targets).to be_empty + end + end + + on_a_type_union_or_interface_type do |type_def_method| + it "does not dump information about `update_targets` based on the subtypes; that info will be dumped on those types" do + metadata = object_type_metadata_for "Thing" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + # Use an alternate `name_in_index` to force `metadata` not to be `nil`. + t.field "workspace_id", "ID!", name_in_index: "wid" + t.index "widgets" + link_subtype_to_supertype(t, "Thing") + t.derive_indexed_type_fields "WidgetWorkspace", from_id: "workspace_id" do |derive| + derive.append_only_set "widget_ids", from: "id" + end + end + + s.object_type "WidgetWorkspace" do |t| + t.field "id", "ID!" + t.index "widget_workspaces" + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + link_subtype_to_supertype(t, "Thing") + t.index "components" + end + + s.public_send type_def_method, "Thing" do |t| + link_supertype_to_subtypes(t, "Widget", "Component") + end + end + + expect(metadata.update_targets).to eq [] + end + + it "dumps info about `update_targets` from the supertype itself" do + metadata = object_type_metadata_for "Thing" do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "workspace_id", "ID!" + link_subtype_to_supertype(t, "Thing") + t.index "widgets" + end + + s.object_type "Component" do |t| + t.field "id", "ID!" + t.field "workspace_id", "ID!" + link_subtype_to_supertype(t, "Thing") + t.index "components" + end + + s.public_send type_def_method, "Thing" do |t| + link_supertype_to_subtypes(t, "Widget", "Component") + t.index "things" + t.derive_indexed_type_fields "ThingWorkspace", from_id: "workspace_id" do |derive| + derive.append_only_set "thing_ids", from: "id" + end + end + + s.object_type "ThingWorkspace" do |t| + t.field "id", "ID!" + t.index "thing_workspaces" + end + end + + expect(metadata.update_targets.size).to eq 1 + expect(metadata.update_targets.first.type).to eq "ThingWorkspace" + expect(metadata.update_targets.first.relationship).to eq nil + expect(metadata.update_targets.first.script_id).to start_with "update_ThingWorkspace_from_Thing" + expect(metadata.update_targets.first.id_source).to eq "workspace_id" + expect(metadata.update_targets.first.data_params).to eq({"id" => dynamic_param_with(source_path: "id", cardinality: :many)}) + expect(metadata.update_targets.first.metadata_params).to eq({}) + end + end + + def standard_metadata_params(relationship:) + {"sourceId" => "id", "sourceType" => "type", "version" => "version"}.transform_values do |source_path| + dynamic_param_with(source_path: source_path, cardinality: :one) + end.merge({ + "relationship" => static_param_with(relationship) + }) + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/runtime_metadata_support.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/runtime_metadata_support.rb new file mode 100644 index 00000000..74d46e4d --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/runtime_metadata_support.rb @@ -0,0 +1,23 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/schema_definition_helpers" +require "elastic_graph/spec_support/runtime_metadata_support" + +module ElasticGraph + module SchemaDefinition + ::RSpec.shared_context "RuntimeMetadata support" do + include_context "SchemaDefinitionHelpers" + include SchemaArtifacts::RuntimeMetadata::RuntimeMetadataSupport + + def define_schema(**options, &block) + super(schema_element_name_form: "snake_case", **options) + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/scalar_types_by_name_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/scalar_types_by_name_spec.rb new file mode 100644 index 00000000..c05d257d --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/scalar_types_by_name_spec.rb @@ -0,0 +1,101 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "runtime_metadata_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "RuntimeMetadata #scalar_types_by_name" do + include_context "RuntimeMetadata support" + + it "dumps the coercion adapter" do + metadata = scalar_type_metadata_for "BigInt" do |s| + s.scalar_type "BigInt" do |t| + t.mapping type: "long" + t.json_schema type: "integer" + t.coerce_with "ExampleScalarCoercionAdapter", defined_at: "support/example_extensions/scalar_coercion_adapter" + end + end + + expect(metadata).to eq scalar_type_with(coercion_adapter_ref: { + "extension_name" => "ExampleScalarCoercionAdapter", + "require_path" => "support/example_extensions/scalar_coercion_adapter" + }) + end + + it "dumps the indexing preparer" do + metadata = scalar_type_metadata_for "BigInt" do |s| + s.scalar_type "BigInt" do |t| + t.mapping type: "long" + t.json_schema type: "integer" + t.prepare_for_indexing_with "ExampleIndexingPreparer", defined_at: "support/example_extensions/indexing_preparer" + end + end + + expect(metadata).to eq scalar_type_with(indexing_preparer_ref: { + "extension_name" => "ExampleIndexingPreparer", + "require_path" => "support/example_extensions/indexing_preparer" + }) + end + + it "verifies the validity of the extension when `coerce_with` is called" do + define_schema do |s| + s.scalar_type "BigInt" do |t| + t.mapping type: "long" + t.json_schema type: "integer" + + expect { + t.coerce_with "NotAValidConstant", defined_at: "support/example_extensions/scalar_coercion_adapter" + }.to raise_error NameError, a_string_including("NotAValidConstant") + end + end + end + + it "verifies the validity of the extension when `indexing_preparer` is called" do + define_schema do |s| + s.scalar_type "BigInt" do |t| + t.mapping type: "long" + t.json_schema type: "integer" + + expect { + t.prepare_for_indexing_with "NotAValidConstant", defined_at: "support/example_extensions/indexing_preparer" + }.to raise_error NameError, a_string_including("NotAValidConstant") + end + end + end + + it "dumps runtime metadata for the all scalar types (including ones described in the GraphQL spec) so that the indexing preparer is explicitly defined" do + dumped_scalar_types = define_schema.runtime_metadata.scalar_types_by_name.keys + + expect(dumped_scalar_types).to include("ID", "Int", "Float", "String", "Boolean") + end + + it "allows `on_built_in_types` to customize scalar runtime metadata" do + metadata = scalar_type_metadata_for "Int" do |s| + s.on_built_in_types do |t| + if t.is_a?(SchemaElements::ScalarType) + t.coerce_with "ExampleScalarCoercionAdapter", defined_at: "support/example_extensions/scalar_coercion_adapter" + end + end + end + + expect(metadata.coercion_adapter_ref).to eq({ + "extension_name" => "ExampleScalarCoercionAdapter", + "require_path" => "support/example_extensions/scalar_coercion_adapter" + }) + end + + def scalar_type_metadata_for(name, &block) + define_schema(&block) + .runtime_metadata + .scalar_types_by_name + .fetch(name) + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/static_script_ids_by_scoped_name_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/static_script_ids_by_scoped_name_spec.rb new file mode 100644 index 00000000..de292e46 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/static_script_ids_by_scoped_name_spec.rb @@ -0,0 +1,24 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "runtime_metadata_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "RuntimeMetadata #static_script_ids_by_scoped_name" do + include_context "RuntimeMetadata support" + + it "has the ids and scoped names from our static scripts" do + static_script_ids_by_scoped_name = define_schema.runtime_metadata.static_script_ids_by_scoped_name + + expect(static_script_ids_by_scoped_name.keys).to include("filter/by_time_of_day") + expect(static_script_ids_by_scoped_name["filter/by_time_of_day"]).to match(/\Afilter_by_time_of_day_\w+\z/) + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/schema_elements/enum_value_namer_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/schema_elements/enum_value_namer_spec.rb new file mode 100644 index 00000000..f75dd234 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/schema_elements/enum_value_namer_spec.rb @@ -0,0 +1,100 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_definition/schema_elements/enum_value_namer" +require "graphql" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + RSpec.describe EnumValueNamer do + describe "#name_for" do + it "echoes back the given value name if no override has been provided for the type" do + namer = EnumValueNamer.new + + expect(namer.name_for("DayOfWeek", "MONDAY")).to eq("MONDAY") + end + + it "echoes back the given value name if the named type has overrides but not for the given value" do + namer = EnumValueNamer.new({"DayOfWeek" => {"TUESDAY" => "TUE"}}) + + expect(namer.name_for("DayOfWeek", "MONDAY")).to eq("MONDAY") + end + + it "returns the override value for the given type and value if one has been provided" do + namer = EnumValueNamer.new({"DayOfWeek" => {"TUESDAY" => "TUE"}}) + + expect(namer.name_for("DayOfWeek", "TUESDAY")).to eq("TUE") + end + + it "allows the overrides to be configured with string or symbol keys" do + namer1 = EnumValueNamer.new({"DayOfWeek" => {"TUESDAY" => "TUE"}}) + namer2 = EnumValueNamer.new({DayOfWeek: {TUESDAY: "TUE"}}) + + expect(namer1).to eq(namer2) + end + + it "does not allow an override to produce an invalid GraphQL name" do + expect { + EnumValueNamer.new({DayOfWeek: {TUESDAY: "The day after Monday"}}) + }.to raise_error Errors::ConfigError, a_string_including( + "Provided `enum_value_overrides_by_type_name` have 1 problem(s)", + "`The day after Monday` (the override for `DayOfWeek.TUESDAY`) is not a valid GraphQL type name. #{GRAPHQL_NAME_VALIDITY_DESCRIPTION}" + ) + end + + it "does not allow two overrides on the same type to map to the same name" do + expect { + EnumValueNamer.new({DayOfWeek: {MONDAY: "MON", TUESDAY: "MON"}}) + }.to raise_error Errors::ConfigError, a_string_including( + "Provided `enum_value_overrides_by_type_name` have 1 problem(s)", + "Multiple `DayOfWeek` enum value overrides (MONDAY, TUESDAY) map to the same name (MON)" + ) + end + + it "keeps track of which overrides have and have not been used" do + namer = EnumValueNamer.new( + DayOfWeek: {MONDAY: "MON", TUESDAY: "TUE"}, + MetricUnit: {GRAM: "G"}, + Other: {} + ) + + expect(namer.unused_overrides).to eq({ + "DayOfWeek" => {"MONDAY" => "MON", "TUESDAY" => "TUE"}, + "MetricUnit" => {"GRAM" => "G"}, + "Other" => {} + }) + expect(namer.used_value_names_by_type_name).to be_empty + + namer.name_for("DayOfWeek", "TUESDAY") + namer.name_for("MetricUnit", "GRAM") + namer.name_for("MetricUnit", "METER") + namer.name_for("SomeEnum", "FOO") + + expect(namer.unused_overrides).to eq({ + "DayOfWeek" => {"MONDAY" => "MON"}, + "Other" => {} + }) + expect(namer.used_value_names_by_type_name).to eq({ + "DayOfWeek" => %w[TUESDAY], + "MetricUnit" => %w[GRAM METER], + "SomeEnum" => %w[FOO] + }) + end + + it "does not allow `used_value_names_by_type_name` to be mutated when a caller queries it" do + namer = EnumValueNamer.new + + expect(namer.used_value_names_by_type_name["Foo"]).to be_empty + expect(namer.used_value_names_by_type_name).to be_empty + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/schema_elements/type_namer_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/schema_elements/type_namer_spec.rb new file mode 100644 index 00000000..0a95ed4e --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/schema_elements/type_namer_spec.rb @@ -0,0 +1,344 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_definition/schema_elements/type_namer" +require "graphql" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + RSpec.describe TypeNamer do + describe "#name_for" do + it "echoes back the given string or symbol as a string if no override has been provided" do + namer = TypeNamer.new + + expect(namer.name_for("SomeName")).to eq "SomeName" + expect(namer.name_for(:SomeName)).to eq "SomeName" + end + + it "returns the configured override if there is one" do + namer = TypeNamer.new(name_overrides: {SomeName: "SomeNameAlt"}) + + expect(namer.name_for(:SomeName)).to eq "SomeNameAlt" + expect(namer.name_for("SomeName")).to eq "SomeNameAlt" + end + + it "allows the overrides to be configured with string or symbol keys" do + namer1 = TypeNamer.new(name_overrides: {SomeName: "SomeNameAlt"}) + namer2 = TypeNamer.new(name_overrides: {"SomeName" => "SomeNameAlt"}) + + expect(namer1).to eq(namer2) + end + + it "does not allow an override to produce an invalid GraphQL name" do + expect { + TypeNamer.new(name_overrides: {SomeName: "Not a valid name"}) + }.to raise_error Errors::ConfigError, a_string_including( + "Provided type name overrides have 1 problem(s)", + "`Not a valid name` (the override for `SomeName`) is not a valid GraphQL type name. #{GRAPHQL_NAME_VALIDITY_DESCRIPTION}" + ) + end + + it "keeps track of which overrides have and have not been used" do + namer = TypeNamer.new(name_overrides: {SomeName: "SomeNameAlt", OtherName: "Foo", FinalName: "Bar"}) + + expect(namer.unused_name_overrides).to eq({"SomeName" => "SomeNameAlt", "OtherName" => "Foo", "FinalName" => "Bar"}) + expect(namer.used_names).to be_empty + + namer.name_for("SomeName") + + expect(namer.unused_name_overrides).to eq({"OtherName" => "Foo", "FinalName" => "Bar"}) + expect(namer.used_names).to contain_exactly("SomeName") + + namer.name_for("FinalName") + namer.name_for("SomeName") + namer.name_for("NameThatHasNoOveride") + + expect(namer.unused_name_overrides).to eq({"OtherName" => "Foo"}) + expect(namer.used_names).to contain_exactly("SomeName", "FinalName", "NameThatHasNoOveride") + end + end + + TypeNamer::TYPES_THAT_CANNOT_BE_OVERRIDDEN.each do |type| + it "does not allow a user to override the `#{type}` since it is part of the GraphQL spec and cannot be changed" do + expect { + TypeNamer.new(name_overrides: {type.to_sym => "#{type}Alt"}) + }.to raise_error Errors::ConfigError, a_string_including( + "Provided type name overrides have 1 problem(s)", + "`#{type}` cannot be overridden because it is part of the GraphQL spec." + ) + end + end + + describe "#revert_override_for" do + it "echoes back the given name if it has not and overridden name" do + namer = TypeNamer.new + + expect(namer.revert_override_for("SomeName")).to eq "SomeName" + end + + it "returns the original name if the given name has an override" do + namer = TypeNamer.new(name_overrides: {SomeName: "SomeNameAlt"}) + + expect(namer.revert_override_for("SomeNameAlt")).to eq "SomeName" + end + + it "avoids ambiguities by disallowing multiple names overrides that map to the same result" do + expect { + TypeNamer.new(name_overrides: {SomeName: "SomeNameAlt", Other: "SomeNameAlt"}) + }.to raise_error Errors::ConfigError, a_string_including( + "Provided type name overrides have 1 problem(s)", + "1. Multiple names (Other, SomeName) map to the same override: SomeNameAlt, which is not supported." + ) + end + end + + describe "generated names" do + tested_formats = ::Set.new + + after(:context) do + untested_formats = TypeNamer::DEFAULT_FORMATS.keys - tested_formats.to_a + + expect(untested_formats).to be_empty, "Expected all `TypeNamer::DEFAULT_FORMATS.keys` to be tested via the " \ + "`a TypeNamer format` shared example group, but #{untested_formats.size} were not: #{untested_formats.to_a.inspect}." + end + + shared_examples_for "a TypeNamer format" do |format_name:, valid_args:, valid_args_result:| + tested_formats << format_name + let(:default_format) { TypeNamer::DEFAULT_FORMATS.fetch(format_name) } + + it "has a default format" do + namer = TypeNamer.new + name = namer.generate_name_for(format_name, **valid_args) + + expect(name).to eq valid_args_result + expect(namer.matches_format?(name, format_name)).to be true + expect(namer.matches_format?(name + "_", format_name)).to be false + end + + it "allows the default format to be overridden" do + namer = TypeNamer.new(format_overrides: {format_name => "#{default_format}Alt"}) + name = namer.generate_name_for(format_name, **valid_args) + + expect(name).to eq "#{valid_args_result}Alt" + expect(namer.matches_format?(name, format_name)).to be true + expect(namer.matches_format?(name + "_", format_name)).to be false + end + + it "does not apply a configured name override, requiring that of the caller, to avoid excessive application of that logic" do + namer = TypeNamer.new(name_overrides: {valid_args_result => "SomethingCompletelyDifferent"}) + name = namer.generate_name_for(format_name, **valid_args) + + expect(name).to eq valid_args_result + end + + it "raises a clear error when an overridden format would produce an invalid GraphQL name" do + expect_instantiation_config_error( + %(1. The #{format_name} format "3#{default_format}" does not produce a valid GraphQL type name. #{GRAPHQL_NAME_VALIDITY_DESCRIPTION}), + format_name => "3#{default_format}" + ) + end + + it "raises a clear error when its override omits a placeholder" do + expect_instantiation_config_error( + %(1. The #{format_name} format "#{format_name}Alt" is missing required placeholders: #{valid_args.keys.join(", ")}. Example valid format: "#{default_format}".), + format_name => "#{format_name}Alt" + ) + end + + it "raises a clear error when its includes an unknown placeholder" do + expect_instantiation_config_error( + %(1. The #{format_name} format "#{default_format}Alt%{extra1}%{extra2}" has excess placeholders: extra1, extra2. Example valid format: "#{default_format}".), + format_name => "#{default_format}Alt%{extra1}%{extra2}" + ) + end + + it "raises a clear error when asked to generate a name with a misspelled argument" do + (first_key, first_value), *rest_args = valid_args.to_a + args = {"#{first_key}2": first_value, **rest_args.to_h} + + expect_generate_name_error(format_name, "omits required key(s): #{first_key}", **args) + end + + it "raises a clear error when asked to generate a name with a missing argument" do + (first_key, _), *rest_args = valid_args.to_a + + expect_generate_name_error(format_name, "omits required key(s): #{first_key}", **rest_args.to_h) + end + + it "raises a clear error when asked to generate a name with an extra argument" do + args = {extra_arg: "Foo", **valid_args} + + expect_generate_name_error(format_name, "contains extra key(s): extra_arg", **args) + end + + if TypeNamer::REQUIRED_PLACEHOLDERS.fetch(format_name) == [:base] + it "extracts the base name from an example name in the default format" do + namer = TypeNamer.new + name = namer.generate_name_for(format_name, base: "SomeTypeName") + + expect(namer.extract_base_from(name, format: format_name)).to eq "SomeTypeName" + end + + it "extracts the base name from an example name in an overridden format" do + namer = TypeNamer.new(format_overrides: {format_name => "Pre#{default_format}Post"}) + name = namer.generate_name_for(format_name, base: "SomeTypeName") + + expect(namer.extract_base_from(name, format: format_name)).to eq "SomeTypeName" + end + + it "returns `nil` if the provided name does not match the format" do + namer = TypeNamer.new + + expect(namer.extract_base_from("", format: format_name)).to eq nil + expect(namer.extract_base_from("Invalid", format: format_name)).to eq nil + end + else + it "raises a clear error indicating this format does not support base extraction since it's format is not limited to a single `base` parameter" do + namer = TypeNamer.new + name = namer.generate_name_for(format_name, **valid_args) + + expect { + namer.extract_base_from(name, format: format_name) + }.to raise_error Errors::InvalidArgumentValueError, "The `#{format_name}` format does not support base extraction." + + expect { + namer.extract_base_from("", format: format_name) + }.to raise_error Errors::InvalidArgumentValueError, "The `#{format_name}` format does not support base extraction." + end + end + end + + [ + :AggregatedValues, + :GroupedBy, + :Aggregation, + :Connection, + :Edge, + :FilterInput, + :ListFilterInput, + :FieldsListFilterInput, + :ListElementFilterInput, + :SortOrder + ].each do |simple_format_name| + describe "the #{simple_format_name} format" do + include_examples "a TypeNamer format", { + format_name: simple_format_name, + valid_args: {base: "Widget"}, + valid_args_result: "Widget#{simple_format_name}" + } + end + end + + describe "the InputEnum format" do + include_examples "a TypeNamer format", { + format_name: :InputEnum, + valid_args: {base: "Month"}, + valid_args_result: "MonthInput" + } + end + + describe "the SubAggregation format" do + include_examples "a TypeNamer format", { + format_name: :SubAggregation, + valid_args: {parent_types: "Team", base: "Player"}, + valid_args_result: "TeamPlayerSubAggregation" + } + end + + describe "the SubAggregations format" do + include_examples "a TypeNamer format", { + format_name: :SubAggregations, + valid_args: {parent_agg_type: "TeamAggregation", field_path: "Player"}, + valid_args_result: "TeamAggregationPlayerSubAggregations" + } + end + + describe "an unknown format" do + it "raises a clear error when asked to generate a name for an unknown format" do + expect { + generate_name_for(:FiltreInput, base: "Widget") + }.to raise_error Errors::ConfigError, "Unknown format name: :FiltreInput. Possible alternatives: :FilterInput." + end + + it "raises a clear error when instantiated with an unknown format override" do + expect_instantiation_config_error( + "Unknown format name: :FiltreInput. Possible alternatives: :FilterInput.", + FiltreInput: "%{base}Filtre" + ) + end + end + + describe "properties of an overall schema (using the full test schema for completeness)" do + attr_reader :schema + before(:context) do + @schema = ::GraphQL::Schema.from_definition(CommonSpecHelpers.stock_schema_artifacts(for_context: :graphql).graphql_schema_string) + end + + let(:input_types) { schema.types.values.select { |type| type.kind.input_object? } } + + it "names all input object types with an `Input` suffix" do + expect(input_types.size).to be > 10 + expect(input_types.map(&:graphql_name).grep_v(/Input\z/)).to be_empty + end + + it "uses an `Input`-suffixed enum type for all enum arguments of an input object type" do + input_object_arguments = input_types.flat_map do |input_type| + input_type.arguments.values + end + + expect_input_suffix_on_all_enum_arg_types(input_object_arguments) + end + + it "uses an `Input`-suffixed enum type for all enum arguments of fields of all return types" do + return_types_with_fields = schema.types.values.select { |type| type.kind.fields? } + return_type_field_arguments = return_types_with_fields.flat_map do |type| + type.fields.values.flat_map { |field| field.arguments.values } + end + + expect_input_suffix_on_all_enum_arg_types(return_type_field_arguments) + end + + def expect_input_suffix_on_all_enum_arg_types(arguments) + enum_args = arguments.select { |arg| arg.type.unwrap.kind.enum? } + expect(enum_args.size).to be > 5 # verify we have some, and not just 1 or 2... + + arg_def_sdls = enum_args.map { |arg| "#{arg.path}: #{arg.type.to_type_signature}" } + expect(arg_def_sdls.grep_v(/Input[!\]]*\z/)).to be_empty + end + end + + def generate_name_for(format, overrides: {}, **args) + namer = TypeNamer.new(format_overrides: overrides) + namer.generate_name_for(format, **args) + end + + def expect_instantiation_config_error(*problems, **overrides) + expect { + TypeNamer.new(format_overrides: overrides) + }.to raise_error Errors::ConfigError, a_string_including( + "Provided derived type name formats have #{problems.size} problem(s)", + *problems + ) + end + + def expect_generate_name_error(format_name, error_suffix, **args) + default_format = TypeNamer::DEFAULT_FORMATS.fetch(format_name) + + expect { + generate_name_for(format_name, **args) + }.to raise_error( + Errors::ConfigError, + %(The arguments (#{args.inspect}) provided for `#{format_name}` format ("#{default_format}") #{error_suffix}.) + ) + end + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/schema_elements/type_reference_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/schema_elements/type_reference_spec.rb new file mode 100644 index 00000000..e7401ee2 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/schema_elements/type_reference_spec.rb @@ -0,0 +1,578 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" +require "elastic_graph/schema_definition/api" +require "elastic_graph/spec_support/schema_definition_helpers" +require "graphql" + +module ElasticGraph + module SchemaDefinition + module SchemaElements + RSpec.describe TypeReference do + let(:type_name_overrides) do + { + "RingAggregatedValues" => "RingValuesAggregated", + "RingSortOrder" => "RingOrderSort", + "RingSortOrderInput" => "RingOrderSortInput" + } + end + + let(:state) do + schema_elements = SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: "snake_case") + API.new(schema_elements, true, type_name_overrides: type_name_overrides).state + end + + describe "#unwrap_list" do + it "unwraps a type that has an outer list wrapping" do + type = type_ref("[Int]") + + expect(type.unwrap_list.name).to eq "Int" + end + + it "unwraps both non-null and list when a type has both" do + type = type_ref("[Int]!") + + expect(type.unwrap_list.name).to eq "Int" + end + + it "only unwraps a single list wrapping" do + type = type_ref("[[Int]]") + + expect(type.unwrap_list.name).to eq "[Int]" + end + + it "leaves an inner non-nullability intact" do + type = type_ref("[Int!]") + + expect(type.unwrap_list.name).to eq "Int!" + end + end + + describe "#fully_unwrapped" do + it "removes all wrappings no matter how many there are" do + expect(type_ref("Int").fully_unwrapped.name).to eq "Int" + expect(type_ref("Int!").fully_unwrapped.name).to eq "Int" + expect(type_ref("[Int]").fully_unwrapped.name).to eq "Int" + expect(type_ref("[Int!]").fully_unwrapped.name).to eq "Int" + expect(type_ref("[Int!]!").fully_unwrapped.name).to eq "Int" + expect(type_ref("[[Int!]!]").fully_unwrapped.name).to eq "Int" + expect(type_ref("[[[[Int!]]]]").fully_unwrapped.name).to eq "Int" + end + end + + describe "#scalar_type_needing_grouped_by_object?" do + let(:type_name_overrides) do + { + "Date" => "MyDate", + "DateTime" => "MyDateTime", + "LocalTime" => "MyLocalTime", + "Widget" => "MyWidget" + } + end + + it "returns true for Date and DateTime types only" do + expect(type_ref("Date").scalar_type_needing_grouped_by_object?).to eq(true) + expect(type_ref("DateTime").scalar_type_needing_grouped_by_object?).to eq(true) + + expect(type_ref("LocalTime").scalar_type_needing_grouped_by_object?).to eq(false) + expect(type_ref("Widget").scalar_type_needing_grouped_by_object?).to eq(false) + end + + it "returns true for Date and DateTime types only when overridden" do + expect(type_ref("MyDate").scalar_type_needing_grouped_by_object?).to eq(true) + expect(type_ref("MyDateTime").scalar_type_needing_grouped_by_object?).to eq(true) + + expect(type_ref("MyLocalTime").scalar_type_needing_grouped_by_object?).to eq(false) + expect(type_ref("MyWidget").scalar_type_needing_grouped_by_object?).to eq(false) + end + end + + describe "#with_reverted_override" do + let(:type_name_overrides) do + { + "Date" => "MyDate", + "DateTime" => "MyDateTime", + "Widget" => "MyWidget" + } + end + + it "returns a new TypeReference with the override reverted" do + expect(type_ref("MyDate").with_reverted_override.name).to eq "Date" + expect(type_ref("MyDateTime").with_reverted_override.name).to eq "DateTime" + expect(type_ref("MyWidget").with_reverted_override.name).to eq "Widget" + end + + it "returns the type if no overrides are present" do + expect(type_ref("Date").with_reverted_override.name).to eq "Date" + expect(type_ref("DateTime").with_reverted_override.name).to eq "DateTime" + expect(type_ref("Widget").with_reverted_override.name).to eq "Widget" + end + end + + shared_context "enforce all STATIC_FORMAT_NAME_BY_CATEGORY entries are tested" do + before(:context) do + @tested_derived_type_formats = ::Set.new + end + + after(:context) do + untested_formats = TypeReference::STATIC_FORMAT_NAME_BY_CATEGORY.values.to_set - @tested_derived_type_formats + expect(untested_formats).to be_empty, "Expected all `TypeReference::STATIC_FORMAT_NAME_BY_CATEGORY` to be covered by the tests " \ + "in the `#{self.class.metadata[:description]}` context, but #{untested_formats.size} were not: #{untested_formats.to_a.inspect}." + end + end + + describe "determining the kind of type" do + include_context "enforce all STATIC_FORMAT_NAME_BY_CATEGORY entries are tested" + + %w[ + AggregatedValues GroupedBy Aggregation AggregationConnection AggregationEdge + Connection Edge FilterInput ListFilterInput FieldsListFilterInput ListElementFilterInput + SubAggregation AggregationSubAggregations + ].each do |suffix| + it "considers a `*#{suffix}` type to be an object instead of a leaf" do + expect_to_be_object suffix + end + end + + it "considers a `*SortOrder` type to be an enum leaf instead of an object" do + expect_to_be_leaf "SortOrder", enum: true, format: :SortOrder + end + + it "considers a `*SortOrderInput` type to be an enum leaf instead of an object" do + expect_to_be_leaf "SortOrderInput", enum: true, format: :SortOrder + end + + it "considers a scalr type to be a scalar leaf instead of an object" do + type = type_ref("JsonSafeLong") + expect(type.object?).to be false + expect(type.leaf?).to be true + expect(type.enum?).to be false + + type = type_ref("JsonSafeLong!") + expect(type.object?).to be false + expect(type.leaf?).to be true + expect(type.enum?).to be false + end + + context "when dealing with a type that has been renamed" do + it "correctly detects a renamed object type" do + type = type_ref("RingValuesAggregated") + expect(type_name_overrides.values).to include(type.name) + + expect(type.object?).to be true + expect(type.leaf?).to be false + expect(type.enum?).to be false + end + + it "correctly detects a renamed enum type" do + type = type_ref("RingOrderSort") + expect(type_name_overrides.values).to include(type.name) + + expect(type.object?).to be false + expect(type.leaf?).to be true + expect(type.enum?).to be true + + type = type_ref("RingOrderSortInput") + expect(type_name_overrides.values).to include(type.name) + + expect(type.object?).to be false + expect(type.leaf?).to be true + expect(type.enum?).to be true + end + end + + it "is unsure about an `*Input` type since it could be an enum or an object" do + type_ref = type_ref_for("Input", format: :InputEnum) + + expect_cannot_resolve_failures_for(type_ref) + end + + it "raises an error if it cannot figure it out due to the type not being defined yet" do + type = type_ref("Unknown") + + expect_cannot_resolve_failures_for(type) + end + + def expect_to_be_object(suffix, format: suffix.to_sym) + type = type_ref_for(suffix, format: format) + expect(type.object?).to be true + expect(type.leaf?).to be false + + type = type_ref_for("#{suffix}!", format: format) + expect(type.object?).to be true + expect(type.leaf?).to be false + expect(type.enum?).to be false + end + + def expect_to_be_leaf(suffix, enum:, format: suffix.to_sym) + type = type_ref_for(suffix, format: format) + expect(type.object?).to be false + expect(type.leaf?).to be true + expect(type.enum?).to be enum + + type = type_ref_for("#{suffix}!", format: format) + expect(type.object?).to be false + expect(type.leaf?).to be true + expect(type.enum?).to be enum + end + + def type_ref_for(suffix, format:) + @tested_derived_type_formats << format + type_ref("MyType#{suffix}") + end + + def expect_cannot_resolve_failures_for(type) + expect { type.object? }.to raise_error a_string_including("Type `#{type.name}` cannot be resolved") + expect { type.leaf? }.to raise_error a_string_including("Type `#{type.name}` cannot be resolved") + expect { type.enum? }.to raise_error a_string_including("Type `#{type.name}` cannot be resolved") + end + end + + describe "derived type names" do + include_context "SchemaDefinitionHelpers" + attr_reader :state, :defined_graphql_types + + before(:context) do + result = define_widget_and_color_schema + @result = result + @state = result.state + schema_string_with_orphaned_types_dropped = ::GraphQL::Schema.from_definition(result.graphql_schema_string).to_definition + @defined_graphql_types = ::GraphQL::Schema.from_definition(schema_string_with_orphaned_types_dropped).types.keys.to_set + end + + describe "#as_parent_aggregation" do + it "raises an exception when `parent_doc_types` is passed as an empty array" do + type = type_ref("Widget") + expect { + type.as_parent_aggregation(parent_doc_types: []) + }.to raise_error Errors::SchemaError, a_string_including("`parent_doc_types` must not be empty") + end + end + + describe "#as_sub_aggregation" do + it "preserves non-nullability when converting to the derived type" do + type_ref = state.type_ref("Widget!") + + expect(type_ref.as_sub_aggregation(parent_doc_types: ["Manufacturer"]).name).to eq "ManufacturerWidgetSubAggregation!" + end + + it "preserves a list wrapping when converting to the derived type" do + type_ref = state.type_ref("[Widget]") + + expect(type_ref.as_sub_aggregation(parent_doc_types: ["Manufacturer"]).name).to eq "[ManufacturerWidgetSubAggregation]" + end + + it "preserves multiple layers of list and non-null wrappings when converting to the derived type" do + type_ref = state.type_ref("[[Widget!]]!") + + expect(type_ref.as_sub_aggregation(parent_doc_types: ["Manufacturer"]).name).to eq "[[ManufacturerWidgetSubAggregation!]]!" + end + end + + describe "#as_aggregation_sub_aggregations" do + it "preserves non-nullability when converting to the derived type" do + type_ref = state.type_ref("Widget!") + + expect(type_ref.as_aggregation_sub_aggregations.name).to eq "WidgetAggregationSubAggregations!" + end + + it "preserves a list wrapping when converting to the derived type" do + type_ref = state.type_ref("[Widget]") + + expect(type_ref.as_aggregation_sub_aggregations.name).to eq "[WidgetAggregationSubAggregations]" + end + + it "preserves multiple layers of list and non-null wrappings when converting to the derived type" do + type_ref = state.type_ref("[[Widget!]]!") + + expect(type_ref.as_aggregation_sub_aggregations.name).to eq "[[WidgetAggregationSubAggregations!]]!" + end + end + + shared_examples_for "a static derived name format" do |source_type:, category:, suffix:, offer_predicate:, suffix_on_defined_type: suffix| + let(:type_ref) { state.type_ref(source_type) } + let(:expected_type_name) { source_type + suffix } + + it "generates the expected type name from `#as_static_derived_type(:#{category})`" do + derived_type_ref = type_ref.as_static_derived_type(category) + + expect(derived_type_ref).to be_a(TypeReference) + expect(derived_type_ref.name).to eq(expected_type_name) + end + + it "offers a `#as_#{category}` method that returns a name like `{source_type}#{suffix}`" do + derived_type_ref = type_ref.public_send(:"as_#{category}") + + expect(derived_type_ref).to be_a(TypeReference) + expect(derived_type_ref.name).to eq(expected_type_name) + end + + it "preserves non-nullability when converting to the derived type" do + type_ref = state.type_ref("#{source_type}!") + + expect(type_ref.as_static_derived_type(category).name).to eq("#{expected_type_name}!") + expect(type_ref.public_send(:"as_#{category}").name).to eq("#{expected_type_name}!") + end + + it "preserves a list wrapping when converting to the derived type" do + type_ref = state.type_ref("[#{source_type}]") + + expect(type_ref.as_static_derived_type(category).name).to eq("[#{expected_type_name}]") + expect(type_ref.public_send(:"as_#{category}").name).to eq("[#{expected_type_name}]") + end + + it "preserves multiple layers of list and non-null wrappings when converting to the derived type" do + type_ref = state.type_ref("[[#{source_type}!]]!") + + expect(type_ref.as_static_derived_type(category).name).to eq("[[#{expected_type_name}!]]!") + expect(type_ref.public_send(:"as_#{category}").name).to eq("[[#{expected_type_name}!]]!") + end + + it "generates a type named `{source_type}#{suffix_on_defined_type}` in the GraphQL schema string" do + expect(defined_graphql_types).to include(source_type + suffix_on_defined_type) + end + + if offer_predicate + it "indicates it is in the `:#{category}` via `##{category}?`" do + derived_type_ref = type_ref.as_static_derived_type(category) + + expect(derived_type_ref.public_send(:"#{category}?")).to be true + expect(type_ref.public_send(:"#{category}?")).to be false + end + else + it "does not offer a `##{category}?` predicate" do + expect(type_ref).not_to respond_to(:"#{category}?") + end + end + end + + TypeReference::STATIC_FORMAT_NAME_BY_CATEGORY.except(:sort_order, :input_enum, :list_filter_input, :list_element_filter_input).each do |category, format_name| + describe "the `:#{category}` derived type" do + include_examples "a static derived name format", + source_type: "Widget", + category: category, + suffix: format_name.to_s, + offer_predicate: false + end + end + + describe "the `:list_filter_input` derived type" do + include_examples "a static derived name format", + source_type: "Widget", + category: :list_filter_input, + suffix: "ListFilterInput", + offer_predicate: true + end + + describe "the `:list_element_filter_input` derived type" do + include_examples "a static derived name format", + source_type: "Color", + category: :list_element_filter_input, + suffix: "ListElementFilterInput", + offer_predicate: true + end + + describe "the `:input_enum` derived type" do + include_examples "a static derived name format", + category: :input_enum, + source_type: "Color", + suffix: "Input", + offer_predicate: false + end + + describe "the `:sort_order` derived type" do + include_examples "a static derived name format", + category: :sort_order, + source_type: "Widget", + suffix: "SortOrder", + suffix_on_defined_type: "SortOrderInput", + offer_predicate: false + end + + describe "#to_final_form" do + it "returns the existing `TypeReference` instance if no customizations apply to it" do + ref = type_ref("MyTypeGroupedBy") + + expect(ref.to_final_form).to be(ref) + end + + it "uses the configured name override if one is configured for this type" do + ref = type_ref("MyTypeGroupedBy", type_name_overrides: {"MyTypeGroupedBy" => "MyTypeGroupedByAlt"}) + + expect(ref.to_final_form.name).to eq "MyTypeGroupedByAlt" + end + + it "preserves wrappings" do + ref = type_ref("[MyTypeGroupedBy!]", type_name_overrides: {"MyTypeGroupedBy" => "MyTypeGroupedByAlt"}) + + expect(ref.to_final_form.name).to eq "[MyTypeGroupedByAlt!]" + end + + it "converts to the input enum form if the type is an enum type and `as_input: true` was passed" do + ref = type_ref("[WidgetSortingOrder!]", derived_type_name_formats: {InputEnum: "%{base}Inp", SortOrder: "%{base}SortingOrder"}) + + expect(ref.to_final_form(as_input: true).name).to eq "[WidgetSortingOrderInp!]" + expect(ref.to_final_form.name).to eq "[WidgetSortingOrder!]" + end + + it "applies a name override to the converted input type when one is configured" do + ref = type_ref( + "[WidgetSortingOrder!]", + derived_type_name_formats: {InputEnum: "%{base}Inp", SortOrder: "%{base}SortingOrder"}, + type_name_overrides: {WidgetSortingOrderInp: "InpSortingWidgetOrder"} + ) + + expect(ref.to_final_form(as_input: true).name).to eq "[InpSortingWidgetOrder!]" + end + + it "raises if it cannot determine if it is an enum with `as_input: true`" do + ref = type_ref("SomeType") + + expect(ref.to_final_form(as_input: false).name).to eq "SomeType" + expect(ref.to_final_form.name).to eq "SomeType" + expect { ref.to_final_form(as_input: true) }.to raise_error a_string_including("Type `SomeType` cannot be resolved") + end + + it "uses a configured name override, regardless of `as_input`, when a type isn't registered under the old name" do + api = api(type_name_overrides: {"OldName" => "NewName"}) + api.object_type("NewName") + + ref = api.state.type_ref("OldName") + expect(ref.to_final_form(as_input: false).name).to eq "NewName" + expect(ref.to_final_form(as_input: true).name).to eq "NewName" + end + + def api(type_name_overrides: {}, derived_type_name_formats: {}) + API.new( + SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: "snake_case"), + true, + type_name_overrides: type_name_overrides, + derived_type_name_formats: derived_type_name_formats + ) + end + + def type_ref(name, type_name_overrides: {}, derived_type_name_formats: {}) + api(type_name_overrides: type_name_overrides, derived_type_name_formats: derived_type_name_formats) + .state + .type_ref(name) + end + end + + context "when we change the derived type formats" do + include_context "enforce all STATIC_FORMAT_NAME_BY_CATEGORY entries are tested" + + before(:context) do + derived_type_name_formats = TypeNamer::DEFAULT_FORMATS.transform_values do |format| + mangle_format(format) + end + + result = define_widget_and_color_schema(derived_type_name_formats: derived_type_name_formats) + @state = result.state + @defined_graphql_types = ::GraphQL::Schema.from_definition(result.graphql_schema_string).types.keys.sort.to_set + end + + shared_examples_for "a static format" do |category:, format:, suffix:, source_type:| + # Note: the test here doesn't really exercise `TypeReference` -- rather, it validates that all the other parts of + # `elasticgraph-schema_definition` use `TypeReference` to generate these derived type names and don't hardcode + # their own derived type suffixes. + it "generates the `:#{category}` type using an alternate format rather than the standard one" do + @tested_derived_type_formats << format + expect(defined_graphql_types).to include("#{source_type}#{mangle_format(suffix)}").and exclude("#{source_type}#{suffix}") + end + end + + TypeReference::STATIC_FORMAT_NAME_BY_CATEGORY.except(:list_element_filter_input, :sort_order, :input_enum).each do |category, format_name| + include_examples "a static format", + category: category, + format: format_name, + suffix: format_name.to_s, + source_type: "Widget" + end + + include_examples "a static format", + category: :list_element_filter_input, + format: :ListElementFilterInput, + suffix: "ListElementFilterInput", + # The `ListElementFilterInput` type is only generated for a leaf type like `Color`, not for object types like `Widget`. + source_type: "Color" + + include_examples "a static format", + category: :input_enum, + format: :InputEnum, + suffix: "Input", + source_type: "Color" + + include_examples "a static format", + category: :sort_order, + format: :SortOrder, + suffix: "SortOrderInput", + source_type: "Widget" + + def mangle_format(format) + # Deal with compound formats... + format = format.sub("Aggregation", "Aggregation2") + format = format.sub("SortOrder", "SortOrder2") + + format.end_with?("2") ? format : "#{format}2" + end + end + + def define_widget_and_color_schema(derived_type_name_formats: {}) + schema_elements = SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: "snake_case") + define_schema_with_schema_elements(schema_elements, derived_type_name_formats: derived_type_name_formats) do |api| + api.enum_type "Color" do |t| + t.values "RED", "GREEN", "BLUE" + end + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.field "name", "String" + t.field "colors", "[Color!]!" + t.paginated_collection_field "colors_paginated", "Color" + t.field "color", "Color" + t.field "cost", "Int" + # Some derived types are only generated when a type has nested fields. + t.field "is_nested", "[IsNested!]!" do |f| + f.mapping type: "nested" + end + t.index "widgets" + end + + api.object_type "IsNested" do |t| + t.field "something", "String" + end + + api.object_type "HasNested" do |t| + t.field "id", "ID" + t.field "string", "String" + t.field "int", "Int" + + # Some derived types are only generated when a type is used by a nested field. + t.field "widgets_nested", "[Widget!]!" do |f| + f.mapping type: "nested" + end + + # Some derived types are only generated when a type is used by a a list of `object` field. + t.field "widgets_object", "[Widget!]!" do |f| + f.mapping type: "object" + end + t.index "has_nested" + end + end + end + end + + def type_ref(name) + state.type_ref(name) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/scripting/file_system_repository_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/scripting/file_system_repository_spec.rb new file mode 100644 index 00000000..b6c6ab1b --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/scripting/file_system_repository_spec.rb @@ -0,0 +1,140 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_definition/scripting/file_system_repository" + +module ElasticGraph + module SchemaDefinition + module Scripting + RSpec.describe FileSystemRepository do + it "loads scripts in multiple supported languages from a directory, treating sub-dirs as script contexts" do + repo = repo_for_fixture_dir("multiple_contexts_and_languages") + + expect(repo.scripts).to contain_exactly( + by_age = Script.new( + name: "by_age", + source: "// Painless code would go here.", + language: "painless", + context: "filter" + ), + using_math = Script.new( + name: "UsingMath", + source: "// Java code would go here.", + language: "java", + context: "filter" + ), + by_edit_distance = Script.new( + name: "by_edit_distance", + source: "// Lucene expression syntax would go here.", + language: "expression", + context: "score" + ), + template1 = Script.new( + name: "template1", + source: "{{! mustache code would go here}}", + language: "mustache", + context: "update" + ) + ) + + expect(repo.script_ids_by_scoped_name).to eq({ + "filter/by_age" => by_age.id, + "filter/UsingMath" => using_math.id, + "score/by_edit_distance" => by_edit_distance.id, + "update/template1" => template1.id + }) + end + + it "memoizes the script state to avoid re-doing the same I/O over again" do + repo = repo_for_fixture_dir("multiple_contexts_and_languages") + + scripts = repo.scripts + script_ids_by_scoped_name = repo.script_ids_by_scoped_name + + expect(repo.scripts).to be(scripts) + expect(repo.script_ids_by_scoped_name).to be(script_ids_by_scoped_name) + end + + it "provides a clear error when a file has an extension it doesn't support" do + repo = repo_for_fixture_dir("unsupported_language") + + expect { + repo.scripts + }.to raise_error Errors::InvalidScriptDirectoryError, a_string_including("unrecognized file extension", ".rb") + end + + it "provides a clear error when the given directory has script files not nested in a context sub-dir" do + repo = repo_for_fixture_dir("unnested_script_files") + + expect { + repo.scripts + }.to raise_error Errors::InvalidScriptDirectoryError, a_string_including("not a context directory as expected", "by_age.painless") + end + + it "provides a clear error when the given directory has script files not nested in a context sub-dir" do + repo = repo_for_fixture_dir("double_nested_script_files") + + expect { + repo.scripts + }.to raise_error Errors::InvalidScriptDirectoryError, a_string_including("extra directory nesting", "/filter/filter") + end + + it "provides a clear error when multiple scripts exist with the same name in the same context (but with a different language extension)" do + repo = repo_for_fixture_dir("duplicate_name_with_different_lang") + + expect { + repo.scripts + }.to raise_error Errors::InvalidScriptDirectoryError, a_string_including("multiple scripts with the same scoped name", "filter/by_age") + end + + it "allows a script name to be re-used for a different context" do + repo = repo_for_fixture_dir("duplicate_name_in_different_contexts") + + expect(repo.scripts).to contain_exactly( + filter_by_age = Script.new( + name: "by_age", + source: "// Painless code would go here.", + language: "painless", + context: "filter" + ), + update_by_age = Script.new( + name: "by_age", + source: "// Painless code would go here.", + language: "painless", + context: "update" + ) + ) + + expect(repo.script_ids_by_scoped_name).to eq({ + "filter/by_age" => filter_by_age.id, + "update/by_age" => update_by_age.id + }) + end + + it "winds up with a different `id` for two scripts that are the same except for the `context`" do + repo = repo_for_fixture_dir("duplicate_name_in_different_contexts") + + expect(repo.scripts.map(&:source)).to all eq("// Painless code would go here.") + expect(repo.scripts.map(&:name)).to all eq("by_age") + expect(repo.scripts.map(&:language)).to all eq("painless") + expect(repo.scripts.map(&:context)).to contain_exactly("filter", "update") + + expect(repo.scripts.first.id).not_to eq(repo.scripts.last.id) + expect(repo.scripts.map(&:id)).to contain_exactly( + a_string_starting_with("update_by_age_"), + a_string_starting_with("filter_by_age_") + ) + end + + def repo_for_fixture_dir(dir_name) + FileSystemRepository.new(::File.join(FIXTURE_DIR, dir_name)) + end + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/static_scripts_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/static_scripts_spec.rb new file mode 100644 index 00000000..96dd47d9 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/static_scripts_spec.rb @@ -0,0 +1,34 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/schema_definition/results" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "Static scripts" do + describe "the `INDEX_DATA_UPDATE_SCRIPT_ID` constant" do + it "matches the current id of our static script" do + expected_id = Results::STATIC_SCRIPT_REPO.script_ids_by_scoped_name.fetch("update/index_data") + + expect(INDEX_DATA_UPDATE_SCRIPT_ID).to eq(expected_id) + end + end + + describe "the `UPDATE_WAS_NOOP_MESSAGE_PREAMBLE` constant" do + it "is used by the `index_data` script at the start of an exception message to indicate a no-op" do + script = Results::STATIC_SCRIPT_REPO.scripts.find { |s| s.scoped_name == "update/index_data" } + + # We care about the "was a no-op" exception starting with UPDATE_WAS_NOOP_MESSAGE_PREAMBLE because + # our indexing logic detects this case by looking for a failure with that at the start of it. + expect(script.source).to include("throw new IllegalArgumentException(\"#{UPDATE_WAS_NOOP_MESSAGE_PREAMBLE}") + end + end + end + end +end diff --git a/elasticgraph-support/.rspec b/elasticgraph-support/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph-support/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-support/.yardopts b/elasticgraph-support/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph-support/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-support/Gemfile b/elasticgraph-support/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph-support/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-support/LICENSE.txt b/elasticgraph-support/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph-support/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-support/README.md b/elasticgraph-support/README.md new file mode 100644 index 00000000..7378410b --- /dev/null +++ b/elasticgraph-support/README.md @@ -0,0 +1,7 @@ +# ElasticGraph::Support + +This gem provides support utilities for the rest of the ElasticGraph gems. As +such, it is not intended to provide any public APIs for ElasticGraph users. + +Importantly, it is intended to have as few dependencies as possible: it currently +only depends on `logger` (which originated in the Ruby standard library). diff --git a/elasticgraph-support/elasticgraph-support.gemspec b/elasticgraph-support/elasticgraph-support.gemspec new file mode 100644 index 00000000..5d962848 --- /dev/null +++ b/elasticgraph-support/elasticgraph-support.gemspec @@ -0,0 +1,23 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :core) do |spec, eg_version| + spec.summary = "ElasticGraph gem providing support utilities to the other ElasticGraph gems." + + # Ruby 3.4 warns about using `logger` being moved out of the standard library, and in Ruby 3.5 + # it'll no longer be available without declaring a dependency. + # + # Note: Logger 1.6.0 has an issue that impacts our ElasticGraph lambdas, but 1.6.1 avoids the issue: + # https://github.com/aws/aws-lambda-ruby-runtime-interface-client/issues/33 + spec.add_dependency "logger", "~> 1.6", ">= 1.6.1" + + spec.add_development_dependency "faraday", "~> 2.12" + spec.add_development_dependency "rake", "~> 13.2" +end diff --git a/elasticgraph-support/lib/elastic_graph/constants.rb b/elasticgraph-support/lib/elastic_graph/constants.rb new file mode 100644 index 00000000..a61f7aac --- /dev/null +++ b/elasticgraph-support/lib/elastic_graph/constants.rb @@ -0,0 +1,259 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# Root namespace for all ElasticGraph code. +module ElasticGraph + # Here we enumerate constants that are used from multiple places in the code. + + # The datastore date format used by ElasticGraph. Matches ISO-8601/RFC-3339. + # See https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html#built-in-date-formats + # @private + DATASTORE_DATE_FORMAT = "strict_date" + + # The datastore date time format used by ElasticGraph. Matches ISO-8601/RFC-3339. + # See https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html#built-in-date-formats + # @private + DATASTORE_DATE_TIME_FORMAT = "strict_date_time" + + # HTTP header that ElasticGraph HTTP implementations (e.g. elasticgraph-rack, elasticgraph-lambda) + # look at to determine a client-specified request timeout. + # @private + TIMEOUT_MS_HEADER = "ElasticGraph-Request-Timeout-Ms" + + # Min/max values for the `Int` type. + # Based on the GraphQL spec: + # + # > If the integer internal value represents a value less than -2^31 or greater + # > than or equal to 2^31, a field error should be raised. + # + # (from http://spec.graphql.org/June2018/#sec-Int) + # @private + INT_MIN = -(2**31).to_int + # @private + INT_MAX = -INT_MIN - 1 + + # Min/max values for our `JsonSafeLong` type. + # Based on https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER + # @private + JSON_SAFE_LONG_MIN = -((2**53) - 1).to_int + # @private + JSON_SAFE_LONG_MAX = -JSON_SAFE_LONG_MIN + + # Min/max values for our `LongString` type. + # This range is derived from the Elasticsearch docs on its longs: + # > A signed 64-bit integer with a minimum value of -2^63 and a maximum value of 2^63 - 1. + # (from https://www.elastic.co/guide/en/elasticsearch/reference/current/number.html) + # @private + LONG_STRING_MIN = -(2**63).to_int + # @private + LONG_STRING_MAX = -LONG_STRING_MIN - 1 + + # When indexing large string values into the datastore, we've observed errors like: + # + # > bytes can be at most 32766 in length + # + # This is also documented on the Elasticsearch docs site, under "Choosing a keyword family field type": + # https://www.elastic.co/guide/en/elasticsearch/reference/8.2/keyword.html#wildcard-field-type + # + # Note that it's a byte limit, but JSON schema's maxLength is a limit on the number of characters. + # UTF8 uses up to 4 bytes per character so to guard against a maliciously crafted payload, we limit + # the length to a quarter of 32766. + # @private + DEFAULT_MAX_KEYWORD_LENGTH = 32766 / 4 + + # Strings indexed as `text` can be much larger than `keyword` fields. In fact, there's no limitation + # on the `text` length, except for the overall size of the HTTP request body when we attempt to index + # a `text` field. By default it's limited to 100MB via the `http.max_content_length` setting: + # + # https://www.elastic.co/guide/en/elasticsearch/reference/8.11/modules-network.html#http-settings + # + # Note: there's no guarantee that `text` values shorter than this will succeed when indexing them--it + # depends on how many other fields and documents are included in the indexing payload, since the limit + # is on the overall payload size, and not on the size of one field. Given that, there's not really a + # discrete value we can use for the max length that guarantees successful indexing. But we know that + # values larger than this will fail, so this is the limit we use. + # @private + DEFAULT_MAX_TEXT_LENGTH = 100 * (2**20).to_int + + # The name of the JSON schema definition for the ElasticGraph event envelope. + # @private + EVENT_ENVELOPE_JSON_SCHEMA_NAME = "ElasticGraphEventEnvelope" + + # For some queries, we wind up needing a pagination cursor for a collection + # that will only ever contain a single value (and has no "key" to speak of + # to encode into a cursor). In those contexts, we'll use this as the cursor value. + # Ideally, we want this to be a value that could never be produced by our normal + # cursor encoding logic. This cursor is encoded from data that includes a UUID, + # which we can trust is unique. + # @private + SINGLETON_CURSOR = "eyJ1dWlkIjoiZGNhMDJkMjAtYmFlZS00ZWU5LWEwMjctZmVlY2UwYTZkZTNhIn0=" + + # Schema artifact file names. + # @private + GRAPHQL_SCHEMA_FILE = "schema.graphql" + # @private + JSON_SCHEMAS_FILE = "json_schemas.yaml" + # @private + DATASTORE_CONFIG_FILE = "datastore_config.yaml" + # @private + RUNTIME_METADATA_FILE = "runtime_metadata.yaml" + + # Name for directory that contains versioned json_schemas files. + # @private + JSON_SCHEMAS_BY_VERSION_DIRECTORY = "json_schemas_by_version" + # Name for field in json schemas files that represents schema "version". + # @private + JSON_SCHEMA_VERSION_KEY = "json_schema_version" + + # String that goes in the middle of a rollover index name, used to mark it as a rollover + # index (and split on to parse a rollover index name). + # @private + ROLLOVER_INDEX_INFIX_MARKER = "_rollover__" + + # @private + DERIVED_INDEX_FAILURE_MESSAGE_PREAMBLE = "Derived index update failed due to bad input data" + + # The current id of our static `index_data` update script. Verified by a test so you can count + # on it being accurate. We expose this as a constant so that we can detect this specific script + # in environments where we can't count on `elasticgraph-schema_definition` (where the script is + # defined) being available, since that gem is usually only used in development. + # + # Note: this constant is automatically kept up-to-date by our `schema_artifacts:dump` rake task. + # @private + INDEX_DATA_UPDATE_SCRIPT_ID = "update_index_data_d577eb4b07ee3c53b59f2f6d6c7b2413" + + # The id of the old version of the update data script before ElasticGraph v0.9. For now, we are maintaining + # backwards compatibility with how it recorded event versions, and we have test coverage for that which relies + # upon this id. + # + # TODO: Drop this when we no longer need to maintain backwards-compatibility. + # @private + OLD_INDEX_DATA_UPDATE_SCRIPT_ID = "update_index_data_9b97090d5c97c4adc82dc7f4c2b89bc5" + + # When an update script has a no-op result we often want to communicate more information about + # why it was a no-op back to ElatsicGraph from the script. The only way to do that is to throw + # an exception with an error message, but, as far as I can tell, painless doesn't let you define + # custom exception classes. To allow elasticgraph-indexer to detect that the script "failed" due + # to a no-op (rather than a true failure) we include this common preamble in the exception message + # thrown from our update scripts for the no-op case. + # @private + UPDATE_WAS_NOOP_MESSAGE_PREAMBLE = "ElasticGraph update was a no-op: " + + # The name used to refer to a document's own/primary source event (that is, the event that has a `type` + # matching the document's type). The name here was chosen to avoid naming collisions with relationships + # defined via the `relates_to_one`/`relates_to_many` APIs. The GraphQL spec reserves the double-underscore + # prefix on field names, which means that users cannot define a relationship named `__self` via the + # `relates_to_one`/`relates_to_many` APIs. + # @private + SELF_RELATIONSHIP_NAME = "__self" + + # This regex aligns with the datastore format of HH:mm:ss || HH:mm:ss.S || HH:mm:ss.SS || HH:mm:ss.SSS + # See https://rubular.com/r/NHjBWrpZvzOTJO for examples. + # @private + VALID_LOCAL_TIME_REGEX = /\A(([0-1][0-9])|(2[0-3])):[0-5][0-9]:[0-5][0-9](\.[0-9]{1,3})?\z/ + + # `VALID_LOCAL_TIME_REGEX`, expressed as a JSON schema pattern. JSON schema supports a subset of + # Ruby Regexp features and is expressed as a String object. Here we convert from the Ruby Regexp + # start-and-end-of-string anchors (\A and \z) and convert them to the JSON schema ones (^ and $). + # + # For more info, see: + # https://json-schema.org/understanding-json-schema/reference/regular_expressions.html + # http://www.rexegg.com/regex-anchors.html + # @private + VALID_LOCAL_TIME_JSON_SCHEMA_PATTERN = VALID_LOCAL_TIME_REGEX.source.sub(/\A\\A/, "^").sub(/\\z\z/, "$") + + # Special hidden field defined in an index where we store the count of elements in each list field. + # We index the list counts so that we can offer a `count` filter operator on list fields, allowing + # clients to query on the count of list elements. + # + # The field name has a leading `__` because the GraphQL spec reserves that prefix for its own use, + # and we can therefore assume that no GraphQL fields have this name. + # @private + LIST_COUNTS_FIELD = "__counts" + + # Character used to separate parts of a field path for the keys in the special `__counts` + # field which contains the counts of the various list fields. We were going to use a dot + # (as you'd expect) but ran into errors like this from the datastore: + # + # > can't merge a non object mapping [seasons.players.__counts.seasons] with an object mapping + # + # When we have a list of `object`, and then a list field on that object type, we want to + # store the count of both the parent list and the child list, but if we use dots then the datastore + # treats it like a nested JSON object, and the JSON entry at the parent path can't both be an integer + # (for the parent list count) and an object containing counts of its child lists. + # + # By using `|` instead of `.`, we avoid this problem. + # @private + LIST_COUNTS_FIELD_PATH_KEY_SEPARATOR = "|" + + # The set of datastore field types which have no `properties` in the mapping, but which + # can be represented as a JSON object at indexing time. + # + # I built this list by auditing the full list of index field mapping types: + # https://www.elastic.co/guide/en/elasticsearch/reference/8.9/mapping-types.html + # @private + DATASTORE_PROPERTYLESS_OBJECT_TYPES = [ + "aggregate_metric_double", # https://www.elastic.co/guide/en/elasticsearch/reference/8.9/aggregate-metric-double.html + "completion", # https://www.elastic.co/guide/en/elasticsearch/reference/8.9/search-suggesters.html#completion-suggester + "flattened", # https://www.elastic.co/guide/en/elasticsearch/reference/8.9/flattened.html + "geo_point", # https://www.elastic.co/guide/en/elasticsearch/reference/8.9/geo-point.html + "geo_shape", # https://www.elastic.co/guide/en/elasticsearch/reference/8.9/geo-shape.html + "histogram", # https://www.elastic.co/guide/en/elasticsearch/reference/8.9/histogram.html + "join", # https://www.elastic.co/guide/en/elasticsearch/reference/8.9/parent-join.html + "percolator", # https://www.elastic.co/guide/en/elasticsearch/reference/8.9/percolator.html + "point", # https://www.elastic.co/guide/en/elasticsearch/reference/8.9/point.html + "range", # https://www.elastic.co/guide/en/elasticsearch/reference/8.9/range.html + "rank_features", # https://www.elastic.co/guide/en/elasticsearch/reference/8.9/rank-features.html + "shape" # https://www.elastic.co/guide/en/elasticsearch/reference/8.9/shape.html + ].to_set + + # This pattern matches the spec for a valid GraphQL name: + # http://spec.graphql.org/June2018/#sec-Names + # + # ...however, it allows additional non-valid characters before and after it. + # @private + GRAPHQL_NAME_WITHIN_LARGER_STRING_PATTERN = /[_A-Za-z][_0-9A-Za-z]*/ + + # This pattern exactly matches a valid GraphQL name, with no extra characters allowed before or after. + # @private + GRAPHQL_NAME_PATTERN = /\A#{GRAPHQL_NAME_WITHIN_LARGER_STRING_PATTERN}\z/ + + # Description in English of the requirements for GraphQL names. (Used in multiple error messages). + # @private + GRAPHQL_NAME_VALIDITY_DESCRIPTION = "Names are limited to ASCII alphanumeric characters (plus underscore), and cannot start with a number." + + # The standard set of scalars that are defined by the GraphQL spec: + # https://spec.graphql.org/October2021/#sec-Scalars + # @private + STOCK_GRAPHQL_SCALARS = %w[Boolean Float ID Int String].to_set.freeze + + # The current variant of JSON schema that we use. + # @private + JSON_META_SCHEMA = "http://json-schema.org/draft-07/schema#" + + # Filter the bulk response payload with a comma separated list using dot notation. + # https://www.elastic.co/guide/en/elasticsearch/reference/7.10/common-options.html#common-options-response-filtering + # + # Note: anytime you change this constant, be sure to check all the comments in the unit specs that mention this constant. + # When stubbing a datastore client test double, it doesn't respect this filtering obviously, so it's up to us + # to accurately mimic the filtering in our stubbed responses. + # @private + DATASTORE_BULK_FILTER_PATH = [ + # The key under `items` names the type of operation (e.g. `index` or `update`) and + # we use a `*` for it since we always use that key, regardless of which operation it is. + "items.*.status", "items.*.result", "items.*.error" + ].join(",") + + # HTTP header set by `elasticgraph-graphql_lambda` to indicate the AWS ARN of the caller. + # @private + GRAPHQL_LAMBDA_AWS_ARN_HEADER = "X-AWS-LAMBDA-CALLER-ARN" + + # TODO(steep): it complains about `define_schema` not being defined but it is defined + # in another file; I shouldn't have to say it's dynamic here. For now this works though. + # @dynamic self.define_schema +end diff --git a/elasticgraph-support/lib/elastic_graph/errors.rb b/elasticgraph-support/lib/elastic_graph/errors.rb new file mode 100644 index 00000000..675d31d0 --- /dev/null +++ b/elasticgraph-support/lib/elastic_graph/errors.rb @@ -0,0 +1,84 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + # @private + module Errors + class Error < StandardError + end + + class CursorEncoderError < Error + end + + class InvalidSortFieldsError < CursorEncoderError + end + + class InvalidCursorError < CursorEncoderError + end + + class CursorEncodingError < CursorEncoderError + end + + class CountUnavailableError < Error + end + + class InvalidArgumentValueError < Error + end + + class InvalidMergeError < Error + end + + class SchemaError < Error + end + + class InvalidGraphQLNameError < SchemaError + end + + class NotFoundError < Error + end + + class SearchFailedError < Error + end + + class RequestExceededDeadlineError < SearchFailedError + end + + class IdentifyDocumentVersionsFailedError < Error + end + + class IndexOperationError < Error + end + + class ClusterOperationError < Error + end + + class InvalidExtensionError < Error + end + + class ConfigError < Error + end + + class ConfigSettingNotSetError < ConfigError + end + + class InvalidScriptDirectoryError < Error + end + + class MissingSchemaArtifactError < Error + end + + class S3OperationFailedError < Error + end + + class MessageIdsMissingError < Error + end + + class BadDatastoreRequest < Error + end + end +end diff --git a/elasticgraph-support/lib/elastic_graph/support/faraday_middleware/msearch_using_get_instead_of_post.rb b/elasticgraph-support/lib/elastic_graph/support/faraday_middleware/msearch_using_get_instead_of_post.rb new file mode 100644 index 00000000..2ec64d1b --- /dev/null +++ b/elasticgraph-support/lib/elastic_graph/support/faraday_middleware/msearch_using_get_instead_of_post.rb @@ -0,0 +1,32 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Support + # @private + module FaradayMiddleware + # Custom Faraday middleware that forces `msearch` calls to use an HTTP GET instead of an HTTP POST. While not + # necessary, it preserves a useful property: all "read" calls made by ElasticGraph use an HTTP GET, and HTTP POST + # requests are "write" calls. This allows the access policy to only grant HTTP GET access from the GraphQL endpoint, + # which leads to a more secure setup (as the GraphQL endpoint can be blocked from performing any writes). + # + # Note: before elasticsearch-ruby 7.9.0, `msearch` used an HTTP GET request, so this simply restores that behavior. + # This results in an HTTP GET with a request body, but it works just fine and its what the Ruby Elasticsearch client + # did for years. + # + # For more info, see: https://github.com/elastic/elasticsearch-ruby/issues/1005 + MSearchUsingGetInsteadOfPost = ::Data.define(:app) do + # @implements MSearchUsingGetInsteadOfPost + def call(env) + env.method = :get if env.url.path.to_s.end_with?("/_msearch") + app.call(env) + end + end + end + end +end diff --git a/elasticgraph-support/lib/elastic_graph/support/faraday_middleware/support_timeouts.rb b/elasticgraph-support/lib/elastic_graph/support/faraday_middleware/support_timeouts.rb new file mode 100644 index 00000000..e574ed3c --- /dev/null +++ b/elasticgraph-support/lib/elastic_graph/support/faraday_middleware/support_timeouts.rb @@ -0,0 +1,37 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/errors" + +module ElasticGraph + module Support + # @private + module FaradayMiddleware + # Faraday supports specifying a timeout at both the client level (when building the Faraday connection) or on a + # per-request basis. We want to specify it on a per-request basis, but unfortunately, the Elasticsearch/OpenSearch + # clients don't provide any per-request API to specify the timeout (it only supports it when instantiating your + # client). + # + # This middleware helps us work around this deficiency by looking for the TIMEOUT_MS_HEADER. If present, it deletes + # it from the headers and instead sets it as the request timeout. + SupportTimeouts = ::Data.define(:app) do + # @implements SupportTimeouts + def call(env) + if (timeout_ms = env.request_headers.delete(TIMEOUT_MS_HEADER)) + env.request.timeout = Integer(timeout_ms) / 1000.0 + end + + app.call(env) + rescue ::Faraday::TimeoutError + raise Errors::RequestExceededDeadlineError, "Datastore request exceeded timeout of #{timeout_ms} ms." + end + end + end + end +end diff --git a/elasticgraph-support/lib/elastic_graph/support/from_yaml_file.rb b/elasticgraph-support/lib/elastic_graph/support/from_yaml_file.rb new file mode 100644 index 00000000..297096f5 --- /dev/null +++ b/elasticgraph-support/lib/elastic_graph/support/from_yaml_file.rb @@ -0,0 +1,56 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "yaml" + +module ElasticGraph + # Provides support utilities for the rest of the ElasticGraph gems. As such, it is not intended + # to provide public APIs for ElasticGraph users. + module Support + # @private + module FromYamlFile + # Factory method that will build an instance from the provided `yaml_file`. + # `datastore_client_customization_block:` can be passed to customize the datastore clients. + # In addition, a block is accepted that can prepare the settings before the object is built + # (e.g. to override specific settings). + def from_yaml_file(yaml_file, datastore_client_customization_block: nil) + parsed_yaml = ::YAML.safe_load_file(yaml_file, aliases: true) + parsed_yaml = yield(parsed_yaml) if block_given? + from_parsed_yaml(parsed_yaml, &datastore_client_customization_block) + end + + # An extension module that provides a `from_yaml_file` factory method on a `RakeTasks` class. + # + # This is designed for a `RakeTasks` class that needs an ElasticGraph component (e.g. an + # `ElasticGraph::GraphQL`, `ElasticGraph::Admin`, or `ElasticGraph::Indexer` instance). + # When the schema artifacts are out of date, loading those components can fail. This gracefully + # handles that for you, giving you clear instructions of what to do when this happens. + # + # This requires the `RakeTasks` class to accept the ElasticGraph component instance via a block + # so that it happens lazily. + class ForRakeTasks < ::Module + # @dynamic from_yaml_file + + def initialize(component_class) + define_method :from_yaml_file do |yaml_file, *args, **options| + __skip__ = new(*args, **options) do + component_class.from_yaml_file(yaml_file) + rescue => e + raise <<~EOS + Failed to load `#{component_class}` with `#{yaml_file}`. This can happen if the schema artifacts are out of date. + Run `rake schema_artifacts:dump` and try again. + + #{e.class}: #{e.message} + EOS + end + end + end + end + end + end +end diff --git a/elasticgraph-support/lib/elastic_graph/support/graphql_formatter.rb b/elasticgraph-support/lib/elastic_graph/support/graphql_formatter.rb new file mode 100644 index 00000000..8e441e0b --- /dev/null +++ b/elasticgraph-support/lib/elastic_graph/support/graphql_formatter.rb @@ -0,0 +1,68 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "json" + +module ElasticGraph + module Support + # Utility module that provides helper methods for generating well-formatted GraphQL syntax. + # + # @private + module GraphQLFormatter + # Formats the given hash as an argument list. If `args` is empty, returns an empty string. + # Otherwise, wraps the args list in parens. This allows the returned string to be appended + # to a field or directive, and it'll correctly use parens (or not) based on if there are args + # or not. + def self.format_args(**args) + return "" if args.empty? + "(#{serialize(args, wrap_hash_with_braces: false)})" + end + + # Formats the given value in GraphQL syntax. This method was derived + # from a similar method from the graphql-ruby gem: + # + # https://github.com/rmosolgo/graphql-ruby/blob/v1.11.4/lib/graphql/language.rb#L17-L33 + # + # We don't want to use that method because it is marked as `@api private`, indicating + # it could be removed in any release of the graphql gem. If we used it, it could hinder + # future upgrades. + # + # Our implementation here differs in a few ways: + # + # - case statement instead of multiple `if value.is_a?` checks (a bit cleaner) + # - `wrap_hash_with_braces` since we do not want to wrap an args hash with braces. + # - Readable spacing has been added so we get `foo: [1, 2], bar: 3` instead of `foo:[1,2],bar:3`. + # - Symbol support has been added. Symbols are converted to strings (with no quotes), allowing + # callers to pass them for GraphQL enums. + # - We've removed the `quirks_mode: true` flag passed to `JSON.generate` since it has been + # deprecated for a while: https://github.com/flori/json/issues/309 + def self.serialize(value, wrap_hash_with_braces: true) + case value + when ::Hash + serialized_hash = value.map do |k, v| + "#{k}: #{serialize v}" + end.join(", ") + + return serialized_hash unless wrap_hash_with_braces + + "{#{serialized_hash}}" + when ::Array + serialized_array = value.map do |v| + serialize v + end.join(", ") + + "[#{serialized_array}]" + when ::Symbol + value.to_s + else + ::JSON.generate(value) + end + end + end + end +end diff --git a/elasticgraph-support/lib/elastic_graph/support/hash_util.rb b/elasticgraph-support/lib/elastic_graph/support/hash_util.rb new file mode 100644 index 00000000..36ceede3 --- /dev/null +++ b/elasticgraph-support/lib/elastic_graph/support/hash_util.rb @@ -0,0 +1,229 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Support + # @private + class HashUtil + # Fetches a key from a hash (just like `Hash#fetch`) but with a more verbose error message when the key is not found. + # The error message indicates the available keys unlike `Hash#fetch`. + def self.verbose_fetch(hash, key) + hash.fetch(key) do + raise ::KeyError, "key not found: #{key.inspect}. Available keys: #{hash.keys.inspect}." + end + end + + # Like `Hash#to_h`, but strict. When the given input has conflicting keys, `Hash#to_h` will happily let + # the last pair when. This method instead raises an exception. + def self.strict_to_h(pairs) + hash = pairs.to_h + + if hash.size < pairs.size + conflicting_keys = pairs.map(&:first).tally.filter_map { |key, count| key if count > 1 } + raise ::KeyError, "Cannot build a strict hash, since input has conflicting keys: #{conflicting_keys.inspect}." + end + + hash + end + + # Like `Hash#merge`, but verifies that the hashes were strictly disjoint (e.g. had no keys in common). + # An error is raised if they do have any keys in common. + def self.disjoint_merge(hash1, hash2) + conflicting_keys = [] # : ::Array[untyped] + merged = hash1.merge(hash2) do |key, v1, _v2| + conflicting_keys << key + v1 + end + + unless conflicting_keys.empty? + raise ::KeyError, "Hashes were not disjoint. Conflicting keys: #{conflicting_keys.inspect}." + end + + merged + end + + # Recursively transforms any hash keys in the given object to string keys, without + # mutating the provided argument. + def self.stringify_keys(object) + recursively_transform(object) do |key, value, hash| + hash[key.to_s] = value + end + end + + # Recursively transforms any hash keys in the given object to symbol keys, without + # mutating the provided argument. + # + # Important note: this should never be used on untrusted input. Symbols are not GCd in + # Ruby in the same way as strings. + def self.symbolize_keys(object) + recursively_transform(object) do |key, value, hash| + hash[key.to_sym] = value + end + end + + # Recursively prunes nil values from the hash, at any level of its structure, without + # mutating the provided argument. Key paths that are pruned are yielded to the caller + # to allow the caller to have awareness of what was pruned. + def self.recursively_prune_nils_from(object, &block) + recursively_prune_if(object, block, &:nil?) + end + + # Recursively prunes nil values or empty hash/array values from the hash, at any level + # of its structure, without mutating the provided argument. Key paths that are pruned + # are yielded to the caller to allow the caller to have awareness of what was pruned. + def self.recursively_prune_nils_and_empties_from(object, &block) + recursively_prune_if(object, block) do |value| + if value.is_a?(::Hash) || value.is_a?(::Array) + value.empty? + else + value.nil? + end + end + end + + # Recursively flattens the provided source hash, converting keys to strings along the way + # with dots used to separate nested parts. For example: + # + # flatten_and_stringify_keys({ a: { b: 3 }, c: 5 }, prefix: "foo") returns: + # { "foo.a.b" => 3, "foo.c" => 5 } + def self.flatten_and_stringify_keys(source_hash, prefix: nil) + # @type var flat_hash: ::Hash[::String, untyped] + flat_hash = {} + prefix = prefix ? "#{prefix}." : "" + # `_ =` is needed by steep because it thinks `prefix` could be `nil` in spite of the above line. + populate_flat_hash(source_hash, _ = prefix, flat_hash) + flat_hash + end + + # Recursively merges the values from `hash2` into `hash1`, without mutating either `hash1` or `hash2`. + # When a key is in both `hash2` and `hash1`, takes the value from `hash2` just like `Hash#merge` does. + def self.deep_merge(hash1, hash2) + # `_ =` needed to satisfy steep--the types here are quite complicated. + _ = hash1.merge(hash2) do |key, hash1_value, hash2_value| + if ::Hash === hash1_value && ::Hash === hash2_value + deep_merge(hash1_value, hash2_value) + else + hash2_value + end + end + end + + # Fetches a list of (potentially) nested value from a hash. The `key_path` is expected + # to be a string with dots between the nesting levels (e.g. `foo.bar`). Returns `[]` if + # the value at any parent key is `nil`. Returns a flat array of values if the structure + # at any level is an array. + # + # Raises an error if the key is not found unless a default block is provided. + # Raises an error if any parent value is not a hash as expected. + # Raises an error if the provided path is not a full path to a leaf in the nested structure. + def self.fetch_leaf_values_at_path(hash, key_path, &default) + do_fetch_leaf_values_at_path(hash, key_path.split("."), 0, &default) + end + + # Fetches a single value from the hash at the given path. The `key_path` is expected + # to be a string with dots between the nesting levels (e.g. `foo.bar`). + # + # If any parent value is not a hash as expected, raises an error. + # If the key at any level is not found, yields to the provided block (which can provide a default value) + # or raises an error if no block is provided. + def self.fetch_value_at_path(hash, key_path) + path_parts = key_path.split(".") + + path_parts.each.with_index(1).reduce(hash) do |inner_hash, (key, num_parts)| + if inner_hash.is_a?(::Hash) + inner_hash.fetch(key) do + missing_path = path_parts.first(num_parts).join(".") + return yield missing_path if block_given? + raise KeyError, "Key not found: #{missing_path.inspect}" + end + else + raise KeyError, "Value at key #{path_parts.first(num_parts - 1).join(".").inspect} is not a `Hash` as expected; " \ + "instead, was a `#{(_ = inner_hash).class}`" + end + end + end + + private_class_method def self.recursively_prune_if(object, notify_pruned_path) + recursively_transform(object) do |key, value, hash, key_path| + if yield(value) + notify_pruned_path&.call(key_path) + else + hash[key] = value + end + end + end + + private_class_method def self.recursively_transform(object, key_path = nil, &hash_entry_handler) + case object + when ::Hash + # @type var initial: ::Hash[key, value] + initial = {} + object.each_with_object(initial) do |(key, value), hash| + updated_path = key_path ? "#{key_path}.#{key}" : key.to_s + value = recursively_transform(value, updated_path, &hash_entry_handler) + hash_entry_handler.call(key, value, hash, updated_path) + end + when ::Array + object.map.with_index do |item, index| + recursively_transform(item, "#{key_path}[#{index}]", &hash_entry_handler) + end + else + object + end + end + + private_class_method def self.populate_flat_hash(source_hash, prefix, flat_hash) + source_hash.each do |key, value| + if value.is_a?(::Hash) + populate_flat_hash(value, "#{prefix}#{key}.", flat_hash) + elsif value.is_a?(::Array) && value.grep(::Hash).any? + raise ArgumentError, "`flatten_and_stringify_keys` cannot handle nested arrays of hashes, but got: #{value.inspect}" + else + flat_hash["#{prefix}#{key}"] = value + end + end + end + + private_class_method def self.do_fetch_leaf_values_at_path(object, path_parts, level_index, &default) + if level_index == path_parts.size + if object.is_a?(::Hash) + raise KeyError, "Key was not a path to a leaf field: #{path_parts.join(".").inspect}" + else + return Array(object) + end + end + + case object + when nil + [] + when ::Hash + key = path_parts[level_index] + if object.key?(key) + do_fetch_leaf_values_at_path(object.fetch(key), path_parts, level_index + 1, &default) + else + missing_path = path_parts.first(level_index + 1).join(".") + if default + Array(default.call(missing_path)) + else + raise KeyError, "Key not found: #{missing_path.inspect}" + end + end + when ::Array + object.flat_map do |element| + do_fetch_leaf_values_at_path(element, path_parts, level_index, &default) + end + else + # Note: we intentionally do not put the value (`current_level_hash`) in the + # error message, as that would risk leaking PII. But the class of the value should be OK. + raise KeyError, "Value at key #{path_parts.first(level_index).join(".").inspect} is not a `Hash` as expected; " \ + "instead, was a `#{object.class}`" + end + end + end + end +end diff --git a/elasticgraph-support/lib/elastic_graph/support/logger.rb b/elasticgraph-support/lib/elastic_graph/support/logger.rb new file mode 100644 index 00000000..aa5ec3b1 --- /dev/null +++ b/elasticgraph-support/lib/elastic_graph/support/logger.rb @@ -0,0 +1,86 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "json" +require "logger" +require "pathname" + +module ElasticGraph + module Support + # @private + module Logger + # Builds a logger instance from the given parsed YAML config. + def self.from_parsed_yaml(parsed_yaml) + Factory.build(config: Config.from_parsed_yaml(parsed_yaml)) + end + + # @private + module Factory + def self.build(config:, device: nil) + ::Logger.new( + device || config.prepared_device, + level: config.level, + formatter: config.formatter + ) + end + end + + # @private + class JSONAwareFormatter + def initialize + @original_formatter = ::Logger::Formatter.new + end + + def call(severity, datetime, progname, msg) + msg = msg.is_a?(::Hash) ? ::JSON.generate(msg, space: " ") : msg + @original_formatter.call(severity, datetime, progname, msg) + end + end + + # @private + class Config < ::Data.define( + # Determines what severity level we log. Valid values are `DEBUG`, `INFO`, `WARN`, + # `ERROR`, `FATAL` and `UNKNOWN`. + :level, + # Determines where we log to. Must be a string. "stdout" or "stderr" are interpreted + # as being those output streams; any other value is assumed to be a file path. + :device, + # Object used to format log messages. Defaults to an instance of `JSONAwareFormatter`. + :formatter + ) + def prepared_device + case device + when "stdout" then $stdout + when "stderr" then $stderr + else + ::Pathname.new(device).parent.mkpath + device + end + end + + def self.from_parsed_yaml(hash) + hash = hash.fetch("logger") + extra_keys = hash.keys - EXPECTED_KEYS + + unless extra_keys.empty? + raise Errors::ConfigError, "Unknown `logger` config settings: #{extra_keys.join(", ")}" + end + + new( + level: hash["level"] || "INFO", + device: hash.fetch("device"), + formatter: ::Object.const_get(hash.fetch("formatter", JSONAwareFormatter.name)).new + ) + end + + EXPECTED_KEYS = members.map(&:to_s) + end + end + end +end diff --git a/elasticgraph-support/lib/elastic_graph/support/memoizable_data.rb b/elasticgraph-support/lib/elastic_graph/support/memoizable_data.rb new file mode 100644 index 00000000..491be456 --- /dev/null +++ b/elasticgraph-support/lib/elastic_graph/support/memoizable_data.rb @@ -0,0 +1,147 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" +require "stringio" + +module ElasticGraph + module Support + # `::Data.define` in Ruby 3.2+ is *very* handy for defining immutable value objects. However, one annoying + # aspect: instances are frozen, which gets in the way of defining a memoized method (e.g. a method that + # caches the result of an expensive computation). While memoization is not always safe (e.g. if you rely + # on an impure side-effect...) it's safe if what you're memoizing is a pure function of the immutable state + # of the value object. We rely on that very heavily in ElasticGraph (and used it with a prior "value objects" + # library we use before upgrading to Ruby 3.2). + # + # This abstraction aims to behave just like `::Data.define`, but with the added ability to define memoized methods. + # It makes this possible by combining `::Data.define` with `SimpleDelegator`: that defines a data class, but then + # wraps instances of it in a `SimpleDelegator` instance which is _not_ frozen. The memoized methods can then be + # defined on the wrapper. + # + # Note: getting this code to typecheck with steep is quite difficult, so we're just skipping it. + __skip__ = + module MemoizableData + # Defines a data class using the provided attributes. + # + # A block can be provided in order to define custom methods (including memoized methods!) or to override + # `initialize` in order to provide field defaults. + def self.define(*attributes, &block) + data_class = ::Data.define(*attributes) + + DelegateClass(data_class) do + # Store a reference to our wrapped data class so we can use it in `ClassMethods` below. + const_set(:DATA_CLASS, data_class) + + # Define default version of` after_initialize`. This is a hook that a user may override. + # standard:disable Lint/NestedMethodDefinition + private def after_initialize + end + # standard:enable Lint/NestedMethodDefinition + + # If a block is provided, we evaluate it so that it can define memoized methods. + if block + original_initialize = instance_method(:initialize) + module_eval(&block) + + # It's useful for the caller to be define `initialize` in order to provide field defaults, as + # shown in the `Data` docs: + # + # https://rubyapi.org/3.2/o/data + # + # However, to make that work, we need the `initialize` definition to be included in the data class, + # rather than in our `DelegateClass` wrapper. + # + # Here we detect when the block defines an `initialize` method. + if instance_method(:initialize) != original_initialize + # To mix the `initialize` override into the data class, we re-evaluate the block in a new module here. + # The module ignores all method definitions except `initialize`. + init_override_module = ::Module.new do + # We want to ignore all methods except the `initialize` method so that this module only contains `initialize`. + def self.method_added(method_name) + remove_method(method_name) unless method_name == :initialize + end + + module_eval(&block) + end + + data_class.include(init_override_module) + end + end + + # `DelegateClass` objects are mutable via the `__setobj__` method. We don't want to allow mutation, so we undefine it here. + undef_method :__setobj__ + + prepend MemoizableData::InstanceMethods + extend MemoizableData::ClassMethods + end + end + + module InstanceMethods + # SimpleDelegator#== automatically defines `==` so that it unwraps the wrapped type and calls `==` on it. + # However, the wrapped type doesn't automatically define `==` when given an equivalent wrapped instance. + # + # For `==` to work correctly we need to unwrap _both_ sides before delegating, which this takes care of. + def ==(other) + case other + when MemoizableData::InstanceMethods + __getobj__ == other.__getobj__ + else + super + end + end + + # `with` is a standard `Data` API that returns a new instance with the specified fields updated. + # + # Since `DelegateClass` delegates all methods to the wrapped object, `with` will return an instance of the + # data class and not our wrapper. To overcome that, we redefine it here so that the new instance is re-wrapped. + def with(**updates) + # Note: we intentionally do _not_ `super` to the `Date#with` method here, because in Ruby 3.2 it has a bug that + # impacts us: `with` does not call `initialize` as it should. Some of our value classes (based on the old `values` gem) + # depend on this behavior, so here we work around it by delegating to `new` after merging the attributes. + # + # This bug is fixed in Ruby 3.3 so we should be able to revert back to an implementation that delegates with `super` + # after we are on Ruby 3.3. For more info, see: + # - https://bugs.ruby-lang.org/issues/19259 + # - https://github.com/ruby/ruby/pull/7031 + self.class.new(**to_h.merge(updates)) + end + end + + module ClassMethods + # `new` on a `SimpleDelegator` class accepts an instance of the wrapped type to wrap. `MemoizableData` is intended to + # hide the wrapping we're doing here, so here we want `new` to accept the direct arguments that `new` on the `Data` class + # would accept. Here we instantiate the data class and the wrap it. + def new(*args, **kwargs) + data_instance = self::DATA_CLASS.new(*args, **kwargs) + + # Here we re-implement `new` (rather than using `super`) because `initialize` may be overridden. + allocate.instance_eval do + # Match `__setobj__` behavior: https://github.com/ruby/ruby/blob/v3_2_2/lib/delegate.rb#L411 + @delegate_dc_obj = data_instance + after_initialize + self + end + end + + # `SimpleDelegator` delegates instance methods but not class methods. This is a standard `Data` class method + # that is worth delegating. + def members + self::DATA_CLASS.members + end + + def method_added(method_name) + return unless method_name == :initialize + + raise "`#{name}` overrides `initialize` in a subclass of `#{MemoizableData.name}`, but that can break things. Instead:\n\n" \ + " 1) If you want to coerce field values or provide default field values, define `initialize` in a block passed to `#{MemoizableData.name}.define`.\n" \ + " 2) If you want to perform some post-initialization setup, define an `after_initialize` method." + end + end + end + end +end diff --git a/elasticgraph-support/lib/elastic_graph/support/monotonic_clock.rb b/elasticgraph-support/lib/elastic_graph/support/monotonic_clock.rb new file mode 100644 index 00000000..3e522ac6 --- /dev/null +++ b/elasticgraph-support/lib/elastic_graph/support/monotonic_clock.rb @@ -0,0 +1,22 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Support + # A simple abstraction that provides a monotonic clock. + # + # @private + class MonotonicClock + # Returns an abstract "now" value in integer milliseconds, suitable for calculating + # a duration or deadline, without being impacted by leap seconds, etc. + def now_in_ms + ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond) + end + end + end +end diff --git a/elasticgraph-support/lib/elastic_graph/support/threading.rb b/elasticgraph-support/lib/elastic_graph/support/threading.rb new file mode 100644 index 00000000..a88cb857 --- /dev/null +++ b/elasticgraph-support/lib/elastic_graph/support/threading.rb @@ -0,0 +1,43 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Support + # @private + module Threading + # Like Enumerable#map, but performs the map in parallel using one thread per list item. + # Exceptions that happen in the threads will propagate to the caller at the end. + # Due to Ruby's GVL, this will never be helpful for pure computation, but can be + # quite helpful when dealing with blocking I/O. However, the cost of threads is + # such that this method should not be used when you have a large list of items to + # map over (say, hundreds or thousands of items or more). + def self.parallel_map(items) + threads = _ = items.map do |item| + ::Thread.new do + # Disable reporting of exceptions. We use `value` at the end of this method, which + # propagates any exception that happened in the thread to the calling thread. If + # this is true (the default), then the exception is also printed to $stderr which + # is quite noisy. + ::Thread.current.report_on_exception = false + + yield item + end + end + + # `value` here either returns the value of the final expression in the thread, or raises + # whatever exception happened in the thread. `join` doesn't propagate the exception in + # the same way, so we always want to use `Thread#value` even if we are just using threads + # for side effects. + threads.map(&:value) + rescue => e + e.set_backtrace(e.backtrace + caller) + raise e + end + end + end +end diff --git a/elasticgraph-support/lib/elastic_graph/support/time_set.rb b/elasticgraph-support/lib/elastic_graph/support/time_set.rb new file mode 100644 index 00000000..494591a1 --- /dev/null +++ b/elasticgraph-support/lib/elastic_graph/support/time_set.rb @@ -0,0 +1,295 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Support + # Models a set of `::Time` objects, but does so using one or more `::Range` objects. + # This is done so that we can support unbounded sets (such as "all times after midnight + # on date X"). + # + # Internally, this is a simple wrapper around a set of `::Range` objects. Those ranges take + # a few different forms: + # + # - ALL: a range with no bounds, which implicitly contains all `::Time`s. (It's like the + # integer set from negative to positive infinity). + # - An open range: a range with only an upper or lower bound (but not the other). + # - A closed range: a range with an upper and lower bound. + # - An empty range: a range that contains no `::Time`s, by virtue of its bounds having no overlap. + # + # @private + class TimeSet < ::Data.define(:ranges) + # Factory method to construct a `TimeSet` using a range with the given bounds. + def self.of_range(gt: nil, gte: nil, lt: nil, lte: nil) + if gt && gte + raise ArgumentError, "TimeSet got two lower bounds, but can have only one (gt: #{gt.inspect}, gte: #{gte.inspect})" + end + + if lt && lte + raise ArgumentError, "TimeSet got two upper bounds, but can have only one (lt: #{lt.inspect}, lte: #{lte.inspect})" + end + + # To be able to leverage Ruby's Range class, we need to convert to the "inclusive" ("or equal") + # form. This cuts down on the number of test cases we need to write and also Ruby's range lets + # you control whether the end of a range is inclusive or exclusive, but doesn't let you control + # the beginning of the range. + # + # This is safe to do because our datastores only work with `::Time`s at millisecond granularity, + # so `> t` is equivalent to `>= (t + 1ms)` and `< t` is equivalent to `<= (t - 1ms)`. + lower_bound = gt&.+(CONSECUTIVE_TIME_INCREMENT) || gte + upper_bound = lt&.-(CONSECUTIVE_TIME_INCREMENT) || lte + + of_range_objects(_ = [RangeFactory.build_non_empty(lower_bound, upper_bound)].compact) + end + + # Factory method to construct a `TimeSet` from a collection of `::Time` objects. + # Internally we convert it to a set of `::Range` objects, one per unique time. + def self.of_times(times) + of_range_objects(times.map { |t| ::Range.new(t, t) }) + end + + # Factory method to construct a `TimeSet` from a previously built collection of + # ::Time ranges. Mostly used internally by `TimeSet` and in tests. + def self.of_range_objects(range_objects) + # Use our singleton EMPTY or ALL instances if we can to save on memory. + return EMPTY if range_objects.empty? + first_range = _ = range_objects.first + return ALL if first_range.begin.nil? && first_range.end.nil? + + new(range_objects) + end + + # Returns a new `TimeSet` containing `::Time`s common to this set and `other_set`. + def intersection(other_set) + # Here we rely on the distributive and commutative properties of set algebra: + # + # https://en.wikipedia.org/wiki/Algebra_of_sets + # A ∩ (B ∪ C) = (A ∩ B) ∪ (A ∩ C) (distributive property) + # A ∩ B = B ∩ A (commutative property) + # + # We can combine these properties to see how the intersection of sets of ranges would work: + # (A₁ ∪ A₂) ∩ (B₁ ∪ B₂) + # = ((A₁ ∪ A₂) ∩ B₁) ∪ ((A₁ ∪ A₂) ∩ B₂) (expanding based on distributive property) + # = (B₁ ∩ (A₁ ∪ A₂)) ∪ (B₂ ∩ (A₁ ∪ A₂)) (rearranging based on commutative property) + # = ((B₁ ∩ A₁) ∪ (B₁ ∩ A₂)) ∪ ((B₂ ∩ A₁) ∪ (B₂ ∩ A₂)) (expanding based on distributive property) + # = (B₁ ∩ A₁) ∪ (B₁ ∩ A₂) ∪ (B₂ ∩ A₁) ∪ (B₂ ∩ A₂) (removing excess parens) + # = union of (intersection of each pair) + intersected_ranges = ranges.to_a.product(other_set.ranges.to_a) + .filter_map { |r1, r2| intersect_ranges(r1, r2) } + + TimeSet.of_range_objects(intersected_ranges) + end + + # Returns a new `TimeSet` containing `::Time`s that are in either this set or `other_set`. + def union(other_set) + TimeSet.of_range_objects(ranges.union(other_set.ranges)) + end + + # Returns true if the given `::Time` is a member of this `TimeSet`. + def member?(time) + ranges.any? { |r| r.cover?(time) } + end + + # Returns true if this `TimeSet` and the given one have a least one time in common. + def intersect?(other_set) + other_set.ranges.any? do |r1| + ranges.any? do |r2| + ranges_intersect?(r1, r2) + end + end + end + + # Returns true if this TimeSet contains no members. + def empty? + ranges.empty? + end + + # Returns a new `TimeSet` containing the difference between this `TimeSet` and the given one. + def -(other) + new_ranges = other.ranges.to_a.reduce(ranges.to_a) do |accum, other_range| + accum.flat_map do |self_range| + if ranges_intersect?(self_range, other_range) + # Since the ranges intersect, `self_range` must be reduced some how. Depending on what kind of + # intersection we have (e.g. exact equality, `self_range` fully inside `other_range`, `other_range` + # fully inside `self_range`, partial overlap where `self_range` begins before `other_range`, or partial + # overlap where `self_range` ends after `other_range`), we may have a part of `self_range` that comes + # before `other_range`, a part of `self_range` that comes after `other_range`, both, or neither. Below + # we build the before and after parts as candidates, but then ignore any resulting ranges that are + # invalid, which leaves us with the correct result, without having to explicitly handle each possible case. + + # @type var candidates: ::Array[timeRange] + candidates = [] + + if (other_range_begin = other_range.begin) + # This represents the parts of `self_range` that come _before_ `other_range`. + candidates << Range.new(self_range.begin, other_range_begin - CONSECUTIVE_TIME_INCREMENT) + end + + if (other_range_end = other_range.end) + # This represents the parts of `self_range` that come _after_ `other_range`. + candidates << Range.new(other_range_end + CONSECUTIVE_TIME_INCREMENT, self_range.end) + end + + # While some of the ranges produced above may be invalid (due to being descending), we don't have to + # filter them out here because `#initialize` takes care of it. + candidates + else + # Since the ranges don't intersect, there is nothing to remove from `self_range`; just return it unmodified. + [self_range] + end + end + end + + TimeSet.of_range_objects(new_ranges) + end + + def negate + ALL - self + end + + private + + private_class_method :new # use `of_range`, `of_times`, or `of_range_objects` instead. + + # To ensure immutability, we override this to freeze the set. For convenience, we allow the `ranges` + # arg to be an array, and convert to a set here. In addition, we take care of normalizing to the most + # optimal form by merging overlapping ranges here, and ignore descending ranges. + def initialize(ranges:) + normalized_ranges = ranges + .reject { |r| descending_range?(r) } + .to_set + .then { |rs| merge_overlapping_or_adjacent_ranges(rs) } + .freeze + + super(ranges: normalized_ranges) + end + + # Returns true if at least one ::Time exists in both ranges. + def ranges_intersect?(r1, r2) + r1.cover?(r2.begin) || r1.cover?(r2.end) || r2.cover?(r1.begin) || r2.cover?(r1.end) + end + + # The amount to add to a time to get the next consecutive time, based + # on the level of granularity we support. According to the Elasticsearch docs[1], + # it only supports millisecond granularity, so that's all we support: + # + # > Internally, dates are converted to UTC (if the time-zone is specified) and + # > stored as a long number representing milliseconds-since-the-epoch. + # + # We want exact precision here, so we are avoiding using a float for this, preferring + # to use a rational instead. + # + # [1] https://www.elastic.co/guide/en/elasticsearch/reference/7.15/date.html + CONSECUTIVE_TIME_INCREMENT = Rational(1, 1000) + + # Returns true if the given ranges are adjacent with no room for any ::Time + # objects to exist between the ranges given the millisecond granularity we operate at. + def adjacent?(r1, r2) + r1.end&.+(CONSECUTIVE_TIME_INCREMENT)&.==(r2.begin) || r2.end&.+(CONSECUTIVE_TIME_INCREMENT)&.==(r1.begin) || false + end + + # Combines the given ranges into a new range that only contains the common subset of ::Time objects. + # Returns `nil` if there is no intersection. + def intersect_ranges(r1, r2) + RangeFactory.build_non_empty( + [r1.begin, r2.begin].compact.max, + [r1.end, r2.end].compact.min + ) + end + + # Helper method that attempts to merge the given set of ranges into an equivalent + # set that contains fewer ranges in it but covers the same set of ::Time objects. + # As an example, consider these two ranges: + # + # - 2020-05-01 to 2020-07-01 + # - 2020-06-01 to 2020-08-01 + # + # These two ranges can safely be merged into a single range of 2020-05-01 to 2020-08-01. + # Technically speaking, this is not required; we can just return a TimeSet containing + # multiple ranges. However, the goal of a TimeSet is to represent a set of Time objects + # as minimally as possible, and to that end it is useful to merge ranges when possible. + # While it adds a bit of complexity to merge ranges like this, it'll simplify future + # calculations involving a TimeSet. + def merge_overlapping_or_adjacent_ranges(all_ranges) + # We sometimes have to apply this merge algorithm multiple times in order to fully merge + # the ranges into their minimal form. For example, consider these three ranges: + # + # - 2020-05-01 to 2020-07-01 + # - 2020-06-01 to 2020-09-01 + # - 2020-08-01 to 2020-10-01 + # + # Ultimately, we can merge these into a single range of 2020-05-01 to 2020-10-01, but + # our algorithm isn't able to do that in a single pass. On the first pass it'll produce + # two merged ranges (2020-05-01 to 2020-09-01 and 2020-06-01 to 2020-10-01); after we + # apply the algorithm again it is then able to produce the final merged range. + # Since we can't predict how many iterations it'll take, we loop here, and break as + # soon as there is no more progress to be made. + # + # While we can't predice how many iterations it'll take, we can put an upper bound on it: + # it should take no more than `all_ranges.size` times, because every iteration should shrink + # `all_ranges` by at least one element--if not, that iteration didn't make any progress + # (and we're done anyway). + all_ranges.size.times do + # Given our set of ranges, any range is potentially mergeable with any other range. + # Here we determine which pairs of ranges are mergeable. + mergeable_range_pairs = all_ranges.to_a.combination(2).select do |r1, r2| + ranges_intersect?(r1, r2) || adjacent?(r1, r2) + end + + # If there are no mergeable pairs, we're done! + return all_ranges if mergeable_range_pairs.empty? + + # For each pair of mergeable ranges, build a merged range. + merged_ranges = mergeable_range_pairs.filter_map do |r1, r2| + RangeFactory.build_non_empty( + nil_or(:min, from: [r1.begin, r2.begin]), + nil_or(:max, from: [r1.end, r2.end]) + ) + end + + # Update `all_ranges` based on the merges performed so far. + unmergeable_ranges = all_ranges - mergeable_range_pairs.flatten + all_ranges = unmergeable_ranges.union(_ = merged_ranges) + end + + all_ranges + end + + # Helper method for `merge_overlapping_or_adjacent_ranges` used to return the most "lenient" range boundary value. + # `nil` is used for a beginless or endless range, so we return that if available; otherwise + # we apply `min_or_max`.` + def nil_or(min_or_max, from:) + return nil if from.include?(nil) + from.public_send(min_or_max) + end + + def descending_range?(range) + # If either edge is `nil` it cannot be descending. + return false if (range_begin = range.begin).nil? + return false if (range_end = range.end).nil? + + # Otherwise we just compare the edges to determine if it's descending. + range_begin > range_end + end + + # An instance in which all `::Time`s fit. + ALL = new([::Range.new(nil, nil)]) + # Singleton instance that's empty. + EMPTY = new([]) + + module RangeFactory + # Helper method for building a range from the given bounds. Returns either + # a built range, or, if the given bounds produce an empty range, returns nil. + def self.build_non_empty(lower_bound, upper_bound) + if lower_bound.nil? || upper_bound.nil? || lower_bound <= upper_bound + ::Range.new(lower_bound, upper_bound) + end + end + end + end + end +end diff --git a/elasticgraph-support/lib/elastic_graph/support/time_util.rb b/elasticgraph-support/lib/elastic_graph/support/time_util.rb new file mode 100644 index 00000000..130d3ee2 --- /dev/null +++ b/elasticgraph-support/lib/elastic_graph/support/time_util.rb @@ -0,0 +1,109 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module Support + # @private + module TimeUtil + NANOS_PER_SECOND = 1_000_000_000 + NANOS_PER_MINUTE = NANOS_PER_SECOND * 60 + NANOS_PER_HOUR = NANOS_PER_MINUTE * 60 + + # Simple helper function to convert a local time string (such as `03:45:12` or `12:30:43.756`) + # to an integer value between 0 and 24 * 60 * 60 * 1,000,000,000 - 1 representing the nano of day + # for the local time value. + # + # This is meant to match the behavior of Java's `LocalTime#toNanoOfDay()` API: + # https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/time/LocalTime.html#toNanoOfDay() + # + # This is specifically useful when we need to work with local time values in a script: by converting + # a local time parameter to nano-of-day, our script can more efficiently compare values, avoiding the + # need to parse the same local time parameters over and over again as it applies the script to each + # document. + # + # Note: this method assumes the given `local_time_string` is well-formed. You'll get an exception if + # you provide a malformed value, but no effort has been put into giving a clear error message. The + # caller is expected to have already validated that the `local_time_string` is formatted correctly. + def self.nano_of_day_from_local_time(local_time_string) + hours_str, minutes_str, full_seconds_str = local_time_string.split(":") + seconds_str, subseconds_str = (_ = full_seconds_str).split(".") + + hours = Integer(_ = hours_str, 10) + minutes = Integer(_ = minutes_str, 10) + seconds = Integer(seconds_str, 10) + nanos = Integer(subseconds_str.to_s.ljust(9, "0"), 10) + + (hours * NANOS_PER_HOUR) + (minutes * NANOS_PER_MINUTE) + (seconds * NANOS_PER_SECOND) + nanos + end + + # Helper method for advancing time. Unfortunately, Ruby's core `Time` type does not directly support this. + # ActiveSupport (from rails) provides this functionality, but we don't depend on rails at all and don't + # want to add such a heavyweight dependency for such a small thing. + # + # Luckily, our needs are quite limited, which makes this a much simpler problem then a general purpose `time.advance(...)` API: + # + # - We only need to support year, month, day, and hour advances. + # - We only ever need to advance a single unit. + # + # This provides a simple, correct implementation for that constrained problem space. + def self.advance_one_unit(time, unit) + case unit + when :year + with_updated(time, year: time.year + 1) + when :month + maybe_next_month = + if time.month == 12 + with_updated(time, year: time.year + 1, month: 1) + else + with_updated(time, month: time.month + 1) + end + + # If the next month has fewer days than the month of `time`, then it can "spill over" to a day + # from the first week of the month following that. For example, if the date of `time` was 2021-01-31 + # and we add a month, it attempts to go to `2021-02-31` but such a date doesn't exist--instead + # `maybe_next_month` will be on `2021-03-03` because of the overflow. Here we correct for that. + # + # Our assumption (which we believe to be correct) is that every time this happens, both of these are true: + # - `time.day` is near the end of its month + # - `maybe_next_month.day` is near the start of its month + # + # ...and furthermore, we do not believe there is any other case where `time.day` and `maybe_next_month.day` can differ. + if time.day > maybe_next_month.day + corrected_date = maybe_next_month.to_date - maybe_next_month.day + with_updated(time, year: corrected_date.year, month: corrected_date.month, day: corrected_date.day) + else + maybe_next_month + end + when :day + next_day = time.to_date + 1 + with_updated(time, year: next_day.year, month: next_day.month, day: next_day.day) + when :hour + time + 3600 + end + end + + private_class_method def self.with_updated(time, year: time.year, month: time.month, day: time.day) + # UTC needs to be treated special here due to an oddity of Ruby's Time class: + # + # > Time.utc(2021, 12, 2, 12, 30, 30).iso8601 + # => "2021-12-02T12:30:30Z" + # > Time.new(2021, 12, 2, 12, 30, 30, 0).iso8601 + # => "2021-12-02T12:30:30+00:00" + # + # We want to preserve the `Z` suffix on the ISO8601 representation of the advanced time + # (if it was there on the original time), so we use the `::Time.utc` method here to do that. + # Non-UTC time must use `::Time.new(...)` with a UTC offset, though. + if time.utc? + ::Time.utc(year, month, day, time.hour, time.min, time.sec.to_r + time.subsec) + else + ::Time.new(year, month, day, time.hour, time.min, time.sec.to_r + time.subsec, time.utc_offset) + end + end + end + end +end diff --git a/elasticgraph-support/lib/elastic_graph/support/untyped_encoder.rb b/elasticgraph-support/lib/elastic_graph/support/untyped_encoder.rb new file mode 100644 index 00000000..20450b5d --- /dev/null +++ b/elasticgraph-support/lib/elastic_graph/support/untyped_encoder.rb @@ -0,0 +1,69 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "json" + +module ElasticGraph + module Support + # Responsible for encoding `Untyped` values into strings. This logic lives here in `elasticgraph-support` + # so that it can be shared between the `Untyped` indexing preparer (which lives in `elasticgraph-indexer`) + # and the `Untyped` coercion adapter (which lives in `elasticgraph-graphql`). It is important that these + # share the same logic so that the string values we attempt to filter on at query time match the string values + # we indexed when given the semantically equivalent untyped data. + # + # Note: change this class with care. Changing the behavior to make `encode` produce different strings may result + # in breaking queries if the `Untyped`s stored in the index were indexed using previous encoding logic. + # A backfill into the datastore will likely be required to avoid this issue. + # + # @private + module UntypedEncoder + # Encodes the given untyped value to a String so it can be indexed in a Elasticsearch/OpenSearch `keyword` field. + def self.encode(value) + return nil if value.nil? + # Note: we use `fast_generate` here instead of `generate`. They basically act the same, except + # `generate` includes an extra check for self-referential data structures. `value` here ultimately + # comes out of a parsed JSON document (e.g. either from an ElasticGraph event at indexing time, or + # as a GraphQL query variable at search time), and JSON cannot express self-referential data + # structures, so we do not have to worry about that happening. + # + # ...but even if it did, we would get an error either way: `JSON.generate` would raise + # `JSON::NestingError` whereas `:JSON.fast_generate` would give us a `SystemStackError`. + ::JSON.fast_generate(canonicalize(value)) + end + + # Decodes a previously encoded Untyped value, returning its original value. + def self.decode(string) + return nil if string.nil? + ::JSON.parse(string) + end + + # Helper method that converts `value` to a canonical form before we dump it as JSON. + # We do this because we index each JSON value as a `keyword` in the index, and we want + # equality filters on a JSON value field to consider equivalent JSON objects to be equal + # even if their normally generated JSON is not the same. For example, we want ElasticGraph + # to treat these two as being equivalent: + # + # {"a": 1, "b": 2} vs {"b": 2, "a": 1} + # + # To achieve this, we ensure JSON objects are generated in sorted order, and we use this same + # logic both at indexing time and also at query time when we are filtering. + private_class_method def self.canonicalize(value) + case value + when ::Hash + value + .sort_by { |k, v| k.to_s } + .to_h { |k, v| [k, canonicalize(v)] } + when ::Array + value.map { |v| canonicalize(v) } + else + value + end + end + end + end +end diff --git a/elasticgraph-support/lib/elastic_graph/version.rb b/elasticgraph-support/lib/elastic_graph/version.rb new file mode 100644 index 00000000..a2d89521 --- /dev/null +++ b/elasticgraph-support/lib/elastic_graph/version.rb @@ -0,0 +1,15 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + # The version of all ElasticGraph gems. + VERSION = "0.18.0.5" + + # Steep weirdly expects this here... + # @dynamic self.define_schema +end diff --git a/elasticgraph-support/sig/elastic_graph/buildable_from_parsed_yaml.rbs b/elasticgraph-support/sig/elastic_graph/buildable_from_parsed_yaml.rbs new file mode 100644 index 00000000..7577ceb7 --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/buildable_from_parsed_yaml.rbs @@ -0,0 +1,14 @@ +module ElasticGraph + # Parsed YAML settings are expected to be nested hash. The outer hash has + # keys which identify the part of ElasticGraph the settings relate to; the + # inner hash is settings for that part of ElasticGraph. + type parsedYamlSettings = ::Hash[::String, ::Hash[::String, untyped]] + + # Defines a simple factory interface that should be adopted by any class that is + # intended to be buildable from YAML config. The class should define a class method + # with this signature and then `extend _BuildableFromParsedYaml[TheClass]` in its RBS + # signatures. + interface _BuildableFromParsedYaml[T] + def from_parsed_yaml: (parsedYamlSettings) ?{ (untyped) -> void } -> T + end +end diff --git a/elasticgraph-support/sig/elastic_graph/common_types.rbs b/elasticgraph-support/sig/elastic_graph/common_types.rbs new file mode 100644 index 00000000..7c02816e --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/common_types.rbs @@ -0,0 +1,4 @@ +module ElasticGraph + type io = ::IO | ::StringIO + type untypedStringHash = ::Hash[::String, untyped] +end diff --git a/elasticgraph-support/sig/elastic_graph/constants.rbs b/elasticgraph-support/sig/elastic_graph/constants.rbs new file mode 100644 index 00000000..1688413b --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/constants.rbs @@ -0,0 +1,39 @@ +module ElasticGraph + DATASTORE_DATE_FORMAT: ::String + DATASTORE_DATE_TIME_FORMAT: ::String + TIMEOUT_MS_HEADER: ::String + INT_MIN: ::Integer + INT_MAX: ::Integer + JSON_SAFE_LONG_MAX: ::Integer + JSON_SAFE_LONG_MIN: ::Integer + LONG_STRING_MIN: ::Integer + LONG_STRING_MAX: ::Integer + DEFAULT_MAX_KEYWORD_LENGTH: ::Integer + DEFAULT_MAX_TEXT_LENGTH: ::Integer + EVENT_ENVELOPE_JSON_SCHEMA_NAME: ::String + SINGLETON_CURSOR: ::String + GRAPHQL_SCHEMA_FILE: ::String + JSON_SCHEMAS_FILE: ::String + DATASTORE_CONFIG_FILE: ::String + RUNTIME_METADATA_FILE: ::String + ROLLOVER_INDEX_INFIX_MARKER: ::String + JSON_SCHEMAS_BY_VERSION_DIRECTORY: ::String + JSON_SCHEMA_VERSION_KEY: ::String + DERIVED_INDEX_FAILURE_MESSAGE_PREAMBLE: ::String + INDEX_DATA_UPDATE_SCRIPT_ID: ::String + OLD_INDEX_DATA_UPDATE_SCRIPT_ID: ::String + UPDATE_WAS_NOOP_MESSAGE_PREAMBLE: ::String + SELF_RELATIONSHIP_NAME: ::String + VALID_LOCAL_TIME_REGEX: ::Regexp + VALID_LOCAL_TIME_JSON_SCHEMA_PATTERN: ::String + LIST_COUNTS_FIELD: ::String + LIST_COUNTS_FIELD_PATH_KEY_SEPARATOR: ::String + DATASTORE_PROPERTYLESS_OBJECT_TYPES: ::Set[::String] + GRAPHQL_NAME_WITHIN_LARGER_STRING_PATTERN: ::Regexp + GRAPHQL_NAME_PATTERN: ::Regexp + GRAPHQL_NAME_VALIDITY_DESCRIPTION: ::String + STOCK_GRAPHQL_SCALARS: ::Set[::String] + JSON_META_SCHEMA: ::String + DATASTORE_BULK_FILTER_PATH: ::String + GRAPHQL_LAMBDA_AWS_ARN_HEADER: ::String +end diff --git a/elasticgraph-support/sig/elastic_graph/errors.rbs b/elasticgraph-support/sig/elastic_graph/errors.rbs new file mode 100644 index 00000000..a5e87219 --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/errors.rbs @@ -0,0 +1,75 @@ +module ElasticGraph + module Errors + class Error < StandardError + end + + class CursorEncoderError < Error + end + + class InvalidSortFieldsError < CursorEncoderError + end + + class InvalidCursorError < CursorEncoderError + end + + class CursorEncodingError < CursorEncoderError + end + + class CountUnavailableError < Error + end + + class InvalidArgumentValueError < Error + end + + class InvalidMergeError < Error + end + + class SchemaError < Error + end + + class InvalidGraphQLNameError < SchemaError + end + + class NotFoundError < Error + end + + class SearchFailedError < Error + end + + class RequestExceededDeadlineError < SearchFailedError + end + + class IdentifyDocumentVersionsFailedError < Error + end + + class IndexOperationError < Error + end + + class ClusterOperationError < Error + end + + class InvalidExtensionError < Error + end + + class ConfigError < Error + end + + class ConfigSettingNotSetError < ConfigError + end + + class InvalidScriptDirectoryError < Error + end + + class MissingSchemaArtifactError < Error + end + + class S3OperationFailedError < Error + end + + class MessageIdsMissingError < Error + end + + class BadDatastoreRequest < Error + end + end +end diff --git a/elasticgraph-support/sig/elastic_graph/support/faraday_middleware/msearch_using_get_instead_of_post.rbs b/elasticgraph-support/sig/elastic_graph/support/faraday_middleware/msearch_using_get_instead_of_post.rbs new file mode 100644 index 00000000..971de5f9 --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/support/faraday_middleware/msearch_using_get_instead_of_post.rbs @@ -0,0 +1,11 @@ +module ElasticGraph + module Support + module FaradayMiddleware + class MSearchUsingGetInsteadOfPost + include ::Faraday::_App + attr_reader app: ::Faraday::_App + def initialize: (::Faraday::_App) -> void + end + end + end +end diff --git a/elasticgraph-support/sig/elastic_graph/support/faraday_middleware/support_timeouts.rbs b/elasticgraph-support/sig/elastic_graph/support/faraday_middleware/support_timeouts.rbs new file mode 100644 index 00000000..7123bc6c --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/support/faraday_middleware/support_timeouts.rbs @@ -0,0 +1,11 @@ +module ElasticGraph + module Support + module FaradayMiddleware + class SupportTimeouts + include ::Faraday::_App + attr_reader app: ::Faraday::_App + def initialize: (::Faraday::_App) -> void + end + end + end +end diff --git a/elasticgraph-support/sig/elastic_graph/support/from_yaml_file.rbs b/elasticgraph-support/sig/elastic_graph/support/from_yaml_file.rbs new file mode 100644 index 00000000..00450c7b --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/support/from_yaml_file.rbs @@ -0,0 +1,15 @@ +module ElasticGraph + module Support + module FromYamlFile[T]: _BuildableFromParsedYaml[T] + def from_yaml_file: ( + ::String, + ?datastore_client_customization_block: (^(untyped) -> void)? + ) ?{ (parsedYamlSettings) -> parsedYamlSettings } -> T + + class ForRakeTasks[T] < ::Module + def initialize: (_BuildableFromParsedYaml[untyped]) -> void + def from_yaml_file: (::String, *untyped, **untyped) -> T + end + end + end +end diff --git a/elasticgraph-support/sig/elastic_graph/support/graphql_formatter.rbs b/elasticgraph-support/sig/elastic_graph/support/graphql_formatter.rbs new file mode 100644 index 00000000..2eade6d2 --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/support/graphql_formatter.rbs @@ -0,0 +1,8 @@ +module ElasticGraph + module Support + module GraphQLFormatter + def self.format_args: (**untyped) -> ::String + def self.serialize: (untyped, ?wrap_hash_with_braces: bool) -> ::String + end + end +end diff --git a/elasticgraph-support/sig/elastic_graph/support/hash_util.rbs b/elasticgraph-support/sig/elastic_graph/support/hash_util.rbs new file mode 100644 index 00000000..c35fb8be --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/support/hash_util.rbs @@ -0,0 +1,47 @@ +module ElasticGraph + module Support + class HashUtil + type primitive = ::String | ::Integer | bool + type key = ::String | ::Symbol + type value = ::Hash[key, value] | ::Array[value] | primitive? + type nonNullableValue = ::Hash[key, nonNullableValue] | ::Array[nonNullableValue] | primitive + + def self.verbose_fetch: [V] (::Hash[::String | ::Symbol, V], ::String | ::Symbol) -> V + + def self.strict_to_h: [K, V] (::Array[[K, V]] | ::Set[[K, V]]) -> ::Hash[K, V] + + def self.disjoint_merge: [K, V] (::Hash[K, V], ::Hash[K, V]) -> ::Hash[K, V] + + def self.stringify_keys: + [V] (::Hash[::String, V]) -> ::Hash[::String, V] | + [V] (::Hash[::Symbol, V]) -> ::Hash[::String, V] | + [V] (::Hash[::String | ::Symbol, V]) -> ::Hash[::String, V] + + def self.symbolize_keys: + [V] (::Hash[::String, V]) -> ::Hash[::Symbol, V] | + [V] (::Hash[::Symbol, V]) -> ::Hash[::Symbol, V] | + [V] (::Hash[::String | ::Symbol, V]) -> ::Hash[::Symbol, V] + + def self.recursively_prune_nils_from: + [V] (::Hash[::Symbol, V?]) ?{ (::String) -> void } -> ::Hash[::Symbol, V] | + [V] (::Hash[::String, V?]) ?{ (::String) -> void } -> ::Hash[::String, V] + + def self.recursively_prune_nils_and_empties_from: + [V] (::Hash[::Symbol, V?]) ?{ (::String) -> void } -> ::Hash[::Symbol, V] | + [V] (::Hash[::String, V?]) ?{ (::String) -> void } -> ::Hash[::String, V] + + def self.flatten_and_stringify_keys: [K] (::Hash[K, untyped], ?prefix: ::String?) -> ::Hash[::String, untyped] + def self.deep_merge: [K, V] (::Hash[K, V], ::Hash[K, V]) -> ::Hash[K, V] + def self.fetch_leaf_values_at_path: (::Hash[::String, untyped], ::String) ?{ (String) -> untyped } -> ::Array[untyped] + def self.fetch_value_at_path: (::Hash[::String, untyped], ::String) ?{ (String) -> untyped } -> untyped + + private + + # Fully expressing the types here without using `untyped` is quite hard! So for now we use `untyped`. + def self.recursively_prune_if: (::Hash[untyped, untyped], (^(::String) -> void)?) { (untyped) -> bool } -> ::Hash[untyped, untyped] + def self.recursively_transform: (untyped, ?::String?) { (key, value, ::Hash[key, value], ::String) -> void } -> untyped + def self.populate_flat_hash: [K] (::Hash[K, untyped], ::String, ::Hash[::String, untyped]) -> void + def self.do_fetch_leaf_values_at_path: (untyped, ::Array[::String], ::Integer) ?{ (String) -> untyped } -> ::Array[untyped] + end + end +end diff --git a/elasticgraph-support/sig/elastic_graph/support/logger.rbs b/elasticgraph-support/sig/elastic_graph/support/logger.rbs new file mode 100644 index 00000000..1a096d7a --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/support/logger.rbs @@ -0,0 +1,45 @@ +module ElasticGraph + module Support + module Logger + extend _BuildableFromParsedYaml[::Logger] + + module Factory + def self.build: (config: Config, ?device: ::Logger::logdev?) -> ::Logger + end + + class JSONAwareFormatter + include ::Logger::_Formatter + @original_formatter: ::Logger::Formatter + + # Steep 1.6 complains about our impl returning `nil` unless we define this as void + def initialize: () -> void + end + + class ConfigSupertype + attr_reader level: ::String + attr_reader device: ::String + attr_reader formatter: ::Logger::_Formatter + + def initialize: ( + level: ::String, + device: ::String, + formatter: ::Logger::_Formatter + ) -> void + + def with: ( + ?level: ::String, + ?device: ::String, + ?formatter: ::Logger::_Formatter + ) -> Config + + def self.members: () -> ::Array[::Symbol] + end + + class Config < ConfigSupertype + extend _BuildableFromParsedYaml[Config] + def prepared_device: () -> (::String | ::IO) + EXPECTED_KEYS: ::Array[::String] + end + end + end +end diff --git a/elasticgraph-support/sig/elastic_graph/support/memoizable_data.rbs b/elasticgraph-support/sig/elastic_graph/support/memoizable_data.rbs new file mode 100644 index 00000000..2e603694 --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/support/memoizable_data.rbs @@ -0,0 +1,16 @@ +module ElasticGraph + module Support + module MemoizableData + def self.define: [KLASS < _MemoizableDataClass] (*Symbol) ?{ () -> void } -> KLASS + + module InstanceMethods + end + + module ClassMethods + end + end + + interface _MemoizableDataClass + end + end +end diff --git a/elasticgraph-support/sig/elastic_graph/support/monotonic_clock.rbs b/elasticgraph-support/sig/elastic_graph/support/monotonic_clock.rbs new file mode 100644 index 00000000..af61b39c --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/support/monotonic_clock.rbs @@ -0,0 +1,7 @@ +module ElasticGraph + module Support + class MonotonicClock + def now_in_ms: () -> ::Integer + end + end +end diff --git a/elasticgraph-support/sig/elastic_graph/support/negatable_set.rbs b/elasticgraph-support/sig/elastic_graph/support/negatable_set.rbs new file mode 100644 index 00000000..fa5024e0 --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/support/negatable_set.rbs @@ -0,0 +1,10 @@ +module ElasticGraph + module Support + # A minimal set interface that also offers a `negate` operation. + interface _NegatableSet[S] + def union: (S) -> S + def intersection: (S) -> S + def negate: () -> S + end + end +end diff --git a/elasticgraph-support/sig/elastic_graph/support/threading.rbs b/elasticgraph-support/sig/elastic_graph/support/threading.rbs new file mode 100644 index 00000000..39ceb7ce --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/support/threading.rbs @@ -0,0 +1,9 @@ +module ElasticGraph + module Support + module Threading + def self.parallel_map: + [K, V, O] (::Hash[K, V]) { ([K, V]) -> O } -> ::Array[O] + | [I, O] (::Enumerable[I]) { (I) -> O } -> ::Array[O] + end + end +end diff --git a/elasticgraph-support/sig/elastic_graph/support/time_set.rbs b/elasticgraph-support/sig/elastic_graph/support/time_set.rbs new file mode 100644 index 00000000..21600a22 --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/support/time_set.rbs @@ -0,0 +1,49 @@ +module ElasticGraph + module Support + class TimeSetSupertype + def initialize: (ranges: ::Set[TimeSet::timeRange]) -> void + attr_reader ranges: ::Set[TimeSet::timeRange] + end + + class TimeSet < TimeSetSupertype + include _NegatableSet[TimeSet] + type timeRange = ::Range[::Time?] + + def self.of_range: ( + ?gt: ::Time?, + ?gte: ::Time?, + ?lt: ::Time?, + ?lte: ::Time?, + ) -> TimeSet + + def self.of_times: (::Enumerable[::Time]) -> TimeSet + def self.of_range_objects: (::Array[timeRange] | ::Set[timeRange]) -> TimeSet + + ALL: TimeSet + EMPTY: TimeSet + + def member?: (::Time?) -> bool + def intersect?: (TimeSet) -> bool + def empty?: () -> bool + def -: (TimeSet) -> TimeSet + + def self.new: (ranges: ::Array[timeRange] | ::Set[timeRange]) -> instance + | (::Array[timeRange] | ::Set[timeRange]) -> instance + + private + + def initialize: (ranges: ::Array[timeRange] | ::Set[timeRange]) -> void + CONSECUTIVE_TIME_INCREMENT: ::Rational + def ranges_intersect?: (timeRange, timeRange) -> bool + def adjacent?: (timeRange, timeRange) -> bool + def intersect_ranges: (timeRange, timeRange) -> timeRange? + def merge_overlapping_or_adjacent_ranges: (::Set[timeRange]) -> ::Set[timeRange] + def nil_or: (:min | :max, from: [::Time?, ::Time?]) -> ::Time? + def descending_range?: (timeRange) -> bool + + module RangeFactory + def self.build_non_empty: (::Time?, ::Time?) -> timeRange? + end + end + end +end diff --git a/elasticgraph-support/sig/elastic_graph/support/time_util.rbs b/elasticgraph-support/sig/elastic_graph/support/time_util.rbs new file mode 100644 index 00000000..53ad60ca --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/support/time_util.rbs @@ -0,0 +1,16 @@ +module ElasticGraph + module Support + module TimeUtil + NANOS_PER_SECOND: ::Integer + NANOS_PER_MINUTE: ::Integer + NANOS_PER_HOUR: ::Integer + + def self.nano_of_day_from_local_time: (::String) -> ::Integer + + type advancementUnit = :year | :month | :day | :hour + def self.advance_one_unit: (::Time, advancementUnit) -> ::Time + + def self.with_updated: (::Time, ?year: ::Integer, ?month: ::Integer, ?day: ::Integer) -> ::Time + end + end +end diff --git a/elasticgraph-support/sig/elastic_graph/support/untyped_encoder.rbs b/elasticgraph-support/sig/elastic_graph/support/untyped_encoder.rbs new file mode 100644 index 00000000..0070e5b3 --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/support/untyped_encoder.rbs @@ -0,0 +1,9 @@ +module ElasticGraph + module Support + module UntypedEncoder + def self.encode: (untyped) -> ::String? + def self.decode: (::String?) -> untyped + def self.canonicalize: (untyped) -> untyped + end + end +end diff --git a/elasticgraph-support/sig/elastic_graph/version.rbs b/elasticgraph-support/sig/elastic_graph/version.rbs new file mode 100644 index 00000000..29fb931b --- /dev/null +++ b/elasticgraph-support/sig/elastic_graph/version.rbs @@ -0,0 +1,3 @@ +module ElasticGraph + VERSION: ::String +end diff --git a/elasticgraph-support/sig/faraday.rbs b/elasticgraph-support/sig/faraday.rbs new file mode 100644 index 00000000..30a1cfe9 --- /dev/null +++ b/elasticgraph-support/sig/faraday.rbs @@ -0,0 +1,9 @@ +module Faraday + class Request + attr_accessor timeout: ::Numeric + end + + interface _App + def call: (Env) -> Env + end +end diff --git a/elasticgraph-support/sig/patches.rbs b/elasticgraph-support/sig/patches.rbs new file mode 100644 index 00000000..d1fd5e27 --- /dev/null +++ b/elasticgraph-support/sig/patches.rbs @@ -0,0 +1,3 @@ +class Array[unchecked out Elem] + def combination: (2) -> ::Enumerator[[Elem, Elem], self] | ... +end diff --git a/elasticgraph-support/sig/yaml.rbs b/elasticgraph-support/sig/yaml.rbs new file mode 100644 index 00000000..90f1206c --- /dev/null +++ b/elasticgraph-support/sig/yaml.rbs @@ -0,0 +1,6 @@ +module YAML + def self.dump: (untyped) -> ::String + def self.safe_load: (::String, ?aliases: bool) -> untyped + def self.load_file: (::String) -> untyped + def self.safe_load_file: (::String, ?aliases: bool) -> untyped +end diff --git a/elasticgraph-support/spec/spec_helper.rb b/elasticgraph-support/spec/spec_helper.rb new file mode 100644 index 00000000..ac85b825 --- /dev/null +++ b/elasticgraph-support/spec/spec_helper.rb @@ -0,0 +1,22 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file is contains RSpec configuration and common support code for `elasticgraph-support`. +# Note that it gets loaded by `spec_support/spec_helper.rb` which contains common spec support +# code for all ElasticGraph test suites. + +# Here we load the version file so that it gets covered. When running the test suite without using bundler (e.g. after setting +# things up with `bundler --standalone`) the version file will not be pre-loaded from the gemspec like it usually is when run +# through bundler so we need to load it for it to be covered. +require "elastic_graph/version" + +module ElasticGraph + module Support + SPEC_ROOT = __dir__ + end +end diff --git a/elasticgraph-support/spec/support/local_time_to_nano_of_day.java b/elasticgraph-support/spec/support/local_time_to_nano_of_day.java new file mode 100755 index 00000000..586090d5 --- /dev/null +++ b/elasticgraph-support/spec/support/local_time_to_nano_of_day.java @@ -0,0 +1,14 @@ +import java.time.LocalTime; + +/** + * Run this java file via `java --source 11 local_time_to_nano_of_day.java [local_time_string]`. + * + * Note: this script is designed for use from: + * elasticgraph-support/spec/unit/elastic_graph/support/time_util_spec.rb + */ +public class LocalTimeToNanoOfDay { + public static void main(String[] args) { + LocalTime time = LocalTime.parse(args[0]); + System.out.println(time.toNanoOfDay()); + } +} diff --git a/elasticgraph-support/spec/unit/elastic_graph/constants_spec.rb b/elasticgraph-support/spec/unit/elastic_graph/constants_spec.rb new file mode 100644 index 00000000..c66d4215 --- /dev/null +++ b/elasticgraph-support/spec/unit/elastic_graph/constants_spec.rb @@ -0,0 +1,21 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "base64" +require "json" +require "elastic_graph/constants" + +module ElasticGraph + RSpec.describe "Constants" do + specify "SINGLETON_CURSOR is a unique value we never expect to get for a normal cursor while still being encoded like a normal cursor" do + encoded_data = ::JSON.parse(::Base64.urlsafe_decode64(SINGLETON_CURSOR)) + + expect(encoded_data).to eq({"uuid" => "dca02d20-baee-4ee9-a027-feece0a6de3a"}) + end + end +end diff --git a/elasticgraph-support/spec/unit/elastic_graph/support/faraday_middleware/msearch_using_get_instead_of_post_spec.rb b/elasticgraph-support/spec/unit/elastic_graph/support/faraday_middleware/msearch_using_get_instead_of_post_spec.rb new file mode 100644 index 00000000..a3adc27d --- /dev/null +++ b/elasticgraph-support/spec/unit/elastic_graph/support/faraday_middleware/msearch_using_get_instead_of_post_spec.rb @@ -0,0 +1,83 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/faraday_middleware/msearch_using_get_instead_of_post" +require "faraday" + +module ElasticGraph + module Support + module FaradayMiddleware + RSpec.describe MSearchUsingGetInsteadOfPost, :no_vcr do + it "converts a POST to a path ending in `_msearch` to a GET since msearch is read-only" do + faraday = stubbed_faraday do |stub| + stub.get("/foo/bar/_msearch") do |env| + text_response("GET msearch") + end + end + + response = faraday.post("/foo/bar/_msearch") do |req| + req.body = "some body" + end + + expect(response.body).to eq "GET msearch" + end + + it "leaves a POST to a non-msearch URL unchanged" do + faraday = stubbed_faraday do |stub| + stub.post("/foo/bar/other") do |env| + text_response("POST other") + end + end + + response = faraday.post("/foo/bar/other") do |req| + req.body = "some body" + end + + expect(response.body).to eq "POST other" + end + + it "leaves a GET to a path ending in `_msearch` unchanged" do + faraday = stubbed_faraday do |stub| + stub.get("/foo/bar/_msearch") do |env| + text_response("GET msearch") + end + end + + response = faraday.get("/foo/bar/_msearch") + + expect(response.body).to eq "GET msearch" + end + + it "does not treat a path like `/foo/bar_msearch` as an msearch path" do + faraday = stubbed_faraday do |stub| + stub.post("/foo/bar_msearch") do |env| + text_response("POST bar_msearch") + end + end + + response = faraday.post("/foo/bar_msearch") do |req| + req.body = "some body" + end + + expect(response.body).to eq "POST bar_msearch" + end + + def stubbed_faraday(&stub_block) + ::Faraday.new do |faraday| + faraday.use MSearchUsingGetInsteadOfPost + faraday.adapter(:test, &stub_block) + end + end + + def text_response(text) + [200, {"Content-Type" => "text/plain"}, text] + end + end + end + end +end diff --git a/elasticgraph-support/spec/unit/elastic_graph/support/faraday_middleware/support_timeouts_spec.rb b/elasticgraph-support/spec/unit/elastic_graph/support/faraday_middleware/support_timeouts_spec.rb new file mode 100644 index 00000000..660ef412 --- /dev/null +++ b/elasticgraph-support/spec/unit/elastic_graph/support/faraday_middleware/support_timeouts_spec.rb @@ -0,0 +1,71 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/faraday_middleware/support_timeouts" +require "faraday" + +module ElasticGraph + module Support + module FaradayMiddleware + RSpec.describe SupportTimeouts, :no_vcr do + it "sets the request timeout if the TIMEOUT_MS_HEADER is present" do + faraday = stubbed_faraday do |stub| + stub.get("/foo/bar") do |env| + expect(env.request.timeout).to eq 10 + text_response("GET bar") + end + end + + response = faraday.get("/foo/bar") do |req| + req.headers[TIMEOUT_MS_HEADER] = "10000" + end + + expect(response.body).to eq "GET bar" + end + + it "does not set the request timeout if the TIMEOUT_MS_HEADER is not present" do + faraday = stubbed_faraday do |stub| + stub.get("/foo/bar") do |env| + expect(env.request.timeout).to be nil + text_response("GET bar") + end + end + + response = faraday.get("/foo/bar") + + expect(response.body).to eq "GET bar" + end + + it "converts a `Faraday::TimeoutError` to a `Errors::RequestExceededDeadlineError`" do + faraday = stubbed_faraday do |stub| + stub.get("/foo/bar") do |env| + raise ::Faraday::TimeoutError + end + end + + expect { + faraday.get("/foo/bar") do |req| + req.headers[TIMEOUT_MS_HEADER] = "10000" + end + }.to raise_error Errors::RequestExceededDeadlineError, "Datastore request exceeded timeout of 10000 ms." + end + + def stubbed_faraday(&stub_block) + ::Faraday.new do |faraday| + faraday.use SupportTimeouts + faraday.adapter(:test, &stub_block) + end + end + + def text_response(text) + [200, {"Content-Type" => "text/plain"}, text] + end + end + end + end +end diff --git a/elasticgraph-support/spec/unit/elastic_graph/support/from_yaml_file_spec.rb b/elasticgraph-support/spec/unit/elastic_graph/support/from_yaml_file_spec.rb new file mode 100644 index 00000000..768d889d --- /dev/null +++ b/elasticgraph-support/spec/unit/elastic_graph/support/from_yaml_file_spec.rb @@ -0,0 +1,133 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/from_yaml_file" +require "json" +require "rake/tasklib" + +module ElasticGraph + module Support + RSpec.describe FromYamlFile, :in_temp_dir do + before do + stub_const "ExampleComponent", component_class + end + + let(:component_class) do + ::Data.define(:sub_settings_by_name, :datastore_client_customization_block) do + extend FromYamlFile + + def self.from_parsed_yaml(parsed_yaml, &datastore_client_customization_block) + new( + sub_settings_by_name: parsed_yaml.fetch("sub_settings_by_name"), + datastore_client_customization_block: datastore_client_customization_block + ) + end + end + end + + context "when extended onto a class that implements `.from_parsed_yaml`" do + it "builds the class using `.from_parsed_yaml` after first loading YAML the file from disk with alias support" do + ::File.write("settings.yaml", <<~EOS) + sub_settings_by_name: + foo: &foo_settings + size: 12 + bar: *foo_settings + EOS + + customization_block = lambda { |block| } + instance = component_class.from_yaml_file("settings.yaml", datastore_client_customization_block: customization_block) + + expect(instance).to be_a(component_class) + expect(instance.sub_settings_by_name).to eq("foo" => {"size" => 12}, "bar" => {"size" => 12}) + expect(instance.datastore_client_customization_block).to be(customization_block) + end + + it "allows the settings to be overridden using the provided block before the object is built" do + ::File.write("settings.yaml", <<~EOS) + sub_settings_by_name: + foo: &foo_settings + size: 12 + bar: *foo_settings + EOS + + instance = component_class.from_yaml_file("settings.yaml") do |settings| + settings.merge("sub_settings_by_name" => settings.fetch("sub_settings_by_name").merge( + "bar" => {"size" => 14} + )) + end + + expect(instance).to be_a(component_class) + expect(instance.sub_settings_by_name).to eq( + "foo" => {"size" => 12}, + "bar" => {"size" => 14} + ) + end + end + + describe FromYamlFile::ForRakeTasks, :rake_task do + let(:rake_tasks_class) do + component = component_class + + Class.new(::Rake::TaskLib) do + extend FromYamlFile::ForRakeTasks.new(component) + + def initialize(output:, &load_component) + desc "Uses the component" + task :use_component do + component = load_component.call + output.puts "Subsettings: #{::JSON.pretty_generate(component.sub_settings_by_name)}" + end + end + end + end + + it "loads the component from the named yaml file and provides it to the rake task library" do + ::File.write("settings.yaml", <<~EOS) + sub_settings_by_name: + foo: + size: 12 + EOS + + output = run_rake "use_component" + + expect(output).to eq(<<~EOS) + Subsettings: { + "foo": { + "size": 12 + } + } + EOS + end + + it "tells the user to regenerate schema artifacts if the component fails to load" do + expect { + run_rake "use_component" + }.to raise_error(a_string_including( + "Failed to load `ExampleComponent` with `settings.yaml`. This can happen if the schema artifacts are out of date.", + "Run `rake schema_artifacts:dump` and try again.", + "No such file or directory" + )) + end + + it "loads the artifacts lazily, allowing tasks to be listed when the artifacts do not exist or are out of date" do + output = run_rake "--tasks" + + expect(output).to eq(<<~EOS) + rake use_component # Uses the component + EOS + end + + def run_rake(task) + super(task) do |output| + rake_tasks_class.from_yaml_file("settings.yaml", output: output) + end + end + end + end + end +end diff --git a/elasticgraph-support/spec/unit/elastic_graph/support/graphql_formatter_spec.rb b/elasticgraph-support/spec/unit/elastic_graph/support/graphql_formatter_spec.rb new file mode 100644 index 00000000..e2174914 --- /dev/null +++ b/elasticgraph-support/spec/unit/elastic_graph/support/graphql_formatter_spec.rb @@ -0,0 +1,43 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/graphql_formatter" + +module ElasticGraph + module Support + RSpec.describe GraphQLFormatter do + describe "#format_args" do + include GraphQLFormatter + + it "formats an empty arg hash as an empty string" do + formatted = GraphQLFormatter.format_args + + expect(formatted).to eq "" + end + + it "formats numbers, booleans, and string args correctly" do + formatted = GraphQLFormatter.format_args(foo: 3, bar: false, bazz: true, quix: "abc") + + expect(formatted).to eq('(foo: 3, bar: false, bazz: true, quix: "abc")') + end + + it "formats nested arrays and nested objects correctly" do + formatted = GraphQLFormatter.format_args(foo: [1, 2], bar: [{a: 1}, {a: 2}], bazz: {c: 12}) + + expect(formatted).to eq("(foo: [1, 2], bar: [{a: 1}, {a: 2}], bazz: {c: 12})") + end + + it "formats symbols as GraphQL enum values" do + formatted = GraphQLFormatter.format_args(foo: :bar, bazz: "bar") + + expect(formatted).to eq('(foo: bar, bazz: "bar")') + end + end + end + end +end diff --git a/elasticgraph-support/spec/unit/elastic_graph/support/hash_util_spec.rb b/elasticgraph-support/spec/unit/elastic_graph/support/hash_util_spec.rb new file mode 100644 index 00000000..46489776 --- /dev/null +++ b/elasticgraph-support/spec/unit/elastic_graph/support/hash_util_spec.rb @@ -0,0 +1,628 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/hash_util" + +module ElasticGraph + module Support + RSpec.describe HashUtil do + describe ".verbose_fetch" do + it "returns the value for the provided key" do + hash = {foo: 1} + + expect(HashUtil.verbose_fetch(hash, :foo)).to eq 1 + + # ...just like Hash#fetch + expect(hash.fetch(:foo)).to eq 1 + end + + it "indicates the available keys in the error message when given a missing key" do + hash = {foo: 1, bar: 2} + + expect { + HashUtil.verbose_fetch(hash, :bazz) + }.to raise_error KeyError, "key not found: :bazz. Available keys: [:foo, :bar]." + + # ...in contrast to Hash#fetch which does not include the available keys + expect { + hash.fetch(:bazz) + }.to raise_error KeyError, "key not found: :bazz" + end + end + + describe ".strict_to_h" do + it "builds a hash from a list of key/value pairs" do + pairs = [[:foo, 1], [:bar, 2]] + + expect(HashUtil.strict_to_h(pairs)).to eq({foo: 1, bar: 2}) + + # ...just like Hash#to_h + expect(pairs.to_h).to eq({foo: 1, bar: 2}) + end + + it "raises an error if there are conflicting keys, unlike `Hash#to_h`" do + pairs = [[:foo, 1], [:bar, 2], [:foo, 3], [:bazz, 4], [:bar, 1]] + + expect { + HashUtil.strict_to_h(pairs) + }.to raise_error KeyError, "Cannot build a strict hash, since input has conflicting keys: [:foo, :bar]." + + # ... in contrast to Hash#to_h, which allows later entries to stomp earlier ones + expect(pairs.to_h).to eq({foo: 3, bazz: 4, bar: 1}) + end + end + + describe ".disjoint_merge" do + it "merges two disjoint hashes" do + hash1 = {foo: 1, bar: 2} + hash2 = {bazz: 3} + + expect(HashUtil.disjoint_merge(hash1, hash2)).to eq({foo: 1, bar: 2, bazz: 3}) + + # ...just like Hash#merge + expect(hash1.merge(hash2)).to eq({foo: 1, bar: 2, bazz: 3}) + end + + it "raises an error if the hashes are not disjoint" do + hash1 = {foo: 1, bar: 2} + hash2 = {foo: 3} + + expect { + HashUtil.disjoint_merge(hash1, hash2) + }.to raise_error KeyError, "Hashes were not disjoint. Conflicting keys: [:foo]." + + # ...in contrast to Hash#merge, which lets the entry from the last hash win. + expect(hash1.merge(hash2)).to eq({foo: 3, bar: 2}) + end + end + + describe ".stringify_keys" do + it "leaves a hash with string keys unchanged" do + expect(HashUtil.stringify_keys({"a" => 1})).to eq({"a" => 1}) + end + + it "replaces symbol keys with string keys" do + expect(HashUtil.stringify_keys({a: 1, b: 2})).to eq({"a" => 1, "b" => 2}) + end + + it "recursively stringifies keys through a deeply nested hash" do + expect(HashUtil.stringify_keys({a: {b: {c: 2}}})).to eq({"a" => {"b" => {"c" => 2}}}) + end + + it "recursively stringifies keys through nested arrays" do + expect(HashUtil.stringify_keys({a: [{b: 1}, {c: 2}]})).to eq({"a" => [{"b" => 1}, {"c" => 2}]}) + end + end + + describe ".symbolize_keys" do + it "leaves a hash with symbol keys unchanged" do + expect(HashUtil.symbolize_keys({a: 1})).to eq({a: 1}) + end + + it "replaces string keys with symbol keys" do + expect(HashUtil.symbolize_keys({"a" => 1, "b" => 2})).to eq({a: 1, b: 2}) + end + + it "recursively symbolizes keys through a deeply nested hash" do + expect(HashUtil.symbolize_keys({"a" => {"b" => {"c" => 2}}})).to eq({a: {b: {c: 2}}}) + end + + it "recursively symbolizes keys through nested arrays" do + expect(HashUtil.symbolize_keys({"a" => [{"b" => 1}, {"c" => 2}]})).to eq({a: [{b: 1}, {c: 2}]}) + end + end + + describe ".recursively_prune_nils_from" do + it "echoes a hash back that has no nils in it" do + result = HashUtil.recursively_prune_nils_from({a: 1, b: {c: 2, d: [1, 2]}}) + expect(result).to eq({a: 1, b: {c: 2, d: [1, 2]}}) + end + + it "removes entries with `nil` values at any level of a nested hash structure" do + result = HashUtil.recursively_prune_nils_from({a: 1, b: {c: 2, d: [1, 2], e: nil}, f: nil}) + expect(result).to eq({a: 1, b: {c: 2, d: [1, 2]}}) + end + + it "recursively applies to hashes inside nested arrays" do + result = HashUtil.recursively_prune_nils_from({a: 1, b: {c: 2, d: [{g: nil, h: 1}, {g: 2, h: nil}]}}) + expect(result).to eq({a: 1, b: {c: 2, d: [{h: 1}, {g: 2}]}}) + end + + it "yields each pruned key path to support so the caller can do things like log warnings" do + expect { |probe| + HashUtil.recursively_prune_nils_from({ + z: nil, + a: 1, + b: { + c: 2, + d: [ + {g: nil, h: 1}, + {g: 2, h: nil} + ], + foo: nil + }, + bar: nil + }, &probe) + }.to yield_successive_args( + "z", + "b.d[0].g", + "b.d[1].h", + "b.foo", + "bar" + ) + end + end + + describe ".recursively_prune_nils_and_empties_from" do + it "echoes a hash back that has no nils or empties in it" do + result = HashUtil.recursively_prune_nils_and_empties_from({a: 1, b: {c: 2, d: [1, 2]}}) + expect(result).to eq({a: 1, b: {c: 2, d: [1, 2]}}) + end + + it "removes entries with `nil` values at any level of a nested hash structure" do + result = HashUtil.recursively_prune_nils_and_empties_from({a: 1, b: {c: 2, d: [1, 2], e: nil}, f: nil}) + expect(result).to eq({a: 1, b: {c: 2, d: [1, 2]}}) + end + + it "removes empty hash or array entries at any level of a nested hash structure" do + result = HashUtil.recursively_prune_nils_and_empties_from({a: 1, b: {c: 2, d: [1, 2], e: []}, f: {}}) + expect(result).to eq({a: 1, b: {c: 2, d: [1, 2]}}) + end + + it "does not remove empty strings" do + result = HashUtil.recursively_prune_nils_and_empties_from({a: ""}) + expect(result).to eq({a: ""}) + end + + it "recursively applies nil pruning to hashes inside nested arrays" do + result = HashUtil.recursively_prune_nils_and_empties_from({a: 1, b: {c: 2, d: [{g: nil, h: 1}, {g: 2, h: nil}]}}) + expect(result).to eq({a: 1, b: {c: 2, d: [{h: 1}, {g: 2}]}}) + end + + it "recursively applies empty object pruning to hashes inside nested arrays" do + result = HashUtil.recursively_prune_nils_and_empties_from({a: 1, b: {c: 2, d: [{g: [], h: 1}, {g: 2, h: {}}]}}) + expect(result).to eq({a: 1, b: {c: 2, d: [{h: 1}, {g: 2}]}}) + end + + it "yields each pruned key path to support so the caller can do things like log warnings" do + expect { |probe| + HashUtil.recursively_prune_nils_and_empties_from({ + z: nil, + a: 1, + b: { + c: 2, + d: [ + {g: nil, h: 1}, + {g: 2, h: {}} + ], + foo: [] + }, + bar: nil + }, &probe) + }.to yield_successive_args( + "z", + "b.d[0].g", + "b.d[1].h", + "b.foo", + "bar" + ) + end + end + + describe ".flatten_and_stringify_keys" do + it "leaves a flat hash with string keys unchanged" do + expect(HashUtil.flatten_and_stringify_keys({"a" => 1, "b.c" => 2})).to eq({"a" => 1, "b.c" => 2}) + end + + it "converts symbol keys to strings" do + expect(HashUtil.flatten_and_stringify_keys({:a => 1, "b.c" => 2})).to eq({"a" => 1, "b.c" => 2}) + end + + it "flattens nested hashes using dot-separated keys" do + expect(HashUtil.flatten_and_stringify_keys({a: {:b => 3, 2 => false, :c => {d: 5}}, h: 9})).to eq( + {"a.b" => 3, "a.2" => false, "a.c.d" => 5, "h" => 9} + ) + end + + it "supports a `prefix` arg" do + expect(HashUtil.flatten_and_stringify_keys({a: {:b => 3, 2 => false, :c => {d: 5}}, h: 9}, prefix: "foo")).to eq( + {"foo.a.b" => 3, "foo.a.2" => false, "foo.a.c.d" => 5, "foo.h" => 9} + ) + end + + it "raises an exception on an array of hashes" do + expect { + HashUtil.flatten_and_stringify_keys({a: [{b: 1}, {b: 2}]}) + }.to raise_error(/cannot handle nested arrays of hashes/) + end + + it "leaves an array of scalars or an empty array unchanged" do + expect(HashUtil.flatten_and_stringify_keys({a: {b: [1, 2], c: ["d", "f"], g: []}})).to eq( + {"a.b" => [1, 2], "a.c" => ["d", "f"], "a.g" => []} + ) + end + end + + describe ".deep_merge" do + it "is a no-op when merging an empty hash into an existing hash" do + hash1 = { + property1: 1, + property2: { + property3: "abc" + } + } + expect(HashUtil.deep_merge(hash1, {})).to eq(hash1) + end + + it "returns a deep copy of hash2 when hash1 is empty" do + hash2 = { + property1: 1, + property2: { + property3: "abc" + } + } + expect(HashUtil.deep_merge({}, hash2)).to eq(hash2) + end + + it "values from hash2 overwrite values from hash1 (just like HashUtil#merge) when both are flat hashes with same keys" do + hash1 = { + property1: 1, + property2: 2 + } + + hash2 = { + property1: 3, + property2: 4 + } + expect(HashUtil.deep_merge(hash1, hash2)).to eq(hash2) + end + + it "values from hash2 overwrite values from hash1 for common keys and copies their unique keys when hash1 and hash2 have different keys" do + hash1 = { + property1: 1, + property2: 2, + property3: 3 + } + + hash2 = { + property1: 3, + property5: 5 + } + expect(HashUtil.deep_merge(hash1, hash2)).to eq({ + property1: 3, + property2: 2, + property3: 3, + property5: 5 + }) + end + + it "merge values between nested hashes with different keys" do + hash1 = { + property1: { + property2: { + property3: 2, + property4: { + property5: nil + }, + property100: 90 + } + } + } + + hash2 = { + property1: { + property2: { + property3: 5, + property4: { + property5: 7, + property6: { + property7: 10 + } + } + }, + property3: { + property4: { + property5: 6 + } + } + }, + property5: 5 + } + expect(HashUtil.deep_merge(hash1, hash2)).to eq({ + property1: { + property2: { + property3: 5, + property4: { + property5: 7, + property6: { + property7: 10 + } + }, + property100: 90 + }, + property3: { + property4: { + property5: 6 + } + } + }, + property5: 5 + }) + end + end + + describe ".fetch_value_at_path" do + it "returns the single value at the given path" do + hash = { + "other" => 1, + "foo.bar.bazz" => "should not be returned", + "foo" => { + "other" => 2, + "bar" => { + "bazz" => 12, + "other" => 3, + "goo" => nil + } + } + } + + expect(HashUtil.fetch_value_at_path(hash, "other")).to eq 1 + expect(HashUtil.fetch_value_at_path(hash, "foo.other")).to eq 2 + expect(HashUtil.fetch_value_at_path(hash, "foo.bar.bazz")).to eq 12 + expect(HashUtil.fetch_value_at_path(hash, "foo.bar.other")).to eq 3 + expect(HashUtil.fetch_value_at_path(hash, "foo.bar.goo")).to eq nil + end + + it "returns an array of values if that's what's at the given path" do + hash = { + "other" => 1, + "foo.bar.bazz" => "should not be returned", + "foo" => { + "other" => 2, + "bar" => { + "bazz" => [12, 3], + "other" => 3 + } + } + } + + expect(HashUtil.fetch_value_at_path(hash, "foo.bar.bazz")).to eq [12, 3] + end + + it "returns a hash of data if that's what's at the given path" do + expect(HashUtil.fetch_value_at_path({"foo" => {"bar" => 3}}, "foo")).to eq({"bar" => 3}) + end + + it "raises a clear error when a key is not found, providing the missing key path in the error" do + hash = { + "other" => 1, + "foo.bar.bazz" => "should not be returned", + "foo" => { + "other" => 2, + "bar" => { + "bazz" => 12, + "other" => 3 + } + } + } + + expect { + HashUtil.fetch_value_at_path(hash, "bar.bazz") + }.to raise_error KeyError, a_string_including('"bar"') + + expect { + HashUtil.fetch_value_at_path(hash, "foo.bazz.bar") + }.to raise_error KeyError, a_string_including('"foo.bazz"') + + expect { + HashUtil.fetch_value_at_path(hash, "foo.bar.bazz2") + }.to raise_error KeyError, a_string_including('"foo.bar.bazz2"') + end + + it "raises a clear error when the value at a parent key is not a hash" do + expect { + HashUtil.fetch_value_at_path({"foo" => {"bar" => 3}}, "foo.bar.bazz") + }.to raise_error KeyError, a_string_including('Value at key "foo.bar" is not a `Hash` as expected; instead, was a `Integer`') + + expect { + HashUtil.fetch_value_at_path({"foo" => 3}, "foo.bar.bazz") + }.to raise_error KeyError, a_string_including('Value at key "foo" is not a `Hash` as expected; instead, was a `Integer`') + end + + it "allows a block to be passed to provide a default value for missing keys" do + hash = { + "other" => 1, + "foo.bar.bazz" => "should not be returned", + "foo" => { + "other" => 2, + "bar" => { + "bazz" => 12, + "other" => 3 + } + } + } + + expect(HashUtil.fetch_value_at_path(hash, "unknown") { 42 }).to eq 42 + expect(HashUtil.fetch_value_at_path(hash, "foo.bar.unknown") { 37 }).to eq 37 + expect(HashUtil.fetch_value_at_path(hash, "foo.bar.unknown.bazz") { |key| "#{key} is missing" }).to eq "foo.bar.unknown is missing" + end + + it "does not use a provided block when for cases where a parent key is not a hash" do + expect { + HashUtil.fetch_value_at_path({"foo" => {"bar" => 3}}, "foo.bar.bazz") { 3 } + }.to raise_error KeyError, a_string_including('Value at key "foo.bar" is not a `Hash` as expected; instead, was a `Integer`') + end + end + + describe ".fetch_leaf_values_at_path" do + it "returns a list of values at the identified key" do + values = HashUtil.fetch_leaf_values_at_path({"foo" => 17}, "foo") + expect(values).to eq [17] + + values = HashUtil.fetch_leaf_values_at_path({"foo" => [17]}, "foo") + expect(values).to eq [17] + end + + it "handles nested dot-separated keys by recursing through a nested hash" do + hash = { + "other" => 1, + "foo.bar.bazz" => "should not be returned", + "foo" => { + "other" => 2, + "bar" => { + "bazz" => 12, + "other" => 3 + } + } + } + + values = HashUtil.fetch_leaf_values_at_path(hash, "foo.bar.bazz") + + expect(values).to eq [12] + end + + it "returns `[]` when a parent key has an explicit `nil` value" do + hash = {"foo" => {"bar" => nil}} + values = HashUtil.fetch_leaf_values_at_path(hash, "foo.bar.bazz") + expect(values).to eq [] + + hash = {"foo" => nil} + values = HashUtil.fetch_leaf_values_at_path(hash, "foo.bar.bazz") + expect(values).to eq [] + end + + it "returns `[]` when a the nested path has an explicit `nil` value" do + hash = {"foo" => {"bar" => nil}} + values = HashUtil.fetch_leaf_values_at_path(hash, "foo.bar") + expect(values).to eq [] + end + + it "returns multiple values when the specified field is a list" do + hash = { + "other" => 1, + "foo.bar.bazz" => "should not be returned", + "foo" => { + "other" => 2, + "bar" => { + "bazz" => [12, 3], + "other" => 3 + } + } + } + + values = HashUtil.fetch_leaf_values_at_path(hash, "foo.bar.bazz") + + expect(values).to eq [12, 3] + end + + it "combines scalar values at the same path under a nested hash list as needed to return a flat list of values" do + hash = { + "foo" => { + "bar" => [ + {"bazz" => 12}, + {"bazz" => 3} + ] + } + } + + values = HashUtil.fetch_leaf_values_at_path(hash, "foo.bar.bazz") + + expect(values).to eq [12, 3] + end + + it "returns a flat list of values regardless of how many arrays are in the nested structure" do + hash = { + "foo" => [ + { + "bar" => [ + {"bazz" => [12, 3]}, + {"bazz" => [4, 7]}, + {"bazz" => []} + ] + }, + {"bar" => []}, + {"bar" => nil}, + { + "bar" => [ + {"bazz" => [1]}, + {"bazz" => [9, 7]} + ] + } + ] + } + + values = HashUtil.fetch_leaf_values_at_path(hash, "foo.bar.bazz") + + expect(values).to eq [12, 3, 4, 7, 1, 9, 7] + end + + it "raises a clear error when a key is not found, providing the missing key path in the error" do + hash = { + "other" => 1, + "foo.bar.bazz" => "should not be returned", + "foo" => { + "other" => 2, + "bar" => { + "bazz" => 12, + "other" => 3 + } + } + } + + expect { + HashUtil.fetch_leaf_values_at_path(hash, "bar.bazz") + }.to raise_error KeyError, a_string_including('"bar"') + + expect { + HashUtil.fetch_leaf_values_at_path(hash, "foo.bazz.bar") + }.to raise_error KeyError, a_string_including('"foo.bazz"') + + expect { + HashUtil.fetch_leaf_values_at_path(hash, "foo.bar.bazz2") + }.to raise_error KeyError, a_string_including('"foo.bar.bazz2"') + end + + it "allows a default value block to be provided just like with `Hash#fetch`" do + hash = { + "other" => 1, + "foo.bar.bazz" => "should not be returned", + "foo" => { + "other" => 2, + "bar" => { + "bazz" => 12, + "other" => 3 + } + } + } + + expect(HashUtil.fetch_leaf_values_at_path(hash, "bar.bazz") { [] }).to eq [] + expect(HashUtil.fetch_leaf_values_at_path(hash, "foo.bazz.bar") { [] }).to eq [] + expect(HashUtil.fetch_leaf_values_at_path(hash, "foo.bazz.bar") { 3 }).to eq [3] + expect(HashUtil.fetch_leaf_values_at_path(hash, "foo.bar.bazz2") { "abc" }).to eq ["abc"] + expect(HashUtil.fetch_leaf_values_at_path(hash, "foo.bar.bazz2") { |missing_key| missing_key }).to eq ["foo.bar.bazz2"] + end + + it "raises a clear error when the value at a parent key is not a hash" do + expect { + HashUtil.fetch_leaf_values_at_path({"foo" => {"bar" => 3}}, "foo.bar.bazz") + }.to raise_error KeyError, a_string_including('Value at key "foo.bar" is not a `Hash` as expected; instead, was a `Integer`') + + expect { + HashUtil.fetch_leaf_values_at_path({"foo" => 3}, "foo.bar.bazz") + }.to raise_error KeyError, a_string_including('Value at key "foo" is not a `Hash` as expected; instead, was a `Integer`') + end + + it "raises a clear error when the key is not a full path to a leaf" do + expect { + HashUtil.fetch_leaf_values_at_path({"foo" => {"bar" => 3}}, "foo") + }.to raise_error KeyError, a_string_including('Key was not a path to a leaf field: "foo"') + end + end + end + end +end diff --git a/elasticgraph-support/spec/unit/elastic_graph/support/logger_spec.rb b/elasticgraph-support/spec/unit/elastic_graph/support/logger_spec.rb new file mode 100644 index 00000000..a71e222a --- /dev/null +++ b/elasticgraph-support/spec/unit/elastic_graph/support/logger_spec.rb @@ -0,0 +1,103 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/logger" +require "stringio" + +module ElasticGraph + module Support + module Logger + RSpec.describe Logger, ".from_parsed_yaml" do + it "builds a logger instance from a parsed config file" do + logger = Logger.from_parsed_yaml(parsed_test_settings_yaml) + + expect(logger).to be_a(::Logger) + end + end + + RSpec.describe Factory, ".build" do + it "respects the configured log level" do + log_io = StringIO.new + + logger = build_logger(device: log_io, config: {"level" => "WARN"}) + logger.info "Some info" + logger.warn "Some warning" + logger.error "Some error" + + expect(log_io.string).to include("Some warning", "Some error").and exclude("Some info") + end + + it "can log any arbitrary string" do + log_io = StringIO.new + logger = build_logger(device: log_io) + + logger.info "some string" + expect(log_io.string.strip).to end_with("some string") + end + + it "logs the message as JSON when given a hash of metadata" do + log_io = StringIO.new + logger = build_logger(device: log_io) + + logger.info("some" => "metadata", "foobar" => 12) + expect(log_io.string.strip).to end_with("{\"some\": \"metadata\",\"foobar\": 12}") + end + + it "logs to `stdout` when so configured" do + expect { + build_logger(config: {"device" => "stdout"}).info "some log message" + }.to output(a_string_including("some log message")).to_stdout + end + + it "logs to `stderr` when so configured" do + expect { + build_logger(config: {"device" => "stderr"}).info "some log message" + }.to output(a_string_including("some log message")).to_stderr + end + + it "lets a formatter be configured (which can do things like ignore messages)" do + log_io = StringIO.new + original_formatter = ::Logger::Formatter.new + formatter = ->(*args, msg) do + original_formatter.call(*args, msg) unless msg.include?("password") + end + + config = Config.new(device: nil, level: :info, formatter: formatter) + logger = Factory.build(device: log_io, config: config) + + logger.info "username: guest" + logger.info "password: s3cr3t" + + expect(log_io.string).to include("username: guest").and exclude("password", "s3cr3t") + end + + context "when `config.device` is set to a file nested in a directory", :in_temp_dir do + it 'creates the directory so we do not get a "No such file or directory" error' do + dir = "some_new_dir" + + expect { + build_logger(config: {"device" => "#{dir}/eg.log"}).info "message" + }.to change { Dir.exist?(dir) }.from(false).to(true) + end + end + + it "raises an error when given an unrecognized config setting" do + expect { + build_logger(config: {"fake_setting" => 23}) + }.to raise_error Errors::ConfigError, a_string_including("fake_setting") + end + + def build_logger(device: nil, config: {}, **options) + logger_config = {"device" => "/dev/null"}.merge(config) + config = Config.from_parsed_yaml("logger" => logger_config) + Factory.build(device: device, config: config, **options) + end + end + end + end +end diff --git a/elasticgraph-support/spec/unit/elastic_graph/support/memoizable_data_spec.rb b/elasticgraph-support/spec/unit/elastic_graph/support/memoizable_data_spec.rb new file mode 100644 index 00000000..20a0bf2e --- /dev/null +++ b/elasticgraph-support/spec/unit/elastic_graph/support/memoizable_data_spec.rb @@ -0,0 +1,251 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/memoizable_data" + +module ElasticGraph + module Support + RSpec.describe MemoizableData do + specify "`::Data.define` does not allow memoized methods to be defined" do + # If this restriction is ever relaxed in a future version of Ruby, we should remove `MemoizableData` + # and just use `Data` directly! + unmemoizable_class = ::Data.define(:x, :y) do + def sum + @sum ||= x + y + end + end + + expect { + unmemoizable_class.new(1, 2).sum + }.to raise_error ::FrozenError + end + + shared_examples_for MemoizableData do + let(:memoizable_class) do + define(:x, :y) do + def sum + @sum ||= x + y + end + end + end + + it "allows us to define memoized methods on value objects to cache expensive, but pure, derived computations" do + expect(memoizable_class.new(1, 2).sum).to eq(3) + end + + specify "the memoization has no impact on the equality semantics of the class" do + m1 = memoizable_class.new(1, 2) + m2 = memoizable_class.new(1, 2) + m2.sum # so that the memoized state changes + + expect(m1 == m2).to be true + expect(m1.eql?(m2)).to be true + expect(m1.equal?(m2)).to be false + expect(m1.hash).to eq(m2.hash) + + # Swap the operands to confirm the same semantics hold... + expect(m2 == m1).to be true + expect(m2.eql?(m1)).to be true + expect(m2.hash).to eq(m1.hash) + expect(m2.equal?(m1)).to be false + + # instantiate an instance that is not equal... + m3 = memoizable_class.new(1, 3) + + expect(m2 == m3).to be false + expect(m2.eql?(m3)).to be false + expect(m2.eql?(m3)).to be false + expect(m2.hash).not_to eq(m3.hash) + + # Swap the operands to confirm the same semantics hold... + expect(m3 == m2).to be false + expect(m3.eql?(m2)).to be false + expect(m3.eql?(m2)).to be false + expect(m3.hash).not_to eq(m2.hash) + + # Verify our ability to compare against objects that aren't of the same type... + expect(m1 == 5).to be false + end + + it "supports creating new instances using `#with`" do + m1 = memoizable_class.new(1, 2) + + expect(m1.sum).to eq(3) + + m2 = m1.with(y: 3) + expect(m2).to be_a(memoizable_class) + expect(m2.y).to eq(3) + expect(m2.sum).to eq(4) + + expect(m1.sum).to eq(3) + expect(m2).not_to eq(m1) + end + + it "returns the outer type from `#itself` rather than the inner wrapped type" do + m1 = memoizable_class.new(1, 2) + + expect(m1.itself).to be_a(memoizable_class) + expect(m1.sum).to eq(3) + end + + it "can be instantiated with keyword or positional args" do + m1 = memoizable_class.new(1, 2) + m2 = memoizable_class.new(x: 1, y: 2) + + expect(m1).to eq(m2) + end + + it "exposes the same `members` as a data class" do + expect(memoizable_class.members).to eq [:x, :y] + + m1 = memoizable_class.new(1, 2) + expect(m1.members).to eq [:x, :y] + end + + it "inspects nicely" do + m1 = memoizable_class.new(1, 2) + + expect(m1.to_s).to eq "#" + expect(m1.inspect).to eq "#" + end + + it "does not allow mutation via `__setobj__` (which `DelegateClass` usually provides)" do + example_delegate_class = DelegateClass(::String) + expect(example_delegate_class.new(1)).to respond_to(:__setobj__) + + m1 = memoizable_class.new(1, 2) + expect(m1).not_to respond_to(:__setobj__) + end + + it "supports an `after_initialize` hook" do + klass = define(:tags) do + private + + def after_initialize + tags.freeze + end + end + + instance = klass.new(tags: ["a", "b"]) + expect(instance.tags).to be_frozen + + instance = instance.with(tags: ["c"]) + expect(instance.tags).to be_frozen + end + end + + context "with methods defined via a passed block" do + def define(...) + MemoizableData.define(...) + end + + include_examples MemoizableData + + it "allows `initialize` to be overridden in the same way as on a data class" do + measure = define(:amount, :unit) do + def initialize(amount:, unit: "unknown") + super(amount: Float(amount), unit:) + end + end + + a_mile = measure.new("5280", "ft") + expect(a_mile.amount).to eq(5280) + expect(a_mile.unit).to eq("ft") + + ten = measure.new(10) + expect(ten.amount).to eq(10) + expect(ten.unit).to eq("unknown") + end + + it "adds the defined methods only to the `MemoizableData`, not to the wrapped data class" do + measure = define(:amount, :unit) do + def initialize(amount:, unit: "unknown") + # :nocov: + super(amount: Float(amount), unit:) + # :nocov: + end + + def description + # :nocov: + @description ||= "#{amount} #{unit}" + # :nocov: + end + end + + expect(measure.method_defined?(:description)).to be true + expect(measure::DATA_CLASS.method_defined?(:description)).to be false + end + + it "allows `initialize` to be used for coercion, using it when `#with` is called" do + klass = define(:tags) do + def initialize(tags:) + super(tags: tags.to_set) + end + end + + k1 = klass.new(tags: ["a", "b"]) + expect(k1.tags).to be_a(::Set) + + k2 = k1.with(tags: ["c"]) + expect(k2.tags).to be_a(::Set) + end + end + + context "with methods defined on a subclass" do + def define(*attrs, &method_def_block) + ::Class.new(MemoizableData.define(*attrs), &method_def_block) + end + + include_examples MemoizableData + + it "raises an error if you attempt to override `initialize` in the subclass since it breaks things" do + expect { + define(:amount, :unit) do + def self.name + "MyData" + end + + def initialize(amount:, unit: "unknown") + # :nocov: + super(amount: Float(amount), unit:) + # :nocov: + end + end + }.to raise_error a_string_including("`MyData` overrides `initialize` in a subclass of `ElasticGraph::Support::MemoizableData`, but that can break things.") + end + end + + specify "`respond_to?` works as expected in a mixin with a method defined on a subclass" do + # Demonstrate that this works on `::Data` + expect(define_subclass_with_can_resolve_mixin(::Data).new(1, 2).can_resolve?(:sum)).to be true + # It should also work with `MemoizableData`. + expect(define_subclass_with_can_resolve_mixin(MemoizableData).new(1, 2).can_resolve?(:sum)).to be true + end + + def define_subclass_with_can_resolve_mixin(data_class) + my_module = ::Module.new do + def can_resolve?(field) + respond_to?(field) + end + end + + klass = data_class.define(:x, :y) do + include my_module + end + + ::Class.new(klass) do + def sum + # :nocov: + x + y + # :nocov: + end + end + end + end + end +end diff --git a/elasticgraph-support/spec/unit/elastic_graph/support/monotonic_clock_spec.rb b/elasticgraph-support/spec/unit/elastic_graph/support/monotonic_clock_spec.rb new file mode 100644 index 00000000..bed1c7a3 --- /dev/null +++ b/elasticgraph-support/spec/unit/elastic_graph/support/monotonic_clock_spec.rb @@ -0,0 +1,26 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/monotonic_clock" + +module ElasticGraph + module Support + RSpec.describe MonotonicClock do + let(:clock) { MonotonicClock.new } + + it "reports a monotonically increasing time value suitable for tracking durations and deadlines without worrying about leap seconds, etc" do + expect { + sleep(0.002) # sleep for 2 ms; ensuring it's > 1 ms so the montonic clock value is guaranteed to change + }.to change { clock.now_in_ms }.by(a_value_between( + 1, # maybe it's possible with rounding for the sleep to be *slightly* less than 2 ms and this only increase by 1 + 1000 # give plenty of time (up to a second) for GC pauses, etc so our test doesn't flicker + )) + end + end + end +end diff --git a/elasticgraph-support/spec/unit/elastic_graph/support/threading_spec.rb b/elasticgraph-support/spec/unit/elastic_graph/support/threading_spec.rb new file mode 100644 index 00000000..1407bf1b --- /dev/null +++ b/elasticgraph-support/spec/unit/elastic_graph/support/threading_spec.rb @@ -0,0 +1,36 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/threading" + +module ElasticGraph + module Support + RSpec.describe Threading do + describe ".parallel_map" do + it "maps over the given array just like `Enumerable#map`" do + result = Threading.parallel_map(%w[a b c], &:next) + expect(result).to eq %w[b c d] + end + + it "propagates exceptions to the calling thread properly, even preserving the calling thread's stacktrace in the exception" do + expected_trace_frames = caller + + expect { + Threading.parallel_map([1, 2, 3]) do |num| + raise "boom" if num.even? + num * 2 + end + }.to raise_error { |ex| + expect(ex.message).to eq "boom" + expect(ex.backtrace).to end_with(expected_trace_frames) + }.and avoid_outputting.to_stdout.and avoid_outputting.to_stderr + end + end + end + end +end diff --git a/elasticgraph-support/spec/unit/elastic_graph/support/time_set_spec.rb b/elasticgraph-support/spec/unit/elastic_graph/support/time_set_spec.rb new file mode 100644 index 00000000..b8efa285 --- /dev/null +++ b/elasticgraph-support/spec/unit/elastic_graph/support/time_set_spec.rb @@ -0,0 +1,1074 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/time_set" +require "time" + +module ElasticGraph + module Support + # Note: the tests in this file were inherited from an earlier implementation where `gt/gte` and `lt/lte` + # were not normalized to the same representation, and as such there are lots of cases in the tests + # covering both. Technically, we don't really need these (and I wouldn't write them from scratch if + # we didn't already have them), but it seems worth keeping. Don't feel like you have to cover all cases + # like that in new tests, though. + RSpec.describe TimeSet do + describe ".of_range" do + it "prevents `gt` and `gte` being used together as that would create two lower builds" do + expect { + TimeSet.of_range(gt: ::Time.at(1), gte: ::Time.at(2)) + }.to raise_error ArgumentError, a_string_including("two lower bound") + end + + it "prevents `lt` and `lte` being used together as that would create two upper builds" do + expect { + TimeSet.of_range(lt: ::Time.at(1000), lte: ::Time.at(2000)) + }.to raise_error ArgumentError, a_string_including("two upper bounds") + end + + it "can build an endless range" do + time_set = TimeSet.of_range(gte: ::Time.iso8601("2021-03-12T12:30:00Z")) + expect(time_set.ranges).to contain_exactly(::Time.iso8601("2021-03-12T12:30:00Z")..) + end + + it "can build a beginless range" do + time_set = TimeSet.of_range(lte: ::Time.iso8601("2021-03-12T12:30:00Z")) + expect(time_set.ranges).to contain_exactly(..::Time.iso8601("2021-03-12T12:30:00Z")) + end + + it "normalizes `gt` to its equivalent `gte` form (based on datastore millisecond time granularity)" do + time_set = TimeSet.of_range(gt: ::Time.iso8601("2021-03-12T12:30:00Z")) + expect(time_set.ranges).to contain_exactly(::Time.iso8601("2021-03-12T12:30:00.001Z")..) + end + + it "normalizes `lt` to its equivalent `lte` form (based on datastore millisecond time granularity)" do + time_set = TimeSet.of_range(lt: ::Time.iso8601("2021-03-12T12:30:00Z")) + expect(time_set.ranges).to contain_exactly(..::Time.iso8601("2021-03-12T12:29:59.999Z")) + end + + it "ignores a descending range as a set of times greater than a timestamp that comes after the less than timestamp is empty" do + time_set = TimeSet.of_range(gte: ::Time.iso8601("2020-07-01T00:00:00Z"), lt: ::Time.iso8601("2020-05-01T00:00:00Z")) + expect(time_set).to be(TimeSet::EMPTY) + end + end + + describe ".of_range_objects" do + it "merges overlapping ranges to normalize to a more efficient (but equivalent) representation" do + time_set = TimeSet.of_range_objects([ + ::Time.iso8601("2020-05-01T00:00:00Z")..::Time.iso8601("2020-07-01T00:00:00Z"), + ::Time.iso8601("2020-06-01T00:00:00Z")..::Time.iso8601("2020-09-01T00:00:00Z"), + ::Time.iso8601("2020-08-01T00:00:00Z")..::Time.iso8601("2020-10-01T00:00:00Z") + ]) + + expect(time_set).to eq(set_of_range(gte: "2020-05-01T00:00:00Z", lte: "2020-10-01T00:00:00Z")) + end + + it "merges adjacent ranges (e.g. with no gap between) to a more efficient (but equivalent) representation" do + time_set = set_of_ranges( + {gte: "2020-05-01T00:00:00Z", lte: "2020-06-30T23:59:59.999Z"}, + {gte: "2020-07-01T00:00:00Z", lt: "2020-09-01T00:00:00Z"} + ) + + expect(time_set).to eq(set_of_range(gte: "2020-05-01T00:00:00Z", lt: "2020-09-01T00:00:00Z")) + end + + it "ignores descending ranges as a set of times greater than a timestamp that comes after the less than timestamp is empty" do + # Note: we must avoid using `set_of_ranges` here because it leverages `TimeSet.of_range` which already had + # some handling for this; here we use `TimeSet.of_range_objects` directly to verify how it works. + time_set = TimeSet.of_range_objects([ + ::Time.iso8601("2020-07-01T00:00:00Z")..::Time.iso8601("2020-05-01T00:00:00Z"), + ::Time.iso8601("2020-10-01T00:00:00Z")..::Time.iso8601("2020-08-01T00:00:00Z"), + ::Time.iso8601("2021-02-01T00:00:00Z")..::Time.iso8601("2021-04-01T00:00:00Z") + ]) + + expect(time_set).to eq(set_of_range(gte: "2021-02-01T00:00:00Z", lte: "2021-04-01T00:00:00Z")) + end + end + + describe ".of_times" do + it "returns the empty set when given an empty list of time values" do + time_set = TimeSet.of_times([]) + expect(time_set).to be_empty + end + + it "returns a set of a single range that only allows the single timestamp value when given a single time" do + time = ::Time.iso8601("2021-03-12T12:30:00Z") + time_set = TimeSet.of_times([time]) + + expect(time_set).not_to be_empty + expect(time_set.member?(time)).to be true + expect(time_set.member?(time + TimeSet::CONSECUTIVE_TIME_INCREMENT)).to be false + expect(time_set.member?(time - TimeSet::CONSECUTIVE_TIME_INCREMENT)).to be false + expect(time_set.ranges).to contain_exactly(time..time) + end + + it "converts multiple times into multiple ranges, removing duplicates in the process" do + time1 = ::Time.iso8601("2021-03-12T12:30:00Z") + time2 = ::Time.iso8601("2021-04-12T12:30:00Z") + time3 = ::Time.iso8601("2021-05-12T12:30:00Z") + time_set = TimeSet.of_times([time1, time2, time1, time3, time1]) + + expect(time_set.member?(time1)).to be true + expect(time_set.member?(time2)).to be true + expect(time_set.member?(time3)).to be true + expect(time_set.member?(time1 + TimeSet::CONSECUTIVE_TIME_INCREMENT)).to be false + expect(time_set.member?(time1 - TimeSet::CONSECUTIVE_TIME_INCREMENT)).to be false + expect(time_set.member?(time2 + TimeSet::CONSECUTIVE_TIME_INCREMENT)).to be false + expect(time_set.member?(time2 - TimeSet::CONSECUTIVE_TIME_INCREMENT)).to be false + expect(time_set.member?(time3 + TimeSet::CONSECUTIVE_TIME_INCREMENT)).to be false + expect(time_set.member?(time3 - TimeSet::CONSECUTIVE_TIME_INCREMENT)).to be false + + expect(time_set.ranges).to contain_exactly( + time1..time1, + time2..time2, + time3..time3 + ) + end + end + + describe "#intersection" do + it "intersects two overlapping closed ranges by shrinking them to the overlapping portion" do + s1 = set_of_range(gte: "2021-01-01T00:00:00Z", lt: "2021-03-01T00:00:00Z") + s2 = set_of_range(gte: "2021-02-01T00:00:00Z", lt: "2021-09-01T00:00:00Z") + + expect_intersection(s1, s2) do + set_of_range(gte: "2021-02-01T00:00:00Z", lt: "2021-03-01T00:00:00Z") + end + end + + it "intersects two closed ranges where one is a subset of the other by returning the inner subset" do + s1 = set_of_range(gte: "2021-01-01T00:00:00Z", lt: "2021-09-01T00:00:00Z") + s2 = set_of_range(gte: "2021-02-01T00:00:00Z", lt: "2021-03-01T00:00:00Z") + + expect_intersection(s1, s2) { s2 } + end + + it "intersects two beginless ranges by shrinking the upper bound to the lower of the two upper bounds" do + s1 = set_of_range(lt: "2021-03-01T00:00:00Z") + s2 = set_of_range(lt: "2021-09-01T00:00:00Z") + + expect_intersection(s1, s2) do + set_of_range(lt: "2021-03-01T00:00:00Z") + end + end + + it "intersects two endless ranges by shrinking the lower bound to the higher of the two lower bounds" do + s1 = set_of_range(gte: "2021-03-01T00:00:00Z") + s2 = set_of_range(gte: "2021-09-01T00:00:00Z") + + expect_intersection(s1, s2) do + set_of_range(gte: "2021-09-01T00:00:00Z") + end + end + + it "intersects an overlapping beginless range and an endless range by using the finite bound from each range" do + s1 = set_of_range(gte: "2021-03-01T00:00:00Z") + s2 = set_of_range(lt: "2021-09-01T00:00:00Z") + + expect_intersection(s1, s2) do + set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-09-01T00:00:00Z") + end + end + + it "returns an empty set when two bounded ranges have no overlapping portion" do + s1 = set_of_range(gte: "2021-01-01T00:00:00Z", lt: "2021-03-01T00:00:00Z") + s2 = set_of_range(gte: "2021-04-01T00:00:00Z", lt: "2021-09-01T00:00:00Z") + + expect_intersection(s1, s2) do + TimeSet::EMPTY + end + end + + it "returns an empty set when a beginless and an endless range have no overlapping portion" do + s1 = set_of_range(lt: "2021-03-01T00:00:00Z") + s2 = set_of_range(gte: "2021-04-01T00:00:00Z") + + expect_intersection(s1, s2) do + TimeSet::EMPTY + end + end + + it "returns an empty set when two ranges have the same boundary but zero overlap" do + s1 = set_of_range(lt: "2021-04-01T00:00:00Z") + s2 = set_of_range(gte: "2021-04-01T00:00:00Z") + + expect_intersection(s1, s2) do + TimeSet::EMPTY + end + end + + it "returns a set of a single time when two ranges have the same boundary with only that single time value as the overlap" do + s1 = set_of_range(lte: "2021-04-01T00:00:00Z") + s2 = set_of_range(gte: "2021-04-01T00:00:00Z") + + expect_intersection(s1, s2) do + set_of_times("2021-04-01T00:00:00Z") + end + end + + it "returns an empty set when either set is empty" do + s1 = set_of_range(gte: "2021-01-01T00:00:00Z", lt: "2021-03-01T00:00:00Z") + + expect_intersection(s1, TimeSet::EMPTY) do + TimeSet::EMPTY + end + end + + it "returns an empty set when both sets are empty" do + expect_intersection(TimeSet::EMPTY, TimeSet::EMPTY) do + TimeSet::EMPTY + end + end + + it "intersects multiple single times and a range to just those times that fall within the range" do + s1 = set_of_times("2021-02-15T00:00:00Z", "2021-03-15T00:00:00Z", "2021-04-15T00:00:00Z", "2021-05-15T00:00:00Z") + s2 = set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-05-01T00:00:00Z") + + expect_intersection(s1, s2) do + set_of_times("2021-03-15T00:00:00Z", "2021-04-15T00:00:00Z") + end + end + + it "can intersect multiple ranges with multiple ranges" do + s1 = set_of_ranges( + {gte: "2020-01-01T00:00:00Z", lte: "2020-03-01T00:00:00Z"}, + {gte: "2020-05-01T00:00:00Z", lte: "2020-06-01T00:00:00Z"}, + {gte: "2020-09-01T00:00:00Z", lte: "2020-11-01T00:00:00Z"}, + {gte: "2021-01-01T00:00:00Z", lte: "2021-02-01T00:00:00Z"} + ) + + s2 = set_of_ranges( + {gte: "2020-02-01T00:00:00Z", lte: "2020-05-01T00:00:00Z"}, + {gte: "2020-06-01T00:00:00Z", lte: "2020-09-01T00:00:00Z"} + ) + + expect_intersection(s1, s2) do + set_of_ranges( + {gte: "2020-02-01T00:00:00Z", lte: "2020-03-01T00:00:00Z"}, + {gte: "2020-05-01T00:00:00Z", lte: "2020-05-01T00:00:00Z"}, + {gte: "2020-06-01T00:00:00Z", lte: "2020-06-01T00:00:00Z"}, + {gte: "2020-09-01T00:00:00Z", lte: "2020-09-01T00:00:00Z"} + ) + end + end + + def expect_intersection(s1, s2) + actual1 = s1.intersection(s2) + actual2 = s2.intersection(s1) + + expect(actual1).to eq(actual2) + expect(actual1).to eq(yield) + end + end + + describe "#union" do + it "unions two overlapping closed ranges by growing them to the outer bounds" do + s1 = set_of_range(gte: "2021-01-01T00:00:00Z", lt: "2021-03-01T00:00:00Z") + s2 = set_of_range(gte: "2021-02-01T00:00:00Z", lt: "2021-09-01T00:00:00Z") + + expect_union(s1, s2) do + set_of_range(gte: "2021-01-01T00:00:00Z", lt: "2021-09-01T00:00:00Z") + end + end + + it "unions two closed ranges where one is a subset of the other by returning the outer set" do + s1 = set_of_range(gte: "2021-01-01T00:00:00Z", lt: "2021-09-01T00:00:00Z") + s2 = set_of_range(gte: "2021-02-01T00:00:00Z", lt: "2021-03-01T00:00:00Z") + + expect_union(s1, s2) do + s1 + end + end + + it "unions two beginless ranges by growing the upper bound to the larger of the two upper bounds" do + s1 = set_of_range(lt: "2021-03-01T00:00:00Z") + s2 = set_of_range(lt: "2021-09-01T00:00:00Z") + + expect_union(s1, s2) do + set_of_range(lt: "2021-09-01T00:00:00Z") + end + end + + it "unions two endless ranges by growing the lower bound to the lower of the two lower bounds" do + s1 = set_of_range(gte: "2021-03-01T00:00:00Z") + s2 = set_of_range(gte: "2021-09-01T00:00:00Z") + + expect_union(s1, s2) do + set_of_range(gte: "2021-03-01T00:00:00Z") + end + end + + it "unions an overlapping beginless range and an endless range to ALL" do + s1 = set_of_range(gte: "2021-03-01T00:00:00Z") + s2 = set_of_range(lt: "2021-09-01T00:00:00Z") + + expect_union(s1, s2) do + TimeSet::ALL + end + end + + it "returns a set containing each range when two bounded ranges have no overlapping portion" do + s1 = set_of_range(gte: "2021-01-01T00:00:00Z", lt: "2021-03-01T00:00:00Z") + s2 = set_of_range(gte: "2021-04-01T00:00:00Z", lt: "2021-09-01T00:00:00Z") + + expect_union(s1, s2) do + set_of_ranges( + {gte: "2021-01-01T00:00:00Z", lt: "2021-03-01T00:00:00Z"}, + {gte: "2021-04-01T00:00:00Z", lt: "2021-09-01T00:00:00Z"} + ) + end + end + + it "returns an set containing each range when a beginless and an endless range have no overlapping portion" do + s1 = set_of_range(gte: "2021-09-01T00:00:00Z") + s2 = set_of_range(lt: "2021-03-01T00:00:00Z") + + expect_union(s1, s2) do + set_of_ranges( + {gte: "2021-09-01T00:00:00Z"}, + {lt: "2021-03-01T00:00:00Z"} + ) + end + end + + it "merges two ranges into one when they have the same boundary and only with only that single time value as the overlap" do + s1 = set_of_range(gte: "2021-01-01T00:00:00Z", lte: "2021-04-01T00:00:00Z") + s2 = set_of_range(gte: "2021-04-01T00:00:00Z", lt: "2021-10-01T00:00:00Z") + + expect_union(s1, s2) do + set_of_range(gte: "2021-01-01T00:00:00Z", lt: "2021-10-01T00:00:00Z") + end + end + + it "merges two ranges into one when they have the same boundary but zero overlap" do + s1 = set_of_range(gte: "2021-01-01T00:00:00Z", lt: "2021-04-01T00:00:00Z") + s2 = set_of_range(gte: "2021-04-01T00:00:00Z", lt: "2021-10-01T00:00:00Z") + + expect_union(s1, s2) do + set_of_range(gte: "2021-01-01T00:00:00Z", lt: "2021-10-01T00:00:00Z") + end + end + + it "returns the other set when one set is empty" do + s1 = set_of_range(gte: "2021-01-01T00:00:00Z", lt: "2021-03-01T00:00:00Z") + + expect_union(s1, TimeSet::EMPTY) do + s1 + end + end + + it "returns an empty set when both sets are empty" do + expect_union(TimeSet::EMPTY, TimeSet::EMPTY) do + TimeSet::EMPTY + end + end + + it "can union multiple ranges with multiple ranges properly" do + s1 = set_of_ranges( + {gte: "2020-01-01T00:00:00Z", lte: "2020-03-01T00:00:00Z"}, + {gte: "2020-05-01T00:00:00Z", lte: "2020-06-01T00:00:00Z"}, + {gte: "2020-09-01T00:00:00Z", lte: "2020-11-01T00:00:00Z"}, + {gte: "2021-01-01T00:00:00Z", lte: "2021-02-01T00:00:00Z"} + ) + + s2 = set_of_ranges( + {gte: "2020-02-01T00:00:00Z", lte: "2020-05-01T00:00:00Z"}, + {gte: "2020-06-01T00:00:00Z", lte: "2020-09-01T00:00:00Z"} + ) + + expect_union(s1, s2) do + set_of_ranges( + {gte: "2020-01-01T00:00:00Z", lte: "2020-11-01T00:00:00Z"}, + {gte: "2021-01-01T00:00:00Z", lte: "2021-02-01T00:00:00Z"} + ) + end + end + + def expect_union(s1, s2) + actual1 = s1.union(s2) + actual2 = s2.union(s1) + + expect(actual1).to eq(actual2) + expect(actual1).to eq(yield) + end + end + + describe "#member?" do + let(:min_time) { ::Time.iso8601("0000-01-01T00:00:00Z") } + let(:max_time) { ::Time.iso8601("9999-12-31T23:59:59.999Z") } + + context "for the ALL set" do + it "returns true regardless of the timestamp" do + expect(TimeSet::ALL.member?(min_time)).to be true + expect(TimeSet::ALL.member?(::Time.now)).to be true + expect(TimeSet::ALL.member?(max_time)).to be true + end + end + + context "for a set of a range with no upper bound" do + it "returns true only for timestamps on or after a >= lower bound" do + range = set_of_range(gte: "2021-05-12T08:00:00Z") + + expect(range.member?(::Time.iso8601("2021-05-14T12:30:00Z"))).to be true + expect(range.member?(max_time)).to be true + expect(range.member?(::Time.iso8601("2021-05-12T08:00:00Z"))).to be true + + expect(range.member?(::Time.iso8601("2021-05-12T07:59:59.999Z"))).to be false + expect(range.member?(min_time)).to be false + end + + it "returns true only for timestamps after a > lower bound" do + range = set_of_range(gt: "2021-05-12T08:00:00Z") + + expect(range.member?(::Time.iso8601("2021-05-14T12:30:00Z"))).to be true + expect(range.member?(max_time)).to be true + + expect(range.member?(::Time.iso8601("2021-05-12T08:00:00Z"))).to be false + expect(range.member?(::Time.iso8601("2021-05-12T07:59:59.999Z"))).to be false + expect(range.member?(min_time)).to be false + end + end + + context "for a set of a range with no lower bound" do + it "returns true only for timestamps on or before a <= upper bound" do + range = set_of_range(lte: "2021-05-12T08:00:00Z") + + expect(range.member?(::Time.iso8601("2021-05-10T12:30:00Z"))).to be true + expect(range.member?(min_time)).to be true + expect(range.member?(::Time.iso8601("2021-05-12T08:00:00Z"))).to be true + + expect(range.member?(::Time.iso8601("2021-05-12T08:00:00.001Z"))).to be false + expect(range.member?(max_time)).to be false + end + + it "returns true only for timestamps before a < upper bound" do + range = set_of_range(lt: "2021-05-12T08:00:00Z") + + expect(range.member?(::Time.iso8601("2021-05-10T12:30:00Z"))).to be true + expect(range.member?(min_time)).to be true + + expect(range.member?(::Time.iso8601("2021-05-12T08:00:00Z"))).to be false + expect(range.member?(::Time.iso8601("2021-05-12T08:00:00.001Z"))).to be false + expect(range.member?(max_time)).to be false + end + end + + context "for a set with a range with upper and lower bounds" do + it "returns true only for timestamps on or within the boundaries when the bounds are >= and <=" do + range = set_of_range(gte: "2021-05-12T08:00:00Z", lte: "2021-06-12T08:00:00Z") + + expect(range.member?(::Time.iso8601("2021-05-14T12:30:00Z"))).to be true + expect(range.member?(::Time.iso8601("2021-05-12T08:00:00Z"))).to be true + expect(range.member?(::Time.iso8601("2021-06-12T08:00:00Z"))).to be true + + expect(range.member?(::Time.iso8601("2021-05-12T07:59:59.999Z"))).to be false + expect(range.member?(::Time.iso8601("2021-06-12T08:00:00.001Z"))).to be false + expect(range.member?(min_time)).to be false + expect(range.member?(max_time)).to be false + end + + it "returns true only for timestamps after a > lower bound" do + range = set_of_range(gt: "2021-05-12T08:00:00Z", lt: "2021-06-12T08:00:00Z") + + expect(range.member?(::Time.iso8601("2021-05-14T12:30:00Z"))).to be true + expect(range.member?(::Time.iso8601("2021-05-12T08:00:00.001Z"))).to be true + expect(range.member?(::Time.iso8601("2021-06-12T07:59:59.999Z"))).to be true + + expect(range.member?(::Time.iso8601("2021-05-12T08:00:00Z"))).to be false + expect(range.member?(::Time.iso8601("2021-06-12T08:00:00Z"))).to be false + expect(range.member?(min_time)).to be false + expect(range.member?(max_time)).to be false + end + + it "returns false for all timestamps for an empty set" do + range = set_of_range(lt: "2021-05-12T08:00:00Z", gt: "2021-06-12T08:00:00Z", expected_empty: true) + + expect(range.member?(min_time)).to be false + expect(range.member?(::Time.iso8601("2021-05-10T12:30:00Z"))).to be false + expect(range.member?(::Time.iso8601("2021-05-14T12:30:00Z"))).to be false + expect(range.member?(::Time.iso8601("2021-05-12T08:00:00.001Z"))).to be false + expect(range.member?(::Time.iso8601("2021-06-12T07:59:59.999Z"))).to be false + expect(range.member?(::Time.iso8601("2021-05-12T08:00:00Z"))).to be false + expect(range.member?(::Time.iso8601("2021-06-12T08:00:00Z"))).to be false + expect(range.member?(::Time.iso8601("2021-06-14T12:30:00Z"))).to be false + expect(range.member?(max_time)).to be false + end + end + end + + describe "#intersect?" do + context "when one of the sets is ALL" do + it "returns true when given any non-empty set" do + expect(TimeSet::ALL).to intersect_with(set_of_range(gt: "2021-05-12T08:00:00Z", lt: "2021-06-12T08:00:00Z")) + expect(TimeSet::ALL).to intersect_with(TimeSet::ALL) + expect(TimeSet::ALL).to intersect_with(set_of_range(lt: "2021-05-12T08:00:00Z")) + end + + it "does not intersect with an empty set" do + expect(TimeSet::ALL).not_to intersect_with(set_of_range(lt: "2021-05-12T08:00:00Z", gt: "2021-06-12T08:00:00Z", expected_empty: true)) + end + end + + context "when both sets have a single range that lack an upper bound" do + it "returns true, regardless of where the bound is" do + expect(set_of_range(lt: "2021-05-12T08:00:00Z")).to intersect_with(set_of_range(lt: "2020-05-12T08:00:00Z")) + expect(set_of_range(lt: "2021-05-12T08:00:00Z")).to intersect_with(set_of_range(lt: "2019-05-12T08:00:00Z")) + expect(set_of_range(lte: "2021-05-12T08:00:00Z")).to intersect_with(set_of_range(lt: "2020-05-12T08:00:00Z")) + expect(set_of_range(lte: "2021-05-12T08:00:00Z")).to intersect_with(set_of_range(lt: "2019-05-12T08:00:00Z")) + expect(set_of_range(lt: "2021-05-12T08:00:00Z")).to intersect_with(set_of_range(lte: "2020-05-12T08:00:00Z")) + expect(set_of_range(lt: "2021-05-12T08:00:00Z")).to intersect_with(set_of_range(lte: "2019-05-12T08:00:00Z")) + end + end + + context "when both sets have a single range that lacks a lower bound" do + it "returns true, regardless of where the bound is" do + expect(set_of_range(gt: "2021-05-12T08:00:00Z")).to intersect_with(set_of_range(gt: "2020-05-12T08:00:00Z")) + expect(set_of_range(gt: "2021-05-12T08:00:00Z")).to intersect_with(set_of_range(gt: "2019-05-12T08:00:00Z")) + expect(set_of_range(gte: "2021-05-12T08:00:00Z")).to intersect_with(set_of_range(gt: "2020-05-12T08:00:00Z")) + expect(set_of_range(gte: "2021-05-12T08:00:00Z")).to intersect_with(set_of_range(gt: "2019-05-12T08:00:00Z")) + expect(set_of_range(gt: "2021-05-12T08:00:00Z")).to intersect_with(set_of_range(gte: "2020-05-12T08:00:00Z")) + expect(set_of_range(gt: "2021-05-12T08:00:00Z")).to intersect_with(set_of_range(gte: "2019-05-12T08:00:00Z")) + end + end + + context "when both sets have a single range that has both bounds" do + it "returns false if one comes completely before the other" do + expect( + set_of_range(gt: "2021-05-12T08:00:00Z", lt: "2021-06-12T08:00:00Z") + ).not_to intersect_with( + set_of_range(gt: "2021-03-12T08:00:00Z", lt: "2021-04-12T08:00:00Z") + ) + end + + it "returns true if they partially overlap" do + expect( + set_of_range(gt: "2021-04-12T08:00:00Z", lt: "2021-06-12T08:00:00Z") + ).to intersect_with( + set_of_range(gt: "2021-03-12T08:00:00Z", lt: "2021-05-12T08:00:00Z") + ) + end + + it "returns true if they exactly overlap" do + expect( + set_of_range(gt: "2021-04-12T08:00:00Z", lt: "2021-06-12T08:00:00Z") + ).to intersect_with( + set_of_range(gt: "2021-04-12T08:00:00Z", lt: "2021-06-12T08:00:00Z") + ) + end + + it "returns true if one fits in the other" do + expect( + set_of_range(gt: "2021-03-12T08:00:00Z", lt: "2021-06-12T08:00:00Z") + ).to intersect_with( + set_of_range(gt: "2021-04-12T08:00:00Z", lt: "2021-05-12T08:00:00Z") + ) + end + + it "returns the correct value if they share a boundary" do + expect( + set_of_range(gt: "2021-03-12T08:00:00Z", lt: "2021-05-12T08:00:00Z") + ).not_to intersect_with( + set_of_range(gt: "2021-05-12T08:00:00Z", lt: "2021-06-12T08:00:00Z") + ) + + expect( + set_of_range(gt: "2021-03-12T08:00:00Z", lte: "2021-05-12T08:00:00Z") + ).not_to intersect_with( + set_of_range(gt: "2021-05-12T08:00:00Z", lt: "2021-06-12T08:00:00Z") + ) + + expect( + set_of_range(gt: "2021-03-12T08:00:00Z", lt: "2021-05-12T08:00:00Z") + ).not_to intersect_with( + set_of_range(gte: "2021-05-12T08:00:00Z", lt: "2021-06-12T08:00:00Z") + ) + + expect( + set_of_range(gt: "2021-03-12T08:00:00Z", lte: "2021-05-12T08:00:00Z") + ).to intersect_with( + set_of_range(gte: "2021-05-12T08:00:00Z", lt: "2021-06-12T08:00:00Z") + ) + end + end + + context "when one set has a double-bounded range and one set has a single bounded range" do + it "returns true if the double bounded range comes before a less than range" do + expect( + set_of_range(gt: "2021-05-12T08:00:00Z", lt: "2021-06-12T08:00:00Z") + ).to intersect_with( + set_of_range(lt: "2021-07-12T08:00:00Z") + ) + end + + it "returns true if the double bounded range comes after a greater than range" do + expect( + set_of_range(gt: "2021-05-12T08:00:00Z", lt: "2021-06-12T08:00:00Z") + ).to intersect_with( + set_of_range(gt: "2021-03-12T08:00:00Z") + ) + end + + it "returns false if the double bounded range comes after a less than range" do + expect( + set_of_range(gt: "2021-05-12T08:00:00Z", lt: "2021-06-12T08:00:00Z") + ).not_to intersect_with( + set_of_range(lt: "2021-04-12T08:00:00Z") + ) + end + + it "returns false if the double bounded range comes before a greater than range" do + expect( + set_of_range(gt: "2021-05-12T08:00:00Z", lt: "2021-06-12T08:00:00Z") + ).not_to intersect_with( + set_of_range(gt: "2021-07-12T08:00:00Z") + ) + end + + it "returns true if the boundary of the single-bounded range is within the double-bounded range" do + expect( + set_of_range(gt: "2021-04-12T08:00:00Z", lt: "2021-06-12T08:00:00Z") + ).to intersect_with( + set_of_range(gt: "2021-05-12T08:00:00Z") + ).and intersect_with( + set_of_range(lt: "2021-05-12T08:00:00Z") + ) + end + + it "returns the correct value if they share a boundary" do + expect( + set_of_range(gt: "2021-03-12T08:00:00Z", lt: "2021-05-12T08:00:00Z") + ).not_to intersect_with( + set_of_range(gt: "2021-05-12T08:00:00Z") + ) + + expect( + set_of_range(gt: "2021-03-12T08:00:00Z", lte: "2021-05-12T08:00:00Z") + ).not_to intersect_with( + set_of_range(gt: "2021-05-12T08:00:00Z") + ) + + expect( + set_of_range(gt: "2021-03-12T08:00:00Z", lt: "2021-05-12T08:00:00Z") + ).not_to intersect_with( + set_of_range(gte: "2021-05-12T08:00:00Z") + ) + + expect( + set_of_range(gt: "2021-03-12T08:00:00Z", lte: "2021-05-12T08:00:00Z") + ).to intersect_with( + set_of_range(gte: "2021-05-12T08:00:00Z") + ) + end + + context "when one set has a single time value" do + it "returns true when the time value is within the range of the other set" do + expect( + set_of_times("2021-03-12T08:00:00Z") + ).to intersect_with( + set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-04-01T00:00:00Z") + ).and intersect_with( + set_of_range(gte: "2021-03-12T08:00:00Z", lt: "2021-04-01T00:00:00Z") + ).and intersect_with( + set_of_range(gte: "2021-03-01T00:00:00Z", lte: "2021-03-12T08:00:00Z") + ) + end + + it "returns true when the time value is on the inclusive side of an open range" do + expect( + set_of_times("2021-03-12T08:00:00Z") + ).to intersect_with( + set_of_range(gte: "2021-03-01T00:00:00Z") + ).and intersect_with( + set_of_range(lt: "2021-04-01T00:00:00Z") + ) + end + + it "returns false when given an empty set" do + expect( + set_of_times("2021-03-12T08:00:00Z") + ).not_to intersect_with(TimeSet::EMPTY) + end + + it "returns false when the time falls outside the range of the other set" do + time_set = set_of_times("2021-03-12T08:00:00Z") + + expect(time_set).not_to intersect_with(set_of_range(gte: "2021-03-13T00:00:00Z", lt: "2021-04-01T00:00:00Z")) + expect(time_set).not_to intersect_with(set_of_range(gte: "2021-02-13T00:00:00Z", lt: "2021-03-11T00:00:00Z")) + expect(time_set).not_to intersect_with(set_of_range(gte: "2021-03-13T00:00:00Z")) + expect(time_set).not_to intersect_with(set_of_range(lt: "2021-03-12T08:00:00Z")) + end + end + + context "when one set has multiple time values" do + it "returns true when all the time values fall within the range of the other set" do + expect( + set_of_times("2021-03-12T08:00:00Z", "2021-03-13T08:00:00Z", "2021-03-14T08:00:00Z") + ).to intersect_with( + set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-04-01T00:00:00Z") + ).and intersect_with( + set_of_range(gte: "2021-03-12T08:00:00Z", lt: "2021-04-01T00:00:00Z") + ).and intersect_with( + set_of_range(gte: "2021-03-01T00:00:00Z", lte: "2021-03-14T08:00:00Z") + ) + end + + it "returns true when only one of the time values falls within the range of the other set" do + expect( + set_of_times("2021-03-12T08:00:00Z", "2020-03-13T08:00:00Z", "2022-03-14T08:00:00Z") + ).to intersect_with( + set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-04-01T00:00:00Z") + ).and intersect_with( + set_of_range(gte: "2021-03-12T08:00:00Z", lt: "2021-04-01T00:00:00Z") + ).and intersect_with( + set_of_range(gte: "2021-03-01T00:00:00Z", lte: "2021-03-14T08:00:00Z") + ) + end + + it "returns false when none of the time values falls within the range of the other set" do + time_set = set_of_times("2021-03-12T08:00:00Z", "2020-03-13T08:00:00Z", "2022-03-14T08:00:00Z") + + expect(time_set).not_to intersect_with(set_of_range(gte: "2023-03-13T00:00:00Z", lt: "2023-04-01T00:00:00Z")) + expect(time_set).not_to intersect_with(set_of_range(gte: "2020-02-13T00:00:00Z", lt: "2020-03-11T00:00:00Z")) + expect(time_set).not_to intersect_with(set_of_range(gte: "2023-03-13T00:00:00Z")) + expect(time_set).not_to intersect_with(set_of_range(lt: "2020-03-12T08:00:00Z")) + end + end + end + + matcher :intersect_with do |range1| + match do |range2| + range1.intersect?(range2) && range2.intersect?(range1) + end + + match_when_negated do |range2| + !range1.intersect?(range2) && !range2.intersect?(range1) + end + end + end + + describe "#-" do + context "when diffing time stamps" do + it "returns the correct set difference" do + s1 = set_of_times("2021-03-12T08:00:00Z", "2020-03-13T08:00:00Z", "2022-03-14T08:00:00Z") + s2 = set_of_times("2021-03-12T08:00:00Z") + + expect(s1 - s2).to eq(set_of_times("2020-03-13T08:00:00Z", "2022-03-14T08:00:00Z")) + expect(s2 - s1).to eq(TimeSet::EMPTY) + end + + it "ignores a time not present in the original set" do + s1 = set_of_times("2021-03-12T08:00:00Z", "2020-03-13T08:00:00Z", "2022-03-14T08:00:00Z") + s2 = set_of_times("2021-04-13T10:00:00Z") + + expect(s1 - s2).to eq(s1) + expect(s2 - s1).to eq(s2) + end + + it "returns the correct difference when one set is empty" do + s1 = set_of_times("2021-03-12T08:00:00Z", "2020-03-13T08:00:00Z", "2022-03-14T08:00:00Z") + s2 = TimeSet::EMPTY + + expect(s1 - s2).to eq(s1) + expect(s2 - s1).to eq(TimeSet::EMPTY) + end + + it "returns an empty `TimeSet` when both times sets contain the same times" do + s1 = set_of_times("2021-03-12T08:00:00Z", "2020-03-13T08:00:00Z", "2022-03-14T08:00:00Z") + s2 = set_of_times("2021-03-12T08:00:00Z", "2020-03-13T08:00:00Z", "2022-03-14T08:00:00Z") + + expect(s1 - s2).to eq(TimeSet::EMPTY) + expect(s2 - s1).to eq(TimeSet::EMPTY) + end + + it "correctly splits ALL to exclude the subtracted time" do + t1 = set_of_times("2021-03-12T00:00:00Z") + + expected_difference = set_of_ranges( + {lt: "2021-03-12T00:00:00Z"}, + {gt: "2021-03-12T00:00:00Z"} + ) + + expect(TimeSet::ALL - t1).to eq(expected_difference) + end + end + + context "when diffing sets of ranges with a single intersection" do + it "correctly splits ALL to exclude the subtracted TimeSet" do + march_and_april = set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-05-01T00:00:00Z") + + expected_difference = set_of_ranges( + {lt: "2021-03-01T00:00:00Z"}, + {gte: "2021-05-01T00:00:00Z"} + ) + + expect(TimeSet::ALL - march_and_april).to eq(expected_difference) + end + + it "correctly truncates the ranges when the ranges overlap" do + march_and_april = set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-05-01T00:00:00Z") + april_and_may = set_of_range(gte: "2021-04-01T00:00:00Z", lt: "2021-06-01T00:00:00Z") + + expect(march_and_april - april_and_may).to eq(set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-04-01T00:00:00Z")) + expect(april_and_may - march_and_april).to eq(set_of_range(gte: "2021-05-01T00:00:00Z", lt: "2021-06-01T00:00:00Z")) + end + + it "correctly truncates the ranges when the ranges have the same lower bound" do + march_and_april = set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-05-01T00:00:00Z") + march = set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-04-01T00:00:00Z") + + expect(march_and_april - march).to eq(set_of_range(gte: "2021-04-01T00:00:00Z", lt: "2021-05-01T00:00:00Z")) + end + + it "correctly truncates the ranges when the ranges have the same upper bound" do + march_and_april = set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-05-01T00:00:00Z") + april = set_of_range(gte: "2021-04-01T00:00:00Z", lt: "2021-05-01T00:00:00Z") + + expect(march_and_april - april).to eq(set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-04-01T00:00:00Z")) + end + + it "returns the original set of ranges when the ranges do not overlap" do + march = set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-04-01T00:00:00Z") + may = set_of_range(gte: "2021-05-01T00:00:00Z", lt: "2021-06-01T00:00:00Z") + + expect(march - may).to eq(march) + expect(may - march).to eq(may) + end + + it "returns the original set of ranges when boundless ranges do not overlap" do + before_march = set_of_range(lt: "2021-03-01T00:00:00Z") + after_april = set_of_range(gte: "2021-04-01T00:00:00Z") + + expect(before_march - after_april).to eq(before_march) + expect(after_april - before_march).to eq(after_april) + end + + it "returns the empty set when fully covered by the subtracted set" do + april = set_of_range(gte: "2021-04-01T00:00:00Z", lt: "2021-05-01T00:00:00Z") + march_through_may = set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-06-01T00:00:00Z") + all1 = TimeSet::ALL + all2 = TimeSet::ALL + + expect(april - march_through_may).to eq(TimeSet::EMPTY) + expect(april - TimeSet::ALL).to eq(TimeSet::EMPTY) + expect(all1 - all2).to eq(TimeSet::EMPTY) + end + + it "correctly splits the TimeSet when one range is in the middle of the other" do + march_through_may = set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-06-01T00:00:00Z") + april = set_of_range(gte: "2021-04-01T00:00:00Z", lt: "2021-05-01T00:00:00Z") + + march_and_may = set_of_ranges( + {gte: "2021-03-01T00:00:00Z", lt: "2021-04-01T00:00:00Z"}, + {gte: "2021-05-01T00:00:00Z", lt: "2021-06-01T00:00:00Z"} + ) + + expect(march_through_may - april).to eq(march_and_may) + end + + it "correctly truncates ALL when the subtracted TimeSet is boundless" do + after_march = set_of_range(gte: "2021-03-01T00:00:00Z") + before_september = set_of_range(lt: "2021-09-01T00:00:00Z") + + expect(TimeSet::ALL - after_march).to eq(set_of_range(lt: "2021-03-01T00:00:00Z")) + expect(TimeSet::ALL - before_september).to eq(set_of_range(gte: "2021-09-01T00:00:00Z")) + end + + it "truncates a beginless range by increasing the lower bound to the upper bound of the subtracted range" do + before_september = set_of_range(lt: "2021-09-01T00:00:00Z") + before_march = set_of_range(lt: "2021-03-01T00:00:00Z") + + expect(before_september - before_march).to eq(set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-09-01T00:00:00Z")) + expect(before_march - before_september).to eq(TimeSet::EMPTY) + end + + it "truncates an endless range by decreasing the upper bound to the lower bound of the subtracted range" do + after_march = set_of_range(gte: "2021-03-01T00:00:00Z") + after_september = set_of_range(gte: "2021-09-01T00:00:00Z") + + expect(after_march - after_september).to eq(set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-09-01T00:00:00Z")) + expect(after_september - after_march).to eq(TimeSet::EMPTY) + end + + it "truncates an overlapping beginless range and an endless range by using the bound from the subtracted range" do + after_march = set_of_range(gte: "2021-03-01T00:00:00Z") + before_september = set_of_range(lt: "2021-09-01T00:00:00Z") + + expect(after_march - before_september).to eq(set_of_range(gte: "2021-09-01T00:00:00Z")) + expect(before_september - after_march).to eq(set_of_range(lt: "2021-03-01T00:00:00Z")) + end + + it "returns an exclusive boundless range when two ranges have the same boundary with only that single time value as the overlap" do + before_april = set_of_range(lte: "2021-04-01T00:00:00Z") + after_april = set_of_range(gte: "2021-04-01T00:00:00Z") + + expect(before_april - after_april).to eq(set_of_range(lt: "2021-04-01T00:00:00Z")) + expect(after_april - before_april).to eq(set_of_range(gt: "2021-04-01T00:00:00Z")) + end + end + + context "when diffing set of ranges that have multiple intersections" do + it "correctly splits the ranges when one TimeSet is a subset of the other" do + march_through_sept = set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-10-01T00:00:00Z") + april_and_july = set_of_ranges( + {gte: "2021-04-01T00:00:00Z", lt: "2021-05-01T00:00:00Z"}, + {gte: "2021-07-01T00:00:00Z", lt: "2021-08-01T00:00:00Z"} + ) + + before_march = {lt: "2021-04-01T00:00:00Z"} + march = {gte: "2021-03-01T00:00:00Z", lt: "2021-04-01T00:00:00Z"} + may_through_june = {gte: "2021-05-01T00:00:00Z", lt: "2021-07-01T00:00:00Z"} + aug_through_sept = {gte: "2021-08-01T00:00:00Z", lt: "2021-10-01T00:00:00Z"} + after_august = {gte: "2021-08-01T00:00:00Z"} + + expected_difference1 = set_of_ranges( + march, + may_through_june, + aug_through_sept + ) + + expected_difference2 = set_of_ranges( + before_march, + may_through_june, + after_august + ) + + expect(march_through_sept - april_and_july).to eq(expected_difference1) + expect(TimeSet::ALL - april_and_july).to eq(expected_difference2) + end + + it "correctly truncates ALL when the subtracted TimeSet has multiple boundless ranges" do + before_april_after_june = set_of_ranges( + {lt: "2021-04-01T00:00:00Z"}, + {gte: "2021-06-01T00:00:00Z"} + ) + + march_through_may = set_of_range(gte: "2021-04-01T00:00:00Z", lt: "2021-06-01T00:00:00Z") + + expect(TimeSet::ALL - before_april_after_june).to eq(march_through_may) + end + + it "correctly truncates the ranges when one TimeSet covers the lower and upper bound of the other" do + march_through_july = set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-08-01T00:00:00Z") + feb_march_and_july_sept = set_of_ranges( + {gte: "2021-02-01T00:00:00Z", lt: "2021-04-01T00:00:00Z"}, + {gte: "2021-07-01T00:00:00Z", lt: "2021-10-01T00:00:00Z"} + ) + + april_through_june = set_of_ranges( + {gte: "2021-04-01T00:00:00Z", lt: "2021-07-01T00:00:00Z"} + ) + + expect(march_through_july - feb_march_and_july_sept).to eq(april_through_june) + end + + it "returns a set of a single time when one TimeSet covers the entire other set excluding a single time" do + march_through_july = set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-08-01T00:00:00Z") + exclude_may_1 = set_of_ranges( + {lt: "2021-05-01T00:00:00Z"}, + {gt: "2021-05-01T00:00:00Z"} + ) + + expect(march_through_july - exclude_may_1).to eq(set_of_times("2021-05-01T00:00:00Z")) + end + + it "returns an empty TimeSet when one TimeSet covers the other TimeSet" do + march_through_july = set_of_range(gte: "2021-03-01T00:00:00Z", lt: "2021-08-01T00:00:00Z") + around_may_1 = set_of_ranges( + {lte: "2021-05-01T00:00:00Z"}, + {gte: "2021-05-01T00:00:00Z"} + ) + + expect(march_through_july - around_may_1).to eq(TimeSet::EMPTY) + end + + it "correctly truncates the ranges when one TimeSet is a subset of the other" do + march_june_and_aug_nov = set_of_ranges( + {gte: "2021-03-01T00:00:00Z", lt: "2021-07-01T00:00:00Z"}, + {gte: "2021-08-01T00:00:00Z", lt: "2021-12-01T00:00:00Z"} + ) + + april_may_and_sept_oct = set_of_ranges( + {gte: "2021-04-01T00:00:00Z", lt: "2021-06-01T00:00:00Z"}, + {gte: "2021-09-01T00:00:00Z", lt: "2021-11-01T00:00:00Z"} + ) + + march_june_aug_nov = set_of_ranges( + {gte: "2021-03-01T00:00:00Z", lt: "2021-04-01T00:00:00Z"}, + {gte: "2021-06-01T00:00:00Z", lt: "2021-07-01T00:00:00Z"}, + {gte: "2021-08-01T00:00:00Z", lt: "2021-09-01T00:00:00Z"}, + {gte: "2021-11-01T00:00:00Z", lt: "2021-12-01T00:00:00Z"} + ) + expect(march_june_and_aug_nov - april_may_and_sept_oct).to eq(march_june_aug_nov) + end + end + + it "returns an empty TimeSet when both sets are empty" do + empty1 = TimeSet::EMPTY + empty2 = TimeSet::EMPTY + + expect(empty1 - empty2).to eq(TimeSet::EMPTY) + end + + it "returns an empty TimeSet when ALL is subtracted from EMPTY" do + expect(TimeSet::EMPTY - TimeSet::ALL).to eq(TimeSet::EMPTY) + end + + it "returns ALL when the EMPTY TimeSet is subtracted from the ALL TimeSet" do + expect(TimeSet::ALL - TimeSet::EMPTY).to eq(TimeSet::ALL) + end + end + + describe "#negate" do + it "returns EMPTY for ALL and vice versa" do + expect(TimeSet::ALL.negate).to be(TimeSet::EMPTY) + expect(TimeSet::EMPTY.negate).to be(TimeSet::ALL) + end + + it "returns the set containing all times that were excluded from the original set" do + march_to_june_and_aug_to_nov = set_of_ranges( + {gte: "2021-03-01T00:00:00Z", lt: "2021-07-01T00:00:00Z"}, + {gte: "2021-08-01T00:00:00Z", lt: "2021-12-01T00:00:00Z"} + ) + + before_march_and_july_and_after_nov = set_of_ranges( + {lt: "2021-03-01T00:00:00Z"}, + {gte: "2021-07-01T00:00:00Z", lt: "2021-08-01T00:00:00Z"}, + {gte: "2021-12-01T00:00:00Z"} + ) + + expect(march_to_june_and_aug_to_nov.negate).to eq before_march_and_july_and_after_nov + expect(before_march_and_july_and_after_nov.negate).to eq march_to_june_and_aug_to_nov + end + end + + def set_of_range(expected_empty: false, **options) + options = options.transform_values { |iso8601_string| ::Time.iso8601(iso8601_string) } + + TimeSet.of_range(**options).tap do |time_set| + expect(time_set.empty?).to eq(expected_empty) + end + end + + def set_of_ranges(*options_for_ranges) + ranges = options_for_ranges.map { |opts| set_of_range(**opts) }.map do |time_set| + expect(time_set.ranges.size).to eq(1) + time_set.ranges.first + end + + TimeSet.of_range_objects(ranges) + end + + def set_of_times(*iso8601_strings) + times = iso8601_strings.map { |s| ::Time.iso8601(s) } + + TimeSet.of_times(times).tap do |time_set| + expect(time_set.empty?).to eq(iso8601_strings.empty?) + end + end + end + end +end diff --git a/elasticgraph-support/spec/unit/elastic_graph/support/time_util_spec.rb b/elasticgraph-support/spec/unit/elastic_graph/support/time_util_spec.rb new file mode 100644 index 00000000..923ef800 --- /dev/null +++ b/elasticgraph-support/spec/unit/elastic_graph/support/time_util_spec.rb @@ -0,0 +1,228 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/time_util" +require "time" + +module ElasticGraph + module Support + RSpec.describe TimeUtil do + describe ".nano_of_day_from_local_time" do + it "converts `00:00:00` to 0" do + expect_nanos("00:00:00", 0) + end + + it "converts single digit hours/minutes/seconds" do + expect_nanos("01:03:02", 3782000000000) + end + + it "works correctly with single digit values that `Integer(...)` requires a base arg for" do + # `Integer(...)` treats a leading `0` as indicating an octal (base 8) number. + # Consequently, `Integer("08")` and `Integer("09")` are not valid unless passing a base arg. + # It's important `nano_of_day_from_local_time` does this so we cover that case here. + expect(Integer("07")).to eq(7) # not needed for values under 08 since 0 to 7 are the same in decimal and octal. + expect { Integer("08") }.to raise_error(a_string_including("invalid value for Integer", "08")) + expect { Integer("09") }.to raise_error(a_string_including("invalid value for Integer", "09")) + expect(Integer("08", 10)).to eq(8) + expect(Integer("09", 10)).to eq(9) + + expect_nanos("08:08:08.08", 29288080000000) + expect_nanos("09:09:09.09", 32949090000000) + end + + it "converts double digit hours/minutes/seconds" do + expect_nanos("14:37:12", 52632000000000) + end + + it "supports milliseconds" do + expect_nanos("12:35:14.123", 45314123000000) + expect_nanos("12:35:14.009", 45314009000000) + end + + it "allows any number of decimal digits" do + expect_nanos("02:35:14.1", 9314100000000) + expect_nanos("02:35:14.10", 9314100000000) + expect_nanos("02:35:14.100", 9314100000000) + expect_nanos("02:35:14.1000", 9314100000000) + end + + it "converts `23:59:59.999`" do + expect_nanos("23:59:59.999", 86399999000000) + end + + it "rejects non-numeric characters in the hours position" do + expect_invalid_value("ab:00:00", "ab") + end + + it "rejects non-numeric characters in the minutes position" do + expect_invalid_value("00:cd:00", "cd") + end + + it "rejects non-numeric characters in the seconds position" do + expect_invalid_value("00:00:ef", "ef") + end + + it "rejects non-numeric characters in the sub-seconds position" do + expect_invalid_value("00:00:00.xyz", "xyz000000") + end + + def expect_nanos(local_time, expected_nanos) + converted = TimeUtil.nano_of_day_from_local_time(local_time) + expect(converted).to eq(expected_nanos) + + # Java has a simple API to convert a local time string to nano-of-day (`LocalTime.parse(str).toNanoOfDay()`) + # So we can nicely use it as a source of truth to verify that our expected nanos are correct. The + # `script/local_time_to_nano_of_day.java` script uses that Java API to do the conversion for us. + # However, it's kinda slow--on my M1 mac it's the difference between these specs taking < 2 ms and + # them taking 2-3 seconds. + # + # We don't usually want these tests to go 1000x slower just to verify that our expected values are in + # fact correct, but it's nice to run that extra check on CI. + # + # If you're ever adding new examples to the above, you may also want to enable this + # (just change to `if true`). + if ENV["CI"] + # :nocov: -- not executed in all environments + nanos_according_to_java = `java --source 11 #{SPEC_ROOT}/support/local_time_to_nano_of_day.java #{local_time}`.strip + expect(expected_nanos.to_s).to eq(nanos_according_to_java) + # :nocov: + end + end + + def expect_invalid_value(local_time_str, bad_part) + expect { + TimeUtil.nano_of_day_from_local_time(local_time_str) + }.to raise_error ArgumentError, a_string_including("invalid value", bad_part.inspect) + end + end + + describe ".advance_one_unit" do + shared_examples_for "advancing time" do + it "can advance a time by one :year" do + initial = parse_time("2021-12-01 15:20:04.36235") + next_year = parse_time("2022-12-01 15:20:04.36235") + + expect(TimeUtil.advance_one_unit(initial, :year)).to exactly_equal next_year + end + + it "can advance a time by one :month (within a year)" do + initial = parse_time("2021-09-01 15:20:04.36235") + next_month = parse_time("2021-10-01 15:20:04.36235") + + expect(TimeUtil.advance_one_unit(initial, :month)).to exactly_equal next_month + end + + it "can advance a time by one :month (across a year boundary)" do + initial = parse_time("2021-12-01 15:20:04.36235") + next_month = parse_time("2022-01-01 15:20:04.36235") + + expect(TimeUtil.advance_one_unit(initial, :month)).to exactly_equal next_month + end + + it "rounds down to the last day of the month when advancing from a day-of-month that doesn't exist on the next month" do + initial = parse_time("2021-01-31 15:20:04.36235") + next_month = parse_time("2021-02-28 15:20:04.36235") + + expect(TimeUtil.advance_one_unit(initial, :month)).to exactly_equal next_month + end + + it "can advance a time by one :day (within a month)" do + initial = parse_time("2021-12-01 15:20:04.36235") + next_day = parse_time("2021-12-02 15:20:04.36235") + + expect(TimeUtil.advance_one_unit(initial, :day)).to exactly_equal next_day + end + + it "can advance a time by one :day (across a month boundary, but within a year)" do + initial = parse_time("2021-02-28 15:20:04.36235") + next_day = parse_time("2021-03-01 15:20:04.36235") + + expect(TimeUtil.advance_one_unit(initial, :day)).to exactly_equal next_day + end + + it "can advance a time by one :day (across a year boundary)" do + initial = parse_time("2021-12-31 15:20:04.36235") + next_day = parse_time("2022-01-01 15:20:04.36235") + + expect(TimeUtil.advance_one_unit(initial, :day)).to exactly_equal next_day + end + + it "can advance a time by one :hour (within a day)" do + initial = parse_time("2021-12-01 15:20:04.36235") + next_hour = parse_time("2021-12-01 16:20:04.36235") + + expect(TimeUtil.advance_one_unit(initial, :hour)).to exactly_equal next_hour + end + + it "can advance a time by one :hour (across a day boundary, but within a month)" do + initial = parse_time("2021-12-01 23:20:04.36235") + next_hour = parse_time("2021-12-02 00:20:04.36235") + + expect(TimeUtil.advance_one_unit(initial, :hour)).to exactly_equal next_hour + end + + it "can advance a time by one :hour (across a month boundary, but within a year)" do + initial = parse_time("2021-11-30 23:20:04.36235") + next_hour = parse_time("2021-12-01 00:20:04.36235") + + expect(TimeUtil.advance_one_unit(initial, :hour)).to exactly_equal next_hour + end + + it "can advance a time by one :hour (across a year boundary)" do + initial = parse_time("2021-12-31 23:20:04.36235") + next_hour = parse_time("2022-01-01 00:20:04.36235") + + expect(TimeUtil.advance_one_unit(initial, :hour)).to exactly_equal next_hour + end + + # Given our `advancementUnit` type, steep complains if we put any logic in + # `advance_one_unit` to have it raise an error on other types, claiming the + # code is unreachable. However, our code coverage check fails if we don't + # cover that case, so we cover it here with a simple test that just shows it + # returns nil. + it "returns `nil` when given any other time units" do + initial = parse_time("2021-12-31 23:20:04.36235") + + expect(TimeUtil.advance_one_unit(initial, :minute)).to be_nil + expect(TimeUtil.advance_one_unit(initial, :decade)).to be_nil + end + end + + context "with a UTC time" do + include_examples "advancing time" do + def parse_time(time_string) + ::Time.parse("#{time_string}Z") + end + end + end + + context "with a non-UTC time" do + include_examples "advancing time" do + def parse_time(time_string) + ::Time.parse("#{time_string} -0800") + end + end + end + + def exactly_equal(time) + # Here we verify both simple time equality and also that the ISO8601 representations are equal. + # This is necessary to deal with an oddity of Ruby's Time class: + # + # > Time.utc(2021, 12, 2, 12, 30, 30).iso8601 + # => "2021-12-02T12:30:30Z" + # > Time.new(2021, 12, 2, 12, 30, 30, 0).iso8601 + # => "2021-12-02T12:30:30+00:00" + # + # We want to preserve the `Z` suffix on the advanced time (if it was there on the original time), + # so the extra check on the ISO8601 repsentation enforces that. + eq(time).and have_attributes(iso8601: time.iso8601) + end + end + end + end +end diff --git a/elasticgraph-support/spec/unit/elastic_graph/support/untyped_encoder_spec.rb b/elasticgraph-support/spec/unit/elastic_graph/support/untyped_encoder_spec.rb new file mode 100644 index 00000000..7433da44 --- /dev/null +++ b/elasticgraph-support/spec/unit/elastic_graph/support/untyped_encoder_spec.rb @@ -0,0 +1,83 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/untyped_encoder" + +module ElasticGraph + module Support + RSpec.describe UntypedEncoder do + it "dumps an integer as a string so it can be indexed as a keyword" do + expect(encode(3)).to eq "3" + end + + it "dumps a float as a string so it can be indexed as a keyword" do + expect(encode(3.14)).to eq "3.14" + end + + it "drops excess zeroes on a float to convert it to a canonical form" do + expect(encode(3.2100000)).to eq("3.21") + end + + it "dumps a boolean as a string so it can be indexed as a keyword" do + expect(encode(true)).to eq "true" + expect(encode(false)).to eq "false" + end + + it "quotes strings so that it is parseable as JSON" do + expect(encode("true")).to eq '"true"' + expect(encode("3")).to eq '"3"' + end + + it "dumps `nil` as `nil` so that the index field remains unset" do + expect(encode(nil)).to eq nil + end + + it "dumps an array as a compact JSON string so it can be indexed as a keyword" do + expect(encode([1, true, "hello"])).to eq('[1,true,"hello"]') + end + + it "dumps an array as a compact JSON string so it can be indexed as a keyword" do + expect(encode([1, true, "hello"])).to eq('[1,true,"hello"]') + end + + it "orders object keys alphabetically when dumping it to normalize to a canonical form" do + expect(encode({"b" => 1, "a" => 2, "c" => 3})).to eq('{"a":2,"b":1,"c":3}') + end + + it "applies the hash key sorting recursively at any level of the structure" do + data = ["f", {"b" => 1, "a" => [{"d" => 3, "c" => 2}]}] + expect(encode(data)).to eq('["f",{"a":[{"c":2,"d":3}],"b":1}]') + end + + it "can apply hash key sorting even when the keys are of different types" do + data = {"a" => 1, 3 => "b", 2 => "d"} + + # JSON doesn't support keys that aren't strings, but JSON.generate converts keys to strings. + expect(::JSON.generate(data)).to eq('{"a":1,"3":"b","2":"d"}') + # 3 and "a" aren't comparable when sorting... + expect { data.keys.sort }.to raise_error(/comparison of String with (2|3) failed/) + # ...but encode is still able to sort them. + # + # Note: JSON objects don't support non-string keys, so we should never actually hit this case, + # but we don't want our sorting canonicalization logic to introduce exceptions, so we handle this + # case (and cover it with a test). + expect(encode(data, validate_roundtrip: false)).to eq('{"2":"d","3":"b","a":1}') + end + + # This helper method enforces an invariant: parsing the resulting JSON string + # should always produce the original value + def encode(original_value, validate_roundtrip: true) + UntypedEncoder.encode(original_value).tap do |prepared_value| + if validate_roundtrip + expect(UntypedEncoder.decode(prepared_value)).to eq(original_value) + end + end + end + end + end +end diff --git a/elasticgraph/.rspec b/elasticgraph/.rspec new file mode 120000 index 00000000..67e6e21b --- /dev/null +++ b/elasticgraph/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph/.yardopts b/elasticgraph/.yardopts new file mode 120000 index 00000000..e11a2057 --- /dev/null +++ b/elasticgraph/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph/Gemfile b/elasticgraph/Gemfile new file mode 120000 index 00000000..26cb2ad9 --- /dev/null +++ b/elasticgraph/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph/LICENSE.txt b/elasticgraph/LICENSE.txt new file mode 100644 index 00000000..ca4fb445 --- /dev/null +++ b/elasticgraph/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph/README.md b/elasticgraph/README.md new file mode 100644 index 00000000..98435140 --- /dev/null +++ b/elasticgraph/README.md @@ -0,0 +1,4 @@ +# ElasticGraph + +ElasticGraph meta-gem that pulls in all the core ElasticGraph gems. Intended for use when all +parts of ElasticGraph are used from the same deployed app. diff --git a/elasticgraph/elasticgraph.gemspec b/elasticgraph/elasticgraph.gemspec new file mode 100644 index 00000000..03fce1f9 --- /dev/null +++ b/elasticgraph/elasticgraph.gemspec @@ -0,0 +1,18 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../gemspec_helper" + +ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :core) do |spec, eg_version| + spec.summary = "ElasticGraph meta-gem that pulls in all the core ElasticGraph gems." + + spec.add_dependency "elasticgraph-admin", eg_version + spec.add_dependency "elasticgraph-graphql", eg_version + spec.add_dependency "elasticgraph-indexer", eg_version + spec.add_dependency "elasticgraph-local", eg_version +end diff --git a/gemspec_helper.rb b/gemspec_helper.rb new file mode 100644 index 00000000..b94e7488 --- /dev/null +++ b/gemspec_helper.rb @@ -0,0 +1,149 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "elasticgraph-support/lib/elastic_graph/version" + +module ElasticGraphGemspecHelper + # Helper methor for defining a gemspec for an elasticgraph gem. + def self.define_elasticgraph_gem(gemspec_file:, category:) + gem_dir = ::File.expand_path(::File.dirname(gemspec_file)) + validate_gem(gem_dir) + + ::Gem::Specification.new do |spec| + spec.name = ::File.basename(gemspec_file, ".gemspec") + spec.version = ElasticGraph::VERSION + spec.authors = ["Myron Marston", "Ben VandenBos", "Block Engineering"] + spec.email = ["myron@squareup.com"] + spec.license = "MIT" + spec.metadata["gem_category"] = category.to_s + + # Specify which files should be added to the gem when it is released. + # The `git ls-files -z` loads the files in the RubyGem that have been added into git. + # We also remove `.rspec` and `Gemfile` because these files are not needed in + # the packaged gem (they are for local development of the gems) and cause a problem + # for some users of the gem due to the fact that they are symlinks to a parent path. + spec.files = ::Dir.chdir(gem_dir) do + `git ls-files -z`.split("\x0").reject do |f| + (f == __FILE__) || f.match(%r{\A(?:(?:test|spec|features|sig)/|\.(?:git|travis|circleci)|appveyor)}) + end - [".rspec", "Gemfile", ".yardopts"] + end + + spec.bindir = "exe" + spec.executables = spec.files.grep(%r{\Aexe/}) { |f| ::File.basename(f) } + spec.require_paths = ["lib"] + spec.required_ruby_version = "~> 3.2" + + # Here we define common development dependencies used for the CI build of most of our gems. + + # Linting and style checking gems. + spec.add_development_dependency "rubocop-factory_bot", "~> 2.26" + spec.add_development_dependency "rubocop-rake", "~> 0.6" + spec.add_development_dependency "rubocop-rspec", "~> 3.1" + spec.add_development_dependency "standard", "~> 1.41.0" + + # Steep is our type checker. Only needed if there's a `sig` directory. + if ::Dir.exist?(::File.join(gem_dir, "sig")) + spec.add_development_dependency "steep", "~> 1.8" + end + + # If the gem has a `spec` directory then it needs our standard set of testing gems. + if ::Dir.exist?(::File.join(gem_dir, "spec")) + spec.add_development_dependency "coderay", "~> 1.1" + spec.add_development_dependency "flatware-rspec", "~> 2.3", ">= 2.3.3" + spec.add_development_dependency "rspec", "~> 3.13" + spec.add_development_dependency "super_diff", "~> 0.13" + spec.add_development_dependency "simplecov", "~> 0.22" + spec.add_development_dependency "simplecov-console", "~> 0.9" + + # In addition, if any specs have the `:uses_datastore` tag then we need to pull in gems used by that tag. + if `git grep -l ":uses_datastore" #{gem_dir}/spec | wc -l`.strip.to_i > 0 + spec.add_development_dependency "httpx", "~> 1.3" + spec.add_development_dependency "method_source", "~> 1.1" + spec.add_development_dependency "rspec-retry", "~> 0.6" + spec.add_development_dependency "vcr", "~> 6.3", ">= 6.3.1" + end + + # In addition, if any specs have the `:uses_datastore` tag then we need to pull in gems used by that tag. + if `git grep -l ":factories" #{gem_dir}/spec | wc -l`.strip.to_i > 0 + spec.add_development_dependency "factory_bot", "~> 6.4" + spec.add_development_dependency "faker", "~> 3.5" + end + + # If any specs use the `spec_support/lambda_function` helper, then pull in the `aws_lambda_ric` gem, + # as it contains code that AWS bootstraps Ruby lambdas with. Note that we don't depend on anything + # specific in this gem, but we want to include it so that our CI build can detect any incompatibilities + # we may have with it. + if `git grep -l "spec_support\/lambda_function" #{gem_dir}/spec | wc -l`.strip.to_i > 0 + spec.add_development_dependency "aws_lambda_ric", "~> 2.0" + end + end + + yield spec, ElasticGraph::VERSION + + if (symlink_files = spec.files.select { |f| ::File.exist?(f) && ::File.ftype(f) == "link" }).any? + raise "#{symlink_files.size} file(s) of the `#{spec.name}` gem are symlinks, but " \ + "symlinks do not work correctly when the gem is packaged. Symlink files: #{symlink_files.inspect}" + end + end + end + + def self.validate_gem(gem_dir) + gem_warnings = validate_symlinked_file(::File.join(gem_dir, ".yardopts")) + + gem_issues = [] + gem_issues.concat(validate_symlinked_file(::File.join(gem_dir, "Gemfile"))) + gem_issues.concat(validate_symlinked_file(::File.join(gem_dir, ".rspec"))) + gem_issues.concat(validate_license(gem_dir)) + + unless gem_warnings.empty? + warn "WARNING: Gem #{::File.basename(gem_dir)} has the following issues:\n\n" + gem_warnings.join("\n") + end + + return if gem_issues.empty? + + abort "Gem #{::File.basename(gem_dir)} has the following issues:\n\n" + gem_issues.join("\n") + end + + def self.validate_symlinked_file(file) + gem_issues = [] + + if ::File.exist?(file) + if ::File.ftype(file) != "link" + gem_issues << "`#{file}` must be a symlink." + end + else + gem_issues << "`#{file}` is missing." + end + + gem_issues + end + + def self.validate_license(gem_dir) + gem_issues = [] + + file = ::File.join(gem_dir, "LICENSE.txt") + if ::File.exist?(file) + if ::File.ftype(file) == "link" + gem_issues << "`#{file}` must not be a symlink." + end + + contents = ::File.read(file) + unless contents.include?("MIT License") + gem_issues << "`#{file}` must contain 'MIT License'." + end + + unless contents.include?("Copyright (c) 2024 Block, Inc.") + gem_issues << "`#{file}` must contain Block copyright notice." + end + else + gem_issues << "`#{file}` is missing." + end + + gem_issues + end +end diff --git a/rbs_collection.yaml b/rbs_collection.yaml new file mode 100644 index 00000000..894e4d6f --- /dev/null +++ b/rbs_collection.yaml @@ -0,0 +1,80 @@ +# Download sources +sources: + - type: git + name: ruby/gem_rbs_collection + remote: https://github.com/ruby/gem_rbs_collection.git + revision: main + repo_dir: gems + +# You can specify local directories as sources also. +# - type: local +# path: path/to/your/local/repository + +# A directory to install the downloaded RBSs +path: .gem_rbs_collection + +gems: + # ffi is a transitive dependency. We don't depend on any types from it, so we want to ignore its types. + - name: ffi + ignore: true + # We don't depend on any types from HTTPX (it's just one of the faraday adapters), but it gets + # pulled in by default because it's in our `Gemfile`, and we get steep type check errors when + # it is included. + - name: httpx + ignore: true + + # Use `ignore: false` to tell rbs collection to pull the RBS signatures from these gems. + - name: aws-sdk-lambda + ignore: false + - name: aws-sdk-sqs + ignore: false + - name: aws-sdk-s3 + ignore: false + - name: faraday + ignore: false + - name: hashdiff + ignore: false + + # We must ignore all ElasticGraph gems because they are declared as dependencies with + # bundler and `rbs collection install` pulls them in. But they they are _also_ directly + # available in this codebase and steep complains about duplicate definitions. + - name: elasticgraph-admin + ignore: true + - name: elasticgraph-admin_lambda + ignore: true + - name: elasticgraph-indexer_autoscaler_lambda + ignore: true + - name: elasticgraph-apollo + ignore: true + - name: elasticgraph-datastore_core + ignore: true + - name: elasticgraph-elasticsearch + ignore: true + - name: elasticgraph-graphql + ignore: true + - name: elasticgraph-graphql_lambda + ignore: true + - name: elasticgraph-health_check + ignore: true + - name: elasticgraph-indexer + ignore: true + - name: elasticgraph-indexer_lambda + ignore: true + - name: elasticgraph-json_schema + ignore: true + - name: elasticgraph-lambda_support + ignore: true + - name: elasticgraph-local + ignore: true + - name: elasticgraph-opensearch + ignore: true + - name: elasticgraph-query_interceptor + ignore: true + - name: elasticgraph-query_registry + ignore: true + - name: elasticgraph-schema_artifacts + ignore: true + - name: elasticgraph-schema_definition + ignore: true + - name: elasticgraph-support + ignore: true diff --git a/script/ci_parts/run_each_gem_spec b/script/ci_parts/run_each_gem_spec new file mode 100755 index 00000000..0b0a0ea3 --- /dev/null +++ b/script/ci_parts/run_each_gem_spec @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# This script runs each gem's test suite individually. This is slower than running the entire suite +# in one pass but is useful as a way to verify that each individual gem's test suite passes on its +# own. In particular, this can surface dependencies that are not correctly declared in the gem's +# `gemspec` file. When a gem depends on a specific gem, but fails to declare the dependency, it can +# still pass when run with all the other gem test suites so long as one of the other gems declares +# that dependency (due to the global nature of Ruby's `require`). However, it will fail when we run +# the gem's test suite in isolation, so this serves as a good way to surface missing dependency +# declarations. + +source "script/ci_parts/setup_env" "test" $1 $2 + +# Note: In our CI build, we want our coverage thresholds to be met by the test suite of each individual gem. +# For example, we don't want a gem to have 100% test coverage only because of some coverage provided by a +# test from another gem. Running the test suite from each gem independently (as we do here) ensures that +# SimpleCov doesn't count this kind of "cross" coverage. +for gem in $gems_to_build_with_datastore_booted; do + script/run_gem_specs $gem +done + +halt_datastore_daemon + +for gem in $gems_to_build_with_datastore_halted; do + script/run_gem_specs $gem +done diff --git a/script/ci_parts/run_misc_checks b/script/ci_parts/run_misc_checks new file mode 100755 index 00000000..51e2d99c --- /dev/null +++ b/script/ci_parts/run_misc_checks @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# We boot the `local` datastore instead of `test` because the `index_fake_data` task below runs against the `local` datastore. +source "script/ci_parts/setup_env" "local" $1 $2 + +script/spellcheck +script/type_check +script/update_readme --verify +script/update_ci_yaml --verify + +bundle _$bundler_version\_ exec standardrb +bundle _$bundler_version\_ exec rake schema_artifacts:check VERBOSE=true + +# verify that we can index fake data locally. Note: this must come _after_ `rake schema_artifacts:check` +# because the `index_fake_data:widgets:local` task depends on `schema_artifacts:dump`. +bundle _$bundler_version\_ exec rake "index_fake_data:widgets[1]" +bundle _$bundler_version\_ exec rspec config/linting + +# Validate the website. We get frozen string errors form liquid if we leave +# frozen string literals enabled, so we have to disable them here. +RUBYOPT=--disable-frozen-string-literal bundle _$bundler_version\_ exec rake site:validate + +# The apollo compatibility tests boot docker containers and need all the resources we can provide them. +# There does not appear to be enough resources on GitHub Actions for them to run while the the datastore +# docker container is still running, so we halt it before running the compatibility tests. +halt_datastore_daemon + +# Test against federation v2.0, v2.3, and v2.6. +elasticgraph-apollo/script/test_compatibility 2.6 +elasticgraph-apollo/script/test_compatibility 2.3 +elasticgraph-apollo/script/test_compatibility 2.0 diff --git a/script/ci_parts/run_most_specs_with_vcr b/script/ci_parts/run_most_specs_with_vcr new file mode 100755 index 00000000..53a99fcd --- /dev/null +++ b/script/ci_parts/run_most_specs_with_vcr @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +source "script/ci_parts/setup_env" "test" $1 $2 + +# Also make sure the test suite passes when VCR is used. For simplicity this runs test suites of all gems all at once. +# One of the gems we use when VCR is loaded (`method_source`) isn't compatible with `--enable-frozen-string-literal` +# so we unset `RUBYOPT` here to disable that. +# +# These ENV vars are set by default on CI so we have to unset them here. +unset NO_VCR +unset RUBYOPT + +# In addition, we want to run the extra GraphQL schema validation when running the whole schema together. +VALIDATE_GRAPHQL_SCHEMAS=1 script/run_most_specs diff --git a/script/ci_parts/run_specs_file_by_file b/script/ci_parts/run_specs_file_by_file new file mode 100755 index 00000000..82744720 --- /dev/null +++ b/script/ci_parts/run_specs_file_by_file @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +source "script/ci_parts/setup_env" "test" $1 $2 + +# We don't want to track coverage when we run each spec file individually, as we don't expect 100% coverage +# of entire gems when just one spec file is run. +unset COVERAGE + +# Setup standalone binstubs for RSpec that we use below. +bundle _$bundler_version\_ install --standalone && bundle _$bundler_version\_ binstubs rspec-core --standalone + +function run_gem_specs_file_by_file() { + gem=$1 + pushd $gem + for file in `find spec -iname '*_spec.rb'`; do + echo "Running $file" + + # Note: here we avoid using `bundle exec`, opting for a binstub instead. + # Our `bundle install` command (with `--standalone` and `binstubs` options) creates the + # rspec binstub in a way where it won't actually load bundler at runtime. This is + # *slightly* faster. Not enough to usually matter, but it adds up when we boot rspec + # once for each spec file as we do here! + ../bin/rspec $file -b --format progress --no-profile + done + popd +} + +echo "Running each spec file, one-by-one..." + +for gem in $gems_to_build_with_datastore_booted; do + run_gem_specs_file_by_file $gem +done + +halt_datastore_daemon + +for gem in $gems_to_build_with_datastore_halted; do + run_gem_specs_file_by_file $gem +done diff --git a/script/ci_parts/setup_env b/script/ci_parts/setup_env new file mode 100755 index 00000000..67139250 --- /dev/null +++ b/script/ci_parts/setup_env @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +# Abort script at first error, when a command exits with non-zero status. +# Verbose form of `set -e`. +set -o errexit + +# Attempt to use undefined variable outputs error message, and forces an exit +# Verbose form of `set -u`. +set -o nounset + +# If set, the return value of a pipeline is the value of the last (rightmost) +# command to exit with a non-zero status, or zero if all commands in the +# pipeline exit successfully. +set -o pipefail + +# Print a trace of simple commands. +# Verbose form of `set -x`. +set -o xtrace + +export CI=true + +# We want to leverage frozen string literals for faster perf. +# This will enforce it on CI. +export RUBYOPT=--enable-frozen-string-literal + +# We use the VCR gem as a local "test accelerator" which caches datastore requests/responses for us. +# But in our CI build we don't want to use it at all, so we disable it here, for all RSpec runs. +export NO_VCR=1 + +# track test coverage for all specs +export COVERAGE=1 + +export bundler_version=$(cat Gemfile.lock | grep "BUNDLED WITH" -A 2 | grep "[0-9]" | xargs) +export all_gems=$(script/list_eg_gems.rb) + +# elasticgraph-local contains the rake tasks that manage our datastores. Running the acceptance tests for it can +# tend to run into conflicts if the datastore is already running--for example, on GitHub Actions we've observed +# that there are not sufficient resources to run those acceptance tests while the original datastore daemon +# is still running. Therefore, we want to run its tests after halting the datastore daemon. +export gems_to_build_with_datastore_halted=(elasticgraph-local) +# Array subtraction technique taken from: https://stackoverflow.com/a/28161520 +export gems_to_build_with_datastore_booted=$(echo ${all_gems[@]} ${gems_to_build_with_datastore_halted[@]} | tr ' ' '\n' | sort | uniq -u) + +# The ci_parts scripts are designed primarily for use on CI, where we need the datastore booted as an initial step. +# However, locally we want to be able to run the ci_parts scripts while already having the datastore booted (we commonly +# leave it running in the background). Here we boot the datastore if the script arg was passed, and otherwise print a +# message and skip. +if [ "$#" -lt 2 ]; then + echo "No datastore argument specified; will assume the datastore is already running." + echo "If you need want the ci_parts script to boot the datastore, specify the datastore as the first argument, like:" + echo "script/ci_parts/run_each_gem_spec elasticsearch:8.13.0" + + function halt_datastore_daemon() { + echo "Skipping the halt of a datastore daemon since the 'setup_env' script didn't boot the daemon." + } +else + boot_env=$1 + datastore=$2 + sleep_after_boot=${3:-0} + datastore_backend=$(echo "${datastore}" | cut -d ":" -f 1) + datastore_version=$(echo "${datastore}" | cut -d ":" -f 2) + + function halt_datastore_daemon() { + bundle _$bundler_version\_ exec rake ${datastore_backend}:${boot_env}:${datastore_version}:halt + } + + bundle _$bundler_version\_ exec rake ${datastore_backend}:${boot_env}:${datastore_version}:daemon + + # Occasionally, we've see transient "Partial shards failure (N shards unavailable)" errors from + # Elasticsearch/OpenSearch on CI from the first tests that run after booting it. Locally that + # never happens--likely because we do not immediately run tests after booting Elasticsearch/OpenSearch. + # + # Here we sleep a specified amount of time (defaulting to 0 to not slow down locally) to try to avoid this issue. + sleep $sleep_after_boot +fi diff --git a/script/list_eg_gems.rb b/script/list_eg_gems.rb new file mode 100755 index 00000000..32d640f9 --- /dev/null +++ b/script/list_eg_gems.rb @@ -0,0 +1,24 @@ +#!/usr/bin/env ruby + +# This script identifies all of the local ElasticGraph gems in this repo. +# It is designed to be usable from Ruby without shelling out (just require +# it and call `ElasticGraphGems.list`) but also callable from a shell script +# (just run this script). +# +# Note that it does assume that all gems are in a direct subdirectory of the +# the repository root. Our glob only looks one level deep to avoid pulling in +# gems that could be installed in `bundle` (e.g. if `bundle --standalone` is +# being used). + +module ElasticGraphGems + def self.list + repo_root = ::File.expand_path("..", __dir__) + ::Dir.glob("#{repo_root}/*/*.gemspec").map do |gemspec| + ::File.basename(::File.dirname(gemspec)) + end + end +end + +if $PROGRAM_NAME == __FILE__ + puts ElasticGraphGems.list.join("\n") +end diff --git a/script/quick_build b/script/quick_build new file mode 100755 index 00000000..110fddb5 --- /dev/null +++ b/script/quick_build @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# This script is meant to run a "quick" build. A "full" build performed on CI +# isn't terribly suitable for engineers running locally: it assumes everything +# has to be setup from scratch, and opts for completeness/strictness over build +# speed to an extreme level that is great for CI but not for local builds. +# For example, it opts in to some extra GraphQL schema validation and runs the +# test suite without VCR, which verifies the test suite passes without VCR +# recordings, but makes the test suite take much longer (e.g. 2.5-3 minutes +# instead 30-45 seconds). Similarly, the full CI build runs the specs "file by file" +# to ensure every spec file passes when run on its own. This fails *very* rarely but +# is quite slow so we do not want to include it here. + +# Abort script at first error, when a command exits with non-zero status. +# Verbose form of `set -e`. +set -o errexit + +# Attempt to use undefined variable outputs error message, and forces an exit +# Verbose form of `set -u`. +set -o nounset + +# If set, the return value of a pipeline is the value of the last (rightmost) +# command to exit with a non-zero status, or zero if all commands in the +# pipeline exit successfully. +set -o pipefail + +# Print a trace of simple commands. +# Verbose form of `set -x`. +set -o xtrace + +script/spellcheck +bundle exec standardrb + +script/type_check + +bundle exec rake schema_artifacts:check +COVERAGE=1 script/run_most_specs --backtrace + +bundle exec rake site:validate + +success=$? +exit $success diff --git a/script/run_gem_specs b/script/run_gem_specs new file mode 100755 index 00000000..7a99b70b --- /dev/null +++ b/script/run_gem_specs @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +source "script/ci_parts/setup_env" + +gem=$1 + +if [ ! -d "$gem/spec" ]; then + echo "Skipping script/run_gem_specs for $gem because it has no spec directory." + exit 0 +fi + +pushd $gem + # When running the test suite for a gem, we want it to run in the context of a bundle that is + # specific to that gem, so that we will get load errors if the gem attempts to load any gems + # that are not declared as dependencies in the gemspec. To achieve that, we're doing a couple + # notable things here: + # + # 1. We copy `Gemfile.lock` from the root `Gemfile.lock`. We do this because we don't want to + # have to re-run `bundle install` here. The `Gemfile.lock` of the repository contains a + # superset of the gems needed for each gem, and bundler will allow us to reuse it when + # dealing with a bundle that contains fewer gems based on the gemspec. Note that `Gemfile.lock` + # in subdirs is git-ignored. We copy instead of symlink it because `bundle exec` will update + # the `Gemfile.lock` based on the smaller set of gems and we don't want to update the root + # `Gemfile.lock`. + # 2. We pass `BUNDLE_GEMFILE=Gemfile` to force it to use the `Gemfile` from the gem subdirectory. + # By default, bundler would use that file, but if the file did not exist, it would fall back + # to the `Gemfile` from the parent directory (which would allow the gem to load any gem that + # is a dependency of any other elasticgraph gem). To ensure that there is a proper `Gemfile` in + # each gem subdirectory, we pass the ENV var here. If the file does not exist, we'll get an error. + cp ../Gemfile.lock Gemfile.lock + BUNDLE_GEMFILE=Gemfile bundle _$bundler_version\_ check || (rm -rf Gemfile.lock && bundle _$bundler_version\_ install) + BUNDLE_GEMFILE=Gemfile bundle _$bundler_version\_ exec rspec --backtrace --format progress +popd diff --git a/script/run_most_specs b/script/run_most_specs new file mode 100755 index 00000000..2e4835fe --- /dev/null +++ b/script/run_most_specs @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# This script runs all specs from all the elasticgraph gems. Any arguments passed to this script +# will be appended to the `rspec` command. + +# Print a trace of simple commands. +# Verbose form of `set -x`. +set -o xtrace + +# Find the spec dirs of all gems, except for ones we exclude due to being too slow (relative to their importance). +# +# - elasticgraph-local: this is used locally to run Elasticsearch/OpenSearch via docker. The tests boot them and +# are quite slow (~10 seconds each). +# +# Note: we use this technique instead of RSpec tagging (e.g. with `:slow`) because: +# - If we exclude individual specs it messes up our code coverage check (excluding whole gems does not!) +# - The specific gems we want to exclude are ones we are OK excluding all of. +spec_dirs=$(script/list_eg_gems.rb | grep -v elasticgraph-local) + +# Avoid a connection issue if a prior run was interrupted. More info: +# https://github.com/briandunn/flatware/issues/68 +rm -f flatware-sink + +# Limit the number of workers to 16. On some really beefy CI worker hosts, we've seen +# flatware default to 96 workers (due to the host having 96 vCPUs!), but that's overkill, +# and using too many workers can perform worse. +# +# Note: `Etc.nprocessors` is what flatware uses internally for the default number of workers. +worker_count=$(ruby -retc -e "puts [16, Etc.nprocessors].min") + +bundle exec flatware rspec $spec_dirs -w $worker_count "$@" diff --git a/script/spellcheck b/script/spellcheck new file mode 100755 index 00000000..f6c7dfa9 --- /dev/null +++ b/script/spellcheck @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +function install_codespell() { + case $(uname) in + "Darwin") + echo "Installing codespell in MacOS environment" + brew install codespell + ;; + "Linux") + echo "Installing codespell in Linux environment" + python3 -m venv tmp/codespell + source tmp/codespell/bin/activate + pip install codespell==2.3.0 + ;; + *) + echo "Unsupported platform: $(uname)" + exit 1 + ;; + esac +} + +if ! command -v codespell &> /dev/null; then + install_codespell +fi + +echo "Checking spelling with codespell..." + +codespell `git ls-files` "$@" --ignore-words-list "upto,reenable,nome,rouge,socio-economic" diff --git a/script/type_check b/script/type_check new file mode 100755 index 00000000..0f790afe --- /dev/null +++ b/script/type_check @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Abort script at first error, when a command exits with non-zero status. +# Verbose form of `set -e`. +set -o errexit + +# Attempt to use undefined variable outputs error message, and forces an exit +# Verbose form of `set -u`. +set -o nounset + +# If set, the return value of a pipeline is the value of the last (rightmost) +# command to exit with a non-zero status, or zero if all commands in the +# pipeline exit successfully. +set -o pipefail + +# Print a trace of simple commands. +# Verbose form of `set -x`. +set -o xtrace + +bundle exec rbs collection update +# Unfortunately, one of steep's dependencies crashes when you use frozen string literals, +# so we disable that flag when running it here. +RUBYOPT=--disable-frozen-string-literal bundle exec steep check diff --git a/script/update_ci_yaml b/script/update_ci_yaml new file mode 100755 index 00000000..c08a4f62 --- /dev/null +++ b/script/update_ci_yaml @@ -0,0 +1,150 @@ +#!/usr/bin/env ruby + +require "erb" +require "pathname" +require "rubygems" +require "yaml" + +module ElasticGraph + class CIYamlRenderer + PROJECT_ROOT = ::Pathname.new(::File.expand_path("..", __dir__)) + + def initialize(template) + @erb = ::ERB.new(template, trim_mode: "-") + end + + def render + @erb.result(binding) + end + + private + + def datastore_versions + datastore_versions_data.flat_map do |backend, versions| + versions.map do |version| + "#{backend}:#{version}" + end + end + end + + def primary_datastore_version + primary_version = datastore_versions_data.fetch("elasticsearch").max_by do |version| + ::Gem::Version.new(version) + end + + "elasticsearch:#{primary_version}" + end + + def datastore_versions_data + @datastore_versions_data ||= ::YAML.safe_load_file(datastore_versions_file) + end + + def relative_this_script + ::Pathname.new(::File.expand_path(__FILE__)).relative_path_from(PROJECT_ROOT) + end + + def datastore_versions_file + PROJECT_ROOT / "config" / "tested_datastore_versions.yaml" + end + + def relative_datastore_versions_file + datastore_versions_file.relative_path_from(PROJECT_ROOT) + end + end +end + +renderer = ElasticGraph::CIYamlRenderer.new(DATA.read) +contents = renderer.render +file_path = ElasticGraph::CIYamlRenderer::PROJECT_ROOT / ".github" / "workflows" / "ci.yaml" + +case ARGV.first +when "--verify" + if file_path.read == contents + puts "✅ #{file_path} is up-to-date." + else + tmp_path = ElasticGraph::CIYamlRenderer::PROJECT_ROOT / "tmp" / "ci.yaml" + tmp_path.write(contents) + + diff = `git diff --no-index #{file_path} #{tmp_path} #{" --color" unless ENV["CI"]}` + + puts "❌ #{file_path} is out-of-date. Run `#{__FILE__}` to update it. Diff:" + puts + puts diff + exit(1) + end +when nil + file_path.write(contents) + puts "#{file_path} updated." +else + raise "Unknown argument: #{ARGV.first}. Expected `--verify` or nothing." +end + +__END__ +# This file is generated by `<%= relative_this_script %>` based on input from `<%= relative_datastore_versions_file %>`. +# To edit it, make changes to the template at the bottom of `<%= relative_this_script %>` and run it. +name: ElasticGraph CI + +on: + push: + branches: + - main + pull_request: + +env: + # It's recommended to run ElasticGraph with this option to get better performance. We want to run + # our CI builds with it to ensure that the option always works. + RUBYOPT: "--enable-frozen-string-literal" + # We use the VCR gem as a local "test accelerator" which caches datastore requests/responses for us. + # But in our CI build we don't want to use it at all, so we disable it here. + NO_VCR: "1" + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + build_part: + - run_each_gem_spec + ruby: + - "3.2" + - "3.3" + datastore: +<% datastore_versions.each do |datastore_version| -%> + - "<%= datastore_version %>" +<% end -%> + include: + # We have 4 build parts. The "primary" one is `run_each_gem_spec`, and we need that to be run on + # every supported Ruby version and against every supported datastore. It's not necessary to run + # these others against every combination of `ruby` and `datastore` so we just run each with one + # configuration here. + - build_part: "run_misc_checks" + ruby: "3.3" + datastore: "<%= primary_datastore_version %>" + - build_part: "run_most_specs_with_vcr" + ruby: "3.3" + datastore: "<%= primary_datastore_version %>" + - build_part: "run_specs_file_by_file" + ruby: "3.3" + datastore: "<%= primary_datastore_version %>" + + steps: + - uses: actions/checkout@v4 + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + + - uses: actions/setup-node@v3 + with: + node-version: "23.x" + + - uses: KengoTODA/actions-setup-docker-compose@main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Note: the `10` argument on the end is a number of seconds to sleep after booting the datastore. + # We've found that there is a minor race condition where the shards aren't fully ready for the tests + # to hit them if we don't wait a bit after booting. + - run: script/ci_parts/${{ matrix.build_part }} ${{ matrix.datastore }} 10 diff --git a/script/update_readme b/script/update_readme new file mode 100755 index 00000000..d0bead04 --- /dev/null +++ b/script/update_readme @@ -0,0 +1,218 @@ +#!/usr/bin/env ruby + +require "erb" +require_relative "list_eg_gems" + +module ElasticGraph + class ReadmeRenderer + PROJECT_ROOT = File.expand_path("..", __dir__) + + def initialize(template) + @erb = ::ERB.new(template, trim_mode: "-") + end + + def render + @erb.result(binding) + end + + def gem_specs + @gem_specs ||= ::ElasticGraphGems.list.map do |gem_name| + ::Gem::Specification.load("#{PROJECT_ROOT}/#{gem_name}/#{gem_name}.gemspec") + end + end + + def gem_categories + @gem_categories ||= gem_specs.group_by do |spec| + spec.metadata.fetch("gem_category") do + raise "No category set for #{spec.name}." + end + end.map do |category, specs| + category_info = CategoryInfo::BY_NAME.fetch(category) do + raise "Unrecognized category: `#{category}` found on gems: #{specs.map(&:name)}." + end + GemCategory.new(category, category_info, specs, gem_specs) + end + end + end + + class GemCategory < ::Data.define(:category_name, :info, :gem_specs, :all_eg_gem_specs) + def mermaid_definition + <<~MERMAID + ```mermaid + graph LR; + #{mermaid_deps.join("\n")} + #{mermaid_styles.join("\n")} + #{mermaid_clicks.join("\n")} + ``` + MERMAID + end + + private + + def mermaid_deps + gem_specs.filter_map do |spec| + unless spec.runtime_dependencies.empty? + deps = spec.runtime_dependencies.map(&:name).join(" & ") + " #{spec.name} --> #{deps}" + end + end + end + + def mermaid_styles + all_nodes = gem_specs.map(&:name) | gem_specs.flat_map { |spec| spec.runtime_dependencies.map(&:name) } + all_nodes.map do |node| + " style #{node} #{style_for(node)};" + end + end + + def style_for(gem_name) + # These colors were chosen to look good on GitHub in both light mode and dark mode. + if gem_specs.any? { |spec| spec.name == gem_name } + "color: DodgerBlue" # Basic "Blue" doesn't look good in dark mode. + elsif all_eg_gem_specs.any? { |spec| spec.name == gem_name } + "color: Green" + else + "color: Red" + end + end + + def mermaid_clicks + gem_specs.flat_map { |spec| spec.runtime_dependencies.map(&:name) }.uniq.sort.filter_map do |gem_name| + unless all_eg_gem_specs.any? { |spec| spec.name == gem_name } + "click #{gem_name} href \"https://rubygems.org/gems/#{gem_name}\"" + end + end + end + end + + class CategoryInfo < ::Data.define(:description, :discussion) + BY_NAME = { + "core" => new("Core Libraries", <<~EOS), + These libraries form the core backbone of ElasticGraph that is designed to run in a production deployment. Every ElasticGraph deployment will need to use all of these. + EOS + + "datastore_adapter" => new("Datastore Adapters", <<~EOS), + These libraries adapt ElasticGraph to your choice of datastore (Elasticsearch or OpenSearch). + EOS + + "extension" => new("Extensions", <<~EOS), + These libraries extend ElasticGraph to provide optional but commonly needed functionality. + EOS + + "lambda" => new("AWS Lambda Integration Libraries", <<~EOS), + These libraries wrap the the core ElasticGraph libraries so that they can be deployed using AWS Lambda. + EOS + + "local" => new("Local Development Libraries", <<~EOS) + These libraries are used for local development of ElasticGraph applications, but are not intended to be deployed to production (except for `elasticgraph-rack`). + `elasticgraph-rack` is used to boot ElasticGraph locally but can also be used to run ElasticGraph in any rack-compatible server (including a Rails application). + EOS + } + end +end + +renderer = ElasticGraph::ReadmeRenderer.new(DATA.read) +contents = renderer.render +readme_path = ::File.join(__dir__, "..", "README.md") + +case ARGV.first +when "--verify" + if ::File.read(readme_path) == contents + puts "✅ README is up-to-date." + else + tmp_path = ::File.join(__dir__, "..", "tmp", "README.md") + ::File.write(tmp_path, contents) + + diff = `git diff --no-index #{readme_path} #{tmp_path} #{" --color" unless ENV["CI"]}` + + puts "❌ README is out-of-date. Run `#{__FILE__}` to update it. Diff:" + puts + puts diff + exit(1) + end +when nil + ::File.write(readme_path, contents) + puts "README.md updated." +else + raise "Unknown argument: #{ARGV.first}. Expected `--verify` or nothing." +end + +__END__ +# ElasticGraph + +ElasticGraph is a general purpose, near real-time data query and search platform that is scalable and performant, +serves rich interactive queries, and dramatically simplifies the creation of complex reports. The platform combines +the power of indexing and search of Elasticsearch or OpenSearch with the query flexibility of GraphQL language. +Optimized for AWS cloud, it also offers scale and reliability. + +ElasticGraph is a naturally flexible framework with many different possible applications. However, the main motivation we have for +building it is to power various data APIs, UIs and reports. These modern reports require filtering and aggregations across a body of ever +growing data sets. Modern APIs allow us to: + +- Minimize network trips to retrieve your data +- Get exactly what you want in a single query. No over- or under-serving the data. +- Push filtering complex calculations to the backend. + +## Libraries + +ElasticGraph is designed to be modular, with a small core, and many built-in extensions that extend that core +for specific use cases. This minimizes exposure to vulnerabilities, reduces bloat, and makes ongoing upgrades +easier. The libraries that ship with ElasticGraph can be broken down into several categories. + +<% gem_categories.each do |category| -%> +### <%= category.info.description %> (<%= category.gem_specs.size %> gems) + +<%= category.info.discussion %> +<% category.gem_specs.each do |spec| -%> +* [<%= spec.name %>](<%= spec.name %>/README.md): <%= spec.summary %> +<% end -%> + +#### Dependency Diagram + +<%= category.mermaid_definition %> +<% end -%> + +## Versioning Policy + +ElasticGraph does _not_ strictly follow the [SemVer](https://semver.org/) spec. We followed that early in the project's life +cycle and realized that it obscures some important compatibility information. + +ElasticGraph's versioning policy is designed to communicate compatibility information related to the following stakeholders: + +* **Application maintainers**: engineers that define an ElasticGraph schema, maintain project configuration, and perform upgrades. +* **Data publishers**: systems that publish data into an ElasticGraph application for ingestion by an ElasticGraph indexer. +* **GraphQL clients**: clients of the GraphQL API of an ElasticGraph application. + +We use the following versioning scheme: + +* Version numbers are in a `0.MAJOR.MINOR.PATCH` format. (The `0.` prefix is there in order to reserve `1.0.0` and all later versions + for after ElasticGraph has been open-sourced). +* Increments to the PATCH version indicate that the new release contains no backwards incompatibilities for any stakeholders. + It may contain bug fixes, new features, internal refactorings, and dependency upgrades, among other things. You can expect that + PATCH level upgrades are always safe--just update the version in your bundle, generate new schema artifacts, and you should be done. +* Increments to the MINOR version indicate that the new release contains some backwards incompatibilities that may impact the + **application maintainers** of some ElasticGraph applications. MINOR releases may include renames to configuration settings, + changes to the schema definition API, and new schema definition requirements, among other things. You can expect that MINOR + level upgrades can usually be done in 30 minutes or less (usually in a single commit!), with release notes and clear errors + from ElasticGraph command line tasks providing guidance on how to upgrade. +* Increments to the MAJOR version indicate that the new release contains some backwards incompatibilities that may impact the + **data publishers** or **GraphQL clients** of some ElasticGraph applications. MAJOR releases may include changes to the GraphQL + schema that require careful migration of **GraphQL clients** or changes to how indexing is done that require a dataset to be + re-indexed from scratch (e.g. by having **data publishers** republish their data into an ElasticGraph indexer running the new + version). You can expect that the release notes will include detailed instructions on how to perform a MAJOR version upgrade. + +Deprecation warnings may be included at any of these levels--for example, a PATCH release may contain a deprecation warning +for a breaking change that may impact **application maintainers** in an upcoming MINOR release, and a MINOR release may +contain deprecation warnings for breaking changes that may impact **data publishers** or **GraphQL clients** in an upcoming +MAJOR release. + +Each version level is cumulative over the prior levels. That is, a MINOR release may include PATCH-level changes in addition +to backwards incompatibilities that may impact **application maintainers**. A MAJOR release may include PATCH-level or +MINOR-level changes in addition to backwards incompatibilities that may impact **data publishers** or **GraphQL clients**. + +Note that this policy was first adopted in the `v0.15.1.0` release. All prior releases aimed (with some occasional mistakes!) +to follow SemVer with a `0.MAJOR.MINOR.PATCH` versioning scheme. + +Note that _all_ gems in this repository share the same version number. Every time we cut a release, we increment the version +for _all_ gems and release _all_ gems, even if a gem has had no changes since the last release. This is simpler to work with +than the alternatives. diff --git a/spec_support/lib/elastic_graph/spec_support/builds_admin.rb b/spec_support/lib/elastic_graph/spec_support/builds_admin.rb new file mode 100644 index 00000000..d01513b8 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/builds_admin.rb @@ -0,0 +1,26 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin" +require "elastic_graph/spec_support/builds_datastore_core" + +module ElasticGraph + module BuildsAdmin + include BuildsDatastoreCore + extend self + extend CommonSpecHelpers + + def build_admin(datastore_core: nil, **options, &customize_datastore_config) + Admin.new(datastore_core: datastore_core || build_datastore_core(for_context: :admin, **options, &customize_datastore_config)) + end + end + + RSpec.configure do |c| + c.include BuildsAdmin, :builds_admin + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/builds_datastore_core.rb b/spec_support/lib/elastic_graph/spec_support/builds_datastore_core.rb new file mode 100644 index 00000000..a846a8c0 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/builds_datastore_core.rb @@ -0,0 +1,138 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core" +require "elastic_graph/datastore_core/config" +require "elastic_graph/schema_artifacts/from_disk" +require "elastic_graph/spec_support/graphql_profiling_logger_decorator" +require "elastic_graph/spec_support/stub_datastore_client" +require "stringio" + +module ElasticGraph + module BuildsDatastoreCore + def build_datastore_core( + for_context:, + client_customization_block: nil, + clients_by_name: nil, + config: nil, + logger: nil, + schema_definition: nil, + schema_element_name_form: :snake_case, + schema_element_name_overrides: {}, + derived_type_name_formats: {}, + enum_value_overrides_by_type: {}, + index_definitions: nil, + clusters: nil, + schema_artifacts_directory: nil, + schema_artifacts: nil, + datastore_backend: nil, + **config_overrides, + &customize_config + ) + config ||= begin + yaml_config = parsed_test_settings_yaml + if datastore_backend + clusters_with_overrides = + yaml_config.fetch("datastore").fetch("clusters").transform_values do |cluster_config| + cluster_config.merge("backend" => datastore_backend.to_s) + end + + yaml_config = yaml_config.merge( + "datastore" => yaml_config.fetch("datastore").merge( + "clusters" => clusters_with_overrides + ) + ) + end + + DatastoreCore::Config.from_parsed_yaml(yaml_config).with(**config_overrides) + end + + schema_artifacts ||= + if schema_definition || schema_element_name_form != :snake_case || schema_element_name_overrides.any? || derived_type_name_formats.any? + generate_schema_artifacts( + schema_element_name_form: schema_element_name_form, + schema_element_name_overrides: schema_element_name_overrides, + derived_type_name_formats: derived_type_name_formats, + enum_value_overrides_by_type: enum_value_overrides_by_type, + &schema_definition + ) + elsif schema_artifacts_directory + # Deal with the relative nature of paths in config, ensuring we can run the specs while being + # in the repo root and also while being in a gem directory. + SchemaArtifacts::FromDisk.new(schema_artifacts_directory.sub("config", "#{CommonSpecHelpers::REPO_ROOT}/config"), for_context) + else + stock_schema_artifacts(for_context: for_context) + end + + if clients_by_name.nil? && respond_to?(:stubbed_datastore_client) + client = datastore_client # a memoized instance of stubbed_datastore_client + clients_by_name = {"main" => client, "other1" => client, "other2" => client, "other3" => client} + end + + # We want the datastore to be as fast as possible in tests, and don't care about data durability. + # To that end, we configure some settings here that attempt to optimize speed but sacrifice + # durability (the data shouldn't hit disk, for example). + optimal_test_setting_overrides = { + "translog.durability" => "async", + "translog.sync_interval" => "999999d", # effectively never + "translog.flush_threshold_size" => "8gb", # effectively never + "number_of_replicas" => 0 + } + + # We require an index definition for every index, so here we can provide one for *any* + # index created by a test by using a hash with a default proc, which lazily provides + # default configuration on demand. Individual tests can still provide their own index + # definitions as desired. + if index_definitions + config = config.with(index_definitions: index_definitions) + else + original_index_defs = config.index_definitions + + config = config.with(index_definitions: Hash.new do |hash, index_def_name| + hash[index_def_name] = config_index_def_of( + query_cluster: "main", + index_into_clusters: ["main"], + ignore_routing_values: [], + setting_overrides: optimal_test_setting_overrides + ) + end) + + # Merge the optimal test settings into our original definitions. + original_index_defs.each do |index_name, index_config| + config.index_definitions[index_name] = index_config.with( + setting_overrides: optimal_test_setting_overrides, + setting_overrides_by_timestamp: index_config.setting_overrides_by_timestamp.transform_values do |overrides| + optimal_test_setting_overrides.merge(overrides) + end, + custom_timestamp_ranges: index_config.custom_timestamp_ranges.map do |range| + range.with(setting_overrides: optimal_test_setting_overrides.merge(range.setting_overrides)) + end + ) + end + end + + config = config.with(clusters: clusters) if clusters + config = customize_config.call(config) if customize_config + + # If clients_by_name is specified, remove any entries from the config that *aren't* present. + config = config.with(clusters: config.clusters.select { |it| clients_by_name.key?(it) }) if clients_by_name + + DatastoreCore.new( + schema_artifacts: schema_artifacts, + config: config, + logger: GraphQLProfilingLoggerDecorator.maybe_wrap(logger || Logger.new(StringIO.new)), + client_customization_block: client_customization_block, + clients_by_name: clients_by_name + ) + end + end + + RSpec.configure do |c| + c.include BuildsDatastoreCore, :builds_datastore_core + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/builds_graphql.rb b/spec_support/lib/elastic_graph/spec_support/builds_graphql.rb new file mode 100644 index 00000000..540296fb --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/builds_graphql.rb @@ -0,0 +1,57 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/builds_datastore_core" +require "elastic_graph/graphql" +require "elastic_graph/graphql/config" + +module ElasticGraph + module BuildsGraphQL + include BuildsDatastoreCore + + def build_graphql( + extension_modules: [], + extension_settings: {}, + slow_query_latency_warning_threshold_in_ms: 30000, + max_page_size: 500, + default_page_size: 50, + datastore_core: nil, + graphql_adapter: nil, + monotonic_clock: nil, + clock: ::Time, + datastore_search_router: nil, + filter_interpreter: nil, + sub_aggregation_grouping_adapter: nil, + client_resolver: nil, + **datastore_core_options, + &customize_datastore_config + ) + GraphQL.new( + datastore_core: datastore_core || build_datastore_core(for_context: :graphql, **datastore_core_options, &customize_datastore_config), + config: GraphQL::Config.new( + max_page_size: max_page_size, + default_page_size: default_page_size, + slow_query_latency_warning_threshold_in_ms: slow_query_latency_warning_threshold_in_ms, + client_resolver: client_resolver || GraphQL::Client::DefaultResolver.new({}), + extension_modules: extension_modules, + extension_settings: extension_settings + ), + graphql_adapter: graphql_adapter, + datastore_search_router: datastore_search_router, + filter_interpreter: filter_interpreter, + sub_aggregation_grouping_adapter: sub_aggregation_grouping_adapter, + monotonic_clock: monotonic_clock, + clock: clock + ) + end + end + + RSpec.configure do |c| + c.include BuildsGraphQL, :builds_graphql + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/builds_indexer.rb b/spec_support/lib/elastic_graph/spec_support/builds_indexer.rb new file mode 100644 index 00000000..ea0a8408 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/builds_indexer.rb @@ -0,0 +1,43 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/builds_datastore_core" +require "elastic_graph/indexer" +require "elastic_graph/indexer/config" + +module ElasticGraph + module BuildsIndexer + include BuildsDatastoreCore + + def build_indexer( + datastore_core: nil, + latency_slo_thresholds_by_timestamp_in_ms: {}, + skip_derived_indexing_type_updates: {}, + datastore_router: nil, + clock: nil, + monotonic_clock: nil, + **datastore_core_options, + &customize_datastore_config + ) + Indexer.new( + datastore_core: datastore_core || build_datastore_core(for_context: :indexer, **datastore_core_options, &customize_datastore_config), + config: Indexer::Config.new( + latency_slo_thresholds_by_timestamp_in_ms: latency_slo_thresholds_by_timestamp_in_ms, + skip_derived_indexing_type_updates: skip_derived_indexing_type_updates.transform_values(&:to_set) + ), + datastore_router: datastore_router, + clock: clock, + monotonic_clock: monotonic_clock + ) + end + end + + RSpec.configure do |c| + c.include BuildsIndexer, :builds_indexer + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/cluster_configuration_manager.rb b/spec_support/lib/elastic_graph/spec_support/cluster_configuration_manager.rb new file mode 100644 index 00000000..a584cf39 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/cluster_configuration_manager.rb @@ -0,0 +1,167 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/admin" +require "elastic_graph/spec_support/builds_admin" +require "elastic_graph/support/hash_util" +require "yaml" + +module ElasticGraph + # Helper class to manage the datastore index configuration for our tests. + # Re-creating the indices, while not a super slow operation, adds 1-2 seconds to + # a test run, and it's something we generally want to avoid doing on each test run + # unless something has actually changed. Our approach here is to store a state + # file in a git-ignored directory. Our `boot_prep_for_tests` rake task (called + # when booting a datastore) deletes the file each time a datastore is booted. + # If the file already exists and matches the desired index state, we can avoid + # recreating the indices. + # + # The assumption here is that the state file will not get out of sync with the state + # of the indices in the datastore. As long as the engineer isn't manually changing + # the index configuration (which we don't support), this should work. + class ClusterConfigurationManager + include CommonSpecHelpers + include BuildsAdmin + attr_reader :admin + attr_accessor :state_file_name + + def initialize(version:, datastore_backend:, admin: nil, state_file_name: "elasticgraph_configured_indices.yaml") + @version = version + @admin = admin || build_admin(datastore_backend: datastore_backend) + self.state_file_name = state_file_name + + # Also make our old datastore scripts available to call from our tests for backwards-compatibility testing. + # We also need to add the `__sourceVersions` field back that some tests rely on but which we don't want + # generated in our `datastore_config.yaml` anymore. + # + # TODO: Drop this when we no longer need to maintain backwards-compatibility. + # standard:disable Lint/NestedMethodDefinition + def (@admin.schema_artifacts).datastore_scripts + super.merge(::YAML.safe_load_file(::File.join(__dir__, "old_datastore_scripts.yaml"))) + end + + def (@admin.schema_artifacts).indices + datastore_config = super + + overrides = datastore_config.transform_values do |index_config| + { + "mappings" => { + "properties" => { + "__sourceVersions" => { + "type" => "object", + "dynamic" => "false" + } + } + } + } + end + + Support::HashUtil.deep_merge(datastore_config, overrides) + end + + def (@admin.schema_artifacts).index_templates + datastore_config = super + + overrides = datastore_config.transform_values do |index_config| + { + "template" => { + "mappings" => { + "properties" => { + "__sourceVersions" => { + "type" => "object", + "dynamic" => "false" + } + } + } + } + } + end + + Support::HashUtil.deep_merge(datastore_config, overrides) + end + # standard:enable Lint/NestedMethodDefinition + end + + def manage_cluster + without_vcr do + admin.datastore_core.clients_by_name.values.each do |client| + client.delete_indices("unique_index_*") + client.delete_index_template("unique_index_*") + end + end + + # :nocov: -- to save time, avoids executing when the indices are already configured correctly + if !File.exist?(state_file_path) || current_cluster_state != File.read(state_file_path) + recreate_index_configuration + ::FileUtils.mkdir_p(File.dirname(state_file_path)) + File.write(state_file_path, current_cluster_state) + end + # :nocov: + + @version + current_cluster_state # return the current state + end + + private + + # :nocov: -- to save time, avoids executing when the indices are already configured correctly + def recreate_index_configuration + without_vcr do + start = ::Time.now + + notify_recreating_cluster_configuration + + admin.datastore_core.clients_by_name.values.each do |client| + index_definitions.each { |index_def| index_def.delete_from_datastore(client) } + end + + admin.cluster_configurator.configure_cluster(StringIO.new) + + notify_recreated_cluster_configuration(::Time.now - start) + end + end + + def notify_recreating_cluster_configuration + print "\nRecreating cluster configuration (for index definitions: #{index_definitions.map(&:name)})..." + end + + def notify_recreated_cluster_configuration(duration) + puts "done in #{RSpec::Core::Formatters::Helpers.format_duration(duration)}." + end + # :nocov: + + STATE_FILE_DIR = "tmp/datastore-state-files" + + def state_file_path + @state_file_path ||= ::File.join(CommonSpecHelpers::REPO_ROOT, STATE_FILE_DIR, state_file_name) + end + + def current_cluster_state + @current_cluster_state ||= YAML.dump({ + "datastore_scripts" => datastore_scripts, + "indices_by_name" => admin.schema_artifacts.indices.merge(admin.schema_artifacts.index_templates) + }) + end + + # :nocov: -- to save time, avoids executing when the indices are already configured correctly. + def index_definitions + @index_definitions ||= admin.datastore_core.index_definitions_by_name.values + end + # :nocov: + + def datastore_scripts + @datastore_scripts ||= admin.schema_artifacts.datastore_scripts + end + + # :nocov: -- to save time, avoids executing when the indices are already configured correctly + def without_vcr + return yield unless defined?(::VCR) # since we support running w/o VCR. + VCR.turned_off { yield } + end + # :nocov: + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/datastore_client_shared_examples.rb b/spec_support/lib/elastic_graph/spec_support/datastore_client_shared_examples.rb new file mode 100644 index 00000000..e98508b1 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/datastore_client_shared_examples.rb @@ -0,0 +1,326 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "httpx/adapters/faraday" +require "json" + +module ElasticGraph + RSpec.shared_examples_for "a datastore client", :no_vcr do + it "respects the `#{TIMEOUT_MS_HEADER}` header" do + client = build_client({get_msearch: "GET msearch"}) + + expect { + client.msearch(body: [], headers: {TIMEOUT_MS_HEADER => "1"}) + }.to raise_error Errors::RequestExceededDeadlineError, "Datastore request exceeded timeout of 1 ms." + end + + it "converts bad request responses into `Errors::BadDatastoreRequest`" do + client = build_client({get_cluster_health: :bad_request}) + + expect { + client.get_cluster_health + }.to raise_error Errors::BadDatastoreRequest + end + + describe "cluster APIs" do + it "supports `get_cluster_health`" do + client = build_client({get_cluster_health: {status: "healthy"}}) + + expect(client.get_cluster_health).to eq({"status" => "healthy"}) + end + + it "supports `get_node_os_stats`" do + client = build_client({get_node_os_stats: "Node stats"}) + + expect(client.get_node_os_stats).to eq "Node stats" + end + + it "supports `get_flat_cluster_settings`" do + client = build_client({get_flat_cluster_settings: "Flat cluster settings!"}) + + expect(client.get_flat_cluster_settings).to eq "Flat cluster settings!" + end + + it "supports `put_persistent_cluster_settings`" do + client = build_client({put_persistent_cluster_settings: :echo_body}) + + expect(client.put_persistent_cluster_settings({foo: 1})).to eq({"persistent" => {"foo" => 1}}) + end + end + + describe "script APIs" do + it "supports `get_script`" do + client = build_client({get_script_123: {script: "hello world"}}) + + expect(client.get_script(id: "123")).to eq({"script" => "hello world"}) + end + + it "returns `nil` from `get_script` when it is not found" do + client = build_client({get_script_123: :not_found}) + + expect(client.get_script(id: "123")).to eq(nil) + end + + it "supports `put_script`" do + client = build_client({put_script_123: "ok"}) + + expect(client.put_script(id: "123", body: "hello world", context: "update")).to eq("ok") + end + + it "supports `delete_script`" do + client = build_client({delete_script_123: "ok"}) + + expect(client.delete_script(id: "123")).to eq("ok") + end + + it "ignores 404s from `delete_script` since that means the script has already been deleted" do + client = build_client({delete_script_123: :not_found}) + + expect(client.delete_script(id: "123")).to eq nil + end + end + + describe "index template APIs" do + it "supports `get_index_template`" do + client = build_client({get_index_template_my_template: {"index_templates" => [{ + "name" => "my_template", + "index_template" => { + "template" => { + "mapping" => "the_mapping", + "settings" => {"foo" => "bar"} + }, + "index_patterns" => ["foo*"] + } + }]}}) + + expect(client.get_index_template("my_template")).to eq({ + "index_patterns" => ["foo*"], + "template" => { + "settings" => {"foo" => "bar"}, + "mapping" => "the_mapping" + } + }) + end + + it "returns `{}` from `get_index_template` when it is not found" do + client = build_client({get_index_template_my_template: :not_found}) + + expect(client.get_index_template("my_template")).to eq({}) + end + + it "supports `put_index_template`" do + client = build_client({put_index_template_my_template: "ok"}) + + expect(client.put_index_template(name: "my_template", body: {"template" => "config"})).to eq("ok") + end + + it "supports `delete_index_template`" do + client = build_client({delete_index_template_my_template: "ok"}) + + expect(client.delete_index_template("my_template")).to eq("ok") + end + + it "ignores 404s when deleting a template since that means its already in the desired state" do + client = build_client({delete_index_template_my_template: :not_found}) + + expect(client.delete_index_template("my_template")).to eq({}) + end + end + + describe "index APIs" do + it "supports `get_index`" do + client = build_client({get_index_my_index: {"my_index" => {"settings" => "config"}}}) + + expect(client.get_index("my_index")).to eq({"settings" => "config"}) + end + + it "supports `list_indices_matching`" do + client = build_client({list_indices_matching_foo: [{"index" => "foo1"}, {"index" => "foo2"}]}) + + expect(client.list_indices_matching("foo*")).to eq(["foo1", "foo2"]) + end + + it "supports `create_index`" do + client = build_client({create_index_my_index: "ok"}) + + expect(client.create_index(index: "my_index", body: {"settings" => "config"})).to eq("ok") + end + + it "supports `put_index_mapping`" do + client = build_client({put_index_mapping_my_index: "ok"}) + + expect(client.put_index_mapping(index: "my_index", body: {"mapping" => "config"})).to eq("ok") + end + + it "supports `put_index_settings`" do + client = build_client({put_index_settings_my_index: "ok"}) + + expect(client.put_index_settings(index: "my_index", body: {"settings" => "config"})).to eq("ok") + end + + it "supports `delete_indices`" do + client = build_client({delete_indices_ind1_ind2: "ok"}) + + expect(client.delete_indices("ind1", "ind2")).to eq("ok") + end + end + + describe "document APIs" do + it "supports `msearch`, using GET instead of POST to support simple permissioning that only allows the GraphQL endpoint to use HTTP GETs" do + client = build_client({get_msearch: "GET msearch"}) + + expect(client.msearch(body: [], headers: {})).to eq "GET msearch" + end + + it "supports `bulk`" do + client = build_client({post_bulk: "POST bulk"}) + + expect(client.bulk(body: [])).to eq "POST bulk" + end + + it "supports `delete_all_documents`" do + client = build_client({delete_all_documents: "ok"}) + + expect(client.delete_all_documents).to eq "ok" + end + + it "allows an index expression to be provided to `delete_all_documents` in order to limit the deletion to documents in a specific scope" do + client = build_client({delete_test_env_7_documents: "ok"}) + + expect(client.delete_all_documents(index: "test_env_7_*")).to eq "ok" + end + end + + describe "the `faraday_adapter` option" do + it "is not required" do + expect { build_unstubbed_client }.not_to raise_error + end + + it "allows it to be set to a valid, available adapter" do + expect { build_unstubbed_client(faraday_adapter: :httpx) }.not_to raise_error + end + + it "immediately raises an error if set to an unsupported value" do + expect { + build_unstubbed_client(faraday_adapter: :unsupported_value) + }.to raise_error a_string_including(":unsupported_value is not registered on Faraday::Adapter") + end + + it "immediately raises an error set to an supported but unavailable value (e.g. due to a missing gem)" do + expect { + build_unstubbed_client(faraday_adapter: :patron) + }.to raise_error a_string_including(":patron is not registered on Faraday::Adapter") + end + end + + describe "retry behavior" do + it "retries on a 500 (Internal Server Error) response since it's transient" do + expect_retries_for(:internal_server_error, "500") + end + + it "retries on a 500 (Bad Gateway) response since it's transient" do + expect_retries_for(:bad_gateway, "502") + end + + it "retries on a 503 (Service Unavailable) response since it's transient" do + expect_retries_for(:service_unavailable, "503") + end + + it "does not retry on a 504 (Gateway Timeout) response since the datastore may be overloaded and retrying could make it worse" do + client = build_client_with_cluster_health_responses([:gateway_timeout, "ok"], retry_on_failure: 4) + + expect { client.get_cluster_health }.to raise_error a_string_including("504") + end + + def expect_retries_for(response, expected_error) + responses = ([response] * 5) + ["ok"] + client = build_client_with_cluster_health_responses(responses, retry_on_failure: 4) + expect { client.get_cluster_health }.to raise_error a_string_including(expected_error) + + client = build_client_with_cluster_health_responses(responses, retry_on_failure: 5) + expect(client.get_cluster_health).to eq "ok" + end + + def build_client_with_cluster_health_responses(cluster_health_responses, retry_on_failure:) + build_client({get_cluster_health: -> { cluster_health_responses.shift }}, retry_on_failure: retry_on_failure) + end + end + + describe "logging", :capture_logs, :expect_warning_logging do + it "logs full traffic details when provided with a `logger`" do + build_client({put_persistent_cluster_settings: "ok"}, logger: logger).put_persistent_cluster_settings({ + "indices.recovery.max_bytes_per_sec" => "50mb" + }) + + # Verify that we include both basic log details and trace details. + expect(logged_output.lines).to include( + a_string_including("INFO -- : PUT http://ignoredhost.com:9200/_cluster/settings"), + a_string_including("INFO -- : curl -X PUT", "/_cluster/settings"), + a_string_including("indices.recovery.max_bytes_per_sec", "50mb") + ) + end + + it "does not log traffic when no `logger` is provided" do + build_client({put_persistent_cluster_settings: "ok"}, logger: nil).put_persistent_cluster_settings({ + "indices.recovery.max_bytes_per_sec" => "50mb" + }) + + expect(logged_output).to be_empty + end + end + + def build_client(stubs_by_name, **options) + described_class.new( + "some-cluster", + faraday_adapter: :test, + url: "http://ignoredhost.com", + **options + ) do |faraday| + faraday.adapter :test do |stub| + define_stubs(stub, stubs_by_name) + end + end + end + + def build_unstubbed_client(**options) + described_class.new("some-cluster", url: "http://ignoredhost.com", **options) + end + + def response_for(body, env) + status, headers, body = + case body + in :echo_body + [200, {"Content-Type" => "application/json"}, env.body] + in :internal_server_error + [500, {"Content-Type" => "application/json"}, "{}"] + in :bad_gateway + [502, {"Content-Type" => "application/json"}, "{}"] + in :service_unavailable + [503, {"Content-Type" => "application/json"}, "{}"] + in :gateway_timeout + [504, {"Content-Type" => "application/json"}, "{}"] + in :not_found + [404, {"Content-Type" => "application/json"}, "{}"] + in :bad_request + [400, {"Content-Type" => "application/json"}, "{}"] + in ::String + [200, {"Content-Type" => "text/plain"}, body] + in ::Proc + response_for(body.call, env) + else + [200, {"Content-Type" => "application/json"}, ::JSON.generate(body)] + end + + # Here we rewrap the body in a new string, because the datastore client attempts to mutate + # the encoding of the body, and when we run on CI with "--enable-frozen-string-literal" we + # get errors if we haven't wrapped it in a new string instance. + [status, headers, ::String.new(body)] + end + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/enable_simplecov.rb b/spec_support/lib/elastic_graph/spec_support/enable_simplecov.rb new file mode 100644 index 00000000..1deda819 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/enable_simplecov.rb @@ -0,0 +1,123 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file exists to enable simplecov for any of the ElasticGraph gems. To use it, set a `COVERAGE` env var: +# +# COVERAGE=1 be rspec path/to/gem/spec +require "simplecov" +require "simplecov-console" + +module ElasticGraph + class FailedCoverageRequirementFormatter + def format(result) + return if result.missed_lines == 0 && result.missed_branches == 0 + + puts <<~EOS + + #{"=" * 100} + Your test run had #{result.missed_lines} lines and #{result.missed_branches} code branches that were not covered by the executed tests. + We do not have a goal of 100% test coverage in ElasticGraph; however, we do have the goal of having + all uncovered code explicitly labeled as such with `# :nocov:` so that it is easy to tell at a + glance what code is uncovered by tests. And we do want a high level of test coverage; Ruby's dynamic + nature means that misspelled variables, method names, etc can usually only be detected at run time, meaning + that every uncovered line of code is a line that our CI build may not be able to detect a breakage in. + + See the table above to see detailed coverage information. For each bit of uncovered code, do one of the following: + + 1) Delete it. If the code is "dead code" (such as a private method that no longer has any callers), + just delete it! + + 2) Add test coverage. (This might require refactoring the code to make it more testable). + + 3) Surround the code with `# :nocov:` comments (on the lines before and after) to mark it as a known + uncovered bit of code. This should only be done for code that you have determined would cost more + to test than the value we would get from the tests. For example, this is sometimes the case for + code that only ever runs locally (e.g. in a rake task) that interacts heavily with the environment. + Note: if you add a `# :nocov:` comment, please leave an explanation for why the code is not being + covered. + #{"=" * 100} + + EOS + end + end + + if defined?(::Flatware) + module SimpleCovPatches + attr_accessor :flatware_main_process_pid + + def wait_for_other_processes + # There's a race condition with SimpleCov and a parallel runner like flatware: + # the final worker process often hasn't written its results when we get here, and + # we need to sleep a bit to give it time to finish. + sleep 1 if flatware_main_process_pid == ::Process.pid + super + end + end + ::SimpleCov.singleton_class.prepend SimpleCovPatches + + ::Flatware.configure do |conf| + # Record the pid of the main process (the one that spawns the workers, and that SimpleCov prints results from). + conf.before_fork { ::SimpleCov.flatware_main_process_pid = ::Process.pid } + end + end +end + +# Identify if we are running a single gem's specs; if so we will only check coverage of that one gem. +spec_files_to_run = RSpec.configuration.files_to_run +gems_being_tested_dirs = spec_files_to_run + .filter_map { |f| Pathname(f).ascend.find { |p| p.glob("*.gemspec").any? } } + .uniq + +gem_dir = gems_being_tested_dirs.first if gems_being_tested_dirs.one? +repo_root = File.expand_path("../../../..", __dir__) +tmp_coverage_dir = "#{repo_root}/tmp/coverage" + +# Don't allow results from a prior run to "contaminate" the current run. +FileUtils.rm_rf(tmp_coverage_dir) + +SimpleCov.enable_for_subprocesses(true) + +SimpleCov.start do + if gems_being_tested_dirs.one? + gem_dir = gems_being_tested_dirs.first + root gem_dir.to_s + command_name gem_dir.basename.to_s + else + root repo_root + command_name "elasticgraph" + end + + coverage_dir tmp_coverage_dir + + add_filter "/bundle" + + # When we use `script/run_most_specs` we avoid running the `elasticgraph-local` specs, but some of the + # elasticgraph-local code gets loaded and used as a dependency. We don't want to consider its coverage + # status if we're not running it's test suite. + add_filter "/elasticgraph-local/" unless spec_files_to_run.any? { |f| f.include?("/elasticgraph-local/") } + + # This version file is loaded from our gemspecs, which can get loaded by bundler before we get here. + # SimpleCov is only able to track coverage of files loaded after it starts, so we need to filter them out if + # their constant is already defined. They don't contain any branching statements or anything so it's ok to + # ignore them here. + add_filter "lib/elastic_graph/version.rb" if defined?(::ElasticGraph::VERSION) + + formatter SimpleCov::Formatter::MultiFormatter.new([ + SimpleCov::Formatter::HTMLFormatter, + SimpleCov::Formatter::Console, + ElasticGraph::FailedCoverageRequirementFormatter + ]) + + gems_being_tested_globs = gems_being_tested_dirs.flat_map { |dir| [dir / "lib/**/*.rb", dir / "spec/**/*.rb"] } + track_files "{#{gems_being_tested_globs.join(",")}}" + + enable_coverage :branch + minimum_coverage line: 100, branch: 100 + + merge_timeout 1800 # 30 minutes. CI jobs can take 15-20 minutes. +end diff --git a/spec_support/lib/elastic_graph/spec_support/factories.rb b/spec_support/lib/elastic_graph/spec_support/factories.rb new file mode 100644 index 00000000..ff8f51e0 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/factories.rb @@ -0,0 +1,40 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/indexer/test_support/converters" +require "elastic_graph/spec_support/factories/teams" +require "elastic_graph/spec_support/factories/widgets" +require "elastic_graph/support/hash_util" + +RSpec.shared_context "factories" do + # we use `prepend_before` so this runs before other `before` hooks. + prepend_before do |ex| + # Make faker usage deterministic, based on the `full_description` string of the + # RSpec example. `String#sum` is being used purely for speed, as we do not + # need a particularly even distribution in random number seeds. + Faker::Config.random = Random.new(ex.full_description.sum) + end + + def build(type, *args) + # Allow callers to do `build(:part, ...)` to randomly (but deterministically) get + # either an `:electrical_part` or a `:mechanical_part`. + type = Faker::Base.sample(%i[electrical_part mechanical_part]) if type == :part + + super(type, *args) + end + + def build_upsert_event(type, **attributes) + record = ElasticGraph::Support::HashUtil.stringify_keys(build(type, **attributes)) + ElasticGraph::Indexer::TestSupport::Converters.upsert_event_for(record) + end +end + +RSpec.configure do |config| + config.include_context "factories", :factories + config.include FactoryBot::Syntax::Methods, :factories +end diff --git a/spec_support/lib/elastic_graph/spec_support/factories/shared.rb b/spec_support/lib/elastic_graph/spec_support/factories/shared.rb new file mode 100644 index 00000000..a83cf5f0 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/factories/shared.rb @@ -0,0 +1,68 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# activesupport 7.2.0 was recently (2024-08-09) released and broke factory_bot. +# To work around it we need to require it here, as per: +# https://github.com/thoughtbot/factory_bot/issues/1685 +# +# TODO: once factory_bot has had a new release with a fix, remove this require (we don't use activesupport). +require "active_support" + +require "factory_bot" +require "faker" + +# A counter that we increment for the `__version` value on each new factory-generated record. +global_version_counter = 0 + +module ElasticGraphSpecSupport + CURRENCIES_BY_CODE = { + "USD" => {symbol: "$", unit: "dollars", name: "United States Dollar", primary_continent: "North America", introduced_on: "1792-04-02"}, + "CAD" => {symbol: "$", unit: "dollars", name: "Canadian Dollar", primary_continent: "North America", introduced_on: "1868-01-01"}, + "GBP" => {symbol: "£", unit: "pounds", name: "British Pound Sterling", primary_continent: "Europe", introduced_on: "0800-01-01"}, + "JPY" => {symbol: "¥", unit: "yen", name: "Japanese Yen", primary_continent: "Asia", introduced_on: "1871-01-01"} + } +end + +FactoryBot.define do + factory :hash_base, class: Hash do + initialize_with do + attributes.except(*__exclude_fields) + end + + transient do + __exclude_fields { [] } + end + end + + factory :indexed_type, parent: :hash_base do + # When building new factory records, we normally expect each new record to automatically "win" + # over previously generated records, so we use a process-level global counter here that we increment + # for each factory-generated record. + # + # For tests that really care about the version, they override it to control this more tightly. + __version { global_version_counter += 1 } + __typename { raise NotImplementedError, "You must supply __typename." } + __json_schema_version { 1 } + id { Faker::Alphanumeric.alpha(number: 20) } + end + + factory :geo_location, parent: :hash_base do + __typename { "GeoLocation" } + # latitude is -90.0 to +90.0 + latitude { Faker::Number.between(from: -90.0, to: 90.0) } + # longitude is -180.0 to +180.0 + longitude { Faker::Number.between(from: -180.0, to: 180.0) } + end + + currencies_by_code = ElasticGraphSpecSupport::CURRENCIES_BY_CODE + factory :money, parent: :hash_base do + __typename { "Money" } + currency { Faker::Base.sample(currencies_by_code.keys) } + amount_cents { Faker::Number.between(from: 100, to: 10000) } + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/factories/teams.rb b/spec_support/lib/elastic_graph/spec_support/factories/teams.rb new file mode 100644 index 00000000..89b7175c --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/factories/teams.rb @@ -0,0 +1,202 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "date" +require "time" +require "elastic_graph/spec_support/factories/shared" + +# Note: it is *essential* that all factories defined here generate records +# deterministically, in order for the request bodies to (and responses from) +# the dataastore to not change between VCR cassettes being recorded and replayed. + +# Note: in our JSON Schema, `__typename` is defined for ALL object types but is only _required_ on abstract object types +# (e.g. for type unions or interfaces). However, clients that publish using egpublisher always include it on every object +# type (because of how the code gen works). +# +# In addition, in the factories here we require it on all indexed types (the ones with `parent: :graph_base`) because +# `Indexer::TestSupport::Converters.upsert_event_for` relies on it in order to build the event envelope. That process strips the +# `__typename` from the `record`. +# +# We intentionally include `__typename` in many non-indexed object types here in order to simulate it being set on them +# (since that's what our publishers do) in order to exercise edge cases related to `__typename`. However, we haven't +# included it in all object types since some publishers could omit it. + +awards = ["MVP", "Rookie of the Year", "Gold Glove", "Cy Young", "Silver Slugger", "Humanitarian", "Biggest Jerk"] +leagues = %w[MLB NBA NFL NHL MLS NCAA OfNations] + +# A fixed date we can use instead of `Date.today` in our factories. +# As mentioned above, our factories are intended to be deterministic, and we need to avoid using `Date.today` (or `Time.now`) to +# ensure determinism. +recent_date = ::Date.new(2023, 11, 25) + +FactoryBot.define do + factory :team, parent: :indexed_type do + __typename { "Team" } + league { Faker::Base.sample(leagues) } + # Limit `formed_on` to a 5 year stretch so that we don't make too many indices in our test environment since `teams` uses a yearly rollover index. + formed_on { Faker::Date.between(from: recent_date - (5 * 365), to: recent_date - 365).iso8601 } + current_name { Faker::Team.name } + details { build :team_details } + stadium_location { build :geo_location } + country_code { Faker::Address.country_code } + + past_names do + Array.new(Faker::Number.between(from: 0, to: 3)) { Faker::Team.name } - [current_name] + end + + won_championships_at do + Array.new(Faker::Number.between(from: 0, to: 3)) do + Faker::Time.between(from: recent_date - 30, to: recent_date).utc.iso8601 + end + end + + forbes_valuations do + Array.new(Faker::Number.between(from: 1, to: 4)) do + Faker::Number.between(from: 10, to: 50000) * 10_000 + end + end + + forbes_valuation_moneys_nested { forbes_valuations.map { |v| build(:money, amount_cents: v, currency: "USD") } } + forbes_valuation_moneys_object { forbes_valuation_moneys_nested } + + current_players_nested { current_players } + current_players_object { current_players } + seasons_nested { seasons } + seasons_object { seasons } + + nested_fields do + { + current_players: current_players_nested, + forbes_valuation_moneys: forbes_valuation_moneys_nested, + seasons: seasons_nested + } + end + + nested_fields2 { nested_fields } + + transient do + sponsors { [] } + current_players do + Array.new(Faker::Number.between(from: 2, to: 8)) { build :player, sponsors: sponsors } + end + + seasons do + Array.new(Faker::Number.between(from: 2, to: 5)) { build :team_season }.uniq { |h| h.fetch(:year) } + end + end + end + + factory :team_details, parent: :hash_base do + uniform_colors do + Array.new(Faker::Base.sample([2, 3])) { Faker::Color.color_name }.uniq + end + + count { Faker::Number.between(from: 0, to: 100) } + end + + factory :player, parent: :hash_base do + __typename { "Player" } + name { Faker::Name.name } + + nicknames do + Array.new(Faker::Number.between(from: 0, to: 3)) { Faker::FunnyName.name } + end + + seasons_nested { seasons } + seasons_object { seasons } + + affiliations { build :affiliations, sponsors: sponsors } + + transient do + sponsors { [] } + seasons do + Array.new(Faker::Number.between(from: 0, to: 3)) { build :player_season }.uniq { |h| h.fetch(:year) } + end + end + end + + factory :team_record, parent: :hash_base do + wins { Faker::Number.between(from: 0, to: 100) } + losses { Faker::Number.between(from: 0, to: 100) } + first_win_on { first_win_on_date.iso8601 } + last_win_on { (first_win_on_date + 120).iso8601 } + + transient do + first_win_on_date { Faker::Date.between(from: recent_date - 3650, to: recent_date - 200) } + end + end + + factory :team_season, parent: :hash_base do + __typename { "TeamSeason" } + year { Faker::Number.between(from: 1950, to: 2023) } + record { build :team_record } + + notes do + Array.new(Faker::Number.between(from: 1, to: 3)) { Faker::TvShows::MichaelScott.quote } + end + + count { Faker::Number.between(from: 0, to: 100) } + + players_nested { players } + players_object { players } + + started_at { Faker::Time.between(from: start_of_year, to: end_of_year).utc.iso8601 } + + won_games_at do + started_at_time = ::Time.iso8601(started_at) + Array.new(Faker::Number.between(from: 2, to: 5)) do + Faker::Time.between(from: started_at_time, to: end_of_year).utc.iso8601 + end + end + + transient do + players do + Array.new(Faker::Number.between(from: 2, to: 8)) { build :player } + end + + start_of_year { ::Time.iso8601("#{year}-01-01T00:00:00Z") } + end_of_year { ::Time.iso8601("#{year}-12-31T23:59:59.999Z") } + end + end + + factory :player_season, parent: :hash_base do + __typename { "PlayerSeason" } + year { Faker::Number.between(from: 1950, to: 2023) } + games_played { Faker::Number.between(from: 1, to: 162) } + + awards do + Array.new(Faker::Number.between(from: 0, to: 3)) { Faker::Base.sample(awards) }.uniq + end + end + + factory :affiliations, parent: :hash_base do + sponsorships_nested { sponsorships } + sponsorships_object { sponsorships } + transient do + sponsors { [] } + sponsorships do + sponsors.map { |sponsor| build(:sponsorship, sponsor: sponsor) } + end + end + end + + factory :sponsorship, parent: :hash_base do + __typename { "Sponsorship" } + annual_total { build :money, amount_cents: Faker::Number.between(from: 10, to: 50000) * 10_000, currency: "USD" } + sponsor_id { sponsor.fetch(:id) } + + transient do + sponsor { [] } + end + end + + factory :sponsor, parent: :indexed_type do + __typename { "Sponsor" } + name { Faker::Company.name } + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/factories/widgets.rb b/spec_support/lib/elastic_graph/spec_support/factories/widgets.rb new file mode 100644 index 00000000..95313b00 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/factories/widgets.rb @@ -0,0 +1,198 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "date" +require "elastic_graph/spec_support/factories/shared" + +# Note: it is *essential* that all factories defined here generate records +# deterministically, in order for the request bodies to (and responses from) +# the datastore to not change between VCR cassettes being recorded and replayed. + +# Note: in our JSON Schema, `__typename` is defined for ALL object types but is only _required_ on abstract object types +# (e.g. for type unions or interfaces). However, clients that publish using egpublisher always include it on every object +# type (because of how the code gen works). +# +# In addition, in the factories here we require it on all indexed types (the ones with `parent: :graph_base`) because +# `Indexer::TestSupport::Converters.upsert_event_for` relies on it in order to build the event envelope. That process strips the +# `__typename` from the `record`. +# +# We intentionally include `__typename` in many non-indexed object types here in order to simulate it being set on them +# (since that's what our publishers do) in order to exercise edge cases related to `__typename`. However, we haven't +# included it in all object types since some publishers could omit it. + +# A fixed date we can use instead of `Date.today` in our factories. +# As mentioned above, our factories are intended to be deterministic, and we need to avoid using `Date.today` (or `Time.now`) to +# ensure determinism. +recent_date = ::Date.new(2023, 11, 25) + +FactoryBot.define do + factory :widget_options, parent: :hash_base do + __typename { "WidgetOptions" } + size { Faker::Base.sample(["SMALL", "MEDIUM", "LARGE"]) } + the_size { size } + color { Faker::Base.sample(["RED", "GREEN", "BLUE"]) } + end + + factory :person, parent: :hash_base do + __typename { "Person" } + name { Faker::Name.name } + nationality { Faker::Nation.nationality } + end + + factory :company, parent: :hash_base do + __typename { "Company" } + name { Faker::Company.name } + stock_ticker { name[0..3].upcase } + end + + factory :position, parent: :hash_base do + # omitting `__typename` here intentionally; a test in elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/upsert_spec.rb + # fails if we include it because the test uses the entire `event["record"]` in an assertion for simplicity. + # + # __typename { "Position" } + + x { Faker::Number.between(from: -100, to: 100) } + y { Faker::Number.between(from: -100, to: 100) } + end + + factory :geo_shape, parent: :hash_base do + type { "Point" } + coordinates { ::Array.new(2) { Faker::Number.between(from: -100, to: 100) } } + end + + factory :address_timestamps, parent: :hash_base do + # omitting `__typename` here intentionally; a test in elasticgraph-graphql/spec/acceptance/datastore_spec.rb + # fails if we include it because the test uses the entire `event["record"]` in an assertion for simplicity. + # + # __typename { "AddressTimestamps" } + created_at { Faker::Time.between(from: recent_date - 30, to: recent_date).utc.iso8601 } + end + + currencies_by_code = ElasticGraphSpecSupport::CURRENCIES_BY_CODE + + factory :workspace_widget, parent: :hash_base do + id { Faker::Alphanumeric.alpha(number: 20) } + created_at { Faker::Time.between(from: recent_date - 30, to: recent_date).utc.iso8601 } + end + + factory :widget, parent: :indexed_type do + __typename { "Widget" } + workspace_id { Faker::Alphanumeric.alpha(number: 6) } + amount_cents { Faker::Number.between(from: 100, to: 10000) } + cost { build(:money, amount_cents: amount_cents, currency: cost_currency) } + cost_currency_unit { cost&.fetch(:currency)&.then { |code| currencies_by_code.dig(code, :unit) } } + cost_currency_name { cost&.fetch(:currency)&.then { |code| currencies_by_code.dig(code, :name) } } + cost_currency_primary_continent { cost&.fetch(:currency)&.then { |code| currencies_by_code.dig(code, :primary_continent) } } + cost_currency_introduced_on { cost&.fetch(:currency)&.then { |code| currencies_by_code.dig(code, :introduced_on) } } + cost_currency_symbol { cost&.fetch(:currency)&.then { |code| currencies_by_code.dig(code, :symbol) } } + name { Faker::Device.model_name } + name_text { name } + created_at { Faker::Time.between(from: recent_date - 30, to: recent_date).utc.iso8601 } + created_at_time_of_day { ::Time.iso8601(created_at).strftime("%H:%M:%S") } + created_on { ::Time.iso8601(created_at).to_date.iso8601 } + release_timestamps { Array.new(Faker::Number.between(from: 0, to: 4)) { Faker::Time.between(from: recent_date - 30, to: recent_date).utc.iso8601 } } + release_dates { release_timestamps.map { |ts| ::Time.iso8601(ts).to_date.iso8601 } } + options { build :widget_options } + the_options { options } + + component_ids do + components.map { |c| c.fetch(:id) } + end + + inventor { build Faker::Base.sample([:person, :company]) } + named_inventor { inventor } + weight_in_ng { Faker::Number.between(from: 2**51, to: (2**53) - 1) } + weight_in_ng_str { Faker::Number.between(from: 2**60, to: 2**61) } + tags { Array.new(Faker::Number.between(from: 0, to: 4)) { Faker::Alphanumeric.alpha(number: 6) } } + amounts { Array.new(Faker::Number.between(from: 0, to: 4)) { Faker::Number.between(from: 100, to: 10000) } } + fees { build_list(:money, Faker::Number.between(from: 0, to: 4)) } + metadata do + selection = Faker::Base.sample([:meta_number, :meta_id, :meta_names, :meta_json]) + send(selection) + end + + transient do + components { [] } + cost_currency { Faker::Base.sample(currencies_by_code.keys) } + meta_number { Faker::Number.between(from: 0, to: 999999999) } + meta_id { Faker::Alphanumeric.alpha(number: 8) } + meta_names { 3.times.map { Faker::Name.name } } + meta_json do + json = Faker::Json.shallow_json(width: 3, options: {key: "Name.first_name", value: "Name.last_name"}) + ::JSON.parse(json) + end + end + end + + factory :widget_workspace, parent: :indexed_type do + __typename { "WidgetWorkspace" } + widget { build(:workspace_widget) } + name { Faker::Job.field } + end + + factory :manufacturer, parent: :indexed_type do + __typename { "Manufacturer" } + name { Faker::Company.name } + created_at { Faker::Time.between(from: recent_date - 30, to: recent_date).utc.iso8601 } + end + + factory :address, parent: :indexed_type do + __typename { "Address" } + full_address { Faker::Address.full_address } + manufacturer_id { manufacturer&.fetch(:id) } + timestamps { build(:address_timestamps, created_at: created_at) } + geo_location { build(:geo_location) } + shapes { [build(:geo_shape)] } + + transient do + manufacturer { nil } + created_at { Faker::Time.between(from: recent_date - 30, to: recent_date).utc.iso8601 } + end + end + + factory :part_base, parent: :indexed_type do + name do + material = Faker::Base.sample(%w[oak pine iron gold steel copper silicon plastic wood maple]) + type = Faker::Base.sample(%w[gasket dowel rod screw clasp zipper snap button]) + "#{material} #{type} #{Faker::Number.between(from: 10000, to: 99999)}" + end + manufacturer_id { manufacturer&.fetch(:id) } + created_at { Faker::Time.between(from: recent_date - 30, to: recent_date).utc.iso8601 } + + transient do + manufacturer { nil } + end + + factory :electrical_part do + __typename { "ElectricalPart" } + voltage { Faker::Base.sample([110, 120, 220, 240]) } + end + + factory :mechanical_part do + __typename { "MechanicalPart" } + material { Faker::Base.sample(["ALLOY", "CARBON_FIBER"]) } + end + end + + factory :component, parent: :indexed_type do + __typename { "Component" } + name { Faker::ElectricalComponents.active } + created_at { Faker::Time.between(from: recent_date - 30, to: recent_date).utc.iso8601 } + position { build :position } + + part_ids do + parts.map { |part| part.fetch(:id) } + end + + tags { Array.new(Faker::Number.between(from: 0, to: 4)) { Faker::Alphanumeric.alpha(number: 6) } } + + transient do + parts { [] } + end + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/graphql_profiling_logger_decorator.rb b/spec_support/lib/elastic_graph/spec_support/graphql_profiling_logger_decorator.rb new file mode 100644 index 00000000..1548fd73 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/graphql_profiling_logger_decorator.rb @@ -0,0 +1,32 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "logger" +require "delegate" + +module ElasticGraph + class GraphQLProfilingLoggerDecorator < DelegateClass(::Logger) + # Profiling is an opt-in thing in our test suite; so here we wrap if profiling is being used. + def self.maybe_wrap(logger) + # :nocov: -- on any given test run, only one side of this conditional will be covered + return logger unless defined?(::ElasticGraphProfiler) + new(logger) + # :nocov: + end + + def info(message) + if message.is_a?(::Hash) && message["message_type"] == "ElasticGraphQueryExecutorQueryDuration" + ::ElasticGraphProfiler.record_raw("graphql_duration", message.fetch("duration_ms").to_f / 1000) + ::ElasticGraphProfiler.record_raw("graphql_datastore_duration", message.fetch("datastore_server_duration_ms").to_f / 1000) + ::ElasticGraphProfiler.record_raw("graphql_elasticgraph_overhead", message.fetch("elasticgraph_overhead_ms").to_f / 1000) + end + + super + end + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/have_readable_to_s_and_inspect_output.rb b/spec_support/lib/elastic_graph/spec_support/have_readable_to_s_and_inspect_output.rb new file mode 100644 index 00000000..bb4a4500 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/have_readable_to_s_and_inspect_output.rb @@ -0,0 +1,67 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +::RSpec::Matchers.define :have_readable_to_s_and_inspect_output do + max_length = 150 + + # :nocov: -- the logic in these blocks isn't covered on every test run (e.g. because we don't have failures each test run) + chain :including do |*inclusions| + @inclusions = inclusions + end + + chain :and_excluding do |*exclusions| + @exclusions = exclusions + end + + match do |object| + to_s = object.to_s + inspect = object.inspect + + @problems = [] + + @problems << "`inspect` and `to_s` did not have the same output" unless to_s == inspect + @problems << "the output is too long (#{to_s.length} vs target max of #{max_length})" if to_s.length > max_length + expected_start = "#<#{object.class.name}" + @problems << "Does not start with `#{expected_start}`" unless to_s.start_with?(expected_start) + @problems << "Does not end with `>`" unless to_s.end_with?(">") + + if @inclusions + @problems << "Does not include #{@inclusions.map { |s| "`#{s}`" }.join(", ")}" unless @inclusions.all? { |s| to_s.include?(s) } + end + + if @exclusions + @problems << "Does not exclude #{@exclusions.map { |s| "`#{s}`" }.join(", ")}" unless @exclusions.none? { |s| to_s.include?(s) } + end + + @problems.empty? + end + + failure_message do |object| + <<~EOS + expected `#{object.class.name}` instance to #{description}, but had #{@problems.size} problem(s): + + #{@problems.map.with_index { |p, i| "#{i + 1}) #{p}" }.join("\n")} + + `to_s`: #{truncate(object.to_s)} + `inspect`: #{truncate(object.inspect)} + EOS + end + + description do + super().sub(" to s ", " `#to_s` ").sub(" inspect ", " `#inspect` ") + end + + define_method :truncate do |str| + if str.length > 3 * max_length + "#{str[0, max_length * 3]}..." + else + str + end + end + # :nocov: +end diff --git a/spec_support/lib/elastic_graph/spec_support/in_sub_process.rb b/spec_support/lib/elastic_graph/spec_support/in_sub_process.rb new file mode 100644 index 00000000..3aee34b6 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/in_sub_process.rb @@ -0,0 +1,74 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + # We use `aggregate_failures: false` here because aggregating failures does not work well with + # child processes. Any failures get aggregated into state in the child process that gets lost + # when the process exits. + ::RSpec.shared_context "in_sub_process", aggregate_failures: false do + # Runs the provided block in a subprocess. Any failures in the sub process get + # caught and re-raised in the parent process. Also, this returns the return value + # of the child process (using `Marshal` to send it across a pipe). + def in_sub_process(&block) + SubProcess.new.run(&block) + end + end + + class SubProcess < ::Data.define(:reader, :writer) + def initialize + reader, writer = ::IO.pipe + super(reader: reader, writer: writer) + end + + def run(&block) + pid = ::Process.fork { in_child_process(&block) } + in_parent_process(pid) + end + + private + + def in_parent_process(pid) + writer.close # We don't write from the parent process + + ::Process.waitpid(pid) + + result, exception = ::Marshal.load(reader.read) + # :nocov: -- which branch is taken depends on if a test is failing. + raise exception if exception + # :nocov: + result + ensure + reader.close + end + + def in_child_process + reader.close # We don't read from the child process + + handle_exceptions do + result = yield + handle_exceptions(" (exception received while marshaling the result)") do + writer.write(::Marshal.dump([result, nil])) + end + end + ensure + writer.close + end + + def handle_exceptions(suffix = "") + yield + rescue ::Exception => ex # standard:disable Lint/RescueException + # :nocov: -- we only get here when there's a problem. + # Not all exceptions can be marshaled (e.g. if they have state that references unmarshable objects such as a proc). + # Here we just use a `StandardError` with the same message and backtrace to ensure it can be marshaled. + replacement_exception = ::StandardError.new("#{ex.class}: #{ex.message}#{suffix}") + replacement_exception.set_backtrace(ex.backtrace) + writer.write(::Marshal.dump([nil, replacement_exception])) + # :nocov: + end + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/lambda_function.rb b/spec_support/lib/elastic_graph/spec_support/lambda_function.rb new file mode 100644 index 00000000..7a1f2567 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/lambda_function.rb @@ -0,0 +1,112 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/hash_util" +require "elastic_graph/spec_support/in_sub_process" +require "json" +require "logger" +require "tmpdir" +require "yaml" + +RSpec.shared_context "lambda function" do |config_overrides_in_yaml: {}| + include_context "in_sub_process" + + around do |ex| + ::Dir.mktmpdir do |dir| + @tmp_dir = dir + @config_dir = dir + with_lambda_env_vars(&ex) + end + end + + let(:base_config) { ::YAML.safe_load_file(ElasticGraph::CommonSpecHelpers.test_settings_file, aliases: true) } + + let(:example_config_yaml_file) do + "#{@config_dir}/config.yaml".tap do |filename| + config = base_config.merge( + "schema_artifacts" => {"directory" => ::File.join(ElasticGraph::CommonSpecHelpers::REPO_ROOT, "config", "schema", "artifacts")} + ) + config = ElasticGraph::Support::HashUtil.deep_merge(config, config_overrides_in_yaml) + + ::File.write(filename, ::YAML.dump(config)) + end + end + + def expect_loading_lambda_to_define_constant(lambda:, const:) + expect(::Object.const_defined?(const)).to be false + + # Loading the lambda function mutates are global set of constants. To isolate our tests, + # we load it in a sub process--that keeps the parent test process "clean", helping to + # prevent order-dependent test results. + new_constants = in_sub_process do + # Here we install and verify that the AWS lambda runtime is compatible with the current bundle of gems. + # Importantly, we do this in a sub-process so that the monkey patches don't "leak" and impact our main test process! + install_aws_lambda_runtime_monkey_patches + + orig_constants = ::Object.constants + + expect { + load lambda + }.to output(/Booting the lambda function/).to_stdout_from_any_process # silence standard logging + + yield ::Object.const_get(const) + + ::Object.constants - orig_constants + end + + expect(new_constants).to include(const) + end + + let(:cluster_test_urls) do + base_config.fetch("datastore").fetch("clusters").transform_values do |cluster| + cluster.fetch("url") + end + end + + define_method :with_lambda_env_vars do |cluster_urls: cluster_test_urls, extras: {}, &block| + lambda_env_vars = { + "ELASTICGRAPH_YAML_CONFIG" => example_config_yaml_file, + "OPENSEARCH_CLUSTER_URLS" => ::JSON.generate(cluster_urls), + "AWS_REGION" => "us-west-2", + "AWS_ACCESS_KEY_ID" => "some-access-key", + "AWS_SECRET_ACCESS_KEY" => "some-secret-key", + "SENTRY_DSN" => "https://something@sentry.io/something" + }.merge(extras) + + with_env(lambda_env_vars, &block) + end + + # With the release of logger 1.6.0, and the release of faraday 2.10.0 (which depends on the `logger` gem for the first time), + # it was discovered during a failed deploy that the AWS lambda Ruby runtime breaks logger 1.6.0 due to how it monkey patches it! + # This caught us off guard since our CI build didn't fail with the same kind of error. + # + # We've fixed it by pinning logger < 1.6.0. To prevent a regression, and to identify future incompatibilities, here we load the + # AWS Lambda Ruby runtime and install its monkey patches. We observed that this lead to the same kind of error as we saw during + # the failed deploy before we pinned the logger version. + # + # Note: this method is only intended to be called from an `in_sub_process` block since it mutates the runtime environment. + def install_aws_lambda_runtime_monkey_patches + require "aws_lambda_ric" + + # The monkey patches are triggered by the act of instantiating this class: + # https://github.com/aws/aws-lambda-ruby-runtime-interface-client/blob/2.0.0/lib/aws_lambda_ric.rb#L136-L147 + AwsLambdaRuntimeInterfaceClient::TelemetryLoggingHelper.new("lambda_logs.log", @tmp_dir) + + # Here we verify that the Logger monkey patch was indeed installed. The installation of the monkey patch + # gets bypassed when certain errors are encountered (which are silently swallowed), so the mere act of + # instantiating the class above doesn't guarantee the monkey patches are active. + # + # Plus, new versions of the `aws_lambda_ric` may change how the monkey patches are installed. + # + # https://github.com/aws/aws-lambda-ruby-runtime-interface-client/blob/2.0.0/lib/aws_lambda_ric.rb#L145-L147 + expect(::Logger.ancestors).to include(::LoggerPatch) + + # Log a message--this is what triggers a `NoMethodError` when logger 1.6.0 is used. + ::Logger.new($stdout).error("test log message") + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/logging.rb b/spec_support/lib/elastic_graph/spec_support/logging.rb new file mode 100644 index 00000000..b2fe4abb --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/logging.rb @@ -0,0 +1,122 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/support/logger" +require "stringio" + +module LogCaptureSupport + def log_device + @log_device ||= StringIO.new + end + + def logger + @logger ||= begin + original_formatter = ElasticGraph::Support::Logger::JSONAwareFormatter.new + ElasticGraph::Support::Logger::Factory.build( + device: log_device, + config: ElasticGraph::Support::Logger::Config.new( + device: "stdout", # ignored given `device:` above, but required to be set + level: "INFO", + formatter: ->(*args, msg) do + # Don't log VCR UnhandledHTTPRequestErrors. The `GraphQL::QueryExecutor` logs any + # exceptions that happen during query execution, and this exception will happen if + # the recorded VCR cassette differs from the datastore requests being made. + # + # Our VCR support will automatically retry the test when it hits this error after deleting + # the VCR cassette. However, since we also assert that there are no logged warnings in + # many tests, if we allow the VCR errors to get written to our logs, tests can fail + # non-deterministically. So here we exclude them from our logs. + # :nocov: -- the `unless` branch isn't usually covered. + original_formatter.call(*args, msg) unless msg.include?("VCR::Errors::UnhandledHTTPRequestError") + # :nocov: + end + ) + ) + end + end + + # this method must be prepended so that we can force `log_device` so + # that any call to `build_datastore_core` in groups tagged with `:capture_logs + # uses our configured log device. + def build_datastore_core(**options, &block) + super(logger: logger, **options, &block) + end + + def logged_output + log_device.string + end + + def logged_jsons + logged_messages.select.filter_map do |log_message| + if log_message.lines.one? && /{".+}\s*$/.match?(log_message) + ::JSON.parse(log_message[log_message.index("{")..]) + end + end + end + + def logged_jsons_of_type(message_type) + logged_jsons.select { |json| json["message_type"] == message_type } + end + + def logged_warnings + # Ruby's standard log format starts each message with a level indicator: + # https://docs.ruby-lang.org/en/master/Logger.html#class-Logger-label-Log+Level + # e.g. "I, ..." for info, `W, ..." for warn, etc. + # Here we want to only consider messages that are warning level or more severe. + # W=WARN, E=ERROR, F=FATAL, A=UNKNOWN + logged_messages.select do |message| + message.start_with?("W, ", "E, ", "F, ", "A, ") + end + end + + def log(string_regex_or_matcher) + @expect_logging = true + change { logged_output }.to(string_regex_or_matcher) + end + + def log_warning(string_regex_or_matcher) + @expect_logging = true + change { logged_warnings.join }.to(string_regex_or_matcher) + end + + def avoid_logging_warnings + maintain { logged_warnings } + end + + def expect_logging? + !!@expect_logging + end + + def flush_logs + log_device.truncate(0) + log_device.rewind + end + + def logged_messages + # Ruby's standard log format starts each message with a level indicator: + # https://docs.ruby-lang.org/en/master/Logger.html#class-Logger-label-Log+Level + # e.g. "I, ..." for info, `W, ..." for warn, etc. + logged_output.split(/(?=^[DIWEFA], )/) + end +end + +RSpec.configure do |c| + c.prepend LogCaptureSupport, :capture_logs + + # For any example where we are capturing logging, add an automatic assertion + # that no logging occurred (as that indicates a warning of some problem, generally), + # unless the example specifically expected logging, via the use of the `log` matcher + # defined above. + c.around(:example, :capture_logs) do |ex| + if ex.metadata[:expect_warning_logging] + ex.run + else + expect(&ex).to avoid_logging_warnings.or change { ex.example.example_group_instance.expect_logging? }.to(true) + end + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/old_datastore_scripts.yaml b/spec_support/lib/elastic_graph/spec_support/old_datastore_scripts.yaml new file mode 100644 index 00000000..61c6d6cd --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/old_datastore_scripts.yaml @@ -0,0 +1,72 @@ +--- +# Copied from config/schema/artifacts/datastore_scripts.yaml#L196-L264 in ElasticGraph v0.8.0.0. +# TODO: Drop this when we no longer need to maintain backwards-compatibility. +update_index_data_9b97090d5c97c4adc82dc7f4c2b89bc5: + context: update + script: + lang: painless + source: |- + Map source = ctx._source; + + // Numbers in JSON appear to be parsed as doubles, but we want the version stored as a long, so we need to cast it here. + long eventVersion = (long) params.version; + + if (source.__versions == null) { + source.__versions = [:]; + } + + if (source.__versions[params.relationship] == null) { + source.__versions[params.relationship] = [:]; + } + + if (source.__sourceVersions == null) { + source.__sourceVersions = [:]; + } + + if (source.__sourceVersions[params.sourceType] == null) { + source.__sourceVersions[params.sourceType] = [:]; + } + + // While the version in `__versions` is going to be used for the doc version in the future, for now + // we need to continue getting it from `__sourceVersions`. Both our old version and this versions of this + // script keep the value in `__sourceVersions` up-to-date, whereas the old script only writes it to + // `__sourceVersions`. Until we have completely migrated off of the old script for all ElasticGraph + // clusters, we need to keep using it. + // + // Later, after the old script is no longer used by any clusters, we'll stop using `__sourceVersions`. + // TODO: switch to `__versions` when we no longer need to maintain compatibility with the old version of the script. + Number _versionForSourceType = source.__sourceVersions.get(params.sourceType)?.get(params.sourceId); + Number _versionForRelationship = source.__versions.get(params.relationship)?.get(params.sourceId); + + // Our JSON schema requires event versions to be non-negative, so we can safely use Long.MIN_VALUE as a stand-in when the value is null. + long versionForSourceType = _versionForSourceType == null ? Long.MIN_VALUE : _versionForSourceType.longValue(); + long versionForRelationship = _versionForRelationship == null ? Long.MIN_VALUE : _versionForRelationship.longValue(); + + // Pick the larger of the two versions as our doc version. Note that `Math.max` didn't work for me here for + // reasons I don't understand, but a simple ternary works fine. + // + // In theory, we could just use `versionForSourceType` as the `docVersion` (and not even check `__versions` at all) + // since both the old version and this version maintain the doc version in `__sourceVersions`. However, that would + // prevent this version of the script from being forward-compatible with the planned next version of this script. + // In the next version, we plan to stop writing to `__sourceVersions`, and as we can't deploy that change atomically, + // this version of the script will continue to run after that has begun to be used. So this version of the script + // must consider which version is greater here, and not simply trust either version value. + long docVersion = versionForSourceType > versionForRelationship ? versionForSourceType : versionForRelationship; + + if (docVersion >= eventVersion) { + throw new IllegalArgumentException("ElasticGraph update was a no-op: [" + + params.id + "]: version conflict, current version [" + + docVersion + "] is higher or equal to the one provided [" + + eventVersion + "]"); + } else { + source.putAll(params.data); + source.id = params.id; + source.__versions[params.relationship][params.sourceId] = eventVersion; + + // To continue to be backwards compatible with the old version of this script, we need to write the version to + // `__sourceVersions` since that's where it looks. In addition, we need to use `params.version` (which can be + // double) rather than `eventVersion` (a long) to mirror how the old version of this script behaved (which didn't + // do any casting). + // TODO: drop this when we no longer need to maintain compatibility with the old version of the script. + source.__sourceVersions[params.sourceType][params.sourceId] = params.version; + } diff --git a/spec_support/lib/elastic_graph/spec_support/optimize_graphql.rb b/spec_support/lib/elastic_graph/spec_support/optimize_graphql.rb new file mode 100644 index 00000000..07b2cbee --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/optimize_graphql.rb @@ -0,0 +1,120 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# Our test suite does _a ton_ of GraphQL parsing, given that `ElasticGraph::GraphQL` +# is instantated from scratch in virtually every `elasticgraph-graphql` test, and the GraphQL schema gets re-parsed +# each time. Using `profiling.rb` we've found that GraphQL parsing takes up a significant portion +# of our test suite runtime. Here we optimize it by simply memoizing the results of parsing a +# GraphQL string (which applies to both schema definitions and GraphQL queries). The GraphQL +# gem provides a nice `GraphQL.default_parser` API we can use to plug in our own parser, which just +# wraps the built-in one with memoization. +# +# As of 2021-12-29, on a local macbook, without this optimization, `script/run_all_specs` reports: +# +# > ======================================================================================================================== +# > Top 10 profiling results: +# > 1) `ElasticGraph::GraphQL::Schema#initialize`: 362 calls in 24.026 sec (0.066 sec avg) +# > Max time: 0.231 sec for `./elasticgraph/spec/acceptance/application_spec.rb[1:2:2]` from `./spec/acceptance/application_spec.rb:18:in `block in call_graphql_query'` +# > ------------------------------------------------------------------------------------------------------------------------ +# > 2) `Class#from_definition`: 370 calls in 23.064 sec (0.062 sec avg) +# > Max time: 0.212 sec for `./elasticgraph/spec/acceptance/application_spec.rb[1:2:2]` from `./spec/acceptance/application_spec.rb:18:in `block in call_graphql_query'` +# > ======================================================================================================================== +# > +# > Finished in 1 minute 1.6 seconds (files took 3.76 seconds to load) +# > 1859 examples, 0 failures +# > script/run_all_specs 53.84s user 5.59s system 89% cpu 1:06.58 total +# +# With this optimization in place, it reports: +# +# > ======================================================================================================================== +# > Top 10 profiling results: +# > 1) `ElasticGraph::GraphQL::Schema#initialize`: 362 calls in 10.688 sec (0.03 sec avg) +# > Max time: 0.192 sec for `./elasticgraph/spec/acceptance/application_spec.rb[1:1:5]` from `./spec/acceptance/application_spec.rb:18:in `block in call_graphql_query'` +# > ------------------------------------------------------------------------------------------------------------------------ +# > 2) `Class#from_definition`: 370 calls in 9.297 sec (0.025 sec avg) +# > Max time: 0.175 sec for `./elasticgraph/spec/acceptance/application_spec.rb[1:1:5]` from `./spec/acceptance/application_spec.rb:18:in `block in call_graphql_query'` +# > ======================================================================================================================== +# > +# > Finished in 47.41 seconds (files took 3.69 seconds to load) +# > 1859 examples, 0 failures +# > script/run_all_specs 39.91s user 5.65s system 87% cpu 52.142 total +# +# The difference is even starker when simplecov is loaded (as it is when you run `script/quick_build`). Without this: +# +# > ======================================================================================================================== +# > Top 10 profiling results: +# > 1) `ElasticGraph::GraphQL::Schema#initialize`: 362 calls in 58.334 sec (0.161 sec avg) +# > Max time: 0.453 sec for `./elasticgraph/spec/acceptance/application_spec.rb[1:1:9:5]` from `./spec/acceptance/application_spec.rb:18:in `block in call_graphql_query'` +# > ------------------------------------------------------------------------------------------------------------------------ +# > 2) `Class#from_definition`: 370 calls in 57.868 sec (0.156 sec avg) +# > Max time: 0.438 sec for `./elasticgraph/spec/acceptance/application_spec.rb[1:1:9:5]` from `./spec/acceptance/application_spec.rb:18:in `block in call_graphql_query'` +# > ======================================================================================================================== +# > +# > Finished in 1 minute 38.8 seconds (files took 4 seconds to load) +# > 1859 examples, 0 failures +# > COVERAGE=1 script/run_all_specs 91.88s user 5.96s system 92% cpu 1:45.30 total +# +# With this: + +# > ======================================================================================================================== +# > Top 10 profiling results: +# > 1) `ElasticGraph::GraphQL::Schema#initialize`: 362 calls in 16.538 sec (0.046 sec avg) +# > Max time: 0.348 sec for `./elasticgraph/spec/acceptance/application_spec.rb[1:2:5]` from `./spec/acceptance/application_spec.rb:18:in `block in call_graphql_query'` +# > ------------------------------------------------------------------------------------------------------------------------ +# > 2) `Class#from_definition`: 370 calls in 15.263 sec (0.041 sec avg) +# > Max time: 0.376 sec for `./elasticgraph/spec/unit/elastic_graph/application_spec.rb[1:9:1]` from `./spec/unit/elastic_graph/application_spec.rb:136:in `block (3 levels) in '` +# > ======================================================================================================================== +# > +# > Finished in 55.28 seconds (files took 3.76 seconds to load) +# > 1859 examples, 0 failures +# > COVERAGE=1 script/run_all_specs 49.01s user 5.63s system 89% cpu 1:01.17 total +# +# That's a 40% reduction in test suite runtime when simplecov is enabled, and a 20% reduction +# when simplecov is not enabled. + +# Guard against this file applying in a CI build since it makes our tests slightly +# less accurate (production doesn't have this kind of memoization). +# :nocov: -- only one branch is covered on any given run +abort "#{__FILE__} should not be loaded in a CI environment but was" if ENV["CI"] +# :nocov: + +module ElasticGraph + class MemoizingGraphQLParser + # :nocov: some of the gem test suites don't trigger calls to these methods. + def self.parse(graphql_string, filename: nil, trace: ::GraphQL::Tracing::NullTrace, max_tokens: nil) + memoized_results[graphql_string] ||= ::GraphQL::Language::Parser.parse( + graphql_string, + filename: filename, + trace: trace, + max_tokens: max_tokens + ) + end + + def self.memoized_results + @memoized_results ||= {} + end + # :nocov: + end +end + +# To use our parser, we need to set `GraphQL.default_parser`. However, we do not want to +# `require "graphql"` here. That would slow down test runs for individual unit tests that +# don't involve GraphQL at all (of which there are many!). Instead, we want to reactively +# set `default_parser` as soon as the `GraphQL` module is defined. To do that, we use the +# tracepoint API (from the Ruby standard library) and set it as soon as the `GraphQL` module +# has been defined. +trace = TracePoint.new(:end) do |tp| + # :nocov: -- for some reason simplecov reports these lines as uncovered (but they do get run). Maybe the tracepoint API interferes? + if tp.path.end_with?("graphql.rb") && tp.self.name == "GraphQL" + trace.disable # once we've set the default parser we don't want this trace running anymore. + ::GraphQL.default_parser = ElasticGraph::MemoizingGraphQLParser + end + # :nocov: +end + +trace.enable diff --git a/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner.rb b/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner.rb new file mode 100644 index 00000000..34632af1 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner.rb @@ -0,0 +1,113 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module ParallelSpecRunner + def self.test_env_number + @test_env_number ||= ENV.fetch("TEST_ENV_NUMBER") + end + + def self.index_prefix + @index_prefix ||= "test_env_#{test_env_number}_" + end + + # Our various parallel spec runner adapters need to be hook into other classes in order to patch how they work to be compatible + # with running concurrently at the same time as specs in another worker process. However, we support running our tests in multiple + # ways, including in the context of a single gem's bundle. When running in a single gem's bundle, some dependencies may be unavailable + # (it depends on what's in the gem's `gemspec` file). In that case, a require performed from one or more of these adapters may fail + # with a `LoadError`. But that's OK: if a given dependency is not available, then it's not being used and we don't have to patch it! + # So we can safely ignore it. + # + # But when we run from the repository root (i.e. in the context of the entire repo bundle) then we don't want to ignore these errors. + def self.safe_require(file) + require file + rescue ::LoadError + # :nocov: -- we don't get here when running the test suite for the entire repo + raise if ::File.expand_path(::Dir.pwd) == ::File.expand_path(CommonSpecHelpers::REPO_ROOT) + # :nocov: + end + + safe_require "elastic_graph/spec_support/parallel_spec_runner/cluster_configuration_manager_adapter" + safe_require "elastic_graph/spec_support/parallel_spec_runner/datastore_core_adapter" + safe_require "elastic_graph/spec_support/parallel_spec_runner/datastore_spec_support_adapter" + safe_require "elastic_graph/spec_support/parallel_spec_runner/elastic_graph_profiler_adapter" + safe_require "elastic_graph/spec_support/parallel_spec_runner/request_tracker_request_adapter" + + ::Flatware.configure do |flatware| + flatware.after_fork do + # :nocov: -- which sides of these conditionals run depends on options used to run the test suite + ::SimpleCov.at_fork.call(test_env_number) if defined?(::SimpleCov) + + unless ENV["NO_VCR"] + require "elastic_graph/spec_support/vcr" + VCR.configure do |config| + # Use different cassette directories for different index prefixes--otherwise the HTTP requests to the datastore will have conflicting + # index names and our tests will frequently need to re-run after deleting a cassette. + config.cassette_library_dir += "/#{index_prefix.delete_suffix("_")}" + end + end + # :nocov: + end + end + + module Overrides + TEST_DOUBLE_REQUIRES_BY_CONSTANT_NAME = { + "ElasticGraph::Elasticsearch::Client" => "elastic_graph/elasticsearch/client", + "ElasticGraph::GraphQL::DatastoreQuery" => "elastic_graph/graphql/datastore_query", + "ElasticGraph::GraphQL::DatastoreQuery::DocumentPaginator" => "elastic_graph/graphql/datastore_query", + "ElasticGraph::GraphQL::DatastoreQuery::Paginator" => "elastic_graph/graphql/datastore_query", + "ElasticGraph::GraphQL::DatastoreSearchRouter" => "elastic_graph/graphql/datastore_search_router", + "ElasticGraph::SchemaArtifacts::FromDisk" => "elastic_graph/schema_artifacts/from_disk", + "ElasticGraph::Support::MonotonicClock" => "elastic_graph/support/monotonic_clock", + "GraphQL::Execution::Lookahead" => "graphql/execution/lookahead" + } + + # Because `flatware` distributes the spec files across workers, we cannot count on doubled constants always being loaded when + # we use the parallel spec runner. To ensure deterministic results, we're overriding `instance_double` here to make it load the + # doubled constant so that RSpec doubled constant verification can proceed. + def instance_double(constant, *args, **options) + if constant.is_a?(String) + file_to_require = TEST_DOUBLE_REQUIRES_BY_CONSTANT_NAME.fetch(constant) do + # :nocov: -- only covered when there's a missing entry in `TEST_DOUBLE_REQUIRES_BY_CONSTANT_NAME`. + fail "`instance_double` was called with `#{constant.inspect}`, but `TEST_DOUBLE_REQUIRES_BY_CONSTANT_NAME` " \ + "does not know what file to require for this. Please update `TEST_DOUBLE_REQUIRES_BY_CONSTANT_NAME` (in `#{__FILE__}`) " \ + "or use the direct constant instead of the string name of the constant." + # :nocov: + end + + require file_to_require + end + + super + end + end + end +end + +RSpec.configure do |config| + config.mock_with :rspec do |mocks| + # Force `verify_doubled_constant_names` to `true`. This option checks our `instance_double` calls to ensure that the named + # constant exists (because RSpec can't check our stubbed methods otherwise). We do not always set this option because it's + # useful to be able to run a single unit test without loading a heavyweight dependency, and always setting this to true + # would force us to always load dependencies that we use test doubles for. + # + # Usually, we only set this to true if there are any `acceptance` specs getting run--that's a signal that we're running end-to-end + # tests and are running tests with everything loaded. However, when we're running parallel tests, we want to force it to `true`, + # for a couple reasons: + # + # 1. This makes it deterministic. Our parallel test runner distributes the spec files across all workers based on the recorded runtimes + # of the specs (to try and balance them). This process is inherently non-deterministic, and can lead to "flickering" situations where + # a unit spec that uses `instance_double` is run with an acceptance spec on one run and not on the next run even though the entire + # suite is being run. + # 2. We generally only use the parallel spec runner when running the entire suite (or at least the entire suite of a single gem), so + # we are already in a context when we're loading most or all things. + mocks.verify_doubled_constant_names = true + end + + config.prepend ElasticGraph::ParallelSpecRunner::Overrides +end diff --git a/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/cluster_configuration_manager_adapter.rb b/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/cluster_configuration_manager_adapter.rb new file mode 100644 index 00000000..965b5cfc --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/cluster_configuration_manager_adapter.rb @@ -0,0 +1,39 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/cluster_configuration_manager" + +module ElasticGraph + module ParallelSpecRunner + module ClusterConfigurationManagerAdapter + def initialize(...) + super + + # We need to track the state of each scoped set of indices separately, so that we can appropriately recreate the indices + # for a particular index prefix at the right times. To achieve that, we prepend the `state_file_name` with the index prefix here. + self.state_file_name = "#{ParallelSpecRunner.index_prefix}_#{state_file_name}" + end + + def notify_recreating_cluster_configuration + # The message that is normally printed here is "part 1" with the `notify_recreated_cluster_configuration` message completing + # the notification and reporting how long it took. During parallel test execution, we don't want the messages to be split + # like that: it leads to noisy, confusing output. Instead, we've merged the normal contents of this notification into + # `notify_recreated_cluster_configuration` below so that we provide a single notification with all the information. + end + + # :nocov: -- whether this is called depends on whether the datastore is already fully configured or not + def notify_recreated_cluster_configuration(duration) + puts "\nRecreated test env #{ParallelSpecRunner.test_env_number} cluster configuration (for index definitions: " \ + "#{index_definitions.map(&:name)}) in #{RSpec::Core::Formatters::Helpers.format_duration(duration)}." + end + # :nocov: + + ClusterConfigurationManager.prepend(self) + end + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/datastore_client_adapter.rb b/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/datastore_client_adapter.rb new file mode 100644 index 00000000..92ca5783 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/datastore_client_adapter.rb @@ -0,0 +1,146 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" + +module ElasticGraph + module ParallelSpecRunner + # To safely run our tests in parallel, we need to make sure that each test worker has independent interactions with the datastore + # that can't interfere with each other. For example, we can't have a test in one worker deleting all documents from all indices + # while a test in another worker is querying the datastore expecting a document to be there. + # + # Our solution is to intercept all calls to the datastore via this adapter, and scope all operations to indices which have a prefix + # based on the `test_env_number`. Specifically: + # + # * In any request which references an index, we prepend the `test_env_#{test_env_number}_` to the index name or expression. + # * On any response which references an index, we remove the prepended index prefix since our tests don't expect it. + # * We've also modified the configuration of `action.auto_create_index`. The `Admin::ClusterConfigurator::ClusterSettingsManager` + # usually disables the auto-creation of our rollover indices, to guard against a race condition that can happen when indexing + # into a rollover index is happening at the same time we're configuring the indices. But if we allow that behavior in our + # parallel tests, then it can cause problems during parallel test runs. + # + # Note: here we use a `SimpleDelegator` rather than a module that we prepend on to the client class because we need to be able to preserve + # the behavior of the existing client class for when we run the unit specs of that client class. Using a `SimpleDelegator` allows us to + # selectively modify the behavior of _some_ instances of the client class (that is, every other instance besides the ones used for + # those unit tests!). + class DatastoreClientAdapter < ::SimpleDelegator + # Cluster APIs + + def put_persistent_cluster_settings(settings) + if (index_patterns = settings["action.auto_create_index"]) && !index_patterns.include?(ROLLOVER_INDEX_INFIX_MARKER) + settings = settings.merge({ + "action.auto_create_index" => index_patterns + ",+*#{ROLLOVER_INDEX_INFIX_MARKER}*" + }) + end + + super(settings) + end + + # Index Template APIs + + def get_index_template(index_template_name) + with_updated_index_patterns_for_test_env( + super(index_expression_for_test_env(index_template_name)) + ) { |pattern| pattern.delete_prefix(ParallelSpecRunner.index_prefix) } + end + + def put_index_template(name:, body:) + name = index_expression_for_test_env(name) + body = with_updated_index_patterns_for_test_env(body) { |pattern| ParallelSpecRunner.index_prefix + pattern } + super(name: name, body: body) + end + + def delete_index_template(index_template_name) + super(index_expression_for_test_env(index_template_name)) + end + + # Index APIs + + def get_index(index_name) + super(index_expression_for_test_env(index_name)) + end + + def list_indices_matching(index_expression) + super(index_expression_for_test_env(index_expression)).map do |index_name| + index_name.delete_prefix(ParallelSpecRunner.index_prefix) + end + end + + def create_index(index:, body:) + super(index: index_expression_for_test_env(index), body: body) + end + + def put_index_mapping(index:, body:) + super(index: index_expression_for_test_env(index), body: body) + end + + def put_index_settings(index:, body:) + super(index: index_expression_for_test_env(index), body: body) + end + + def delete_indices(*index_names) + super(*index_names.map { |index_name| index_expression_for_test_env(index_name) }) + end + + # Document APIs + + def msearch(body:, headers: nil) + body = body.each_slice(2).flat_map do |(search_metadata, search_body)| + search_metadata = search_metadata.merge(index: index_expression_for_test_env(search_metadata.fetch(:index))) + [search_metadata, search_body] + end + + msearch_response = super(body: body, headers: headers) + + responses = msearch_response.fetch("responses").map do |response| + if (hits_hits = response.dig("hits", "hits")) + hits_hits = hits_hits.map do |hit| + hit.merge("_index" => hit.fetch("_index").delete_prefix(ParallelSpecRunner.index_prefix)) + end + + response.merge("hits" => response.fetch("hits").merge("hits" => hits_hits)) + else + response + end + end + + msearch_response.merge({"responses" => responses}) + end + + def bulk(body:, refresh: false) + body = body.each_slice(2).flat_map do |(op_metadata, op_body)| + op_key, op_meta = op_metadata.to_a.first + op_meta = op_meta.merge(_index: index_expression_for_test_env(op_meta.fetch(:_index))) + + [{op_key => op_meta}, op_body] + end + + super(body: body, refresh: refresh) + end + + def delete_all_documents + super(index: index_expression_for_test_env("*")) + end + + private + + def index_expression_for_test_env(index_expression) + index_expression.split(",").map do |index_sub_expression| + prefix = index_sub_expression.start_with?("-") ? "-" : "" + "#{prefix}#{ParallelSpecRunner.index_prefix}#{index_sub_expression.delete_prefix("-")}" + end.join(",") + end + + def with_updated_index_patterns_for_test_env(body, &adjust_pattern) + return body if body.empty? + patterns = body.fetch("index_patterns").map(&adjust_pattern) + body.merge({"index_patterns" => patterns}) + end + end + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/datastore_core_adapter.rb b/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/datastore_core_adapter.rb new file mode 100644 index 00000000..5c7f938e --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/datastore_core_adapter.rb @@ -0,0 +1,26 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/datastore_core" +require "elastic_graph/spec_support/parallel_spec_runner/datastore_client_adapter" + +module ElasticGraph + module ParallelSpecRunner + # This adapter hooks into the instantiation of new datastore clients from `DatastoreCore` and wraps them with + # `DatastoreClientAdapter` so that all datastore clients have the patched behavior necessary for parallel spec runs. + module DatastoreCoreAdapter + def clients_by_name + @clients_by_name ||= super.transform_values do |client| + DatastoreClientAdapter.new(client) + end + end + + DatastoreCore.prepend(self) + end + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/datastore_spec_support_adapter.rb b/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/datastore_spec_support_adapter.rb new file mode 100644 index 00000000..c85ec34a --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/datastore_spec_support_adapter.rb @@ -0,0 +1,24 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/parallel_spec_runner/datastore_client_adapter" +require "elastic_graph/spec_support/uses_datastore" + +module ElasticGraph + module ParallelSpecRunner + # This adapter hooks into the instantiation of new datastore clients from `DatastoreSpecSupport` and wraps them with + # `DatastoreClientAdapter` so that all datastore clients have the patched behavior necessary for parallel spec runs. + module DatastoreSpecSupportAdapter + def new_datastore_client(...) + DatastoreClientAdapter.new(super) + end + + DatastoreSpecSupport.prepend(self) + end + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/elastic_graph_profiler_adapter.rb b/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/elastic_graph_profiler_adapter.rb new file mode 100644 index 00000000..dcff8ec6 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/elastic_graph_profiler_adapter.rb @@ -0,0 +1,30 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/profiling" + +module ElasticGraph + module ParallelSpecRunner + module ElasticGraphProfilerAdapter + def record(...) + yield # don't waste time recording anything since we silence the reporting below. + end + + def record_raw(...) + # don't waste time recording anything since we silence the reporting below. + end + + # If we're using a parallel test runner we don't want this output to show up at various times as worker + # processes exit, so we override this to be a no-op. + def report_results + end + + ElasticGraphProfiler.singleton_class.prepend(self) + end + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/request_tracker_request_adapter.rb b/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/request_tracker_request_adapter.rb new file mode 100644 index 00000000..50add633 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/request_tracker_request_adapter.rb @@ -0,0 +1,28 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/spec_support/uses_datastore" + +module ElasticGraph + module ParallelSpecRunner + # Used to patch `RequestTracker::Request` to remove the index prefix that gets added to our index names when running specs in parallel. + # Some tests assert on tracked requests and don't expect the index prefix, so it must be removed. + module RequestTrackerRequestAdapter + def initialize(http_method:, url:, body:, timeout:) + super( + http_method: http_method, + url: ::URI.parse(url.to_s.gsub(ParallelSpecRunner.index_prefix, "")), + body: body&.gsub(ParallelSpecRunner.index_prefix, ""), + timeout: timeout + ) + end + + RequestTracker::Request.prepend(self) + end + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/profiling.rb b/spec_support/lib/elastic_graph/spec_support/profiling.rb new file mode 100644 index 00000000..a4381ea8 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/profiling.rb @@ -0,0 +1,98 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# :nocov: -- when running with the parallel spec runner we patch this to disable it, so things here are uncovered. +module ElasticGraphProfiler + def self.results + @results ||= Hash.new { |h, k| h[k] = [] } + end + + def self.record(id, skip_frames: 1) + start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + return_value = yield + stop = Process.clock_gettime(Process::CLOCK_MONOTONIC) + + record_raw( + id, + stop - start, + # Record some caller frames for use later on. We skip the first few frames + # (to skip the profiling logic) as we really want to see the caller from the + # spec file, and 50 frames should be good enough to be able to identify the callsite. + # We don't get all caller frames because that can be expensive. + caller_frames: caller(skip_frames, 50) + ) + + return_value + end + + def self.record_raw(id, duration, caller_frames: []) + results[id] << { + duration: duration, + example: RSpec.current_example, + # Record some caller frames for use later on. We skip the first few frames + # (to skip the profiling logic) as we really want to see the caller from the + # spec file, and 50 frames should be good enough to be able to identify the callsite. + # We don't get all caller frames because that can be expensive. + caller_frames: caller_frames + } + end + + def self.report_results + computed_results = results.map do |id, results_for_id| + durations = results_for_id.map { |h| h.fetch(:duration) } + total = durations.sum + count = durations.count + max_result = results_for_id.max_by { |h| h.fetch(:duration) } + + # Identify where in the spec this method was called--preferrably + # from a spec file itself, but if we can't find it there, a support + # file in the spec directory is also OK. + callsite = max_result.fetch(:caller_frames).find do |line| + line.include?("/spec/") && line =~ /_spec\.rb:\d+/ + end || max_result.fetch(:caller_frames).find do |line| + line.include?("/spec/") + end || "(can't identify callsite)" + + # shorten it to a relative path instead of an absolute one. + callsite = callsite.sub(/.*\/spec\//, "./spec/") + + { + id: id, + count: count, + max: max_result.fetch(:duration).round(3), + total: total.round(3), + avg: (total / count).round(3), + example: max_result.fetch(:example), + callsite: callsite + } + end + + top_results = computed_results + .sort_by { |result| result.fetch(:total) } + .last(6) + .reverse + + puts + puts "=" * 120 + puts "Top #{top_results.size} profiling results:" + + top_results.each_with_index do |result, index| + puts "#{index + 1}) `#{result[:id]}`: #{result[:count]} calls in #{result[:total]} sec (#{result[:avg]} sec avg)" + puts "Max time: #{result.fetch(:max)} sec for `#{result[:example]&.id || "(outside of an example)"}` from `#{result[:callsite]}`" + puts "-" * 120 + end + puts "=" * 120 + end +end + +RSpec.configure do |c| + c.after(:suite) do + ElasticGraphProfiler.report_results if c.profile_examples? + end +end +# :nocov: diff --git a/spec_support/lib/elastic_graph/spec_support/rake_task.rb b/spec_support/lib/elastic_graph/spec_support/rake_task.rb new file mode 100644 index 00000000..f6017d03 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/rake_task.rb @@ -0,0 +1,71 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "stringio" +require "rake" + +module ElasticGraph + module RakeTaskSupport + # Runs rake with the given CLI args, returning the string output of running rake. + # The caller should pass a block that defines the tasks (using the provided `output` object). + def run_rake(*cli_args) + output = StringIO.new + + rake_app = ::Rake::Application.new + + # Rake truncates output when it detects a TTY output, and the truncation is dynamic + # based on the detected width of the terminal. To prevent a narrow terminal from causing + # test failures, we set this to disable truncation. + rake_app.tty_output = false + + # Stop Rake from attempting to load a rakefile. Instead, when we yield the caller will define the tasks + # by instantiating a class that inherits from `Rake::TaskLib`. + def rake_app.load_rakefile + end + + rake_app.options.trace_output = output + + # Neutralize the standard exception handling. Otherwise an exception (which we want to allow to be propagated into + # the test itself) can cause Rake to call `exit` and end the Ruby process. + def rake_app.standard_exception_handling + yield + end + + # Ensure any print done by the rake application goes to `output`. + rake_app.define_singleton_method(:printf) do |*args, **options, &block| + output.printf(*args, **options, &block) + end + + # The `--dry-run` flag mutates some global Rake state. Here we restore that state to its original values. + # Otherwise, a test running with `--dry-run` can impact later tests that run. + ::Rake.nowrite(RakeTaskSupport.rake_orig_nowrite) + ::Rake.verbose(RakeTaskSupport.rake_orig_verbose) + + ::Rake.with_application(rake_app) do + # Necessary so that testing `--tasks` works. + ::Rake::TaskManager.record_task_metadata = true + + yield output # to define the tasks. Caller should inject `output` in their rake task definitions. + rake_app.run(cli_args) + end + + output.string + end + + class << self + attr_reader :rake_orig_nowrite, :rake_orig_verbose + end + + @rake_orig_nowrite = ::Rake.nowrite + @rake_orig_verbose = ::Rake.verbose + end +end + +RSpec.configure do |c| + c.include ElasticGraph::RakeTaskSupport, :rake_task +end diff --git a/spec_support/lib/elastic_graph/spec_support/runtime_metadata_support.rb b/spec_support/lib/elastic_graph/spec_support/runtime_metadata_support.rb new file mode 100644 index 00000000..6431c52d --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/runtime_metadata_support.rb @@ -0,0 +1,173 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + module RuntimeMetadataSupport + def schema_with( + object_types_by_name: {}, + scalar_types_by_name: {}, + enum_types_by_name: {}, + index_definitions_by_name: {}, + schema_element_names: SchemaElementNames.new(form: :snake_case), + graphql_extension_modules: [], + static_script_ids_by_scoped_name: {} + ) + Schema.new( + object_types_by_name: object_types_by_name, + scalar_types_by_name: scalar_types_by_name, + enum_types_by_name: enum_types_by_name, + index_definitions_by_name: index_definitions_by_name, + schema_element_names: schema_element_names, + graphql_extension_modules: graphql_extension_modules, + static_script_ids_by_scoped_name: static_script_ids_by_scoped_name + ) + end + + def object_type_with( + update_targets: [], + index_definition_names: [], + graphql_fields_by_name: {}, + elasticgraph_category: nil, + source_type: nil, + graphql_only_return_type: false + ) + ObjectType.new( + index_definition_names: index_definition_names, + update_targets: update_targets, + graphql_fields_by_name: graphql_fields_by_name, + elasticgraph_category: elasticgraph_category, + source_type: source_type, + graphql_only_return_type: graphql_only_return_type + ) + end + + def derived_indexing_update_target_with( + type: "DerivedIndexingUpdateTarget", + relationship: nil, + script_id: "some_script_id", + id_source: "source_id", + routing_value_source: "routing_value_source", + rollover_timestamp_value_source: "rollover_timestamp_value_source", + data_params: {}, + metadata_params: {} + ) + UpdateTarget.new( + type: type, + relationship: relationship, + script_id: script_id, + id_source: id_source, + routing_value_source: routing_value_source, + rollover_timestamp_value_source: rollover_timestamp_value_source, + data_params: data_params, + metadata_params: metadata_params + ) + end + + def normal_indexing_update_target_with( + type: "UpdateTarget", + relationship: SELF_RELATIONSHIP_NAME, + id_source: "source_id", + routing_value_source: "routing_value_source", + rollover_timestamp_value_source: "rollover_timestamp_value_source", + data_params: {}, + metadata_params: {} + ) + UpdateTarget.new( + type: type, + relationship: relationship, + script_id: INDEX_DATA_UPDATE_SCRIPT_ID, + id_source: id_source, + routing_value_source: routing_value_source, + rollover_timestamp_value_source: rollover_timestamp_value_source, + data_params: data_params, + metadata_params: metadata_params + ) + end + + def dynamic_param_with(source_path: "some_field", cardinality: :one) + DynamicParam.new(source_path: source_path, cardinality: cardinality) + end + + def static_param_with(value) + StaticParam.new(value: value) + end + + def index_definition_with(route_with: nil, rollover: nil, default_sort_fields: [], current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}) + IndexDefinition.new( + route_with: route_with, + rollover: rollover, + default_sort_fields: default_sort_fields, + current_sources: current_sources, + fields_by_path: fields_by_path + ) + end + + def index_field_with(source: SELF_RELATIONSHIP_NAME) + IndexField.new(source: source) + end + + def enum_type_with(values_by_name: {}) + Enum::Type.new(values_by_name: values_by_name) + end + + def sort_field_with(field_path: "path.to.some.field", direction: :asc) + SortField.new( + field_path: field_path, + direction: direction + ) + end + + def relation_with(foreign_key: "some_id", direction: :asc, additional_filter: {}, foreign_key_nested_paths: []) + Relation.new(foreign_key: foreign_key, direction: direction, additional_filter: additional_filter, foreign_key_nested_paths: foreign_key_nested_paths) + end + + def graphql_field_with(name_in_index: "name_index", relation: nil, computation_detail: nil) + GraphQLField.new( + name_in_index: name_in_index, + relation: relation, + computation_detail: computation_detail + ) + end + + def scalar_type_with( + coercion_adapter_ref: ScalarType::DEFAULT_COERCION_ADAPTER_REF, + indexing_preparer_ref: ScalarType::DEFAULT_INDEXING_PREPARER_REF + ) + ScalarType.new( + coercion_adapter_ref: coercion_adapter_ref, + indexing_preparer_ref: indexing_preparer_ref + ) + end + + def scalar_coercion_adapter1 + Extension.new(ScalarCoercionAdapter1, "support/example_extensions/scalar_coercion_adapters", {}) + end + + def scalar_coercion_adapter2 + Extension.new(ScalarCoercionAdapter2, "support/example_extensions/scalar_coercion_adapters", {}) + end + + def indexing_preparer1 + Extension.new(IndexingPreparer1, "support/example_extensions/indexing_preparers", {}) + end + + def indexing_preparer2 + Extension.new(IndexingPreparer2, "support/example_extensions/indexing_preparers", {}) + end + + def graphql_extension_module1 + Extension.new(GraphQLExtensionModule1, "support/example_extensions/graphql_extension_modules", {}) + end + end + end + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/schema_definition_helpers.rb b/spec_support/lib/elastic_graph/spec_support/schema_definition_helpers.rb new file mode 100644 index 00000000..cc8cc99d --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/schema_definition_helpers.rb @@ -0,0 +1,37 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_definition/test_support" + +# Combines `:capture_logs` with `ElasicGraph::SchemaDefinition::TestSupport` in order +# to silence log output and fail if any tests result in logged warnings. +::RSpec.shared_context "SchemaDefinitionHelpers", :capture_logs do + include ::ElasticGraph::SchemaDefinition::TestSupport + + def define_schema_with_schema_elements( + schema_elements, + index_document_sizes: true, + json_schema_version: 1, + extension_modules: [], + derived_type_name_formats: {}, + type_name_overrides: {}, + enum_value_overrides_by_type: {}, + output: nil + ) + super( + schema_elements, + index_document_sizes: index_document_sizes, + json_schema_version: json_schema_version, + extension_modules: extension_modules, + derived_type_name_formats: derived_type_name_formats, + type_name_overrides: type_name_overrides, + enum_value_overrides_by_type: enum_value_overrides_by_type, + output: output || log_device + ) + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/stub_datastore_client.rb b/spec_support/lib/elastic_graph/spec_support/stub_datastore_client.rb new file mode 100644 index 00000000..fdca16ea --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/stub_datastore_client.rb @@ -0,0 +1,34 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module StubDatastoreClient + def datastore_client + @datastore_client ||= stubbed_datastore_client + end + + def stubbed_datastore_client(**additional_stubs) + instance_double( + "ElasticGraph::Elasticsearch::Client", + # Here we stub methods that are called from `DatastoreCore` index definitions, so that our unit specs + # can use index definitions without worrying about datastore calls they will make. + list_indices_matching: [], + # `searches_could_hit_incomplete_docs?` calls these. It's tricky to provide an accurate full configuration + # here but it only looks at a tiny part of the config, and falls back to `current_sources` so we can safely + # stub it with an empty hash. + get_index: {}, + get_index_template: {}, + **additional_stubs + ) + end + end + + RSpec.configure do |c| + c.include StubDatastoreClient, :stub_datastore_client + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/uses_datastore.rb b/spec_support/lib/elastic_graph/spec_support/uses_datastore.rb new file mode 100644 index 00000000..8004a3a4 --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/uses_datastore.rb @@ -0,0 +1,547 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "digest/md5" +require "elastic_graph/elasticsearch/client" +require "elastic_graph/indexer/test_support/converters" +require "elastic_graph/indexer/operation/update" +require "elastic_graph/support/hash_util" +require "logger" +require "yaml" + +require_relative "cluster_configuration_manager" +require_relative "profiling" + +datastore_url = YAML.safe_load_file(ElasticGraph::CommonSpecHelpers::TEST_SETTINGS_FILE_TEMPLATE, aliases: true).fetch("datastore").fetch("clusters").fetch("main").fetch("url") +datastore_logs = "#{ElasticGraph::CommonSpecHelpers::REPO_ROOT}/log/datastore_client.test.log" + +module ElasticGraph + # A `Logger` implementation that internally delegates to multiple underlying loggers so that we can both observe/assert on the logs + # and still produce the logs to an actual log file. + class SplitLogger < ::BasicObject + def initialize(test_in_memory_logger, file_logger) + @test_in_memory_logger = test_in_memory_logger + @file_logger = file_logger + end + + [:debug, :info, :warn, :error, :fatal].each do |level| + define_method level do |message, &block| + # The Elasticsearch and OpenSearch clients log 404 responses as warnings, but 404s are often completely expected: our + # idempotent delete logic expects to get a 404 when the resource it's deleting has previously been deleted. We don't want + # these messages to get logged to `@test_in_memory_logger` because that would cause them to show up in `logged_warnings`, + # which our tests assert on. It simplifies everything if we just ignore/exclude these messages from the test in memory logger. + unless message.to_s.include?("resource_not_found_exception") || message.to_s.include?('"found":false') + @test_in_memory_logger.public_send(level, message, &block) + end + + @file_logger.public_send(level, message, &block) + end + + define_method :"#{level}?" do + @test_in_memory_logger.public_send(:"#{level}?") + end + end + end +end + +class RequestTracker + def initialize(app, request_accumulator) + @app = app + @request_accumulator = request_accumulator + end + + def call(env) + request = Request.new(env.method, env.url, env.body, env.request.timeout) + @request_accumulator << request + + ElasticGraphProfiler.record("datastore (overall)", skip_frames: 3) do + ElasticGraphProfiler.record(request.profiling_id, skip_frames: 4) do + @app.call(env) + end + end + end + + Request = ::Data.define(:http_method, :url, :body, :timeout) do + def profiling_id + # The time a request takes is entirely different if VCR is involved, so mention it + # in the profiling id if we are planing back. + # :nocov: -- which branch executes depends on `NO_VCR` env var, and a single test run will never cover all branches. + vcr_playing_back = + if !defined?(::VCR) + false + elsif (cassette = ::VCR.current_cassette) + !cassette.recording? + else + false + end + # :nocov: + + # Use the first path segment to identify the Datastore action into a broad category. + # (Note that due to the leading `/` we have to take the first 2 split parts to actually get + # the first path segment). + # Also, group all `unique_index` resources as one since the unique index name is different every time. + resource = url.path.split("/").first(2).join("/").sub(/unique_index_\w+/, "unique_index_*") + + # :nocov: -- `vcr_playing_back` depends on `NO_VCR` env var, and a single test run will never cover all branches. + "datastore--#{http_method.upcase} #{resource}#{" (w/ VCR playback)" if vcr_playing_back}" + # :nocov: + end + + def description + # we don't care about protocol, host, port, etc. + path_and_query = url.to_s.sub(%r{^https?://}, "").sub(%r{[^/]*}, "") + + "#{http_method.to_s.upcase} #{path_and_query}" + end + end +end + +UnsupportedAWSOpenSearchOperationError = Class.new(StandardError) + +# Unfortunately, managed AWS OpenSearch does not support all operations that the open source OpenSearch +# distribution we use in our test suite does. Since ElasticGraph supports using managed AWS OpenSearch, +# we want to limit the operations we use to the ones it supports. +# +# AWS documents the supported operations here: +# https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-operations.html#version_opensearch_2.11 +# +# We have used this page to generate the `SUPPORTED_OPERATIONS` set below, using console javascript like: +# document.querySelectorAll("table#w569aac27c13b9b5 td li code").forEach(x => { if (x.innerText.startsWith("/_")) console.log(x.innerText) }) +# +# If you get an `UnsupportedAWSOpenSearchOperationError`, you can check the AWS docs to see if +# it is an operation that newer releases support. Please update the set below as needed. +class DisallowUnsupportedAWSOperations + SUPPORTED_OPERATIONS = %w[ + /_alias /_aliases /_all /_analyze /_bulk /_cat /_cat/nodeattrs /_cluster/allocation/explain /_cluster/health + /_cluster/pending_tasks /_cluster/settings /_cluster/state /_cluster/stats /_count /_delete_by_query /_explain + /_field_caps /_field_stats /_flush /_index_template /_ingest/pipeline /_mapping /_mget /_msearch /_mtermvectors /_nodes + /_opendistro/_alerting /_opendistro/_anomaly_detection /_opendistro/_ism /_opendistro/_ppl /_opendistro/_security /_opendistro/_sql + /_percolate /_plugin/kibana /_rank_eval /_refresh /_reindex /_render /_resolve/index /_rollover /_scripts /_search /_search profile + /_shard_stores /_shrink /_snapshot /_split /_stats /_status /_tasks /_template /_update_by_query /_validate + ].to_set + + def initialize(app) + @app = app + end + + def call(env) + # :nocov: -- normally only the `supported_operation?` branch is taken. + return @app.call(env) if supported_operation?(env.url.path) + raise UnsupportedAWSOpenSearchOperationError, "Operation is unsupported on AWS: #{env.url.path}." + # :nocov: + end + + def supported_operation?(path) + # paths that do not start with `/_` are operations on specific indices, and are automatically supported. + return true unless path.start_with?("/_") + # ...otherwise we need to see if the SUPPORTED_OPERATIONS set contains the operation. + path_parts = path.split("/") + + # The supported operations have different numbers of parts. The fewest parts is two + # (e.g. "/_alias" translates into parts like: ["", "_alias"]), so we start there + # and continue checking more parts until we've checked the entire path. + 2.upto(path_parts.size).any? do |num_parts| + SUPPORTED_OPERATIONS.include?(path_parts.first(num_parts).join("/")) + end + end +end + +RSpec.shared_context "datastore support", :capture_logs do + # Provides a unique index name, so that a test can have an index that no other test interacts with. + # The index name is derived from the unique id RSpec assigns to each example. + # + # This can allow us to avoid the need to frequently delete indices, leading to faster test runs. + # + # This can also be used as a prefix for an index name, and tests that use this to create a new + # index or template do not have to cleanup after themselves; all indices that use this will be cleaned + # up at the start of each test run automatically. + let(:unique_index_name) do |example| + # `example.id` is a string like: + # "./spec/path/to/file_spec.rb[1:2:1]". + # Here we turn it into a valid index name. + "unique_index_#{example.id.gsub(/\W+/, "_").delete_prefix("_").delete_suffix("_")}" + end + + # tracks datastore requests so we can assert on them. + def datastore_requests_by_cluster_name + @datastore_requests_by_cluster_name ||= Hash.new { |h, k| h[k] = [] } + end + + let(:main_datastore_client) { new_datastore_client("main") } + let(:other1_datastore_client) { new_datastore_client("other1") } + let(:other2_datastore_client) { new_datastore_client("other2") } + let(:other3_datastore_client) { new_datastore_client("other3") } + + prepend_before do |ex| + if ex.metadata[:type] == :unit + # :nocov: -- on a successful test run this'll never get executed. + fail "`:uses_datastore` is only appropriate on integration and acceptance tests, but it is being used by a unit test. Please move the test to `spec/integration` or remove the `:uses_datastore` tag." + # :nocov: + end + + # flush everything to give us a clean slate between tests + # We do this in a `prepend_before` hook to ensure it runs _before_ + # other `before` hooks (which may index data intended to be in the datastore for the test) + # Note: we do not need to use `other_datastore_client` since it talks to the same datastore server. + main_datastore_client.delete_all_documents + end + + def datastore_requests(cluster_name) + datastore_requests_by_cluster_name[cluster_name] + end + + def datastore_write_requests(cluster_name) + datastore_requests(cluster_name).reject do |req| + req.http_method == :get || req.http_method == :head + end + end + + def datastore_msearch_requests(cluster_name) + datastore_requests(cluster_name).select do |req| + req.url.path.end_with?("/_msearch") + end + end + + def count_of_searches_in(msearch_request) + # Each search within an msearch request uses 2 lines: header line and body line. + msearch_request.body.split("\n").size / 2 + end + + def performed_search_metadata(cluster_name) + datastore_msearch_requests(cluster_name).flat_map do |req| + req.body.split("\n").each_slice(2).map do |header_line, _body_line| + JSON.parse(header_line) + end + end + end + + def index_search_expressions_from_queries(cluster_name) + performed_search_metadata(cluster_name).map do |headers| + headers.fetch("index") + end + end + + def indices_excluded_from_searches(cluster_name) + index_search_expressions_from_queries(cluster_name).map do |index_search_expression| + index_search_expression.split(",").select do |index_expression| + index_expression.start_with?("-") + end.map do |index_expression| + index_expression.delete_prefix("-") + end + end + end + + def unrefreshed_bulk_calls(cluster_name) + datastore_write_requests(cluster_name) + .select { |req| req.url.path == "/_bulk" } + .reject { |req| req.url.query.to_s.include?("refresh=true") } + end + + after do |ex| + # We need to be careful with our use of the `routing` feature. If we ever + # pass a `routing` option on a search when it's not correct to do so, the + # search response might silently be missing documents that it should have + # included. However, our existing test suite alone isn't sufficient to guard + # against this, for a couple reasons: + # + # 1) Most of our indices have only 1 shard, in which case the routing value + # effectively has no impact on search behavior, as all routing values (or + # none!) would route to the one and only shard. + # 2) For indices where we have multiple shards, we can't control what documents + # wind up on which shards (it's an internal detail of the datastore). Therefore, + # it's entirely possible for a test that runs a particular search to get "lucky" + # and get the expected documents in the response even if a routing value was + # used when it should not have been. + # + # Given these issues, we want to be careful to ensure that `routing` is only used + # when we are sure that it should be. To assist with this, this `after` hook automatically + # adds an assertion to every test that uses the datastore that `routing` was not used on + # any searches. For the few tests that are meant to include searches that do use `routing`, + # they can opt-out of this check by tagging themselves with `:expect_search_routing` (and + # the test should then assert on what routing was used). + unless ex.metadata[:expect_search_routing] + expect(performed_search_metadata("main")).to all exclude("routing") + expect(performed_search_metadata("other1")).to all exclude("routing") + expect(performed_search_metadata("other2")).to all exclude("routing") + expect(performed_search_metadata("other3")).to all exclude("routing") + end + + # Similarly, we have to be careful with index exclusions: it's possible to have a query that + # wrongly excludes indices and have a passing test by "luck", so here we force tests that expect + # index exclusions to be tagged with `:expect_index_exclusions`. They can (and should) use + # `expect_to_have_excluded_indices` to specify what the expected exclusions are. + unless ex.metadata[:expect_index_exclusions] + expect(indices_excluded_from_searches("main").flatten).to eq [] + expect(indices_excluded_from_searches("other1").flatten).to eq [] + expect(indices_excluded_from_searches("other2").flatten).to eq [] + expect(indices_excluded_from_searches("other3").flatten).to eq [] + end + + # Verify that all queries specify what indices to search. If the index search expression is `""`, + # the datastore will search ALL indices which is undesirable. + expect(index_search_expressions_from_queries("main")).to exclude("") + expect(index_search_expressions_from_queries("other1")).to exclude("") + expect(index_search_expressions_from_queries("other2")).to exclude("") + expect(index_search_expressions_from_queries("other3")).to exclude("") + + expect( + unrefreshed_bulk_calls("main") + + unrefreshed_bulk_calls("other1") + + unrefreshed_bulk_calls("other2") + + unrefreshed_bulk_calls("other3") + ).to be_empty, "One or more `/_bulk` calls made from this test lack `?refresh=true`, but it is required on " \ + "all `/_bulk` calls from tests to prevent the non-deterministic leaking of data between tests. Our " \ + "strategy for giving each test a clean slate (empty indices) is to use a `/_delete_by_query` call to delete " \ + "all documents that are findable by a query. If a `/_bulk` call is made without `?refresh=true`, the indexed " \ + "documents may not be visible in the index until AFTER the next `/_delete_by_query` call (but BEFORE the next " \ + "test runs a query), leading to the documents polluting the results of the next test that runs. Please pass " \ + "`refresh: true` in the `/_bulk` call." + end + + def index_into(indexer, *records) + # to aid in migrating existing tests, allow them to pass their graphql instance here... + if indexer.is_a?(ElasticGraph::GraphQL) + indexer = build_indexer(datastore_core: indexer.datastore_core) + end + + operations = Faker::Base.shuffle(records).flat_map do |record| + event = ElasticGraph::Indexer::TestSupport::Converters.upsert_event_for(ElasticGraph::Support::HashUtil.stringify_keys(record)) + update_target = indexer + .schema_artifacts + .runtime_metadata + .object_types_by_name + .fetch(event.fetch("type")) + .update_targets + .find { |t| t.type == event.fetch("type") } + + indexer.datastore_core.index_definitions_by_graphql_type.fetch(event.fetch("type")).map do |index_def| + if !index_def.name.include?(unique_index_name) && index_def.rollover_index_template? + expect(index_def.frequency).to eq(:yearly), + "Expected #{index_def} to have :yearly rollover frequency, but had #{index_def.frequency}. " \ + ":yearly frequency is required when indexing documents so that the set of indices is deterministic " \ + "and consistent--we don't want individual tests to dynamically create indices which could interact " \ + "with later tests that run." + end + + ElasticGraph::Indexer::Operation::Update.new( + event: event, + prepared_record: indexer.record_preparer_factory.for_latest_json_schema_version.prepare_for_index( + event.fetch("type"), + event.fetch("record") + ), + destination_index_def: index_def, + update_target: update_target, + doc_id: event.fetch("id"), + destination_index_mapping: indexer.schema_artifacts.index_mappings_by_index_def_name.fetch(index_def.name) + ) + end + end + + # we `refresh: true` so the newly indexed records are immediately available in the search index, + # so our tests can deterministically search for them. + indexer.datastore_router.bulk(operations, refresh: true) + records + end + + def edges_of(*edges) + {"edges" => edges} + end + + def node_of(*args, **options) + {"node" => string_hash_of(*args, **options)} + end + + def string_hash_of(source_hash, *direct_fields, **fields_with_values) + source_hash = ElasticGraph::Support::HashUtil.stringify_keys(source_hash) + + {}.tap do |hash| + direct_fields.each do |field| + hash[field.to_s] = source_hash.fetch(field.to_s) + end + + fields_with_values.each do |key, value| + hash[key.to_s] = value + end + end + end + + def query_datastore(cluster_name, n) + change { datastore_requests(cluster_name).count }.by(n).tap do |matcher| + matcher.extend QueryDatastoreMatcherFluency + end + end + + def expect_to_have_routed_to_shards_with(cluster_name, *index_routing_value_pairs) + expect(performed_search_metadata(cluster_name).last(index_routing_value_pairs.size).map { |m| [m["index"].gsub("_camel", ""), m["routing"]] }).to eq(index_routing_value_pairs) + end + + def expect_to_have_excluded_indices(cluster_name, *excluded_indices_for_last_n_queries) + expect(indices_excluded_from_searches(cluster_name).last(excluded_indices_for_last_n_queries.size)).to eq(excluded_indices_for_last_n_queries) + end + + def make_datastore_calls(cluster_name, *methods_and_paths) + datastore_requests(cluster_name).clear + change { datastore_requests(cluster_name).map(&:description) }.from([]) + end + + def make_no_datastore_calls(cluster_name) + maintain { datastore_requests(cluster_name).map(&:description) } + end + + def make_no_datastore_write_calls(cluster_name) + maintain { datastore_write_requests(cluster_name).map(&:description) } + end + + def make_datastore_write_calls(cluster_name, *request_descriptions) + datastore_requests(cluster_name).clear + change { datastore_write_requests(cluster_name).map(&:description) } + .from([]) + .to(a_collection_containing_exactly(*request_descriptions)) + end + + def index_records(*records) + events = ElasticGraph::Indexer::TestSupport::Converters.upsert_events_for_records(records) + indexer.processor.process(events, refresh_indices: true) + events + end + + # Helper method that forces the `known_related_query_rollover_indices` and `searches_could_hit_incomplete_docs?` + # to be computed and cached. This is useful since some tests strictly verify what datastore requests are + # made and `known_related_query_rollover_indices` is called as part of preparing to query the datastore. Since + # it caches the result it can non-determnistically trigger a new datastore request in the middle of a test + # that is unexpected. We can use this in such a test to make it deterministic. + def pre_cache_index_state(graphql) + graphql.datastore_core.index_definitions_by_name.values.each do |i| + # :nocov: -- which side of the conditional is executed depends on the order the tests run in. + i.remove_instance_variable(:@known_related_query_rollover_indices) if i.instance_variable_defined?(:@known_related_query_rollover_indices) + i.remove_instance_variable(:@search_could_hit_incomplete_docs) if i.instance_variable_defined?(:@search_could_hit_incomplete_docs) + # :nocov: + + i.known_related_query_rollover_indices + i.searches_could_hit_incomplete_docs? + end + end +end + +module QueryDatastoreMatcherFluency + def times + self + end + + def time + self + end +end + +module DatastoreSpecSupport + # this method must be prepended so that we can force `main_datastore_client` so + # that any call to `build_datastore_core` in groups tagged with `:uses_datastore + # uses our configured datastore client. + def build_datastore_core(**options) + clients_by_name = options.fetch(:clients_by_name) do + { + "main" => main_datastore_client, + "other1" => other1_datastore_client, + "other2" => other2_datastore_client, + "other3" => other3_datastore_client + } + end + + super(clients_by_name: clients_by_name, **options) + end +end + +RSpec.configure do |config| + curl_output = `curl -is #{datastore_url}` + version = nil + backend = nil + + # :nocov: -- only executed when the datastore isn't running. + unless /200 OK/.match?(curl_output) + abort <<~EOS + The datastore does not appear to be running at `#{datastore_url}`. Correct this by running one of these: + + - bundle exec rake elasticsearch:test:boot + - bundle exec rake opensearch:test:boot + + ...and then try running the test suite again. + EOS + end + # :nocov: + + version_info = JSON.parse(curl_output.sub(/\A[^{]+/, "")).fetch("version") + version = version_info.fetch("number") + backend = version_info.fetch("distribution") { "elasticsearch" }.to_sym + require "elastic_graph/#{backend}/client" + + # Force the datastore backend used for this test suite run, so that it matches the datastore that is running. + ElasticGraph::CommonSpecHelpers.datastore_backend = backend + + config.before(:suite) do |ex| + datastore_state = ElasticGraph::ClusterConfigurationManager + .new(version: version, datastore_backend: backend) + .manage_cluster + + # :nocov: -- only executes if VCR is loaded, which is optional + if defined?(::VCR) + # Add suffix to the VCR cassette library directory based on the state of the datastore + # index configuration. This ensures that we don't playback VCR cassettes that were recorded + # against a datastore with a different configuration. If we allowed that kind of playback, + # it could lead to false confidence, where the tests pass because of responses recorded against + # a different datastore configuration, but do not pass against the configuration we are + # currently using. + VCR.configuration.cassette_library_dir += "/#{Digest::MD5.hexdigest(datastore_state)[0..7]}" + + puts "Using VCR cassette directory: #{VCR.configuration.cassette_library_dir}." + end + # :nocov: + end + + DatastoreSpecSupport.module_eval do + define_method(:datastore_backend) { backend } + define_method(:datastore_version) { version } + + define_method :manage_cluster_for do |**args| + ElasticGraph::ClusterConfigurationManager.new( + version: version, + datastore_backend: datastore_backend, + **args + ).manage_cluster + end + + define_method :new_datastore_client do |name| # use `define_method` so we have access to `datastore_url` and `datastore_logs` locals. + # :nocov: -- on a given test run only one side of this ternary gets covered. + client_class = (datastore_backend == :opensearch) ? ElasticGraph::OpenSearch::Client : ElasticGraph::Elasticsearch::Client + # :nocov: + + client_class.new(name, faraday_adapter: :httpx, url: datastore_url, logger: ElasticGraph::SplitLogger.new(logger, Logger.new(datastore_logs))) do |conn| + conn.use DisallowUnsupportedAWSOperations + conn.use RequestTracker, datastore_requests_by_cluster_name[name] + end + end + end + + config.prepend DatastoreSpecSupport, :uses_datastore + config.include_context "datastore support", :uses_datastore + + # The datastore is quite slow in tests--it takes a couple seconds to store _anything_ + # in it within a test, for reasons I don't understand. Luckily, it speaks HTTP, so we can + # use VCR to cache responses and speed things up. + # + # Here we hook up VCR to automatically wrap any example tagged with `:uses_datastore` + # so that any examples that use the datastore automatically get a speed up when you + # re-run them. + # + # See `support/vcr.rb` for more on the VCR setup. + config.define_derived_metadata(:uses_datastore) do |meta| + # Note: we MUST consider the `body` when matching requests, because the body is + # part of the core identity of requests to the datastore. + meta[:vcr] = {match_requests_on: [:method, :uri, :body_ignoring_bulk_version]} unless meta.key?(:vcr) + meta[:builds_indexer] = true + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/validate_graphql_schemas.rb b/spec_support/lib/elastic_graph/spec_support/validate_graphql_schemas.rb new file mode 100644 index 00000000..b0e3b8ec --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/validate_graphql_schemas.rb @@ -0,0 +1,58 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# During the upgrade of the GraphQL gem from version 2.0.15 to 2.0.16, we discovered that +# some of the schemas generated in our tests were not parseable by the GraphQL gem. Allowing +# invalid GraphQL schemas can hide latent issues (or lead to issues later on when we are forced +# to correct it) so we'd like to prevent invalid schemas from being generated in the first place. +# +# Here we define a method that uses the GraphQL gem to enforce the validity of our generated +# schemas. However, parsing all our test schemas makes our tests 2-3 times slower. We don't +# want to slow down every local test run to add this validation, so it's something that you can +# opt in to via the `VALIDATE_GRAPHQL_SCHEMAS` env var. We also pass this env var from our CI +# build where it's ok if the test suite is slower. +# +# :nocov: -- only one of the two branches gets run on any test run. +return unless ENV["VALIDATE_GRAPHQL_SCHEMAS"] + +require "elastic_graph/schema_definition/test_support" +require "graphql" + +module ElasticGraph + module ValidateGraphQLSchemas + def define_schema_with_schema_elements(...) + super.tap do |results| + ValidateGraphQLSchemas.validate_graphql_schema!(results) + end + end + + # Hook in to the API tests use to define schemas so that we can automatically validate each schema. + SchemaDefinition::TestSupport.prepend(self) + + def self.validate_graphql_schema!(results) + # Allow an example to opt-out of this validation by tagging it with `:dont_validate_graphql_schema`. + # This can be useful when an example defines an intentionally invalid schema. + rspec_example_meta = ::RSpec.current_example&.metadata || {} + return if rspec_example_meta[:dont_validate_graphql_schema] + + ::GraphQL::Schema.from_definition(results.graphql_schema_string) + rescue ::ElasticGraph::Errors::Error => e + raise e # re-raise intentional errors raised by ElasticGraph itself. + rescue => e + raise(::RuntimeError, <<~EOS, e.backtrace) + This test generated SDL that can't be parsed by the GraphQL gem. The error[^1] is shown below. + Note that the extra GraphQL gem parsing validation is not applied by default when you run tests locally. + The extra validation runs on CI (where we are OK with the slow down that produces), and you can opt into + it by passing `VALIDATE_GRAPHQL_SCHEMAS=1` when running your tests. + + [^1]: #{e.message} + EOS + end + end +end +# :nocov: diff --git a/spec_support/lib/elastic_graph/spec_support/vcr.rb b/spec_support/lib/elastic_graph/spec_support/vcr.rb new file mode 100644 index 00000000..7412788e --- /dev/null +++ b/spec_support/lib/elastic_graph/spec_support/vcr.rb @@ -0,0 +1,157 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# On CI, we do want to be able to avoid loading VCR (we care more about accuracy than speed), so +# we support using `NO_VCR=1 rspec` to skip VCR. +# :nocov: -- we avoid loading/using VCR when these ENV vars are set +return if ENV["NO_VCR"] + +require "vcr" +require "rspec/retry" +require "method_source" # needed by `aggregate_failures_meta_value_for` method below. + +module VCRSupport + extend self + + def aggregate_failures_meta_value_for(meta) + !(meta[:uses_datastore] && meta[:block]&.source&.include?("raise_error")) + end + + def name_for(metadata) + description = metadata[:description] + + example_group = metadata.fetch(:example_group) do + metadata.fetch(:parent_example_group) do + return description + end + end + + [name_for(example_group), description].join("/") + end + + def ignoring_bulk_body_version(request) + return request.body unless /_bulk/.match?(request.uri) + # https://rubular.com/r/wwUZ44xMJpz2m0 + request.body.gsub(/,"version":\d+/, "") + end + + # Used to match against `VCR::Errors::UnhandledHTTPRequestError` exceptions + # and any exceptions that are caused by that type of exception (as indicated + # by its presence as the `cause` or it being mentioned in the `message). + # + # This is needed to deal with VCR errors that happen in blocks that we apply + # `raise_error` matchers too, since RSpec rescues the error and transforms it to + # an RSpec failure instead of a VCR error. + module ExceptionCausedByVCRUnhandledHTTPRequestError + def self.===(exception) + if VCR::Errors::UnhandledHTTPRequestError === exception || + exception.message.include?(VCR::Errors::UnhandledHTTPRequestError.name) + return true + end + + return self === exception.cause if exception.cause + false + end + end +end + +VCR.configure do |config| + # We use a directory in `tmp` for cassettes. We do not want them to ever be committed to + # source control. We are using VCR just to speed things up (as a smart cache) and do + # not want any tests to rely on VCR to pass. + config.cassette_library_dir = "#{ElasticGraph::CommonSpecHelpers::REPO_ROOT}/tmp/vcr_cassettes" + config.hook_into :faraday # the datastore client is built on faraday + + # Do not record when the example fails. In the past we've occasionally had + # confusing situations where we've screwed up our running datastore node + # (e.g. by deleting its working directory while its running), and when that + # happened, VCR recorded the failed response returned by the datastore. After + # restarting the datastore to fix the issue, specs continued to fail with the + # same failure because VCR recorded the temporary response error, and is playing + # it back on a later test run. This led us to scratch our heads and waste time. + # + # To avoid this situation, we want to entirely avoid recording when an example + # fails, as there is no significant benefit to doing so. Note that this line + # is only part of what makes this work; we also have to call `cassette.run_failed!` + # in our `VCR.use_cassette` block below, because RSpec handles exceptions (so it can + # print the failures, etc), which means that a failure in an example isn't propagated + # to VCR unless we manually call `cassette.run_failed!`. + config.default_cassette_options[:record_on_error] = false + + config.register_request_matcher :body_ignoring_bulk_version do |request_1, request_2| + VCRSupport.ignoring_bulk_body_version(request_1) == VCRSupport.ignoring_bulk_body_version(request_2) + end +end + +RSpec.configure do |config| + console_codes = RSpec::Core::Formatters::ConsoleCodes + + # Here we hook up rspec/retry for examples that use VCR. During playback, + # when VCR encounters a request for which it does not have a recorded response, + # a VCR::Errors::UnhandledHTTPRequestError exception will get raised. When this + # happens, we delete the cassette and try again (which will automatically re-record). + config.retry_callback = ->(ex) { + file = ex.metadata.fetch(:cassette_file) + puts console_codes.wrap( + "Got a VCR unhandled request exception. Deleting the cassette (#{file}) and rerunning the example.", + :yellow + ) + + # Work around a bug in rspec-retry. It clears state stored using `let`, but not any + # other state. Unfortunately, that allows state from the first run attempt to leak + # into the retry, which leads to bugs on retry. The simple (but hacky) fix is to + # remove all instance variables set in the scope of the example before re-running, + # with the exception of `@__memoized`, which RSpec sets eagerly (not lazily), and + # which rspec-retry handles for us. All other instance variables are set while the + # example is running, and removing the variable here will allow it to re-evaluate + # to a new value as needed when the example re-runs. + ex.example_group_instance.instance_eval do + instance_variables.each do |ivar| + next if ivar == :@__memoized + remove_instance_variable(ivar) + end + end + + File.delete(file) + + # Examples tagged with `:in_temp_dir` get run in a temporary directory using an `around` hook. + # However, `run_with_retry` does not re-run the `around` hook for some reason (seems like a bug). + # That can cause a problem because the tmp dir won't be empty as expected when the example is + # retried. We work around that here by deleting all the files in the current temp dir in this case. + FileUtils.rm_rf(".") if ex.metadata[:in_temp_dir] && Dir.pwd.include?(Dir.tmpdir) + } + + exceptions_to_retry = [VCRSupport::ExceptionCausedByVCRUnhandledHTTPRequestError] + + config.around(:example, :vcr) do |ex| + ex.run_with_retry(retry: 2, exceptions_to_retry: exceptions_to_retry) + end + + # Note: + # + # - This uses an `around` hook instead of a `before`/`after` hook (or + # VCR's `configure_rspec_metadata!`) because some datastore HTTP requests + # are made in a `before` hook (such as clearing the indices) and we must wrap + # all datastore requests with VCR. Around hooks wrap before/after hooks. + # - This must be defined after the `around` hook with `ex.run_with_retry` defined + # above, because it is important the `run_with_retry` wraps the VCR cassette, so + # that the cassette is ejected, then re-inserted for the next attempt, rather + # than wrapping multiple attempts. + config.around(:example, :vcr, no_vcr: false) do |ex| + vcr_options = ex.metadata[:vcr].is_a?(Hash) ? ex.metadata[:vcr] : {} + VCR.use_cassette(VCRSupport.name_for(ex.metadata), **vcr_options) do |cassette| + # store the cassette file for use in our retry callback. + ex.metadata[:cassette_file] = cassette.file + ex.run + + # Notify the cassette if we got an error, so it honors `record_on_error: false`. + cassette.run_failed! if ex.exception && exceptions_to_retry.none? { |ex_class| ex_class === ex.exception } + end + end +end +# :nocov: diff --git a/spec_support/spec_helper.rb b/spec_support/spec_helper.rb new file mode 100644 index 00000000..ef24be90 --- /dev/null +++ b/spec_support/spec_helper.rb @@ -0,0 +1,453 @@ +# Copyright 2024 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# Put the lib dir of `spec_support` on the load path so we can load things from it. +$LOAD_PATH.unshift("#{__dir__}/lib") + +# For simplecov to measure coverage of all files, it MUST be loaded and started before we load any of our +# files (so this must be the first thing here!). It makes our tests a bit slower, so it's setup as an +# "opt-in" feature of our test suite. To enable it, run the test suite with the `COVERAGE` ENV var set, e.g.: +# +# ``` +# $ COVERAGE=1 bundle exec rspec path/to/gem/spec +# ``` +require "elastic_graph/spec_support/enable_simplecov" if ENV["COVERAGE"] + +# super_diff generally provides much better diffs than RSpec provides on its own. However, sometimes it makes the failure output worse, +# and we've also run into bugs with it. We've been impacted by all of these issues: +# +# https://github.com/mcmire/super_diff/issues/160 +# https://github.com/mcmire/super_diff/issues/250 +# https://github.com/mcmire/super_diff/issues/252 +# https://github.com/mcmire/super_diff/issues/253 +# +# So for now we keep super_diff disabled by default. But it's available to opt into by passing `SUPERDIFF=1` at the command line. +# This can be useful when dealing with a spec with hard-to-parse failure output--try it with `SUPERDIFF=1` and see if it helps. +# +# :nocov: -- only one side of this conditional gets executed on any given test run. +if ENV["SUPERDIFF"] + require "super_diff/rspec" + + SuperDiff.configure do |config| + config.actual_color = :green + config.expected_color = :red + config.border_color = :yellow + config.header_color = :yellow + end +end +# :nocov: + +# This file was generated by the `rspec --init` command. Conventionally, all +# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. +# The generated `.rspec` file contains `--require spec_helper` which will cause +# this file to always be loaded, without a need to explicitly require it in any +# files. +# +# Given that it is always loaded, you are encouraged to keep this file as +# light-weight as possible. Requiring heavyweight dependencies from this file +# will add to the boot time of your test suite on EVERY test run, even for an +# individual file that may not need all of that loaded. Instead, consider making +# a separate helper file that requires the additional dependencies and performs +# the additional setup, and require it from the spec files that actually need +# it. +# +# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration +RSpec.configure do |config| + # We don't want to skip the majority of the tests due to misuse of `fit` + if ENV["CI"] + config.before(:example, :focus) do + abort "Focusing examples in a CI environment is not allowed. Was `fit`, `fdescribe`, or `fcontext` left somewhere?" + end + end + + # We intentionally use `abort` in some places, but do not want the RSpec runner to abort, so here we rescue that + # exception and fail with a clear error indicating `abort_with` should be used. + config.around(:example) do |ex| + ex.run + rescue ::SystemExit => e + # :nocov: -- we only get here when we forget to use `abort_with`, so this is not covered on a normal, passing test suite run. + fail <<~EOS + The example attempted to exit with the following `SystemExit` error. If the implementation uses `abort` to cleanly notify the user of an issue, use the `abort_with` matcher to specify it in the spec. + #{"-" * 120} + + #{e.message} + + #{e.backtrace.join("\n")}" + EOS + # :nocov: + end + + # rspec-expectations config goes here. You can use an alternate + # assertion/expectation library such as wrong or the stdlib/minitest + # assertions if you prefer. + config.expect_with :rspec do |expectations| + # This option will default to `true` in RSpec 4. It makes the `description` + # and `failure_message` of custom matchers include text for helper methods + # defined using `chain`, e.g.: + # be_bigger_than(2).and_smaller_than(4).description + # # => "be bigger than 2 and smaller than 4" + # ...rather than: + # # => "be bigger than 2" + expectations.include_chain_clauses_in_custom_matcher_descriptions = true + + # RSpec truncates the output when it formats our arbitrary objects in its + # failure messages to ensure there isn't a giant wall of text. It truncates + # it at a couple hundered characters, which is shorter than we'd like and + # omits useful detail so here we make it larger. + expectations.max_formatted_output_length = 10_000 + end + + # rspec-mocks config goes here. You can use an alternate test double + # library (such as bogus or mocha) by changing the `mock_with` option here. + config.mock_with :rspec do |mocks| + # Prevents you from mocking or stubbing a method that does not exist on + # a real object. This is generally recommended, and will default to + # `true` in RSpec 4. + mocks.verify_partial_doubles = true + end + + %i[unit integration acceptance].each do |type| + config.define_derived_metadata(file_path: %r{/spec/#{type}/}) do |meta| + meta[:type] ||= type + end + end + + # This turns on the aggregate_failures feature of RSpec 3.3 by default. + # + # Within the aggregate_failures block, expectations failures do not cause + # the example to abort. Instead, a single aggregate exception will be raised at + # the end containing multiple sub-failures which RSpec will format nicely for you. + # + # The unless meta.key?(:aggregate_failures) bit allows you to opt out individual + # examples or groups by tagging them with aggregate_failures: false. + # + # See more here: http://rspec.info/blog/2015/06/rspec-3-3-has-been-released/ + config.define_derived_metadata do |meta| + # We generally want to set `aggregate_failures: true` for nicer RSpec output when + # an example has multiple expectations. However, we do not want to set it to `true` + # in a few cases: + # + # - When the example has explicitly set `aggregate_failures: false`. + # - When dealing with an example that accesses the datastore and uses and `raise_error`. + # We use rspec-retry to auto-retry examples that fail due to VCR::Errors::UnhandledHTTPRequestError. + # VCR is always enabled for examples that set `:uses_datastore`. If a VCR error is raised in a + # `raise_error` block, it gets caught and RSpec notifies of the failed expectation. When + # `aggregate_failures` is set to `true`, RSpec notifies failures at the end of the example, after + # rspec-retry checks the example exception to see if it should retry. To avoid problems here, we need + # to set `aggregate_failures` to `false` in that case. + unless meta.key?(:aggregate_failures) + # example groups have a `:parent_example_group` key instead of `:example_group` + # And we only want to set `aggregate_failures` on examples. + if meta.key?(:example_group) + meta[:aggregate_failures] = !defined?(::VCRSupport) || ::VCRSupport.aggregate_failures_meta_value_for(meta) + end + end + end + + # This option will default to `:apply_to_host_groups` in RSpec 4 (and will + # have no way to turn it off -- the option exists only for backwards + # compatibility in RSpec 3). It causes shared context metadata to be + # inherited by the metadata hash of host groups and examples, rather than + # triggering implicit auto-inclusion in groups with matching metadata. + config.shared_context_metadata_behavior = :apply_to_host_groups + + # This allows you to limit a spec run to individual examples or groups + # you care about by tagging them with `:focus` metadata. When nothing + # is tagged with `:focus`, all examples get run. RSpec also provides + # aliases for `it`, `describe`, and `context` that include `:focus` + # metadata: `fit`, `fdescribe` and `fcontext`, respectively. + config.filter_run_when_matching :focus + + # Allows RSpec to persist some state between runs in order to support + # the `--only-failures` and `--next-failure` CLI options. We recommend + # you configure your source control system to ignore this file. + config.example_status_persistence_file_path = "#{__dir__}/../tmp/rspec/stats.txt" + + # Limits the available syntax to the non-monkey patched syntax that is + # recommended. For more details, see: + # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/ + # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ + # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode + config.disable_monkey_patching! + + # Many RSpec users commonly either run the entire suite or an individual + # file, and it's useful to allow more verbose output when running an + # individual spec file. + # Use the documentation formatter for detailed output when running one + # file, or the progress formatter when running many files. The user can + # still pick a formatter directly via command-line flag. + config.default_formatter = config.files_to_run.one? ? "doc" : "progress" + + # Print the slowest examples and example groups at the + # end of the spec run, to help surface which specs are running + # particularly slow. If we are running only one file, cut the + # number we print to 3. + config.profile_examples = config.files_to_run.one? ? 3 : 10 + + # Run specs in random order to surface order dependencies. If you find an + # order dependency and want to debug it, you can fix the order by providing + # the seed, which is printed after each run. + # --seed 1234 + config.order = :random + + # Seed global randomization in this process using the `--seed` CLI option. + # Setting this allows you to use `--seed` to deterministically reproduce + # test failures related to randomization by passing the same `--seed` value + # as the one that triggered the failure. + Kernel.srand config.seed + + config.raise_on_warning = true + config.raise_errors_for_deprecations! + + config.when_first_matching_example_defined(:uses_datastore) { require "elastic_graph/spec_support/uses_datastore" } + config.when_first_matching_example_defined(:builds_admin) { require "elastic_graph/spec_support/builds_admin" } + config.when_first_matching_example_defined(:builds_datastore_core) { require "elastic_graph/spec_support/builds_datastore_core" } + config.when_first_matching_example_defined(:builds_indexer) { require "elastic_graph/spec_support/builds_indexer" } + config.when_first_matching_example_defined(:builds_graphql) { require "elastic_graph/spec_support/builds_graphql" } + config.when_first_matching_example_defined(:capture_logs) { require "elastic_graph/spec_support/logging" } + config.when_first_matching_example_defined(:factories) { require "elastic_graph/spec_support/factories" } + config.when_first_matching_example_defined(:rake_task) { require "elastic_graph/spec_support/rake_task" } + config.when_first_matching_example_defined(:stub_datastore_client) { require "elastic_graph/spec_support/stub_datastore_client" } + config.when_first_matching_example_defined(:vcr) { require "elastic_graph/spec_support/vcr" } + + config.when_first_matching_example_defined(:in_temp_dir) { require "tmpdir" } + config.around(:example, :in_temp_dir) do |ex| + Dir.mktmpdir { |tmp_dir| Dir.chdir(tmp_dir, &ex) } + end + + # Here we define support for `:no_vcr` which can be used to tag groups or examples where we don't + # want VCR to be used for some reason. `no_vcr: false` is added by default as metadata to every + # example (so that we can filter on `:vcr, no_vcr: false` in `spec_support/vcr.rb`), but when + # an example is tagged with `:no_vcr` we disable VCR while the test runs. + config.around(:example, :no_vcr) { |ex| without_vcr(&ex) } + config.define_derived_metadata do |meta| + meta[:no_vcr] = false unless meta.key?(:no_vcr) + end + + config.when_first_matching_example_defined(type: :acceptance) do + config.mock_with :rspec do |mocks| + # Guard against typos in string constant names used with verifying doubles + # when we have loaded the entire application (as indicated by the fact that + # an acceptance spec has been loaded). + mocks.verify_doubled_constant_names = true + end + end +end + +RSpec::Matchers.define_negated_matcher :exclude, :include +RSpec::Matchers.define_negated_matcher :excluding, :including +RSpec::Matchers.define_negated_matcher :a_hash_excluding, :a_hash_including +RSpec::Matchers.define_negated_matcher :a_string_excluding, :a_string_including +RSpec::Matchers.define_negated_matcher :a_collection_excluding, :a_collection_including +RSpec::Matchers.define_negated_matcher :maintain, :change +RSpec::Matchers.define_negated_matcher :raise_no_error, :raise_error +RSpec::Matchers.define_negated_matcher :avoid_outputting, :output +RSpec::Matchers.define_negated_matcher :have_never_received, :have_received + +module ElasticGraph + module CommonSpecHelpers + REPO_ROOT = File.expand_path("../..", __FILE__) + TEST_SETTINGS_FILE_TEMPLATE = File.join(REPO_ROOT, "config", "settings", "test.yaml.template") + + def self.datastore_backend=(backend) + # :nocov: -- the `raise` is only hit when the method is called more than once, which should never happen. + raise "Cannot set `datastore_backend` more than once, but it has already been set." if @test_settings_file + # :nocov: + + require "tempfile" + @test_settings_file = ::Tempfile.new("test.yaml") + template = ::File.read(TEST_SETTINGS_FILE_TEMPLATE) + ::File.write(@test_settings_file, template % {datastore_backend: backend.to_s}) + + puts "\n\n**** Using the #{backend} datastore client for this test run." + end + + # The test settings file path is always the same. However, the contents of that file must be different when we are running the + # test suite against OpenSearch vs Elasticsearch, and `datastore_backend=` generates the test settings file with appropriately + # different contents in tbhose two cases. In cases where the specific backend used really matters, `datastore_backend=` gets + # eagerly called to force which one is used. Otherwise, we lazily pick one of the two backends here. In that case, which we use + # shouldn't matter. To ensure we don't assume one or the other, we randomly pick one of the two backends. + # + # We've made `test_settings_file` a method instead of a constant so that we can lazily generate the test settings file when needed. + def self.test_settings_file + self.datastore_backend = [:elasticsearch, :opensearch].sample unless @test_settings_file + @test_settings_file + end + + def self.stock_schema_artifacts(for_context:) + @stock_schema_artifacts ||= {} + @stock_schema_artifacts[for_context] ||= begin + require "elastic_graph/schema_artifacts/from_disk" + SchemaArtifacts.from_parsed_yaml(parsed_test_settings_yaml, for_context: for_context) + end + end + + def self.parsed_test_settings_yaml + @parsed_test_settings_yaml ||= begin + require "yaml" + parsed_yaml = ::YAML.safe_load_file(test_settings_file, aliases: true) + schema_artifacts = parsed_yaml.fetch("schema_artifacts") + schema_artifacts_directory = schema_artifacts.fetch("directory") + + # To allow the spec suite of each gem to pass when run from either the repository root or + # from the gem directory, we need to replace the schema artifacts directory relative path + # with an absolute path. + parsed_yaml.merge( + "schema_artifacts" => schema_artifacts.merge( + "directory" => schema_artifacts_directory.sub("config", File.join(REPO_ROOT, "config")) + ) + ) + end + end + + # Like `raise_error` matcher but for use when the implementation code uses `abort` instead of `raise`. Note: + # + # - `abort` prints the exception message to `stderr` in addition to including it in the raised `SystemExit` exception. + # We don't want it printed in our spec output so we silence that by using the `output` matcher. + # - Care must be taken when passing a block to this method, and curly braces should be used. + # If you do `expect { }.to abort_with do |error|...`, the block binds to the `to` method rather than + # `abort_with` and the block is silently ignored. When curly braces are used instead, the block binds to + # the `abort_with` method and works as expected. + def abort_with(*args, &block) + raise_error(::SystemExit, *args, &block).and output(/./).to_stderr + end + + def stock_schema_artifacts(for_context:) + CommonSpecHelpers.stock_schema_artifacts(for_context: for_context) + end + + def parsed_test_settings_yaml + CommonSpecHelpers.parsed_test_settings_yaml + end + + module_function + + def expect_to_return_non_nil_values_from_all_attributes(object) + exposed_attributes = object + .public_methods(false) + .select { |m| object.method(m).arity.zero? } + + exposed_attributes.each do |attr| + expect(object.public_send(attr)).not_to be_nil, "Expected `#{object}.#{attr}` not to be nil but was" + end + end + + def a_boolean + an_object_eq_to(true).or an_object_eq_to(false) + end + + # :nocov: -- any given run either runs with VCR loaded or w/o it loaded, and won't cover all branches here. + def without_vcr + return yield unless defined?(::VCR) # since we support running w/o VCR. + VCR.eject_cassette + VCR.turned_off { yield } + end + # :nocov: + + def generate_schema_artifacts( + schema_element_name_form: :snake_case, + schema_element_name_overrides: {}, + derived_type_name_formats: {}, + enum_value_overrides_by_type: {} + ) + require "elastic_graph/schema_definition/test_support" + require "stringio" + + output = ::StringIO.new # to silence warnings. + ::ElasticGraph::SchemaDefinition::TestSupport.define_schema( + schema_element_name_form: schema_element_name_form, + schema_element_name_overrides: schema_element_name_overrides, + derived_type_name_formats: derived_type_name_formats, + enum_value_overrides_by_type: enum_value_overrides_by_type, + output: output + ) do |schema| + if block_given? + yield schema + else + schema.as_active_instance { load File.join(REPO_ROOT, "config", "schema.rb") } + end + end + end + + # Helper method to assist in building a `Config::IndexDefinition`. Provides defaults so that + # a test can just specify the details that matter to it. + def config_index_def_of( + query_cluster: "main", + index_into_clusters: ["main"], + ignore_routing_values: [], + setting_overrides: {}, + setting_overrides_by_timestamp: {}, + custom_timestamp_ranges: [], + use_updates_for_indexing: true + ) + require "elastic_graph/datastore_core/configuration/index_definition" + DatastoreCore::Configuration::IndexDefinition.from( + query_cluster: query_cluster, + index_into_clusters: index_into_clusters, + ignore_routing_values: ignore_routing_values, + setting_overrides: setting_overrides, + setting_overrides_by_timestamp: setting_overrides_by_timestamp, + custom_timestamp_ranges: custom_timestamp_ranges, + use_updates_for_indexing: use_updates_for_indexing + ) + end + + def cluster_of(url: "https://some/url", backend: nil, settings: {}) + backend ||= respond_to?(:datastore_backend) ? datastore_backend : :elasticsearch + + require "elastic_graph/datastore_core/configuration/cluster_definition" + DatastoreCore::Configuration::ClusterDefinition.from_hash({ + "url" => url, + "backend" => backend.to_s, + "settings" => settings + }) + end + + def with_env(overrides) + env = ENV.to_h + ENV.update(overrides) + + begin + yield + ensure + ENV.replace(env) + end + end + end +end + +RSpec.configure do |c| + c.define_derived_metadata(type: :unit) { |m| m[:stub_datastore_client] = true unless m.key?(:stub_datastore_client) } + c.include ElasticGraph::CommonSpecHelpers +end + +# If we're using the parallel test runner, we have to load our adapters which patch various bits of behavior +# to make it safe to run tests in parallel. +require "elastic_graph/spec_support/parallel_spec_runner" if defined?(::Flatware) + +# optimize_graphql.rb makes our tests faster but also makes them slightly less "realistic" +# since the optimization that implements won't be in place in production. Locally, we care +# a lot about test speed and are willing to make that tradeoff, but on CI we optimize for +# greater "test accuracy" even if it makes the tests a bit slower, so we don't want this +# loaded there. +require "elastic_graph/spec_support/optimize_graphql" unless ENV["CI"] +require "elastic_graph/spec_support/validate_graphql_schemas" +require "pathname" + +# Identify the gem directories from which we are running specs... +gem_dirs = RSpec.configuration + .files_to_run + .filter_map { |f| Pathname(f).ascend.find { |p| p.glob("*.gemspec").any? } } + .map(&:expand_path) + .uniq + +# ...and then load the `spec_helper` file for each, and put `spec` on the load path. +gem_dirs.each do |gem_dir| + $LOAD_PATH.unshift((gem_dir / "spec").to_s) + require((gem_dir / "spec" / "spec_helper.rb").to_s) +end diff --git a/spec_support/subdir_dot_rspec b/spec_support/subdir_dot_rspec new file mode 100644 index 00000000..eadcd832 --- /dev/null +++ b/spec_support/subdir_dot_rspec @@ -0,0 +1,6 @@ +# This file lives in `spec_support/subdir_dot_rspec` but is symlinked from `.rspec` +# in each of the `elasticgraph-*` gem directories. +# +# Here we just load the common `spec_helper` file, for anytime we run specs from +# a specific gem directory. +--require ../spec_support/spec_helper