index resources #400
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: index resources | |
| on: | |
| # Run at ~4am UTC time which is (± an hour) 4am UK, 5am Switzerland, midnight | |
| # US east coast, 9pm US west coast so that for most users (and most | |
| # developers) the index regenerates overnight | |
| schedule: | |
| - cron: '0 4 * * *' | |
| # Manually triggered using GitHub's UI | |
| workflow_dispatch: | |
| workflow_call: | |
| inputs: | |
| is-workflow-call: | |
| description: | | |
| To distinguish workflow_call from other events. | |
| Unable to use `GITHUB_EVENT_NAME` because it points to the caller workflow's github context | |
| type: boolean | |
| default: true | |
| required: false | |
| defaults: | |
| run: | |
| # This is the same as GitHub Action's `bash` keyword as of 20 June 2023: | |
| # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell | |
| # | |
| # Completely spelling it out here so that GitHub can't change it out from under us | |
| # and we don't have to refer to the docs to know the expected behavior. | |
| shell: bash --noprofile --norc -eo pipefail {0} | |
| jobs: | |
| build-ref-matrix: | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| ref-matrix: ${{ steps.ref-matrix.outputs.ref-matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version-file: 'package.json' | |
| - id: ref-matrix | |
| name: Create ref matrix | |
| env: | |
| HEROKU_TOKEN: ${{ secrets.HEROKU_TOKEN_READ_PROTECTED }} | |
| IS_WORKFLOW_CALL: ${{ inputs.is-workflow-call || false }} | |
| # Ensure that we are using the production configs to match Heroku apps | |
| CONFIG_FILE: ./env/production/config.json | |
| run: | | |
| # Assign to variable before output to ensure error exit code propagates to GH Action job | |
| ref_matrix=$(./scripts/get-resource-index-ref-matrix) | |
| echo "ref-matrix=$ref_matrix" | tee -a "$GITHUB_OUTPUT" | |
| rebuild-index: | |
| needs: [build-ref-matrix] | |
| strategy: | |
| matrix: | |
| include: ${{ fromJson(needs.build-ref-matrix.outputs.ref-matrix) }} | |
| # Only allow one run of the job per resource index | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ matrix.resource_index }} | |
| env: | |
| RESOURCE_INDEX: ${{ matrix.resource_index }} | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| id-token: write # needed to interact with GitHub's OIDC Token endpoint | |
| contents: read | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ matrix.ref }} | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version-file: 'package.json' | |
| - run: npm ci | |
| - uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: us-east-1 | |
| role-to-assume: arn:aws:iam::827581582529:role/GitHubActionsRoleResourceIndexer | |
| - name: Rebuild the index | |
| run: | | |
| node resourceIndexer/main.js \ | |
| --gzip --output resources.json.gz \ | |
| --resourceTypes dataset --collections core staging | |
| - name: Upload the new index, overwriting the existing index | |
| if: ${{ startsWith(env.RESOURCE_INDEX, 's3://') }} | |
| run: | | |
| aws s3 cp resources.json.gz "$RESOURCE_INDEX" |