update-example-datasets #649
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: update-example-datasets | |
| on: | |
| schedule: | |
| - cron: "45 0 * * *" | |
| workflow_dispatch: {} | |
| concurrency: update-example-datasets | |
| env: | |
| # Dataset pull credentials | |
| ETHERSCAN_API_KEY: ${{ secrets.ETHERSCAN_API_KEY }} | |
| jobs: | |
| update: | |
| name: Update example datasets | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| steps: | |
| ########################################### | |
| # Install dependencies | |
| ########################################### | |
| - name: Install kamu | |
| shell: bash | |
| run: | | |
| curl -s "https://get.kamu.dev" | sh | |
| - name: Print info | |
| shell: bash | |
| run: | | |
| echo "Work dir: $PWD" | |
| echo "Podman version:" | |
| podman version | |
| echo "AWS CLI version:" | |
| aws --version | |
| echo "Kamu version:" | |
| kamu --version | |
| ########################################### | |
| # Caching | |
| ########################################### | |
| - uses: actions/cache@v3 | |
| with: | |
| path: | | |
| workspace | |
| # TODO: Currently cache action will not update the cache upon hit and there is no option to do so | |
| # Below uses a workaround suggested here: https://github.com/actions/cache/issues/342#issuecomment-673371329 | |
| key: example-datasets-st-${{ github.run_id }} | |
| restore-keys: | | |
| example-datasets-st- | |
| ########################################### | |
| # Checkout repo | |
| ########################################### | |
| - name: Checkout kamu-contrib | |
| uses: actions/checkout@v3 | |
| with: | |
| path: kamu-contrib | |
| ########################################### | |
| # Update | |
| ########################################### | |
| - uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| audience: sts.amazonaws.com | |
| aws-region: us-west-2 | |
| role-to-assume: ${{ secrets.AWS_IAM_ROLE }} | |
| - name: Init workspace from S3 | |
| shell: bash | |
| run: | | |
| mkdir -p workspace | |
| cd workspace | |
| ../kamu-contrib/.ci/example-datasets/init-from-s3.py | |
| echo "State before update:" | |
| kamu list -w | |
| - name: Update datasets | |
| shell: bash | |
| run: | | |
| cd workspace | |
| ../kamu-contrib/.ci/example-datasets/update.py | |
| echo "State after update:" | |
| kamu list -w | |
| - name: Push to S3 | |
| shell: bash | |
| run: | | |
| cd workspace | |
| ../kamu-contrib/.ci/example-datasets/push-to-s3.py |