Build images for Implement Dataflow operators https://github.com/lwyszomi/airflow/pull/172 #858
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # | |
| --- | |
| name: "Build Images" | |
| run-name: > | |
| Build images for ${{ github.event.pull_request.title }} ${{ github.event.pull_request._links.html.href }} | |
| on: # yamllint disable-line rule:truthy | |
| pull_request_target: | |
| permissions: | |
| # all other permissions are set to none | |
| contents: read | |
| pull-requests: read | |
| packages: read | |
| env: | |
| ANSWER: "yes" | |
| DB_RESET: "true" | |
| VERBOSE: "true" | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_USERNAME: ${{ github.actor }} | |
| # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the | |
| # Airflow one is going to be used | |
| CONSTRAINTS_GITHUB_REPOSITORY: >- | |
| ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && | |
| secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} | |
| # This token is WRITE one - pull_request_target type of events always have the WRITE token | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}" | |
| USE_SUDO: "true" | |
| INCLUDE_SUCCESS_OUTPUTS: "true" | |
| concurrency: | |
| group: build-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-info: | |
| timeout-minutes: 10 | |
| name: "Build Info" | |
| runs-on: 'ubuntu-22.04' | |
| env: | |
| TARGET_BRANCH: ${{ github.event.pull_request.base.ref }} | |
| outputs: | |
| python-versions: "${{ steps.selective-checks.python-versions }}" | |
| upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} | |
| all-python-versions-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} | |
| default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} | |
| run-tests: ${{ steps.selective-checks.outputs.run-tests }} | |
| run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} | |
| ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} | |
| prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} | |
| cache-directive: ${{ steps.selective-checks.outputs.cache-directive }} | |
| default-branch: ${{ steps.selective-checks.outputs.default-branch }} | |
| default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} | |
| runs-on: ${{steps.selective-checks.outputs.runs-on}} | |
| is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }} | |
| is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }} | |
| is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }} | |
| is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} | |
| is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} | |
| is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} | |
| target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha || | |
| github.event.pull_request.head.sha || | |
| github.sha | |
| }}" | |
| if: github.repository == 'apache/airflow' | |
| steps: | |
| - name: Discover PR merge commit | |
| id: discover-pr-merge-commit | |
| run: | | |
| TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)" | |
| echo "TARGET_COMMIT_SHA=$TARGET_COMMIT_SHA" >> ${GITHUB_ENV} | |
| echo "target-commit-sha=${TARGET_COMMIT_SHA}" >> ${GITHUB_OUTPUT} | |
| if: github.event_name == 'pull_request_target' | |
| # The labels in the event aren't updated when re-triggering the job, So lets hit the API to get | |
| # up-to-date values | |
| - name: Get latest PR labels | |
| id: get-latest-pr-labels | |
| run: | | |
| echo -n "pull-request-labels=" >> ${GITHUB_OUTPUT} | |
| gh api graphql --paginate -F node_id=${{github.event.pull_request.node_id}} -f query=' | |
| query($node_id: ID!, $endCursor: String) { | |
| node(id:$node_id) { | |
| ... on PullRequest { | |
| labels(first: 100, after: $endCursor) { | |
| nodes { name } | |
| pageInfo { hasNextPage endCursor } | |
| } | |
| } | |
| } | |
| }' --jq '.data.node.labels.nodes[]' | jq --slurp -c '[.[].name]' >> ${GITHUB_OUTPUT} | |
| if: github.event_name == 'pull_request_target' | |
| # Retrieve it to be able to determine which files has changed in the incoming commit of the PR | |
| # we checkout the target commit and its parent to be able to compare them | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.TARGET_COMMIT_SHA }} | |
| persist-credentials: false | |
| fetch-depth: 2 | |
| - name: "Setup python" | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: "Retrieve defaults from branch_defaults.py" | |
| # We cannot "execute" the branch_defaults.py python code here because that would be | |
| # a security problem (we cannot run any code that comes from the sources coming from the PR. | |
| # Therefore, we extract the branches via embedded Python code | |
| # we need to do it before next step replaces checked-out breeze and scripts code coming from | |
| # the PR, because the PR defaults have to be retrieved here. | |
| id: defaults | |
| run: | | |
| python - <<EOF >> ${GITHUB_ENV} | |
| from pathlib import Path | |
| import re | |
| import sys | |
| DEFAULTS_CONTENT = Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text() | |
| BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$' | |
| CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "(.*)"$' | |
| branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) | |
| constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) | |
| output = f""" | |
| DEFAULT_BRANCH={branch} | |
| DEFAULT_CONSTRAINTS_BRANCH={constraints_branch} | |
| """.strip() | |
| print(output) | |
| # Stdout is redirected to GITHUB_ENV but we also print it to stderr to see it in ci log | |
| print(output, file=sys.stderr) | |
| EOF | |
| - name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there. | |
| uses: actions/checkout@v4 | |
| with: | |
| path: "target-airflow" | |
| ref: ${{ github.base_ref }} | |
| persist-credentials: false | |
| - name: > | |
| Override "scripts/ci", "dev" and ".github/actions" with the target branch | |
| so that the PR does not override it | |
| # We should not override those scripts which become part of the image as they will not be | |
| # changed in the image built - we should only override those that are executed to build | |
| # the image. | |
| shell: bash | |
| run: | | |
| rm -rfv "scripts/ci" | |
| mv -v "target-airflow/scripts/ci" "scripts" | |
| rm -rfv "dev" | |
| mv -v "target-airflow/dev" "." | |
| rm -rfv ".github/actions" | |
| mv -v "target-airflow/.github/actions" ".github" | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: Selective checks | |
| id: selective-checks | |
| env: | |
| PR_LABELS: "${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}" | |
| COMMIT_REF: "${{ env.TARGET_COMMIT_SHA }}" | |
| VERBOSE: "false" | |
| run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} | |
| - name: env | |
| run: printenv | |
| env: | |
| PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pull-request-labels }} | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| build-ci-images: | |
| permissions: | |
| contents: read | |
| packages: write | |
| timeout-minutes: 80 | |
| name: > | |
| Build CI images ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info] | |
| if: | | |
| needs.build-info.outputs.ci-image-build == 'true' && | |
| github.event.pull_request.head.repo.full_name != 'apache/airflow' && | |
| github.repository == 'apache/airflow' | |
| env: | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| RUNS_ON: "${{ needs.build-info.outputs.runs-on }}" | |
| BACKEND: sqlite | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.build-info.outputs.target-commit-sha }} | |
| persist-credentials: false | |
| - name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there. | |
| uses: actions/checkout@v4 | |
| with: | |
| path: "target-airflow" | |
| ref: ${{ github.base_ref }} | |
| persist-credentials: false | |
| - name: > | |
| Override "scripts/ci", "dev" and ".github/actions" with the target branch | |
| so that the PR does not override it | |
| # We should not override those scripts which become part of the image as they will not be | |
| # changed in the image built - we should only override those that are executed to build | |
| # the image. | |
| shell: bash | |
| run: | | |
| rm -rfv "scripts/ci" | |
| mv -v "target-airflow/scripts/ci" "scripts" | |
| rm -rfv "dev" | |
| mv -v "target-airflow/dev" "." | |
| rm -rfv ".github/actions" | |
| mv -v "target-airflow/.github/actions" ".github" | |
| - name: > | |
| Build CI Images ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/build-ci-images | |
| env: | |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} | |
| BUILD_TIMEOUT_MINUTES: 70 | |
| - name: "Source constraints" | |
| shell: bash | |
| run: > | |
| breeze release-management generate-constraints --run-in-parallel | |
| --airflow-constraints-mode constraints-source-providers | |
| env: | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} | |
| - name: "Upload constraint artifacts" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: constraints | |
| path: ./files/constraints-*/constraints-*.txt | |
| retention-days: 7 | |
| build-prod-images: | |
| permissions: | |
| contents: read | |
| packages: write | |
| timeout-minutes: 80 | |
| name: > | |
| Build PROD images | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, build-ci-images] | |
| if: | | |
| needs.build-info.outputs.prod-image-build == 'true' && | |
| github.event.pull_request.head.repo.full_name != 'apache/airflow' && | |
| github.repository == 'apache/airflow' | |
| env: | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| RUNS_ON: "${{ needs.build-info.outputs.runs-on }}" | |
| BACKEND: sqlite | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.build-info.outputs.target-commit-sha }} | |
| persist-credentials: false | |
| - name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there. | |
| uses: actions/checkout@v4 | |
| with: | |
| path: "target-airflow" | |
| ref: ${{ github.base_ref }} | |
| persist-credentials: false | |
| - name: > | |
| Override "scripts/ci", "dev" and ".github/actions" with the target branch | |
| so that the PR does not override it | |
| # We should not override those scripts which become part of the image as they will not be | |
| # changed in the image built - we should only override those that are executed to build | |
| # the image. | |
| shell: bash | |
| run: | | |
| rm -rfv "scripts/ci" | |
| mv -v "target-airflow/scripts/ci" "scripts" | |
| rm -rfv "dev" | |
| mv -v "target-airflow/dev" "." | |
| rm -rfv ".github/actions" | |
| mv -v "target-airflow/.github/actions" ".github" | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: > | |
| Pull CI image for PROD build: | |
| ${{ steps.selective-checks.outputs.default-python-version }}:${{ env.IMAGE_TAG }}" | |
| shell: bash | |
| run: breeze ci-image pull --tag-as-latest | |
| env: | |
| PYTHON_MAJOR_MINOR_VERSION: ${{ steps.selective-checks.outputs.default-python-version }} | |
| - name: > | |
| Build PROD Images | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/build-prod-images | |
| with: | |
| build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} | |
| env: | |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} | |
| build-ci-images-arm: | |
| timeout-minutes: 50 | |
| name: "Build ARM CI images ${{needs.build-info.outputs.all-python-versions-list-as-string}}" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, build-prod-images] | |
| # We can change the job to run on ASF ARM runners and do not start our instance once we enable ASF runners | |
| if: | | |
| needs.build-info.outputs.ci-image-build == 'true' && | |
| needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' && | |
| github.event.pull_request.head.repo.full_name != 'apache/airflow' && | |
| needs.build-info.outputs.is-self-hosted-runner == 'true' && | |
| needs.build-info.outputs.is-airflow-runner == 'true' && | |
| github.repository == 'apache/airflow' | |
| env: | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| RUNS_ON: "${{ needs.build-info.outputs.runs-on }}" | |
| BACKEND: sqlite | |
| outputs: ${{toJSON(needs.build-info.outputs) }} | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.build-info.outputs.target-commit-sha }} | |
| persist-credentials: false | |
| - name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there. | |
| uses: actions/checkout@v4 | |
| with: | |
| path: "target-airflow" | |
| ref: ${{ github.base_ref }} | |
| persist-credentials: false | |
| - name: > | |
| Override "scripts/ci", "dev" and ".github/actions" with the target branch | |
| so that the PR does not override it | |
| # We should not override those scripts which become part of the image as they will not be | |
| # changed in the image built - we should only override those that are executed to build | |
| # the image. | |
| shell: bash | |
| run: | | |
| rm -rfv "scripts/ci" | |
| mv -v "target-airflow/scripts/ci" "scripts" | |
| rm -rfv "dev" | |
| mv -v "target-airflow/dev" "." | |
| rm -rfv ".github/actions" | |
| mv -v "target-airflow/.github/actions" ".github" | |
| - name: "Start ARM instance" | |
| run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: > | |
| Build ARM CI images ${{ env.IMAGE_TAG }} | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| run: > | |
| breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64" | |
| env: | |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| - name: "Stop ARM instance" | |
| run: ./scripts/ci/images/ci_stop_arm_instance.sh | |
| if: always() | |
| - name: "Fix ownership" | |
| run: breeze ci fix-ownership | |
| if: always() |