Skip to content

Commit

Permalink
fix up publishing
Browse files Browse the repository at this point in the history
  • Loading branch information
@emmyoop
emmyoop committed Jan 17, 2024
1 parent 661eda3 commit 7034ccc
Show file tree
Hide file tree
Showing 4 changed files with 476 additions and 149 deletions.
140 changes: 8 additions & 132 deletions .github/workflows/build_hatch.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@
# │ ├── dbt-*.whl
# └── <release_notes>.md
#
# Build artifacts get stored in S3 to a bucket with the following directory structure:
# "s3://<s3_bucket>/<org>/<repo>/<artifact_folder>/<version>/<commit>/"
#
# Notes:
# <artifact_folder> - resolves based on `test_run` and `nightly_release` inputs.
Expand All @@ -27,7 +25,6 @@
# sha: The commit to attach to this release
# version_number: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0)
# changelog_path: Path to the changelog file for release notes
# s3_bucket_name: AWS S3 bucket name
# package_test_command: Command to use to check package runs
# test_run: Test run (Bucket to upload the artifact)
# nightly_release: Identifier that this is nightly release
Expand All @@ -42,9 +39,7 @@
# Validation Checks
#
# 1. Make sure the sha has a changelog entry for this version and the version bump has been completed.
# 2. Check if build already exists in AWS s3 bucket. It will live in a bucket following the env.s3 naming convention below.
# If it does exist, upload it to the GitHub artifacts and skip the rest of the workflow.
# 3. Only upload artifacts and changelog to S3 if tests pass
# 2. Upload artifacts

name: Build

Expand All @@ -60,10 +55,6 @@ on:
changelog_path:
required: true
type: string
s3_bucket_name:
required: true
default: "core-team-artifacts"
type: string
package_test_command:
required: true
default: "dbt --version"
Expand Down Expand Up @@ -107,7 +98,6 @@ jobs:
echo The last commit sha in the release: ${{ inputs.sha }}
echo The release version number: ${{ inputs.version_number }}
echo The changelog path: ${{ inputs.changelog_path }}
echo The s3 bucket name: ${{ inputs.s3_bucket_name }}
echo The package test command: ${{ inputs.package_test_command }}
echo Test run: ${{ inputs.test_run }}
echo Nightly release: ${{ inputs.nightly_release }}
Expand All @@ -117,33 +107,6 @@ jobs:
echo Python target version: ${{ env.PYTHON_TARGET_VERSION }}
echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }}
resolve-aws-bucket:
runs-on: ubuntu-latest
outputs:
aws-s3-bucket: ${{ steps.bucket_path.outputs.path }}

steps:
- name: "Resolve S3 Bucket Path"
id: bucket_path
run: |
# Resolve folder to upload/check build artifact
artifact_folder="artifacts"
if [[ ${{ inputs.nightly_release }} == true ]]
then
artifact_folder="nightly-releases"
elif [[ ${{ inputs.test_run }} == true ]]
then
artifact_folder="artifacts_testing"
fi
# Generate path for build artifact.
# Include commit in path in case release commit gets updates on subsequent runs
bucket_path="s3://${{ inputs.s3_bucket_name }}/${{ github.repository }}/$artifact_folder/${{ inputs.version_number }}/${{ inputs.sha }}"
echo "path=$bucket_path" >> $GITHUB_OUTPUT
# Send notification
title="S3 Bucket Path"
echo "$title: $bucket_path"
echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$bucket_path"
audit-version-changelog:
# Make sure the changelog has been generated and the version is up to date
runs-on: ubuntu-latest
Expand All @@ -154,6 +117,11 @@ jobs:
with:
ref: ${{ inputs.sha }}

- name: "Set up Python & Hatch - ${{ env.PYTHON_TARGET_VERSION }}"
uses: ./.github/actions/setup-python-env
with:
python-version: "3.11"

- name: "Audit Version And Parse Into Parts"
id: semver
uses: dbt-labs/actions/[email protected]
Expand Down Expand Up @@ -190,75 +158,10 @@ jobs:
exit 1
fi
check-build-exists:
runs-on: ubuntu-latest
needs: [audit-version-changelog, resolve-aws-bucket]

outputs:
is_exists: ${{ steps.artifact_exists.outputs.is_exists }}

steps:
- name: "Configure AWS Credentials"
uses: aws-actions/configure-aws-credentials@v2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_REGION }}

- name: "Copy Artifact From S3 Via CLI"
run: |
aws s3 cp ${{ needs.resolve-aws-bucket.outputs.aws-s3-bucket }} . --recursive # since it's an entire directory
- name: "[DEBUG] Display Structure Of All Downloaded Files"
run: ls -R

- name: "Check Artifact Integrity"
id: artifact_integrity
uses: andstor/file-existence-action@v2
with:
files: "${{ inputs.changelog_path }}, dist/*.tar.gz, dist/*.whl"

# upload the files downloaded from S3 to artifacts so we don't have to keep
# downloading from S3
- name: "Upload Artifact From S3 To GitHub"
if: ${{ steps.artifact_integrity.outputs.files_exists == 'true' }}
uses: actions/upload-artifact@v3
with:
name: ${{ inputs.version_number }}
path: |
${{ inputs.changelog_path }}
dist/
if-no-files-found: error
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}

- name: "[Notification] Upload Artifact From S3 To GitHub"
if: ${{ steps.artifact_integrity.outputs.files_exists == 'true' }}
run: |
title="Artifact ${{ inputs.version_number }} uploaded from S3 To GitHub"
message="The build artifact is pulled from the S3 bucket and uploaded to the GitHub artifact storage."
echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
- name: "Set Artifact Existence For Subsequent Jobs"
id: artifact_exists
run: echo "is_exists=${{ steps.artifact_integrity.outputs.files_exists }}" >> $GITHUB_OUTPUT

skip-build:
runs-on: ubuntu-latest
needs: [check-build-exists]
if: ${{ needs.check-build-exists.outputs.is_exists == 'true' }}

steps:
- name: "Build Exists, Skip To Test"
run: |
title="Build Exists in AWS S3 bucket"
message="A build already exists for version ${{ inputs.version_number }}, skipping build job."
echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
unit:
name: Unit Test
runs-on: ubuntu-latest
needs: [audit-version-changelog, check-build-exists]
if: ${{ needs.check-build-exists.outputs.is_exists == 'false' }}
needs: [audit-version-changelog]

steps:
- name: "Checkout ${{ github.repository }} Commit ${{ inputs.sha }}"
Expand Down Expand Up @@ -298,6 +201,7 @@ jobs:
run: |
hatch build
# upload artifact in case something fails in verification so we can look at it
- name: "Upload Build Artifact - ${{ inputs.version_number }}"
uses: actions/upload-artifact@v3
with:
Expand Down Expand Up @@ -365,31 +269,3 @@ jobs:
- name: "Check source distributions"
run: |
pip freeze | grep dbt-common
upload-artifacts-aws:
runs-on: ubuntu-latest
needs: [test-build, resolve-aws-bucket]

steps:
- name: "Download Artifact ${{ inputs.version_number }}"
uses: actions/download-artifact@v3
with:
name: ${{ inputs.version_number }}
path: .

- name: "Display Structure Of All Downloaded Files"
run: ls -R

- name: "Configure Aws Credentials"
uses: aws-actions/configure-aws-credentials@v2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_REGION }}

- name: "Upload Artifact To S3 Via CLI"
run: |
aws s3 cp . ${{ needs.resolve-aws-bucket.outputs.aws-s3-bucket }} --recursive # since it's an entire directory
title="Artifact ${{ inputs.version_number }} uploaded to AWS S3 bucket"
message="S3 path: ${{ needs.resolve-aws-bucket.outputs.aws-s3-bucket }}"
echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
Loading

0 comments on commit 7034ccc

Please sign in to comment.