From d4f284a31345f174ae73871985b471a16be9223a Mon Sep 17 00:00:00 2001
From: danielfromearth <daniel.kaufman@nasa.gov>
Date: Mon, 22 Apr 2024 14:23:31 -0400
Subject: [PATCH 1/2] add snyk and pypi steps to build workflow

---
 .github/workflows/build-pipeline.yml  | 324 +++++++++++++-------------
 .github/workflows/release-created.yml |  38 ++-
 .github/workflows/wait-for-pypi.py    |  46 ++++
 3 files changed, 225 insertions(+), 183 deletions(-)
 create mode 100644 .github/workflows/wait-for-pypi.py

diff --git a/.github/workflows/build-pipeline.yml b/.github/workflows/build-pipeline.yml
index 19d781c..4387a4b 100644
--- a/.github/workflows/build-pipeline.yml
+++ b/.github/workflows/build-pipeline.yml
@@ -23,172 +23,170 @@ jobs:
     secrets:
       codecov_token: ${{ secrets.CODECOV_TOKEN }}
 
-  bump_version:
+  build:
     needs: run_tests
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
-
-    - uses: actions/setup-python@v5
-      with:
-        python-version: ${{ env.PYTHON_VERSION }}
-
-    - name: Install Poetry
-      uses: abatilo/actions-poetry@v3.0.0
-      with:
-        poetry-version: ${{ env.POETRY_VERSION }}
-
-    - name: Get version
-      id: get-version
-      run: |
-        echo "current_version=$(poetry version | awk '{print $2}')" >> $GITHUB_OUTPUT
-        echo "pyproject_name=$(poetry version | awk '{print $1}')" >> $GITHUB_ENV
-
-    - name: Bump pre-alpha version
-      # If triggered by push to a feature branch
-      if: |
-        ${{ startsWith(github.ref, 'refs/heads/issue') }}      ||
-        ${{ startsWith(github.ref, 'refs/heads/dependabot/') }} ||
-        ${{ startsWith(github.ref, 'refs/heads/feature/') }}
-      run: |
-        new_ver="${{ steps.get-version.outputs.current_version }}+$(git rev-parse --short ${GITHUB_SHA})"
-        poetry version $new_ver
-        echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
-
-    - name: Bump alpha version
-      # If triggered by push to the develop branch
-      if: ${{ github.ref == 'refs/heads/develop' }}
-      run: |
-        poetry version prerelease
-        echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
-        echo "venue=sit" >> $GITHUB_ENV
-
-    - name: Bump rc version
-      # If triggered by push to a release branch
-      if: ${{ startsWith(github.ref, 'refs/heads/release/') }}
-      env:
-        # True if the version already has a 'rc' pre-release identifier
-        BUMP_RC: ${{ contains(steps.get-version.outputs.current_version, 'rc') }}
-      run: |
-        if [ "$BUMP_RC" = true ]; then
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - name: Retrieve repository
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install Poetry
+        uses: abatilo/actions-poetry@v3.0.0
+        with:
+          poetry-version: ${{ env.POETRY_VERSION }}
+      - name: Get version
+        id: get-version
+        run: |
+          echo "current_version=$(poetry version | awk '{print $2}')" >> $GITHUB_OUTPUT
+          echo "pyproject_name=$(poetry version | awk '{print $1}')" >> $GITHUB_ENV
+
+      # Bumps the version, based on which branch is the target.
+      - name: Bump pre-alpha version
+        # If triggered by push to a feature branch
+        if: |
+          ${{ startsWith(github.ref, 'refs/heads/issue') }}      ||
+          ${{ startsWith(github.ref, 'refs/heads/dependabot/') }} ||
+          ${{ startsWith(github.ref, 'refs/heads/feature/') }}
+        run: |
+          new_ver="${{ steps.get-version.outputs.current_version }}+$(git rev-parse --short ${GITHUB_SHA})"
+          poetry version $new_ver
+          echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
+      - name: Bump alpha version
+        # If triggered by push to the develop branch
+        if: ${{ github.ref == 'refs/heads/develop' }}
+        run: |
           poetry version prerelease
-        else
-          poetry version ${GITHUB_REF#refs/heads/release/}rc1
-        fi
-        echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
-        echo "venue=uat" >> $GITHUB_ENV
-
-    - name: Release version
-      # If triggered by push to the main branch
-      if: ${{ startsWith(github.ref, 'refs/heads/main') }}
-      env:
-        CURRENT_VERSION: ${{ steps.get-version.outputs.current_version }}
-        # True if the version already has a 'rc' pre-release identifier
-        BUMP_RC: ${{ contains(steps.get-version.outputs.current_version, 'rc') }}
-        # True if the version already has an 'alpha' pre-release identifier
-        BUMP_A: ${{ contains(steps.get-version.outputs.current_version, 'a') }}
-        # True if the version already has a 'beta' pre-release identifier
-        BUMP_B: ${{ contains(steps.get-version.outputs.current_version, 'b') }}
-      # Remove rc* from the end of version string
-      # The ${string%%substring} syntax below deletes the longest match of $substring from back of $string.
-      run: |
-        if [ "$BUMP_RC" = true ]; then
+          echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
+          echo "venue=sit" >> $GITHUB_ENV
+      - name: Bump rc version
+        # If triggered by push to a release branch
+        if: ${{ startsWith(github.ref, 'refs/heads/release/') }}
+        env:
+          # True if the version already has a 'rc' pre-release identifier
+          BUMP_RC: ${{ contains(steps.get-version.outputs.current_version, 'rc') }}
+        run: |
+          if [ "$BUMP_RC" = true ]; then
+            poetry version prerelease
+          else
+            poetry version ${GITHUB_REF#refs/heads/release/}rc1
+          fi
+          echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
+          echo "venue=uat" >> $GITHUB_ENV
+      - name: Release version
+        # If triggered by push to the main branch
+        if: ${{ startsWith(github.ref, 'refs/heads/main') }}
+        env:
+          CURRENT_VERSION: ${{ steps.get-version.outputs.current_version }}
+        # Remove rc* from the end of version string
+        # The ${string%%substring} syntax below deletes the longest match of $substring from back of $string.
+        run: |
           poetry version ${CURRENT_VERSION%%rc*}
-        elif [ "$BUMP_B" = true ]; then
-          poetry version ${CURRENT_VERSION%%b*}
-        elif [ "$BUMP_A" = true ]; then
-          poetry version ${CURRENT_VERSION%%a*}
-        fi
-        echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
-        echo "venue=ops" >> $GITHUB_ENV
-
-    - name: Log in to the Container registry
-      if: ${{ !startsWith(github.ref, 'refs/heads/feature/') }}
-      uses: docker/login-action@v3
-      with:
-        registry: ${{ env.REGISTRY }}
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Extract metadata (tags, labels) for Docker
-      if: ${{ !startsWith(github.ref, 'refs/heads/feature/') }}
-      id: meta
-      uses: docker/metadata-action@v5
-      with:
-        images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-        tags: |
-          type=raw,pattern={{version}},value=${{ env.software_version }}
-          type=raw,value=${{ env.venue }}
-
-#    - name: Wait for package
-##      if: ${{ !startsWith(github.ref, 'refs/heads/feature') }}
-#      if: ${{ startsWith(github.ref, 'refs/heads/feature/') }}
-#      run: |
-#        pip install tenacity
-#        ${GITHUB_WORKSPACE}/.github/workflows/wait-for-pypi.py ${{env.pyproject_name}}[harmony]==${{ env.software_version }}
-
-    - name: Build and push Docker image
-      if: ${{ !startsWith(github.ref, 'refs/heads/feature/') }}
-      id: docker-push
-      uses: docker/build-push-action@v5
-      with:
-        context: .
-        file: Dockerfile
-        build-args: |
-          SOURCE=${{env.pyproject_name}}[harmony]==${{ env.software_version }}
-        push: true
-        pull: true
-        tags: ${{ steps.meta.outputs.tags }}
-        labels: ${{ steps.meta.outputs.labels }}
-#
-#    - name: Commit Version Bump
-#      # If building develop, a release branch, or main then we commit the version bump back to the repo
-#      if: |
-#        github.ref == 'refs/heads/develop' ||
-#        github.ref == 'refs/heads/main'    ||
-#        startsWith(github.ref, 'refs/heads/release')
-#      run: |
-#        git config --global user.name 'batchee bot'
-#        git config --global user.email 'batchee@noreply.github.com'
-#        git commit -am "/version ${{ env.software_version }}"
-#        git push
-#
-#    - name: Push Tag
-#      if: |
-#        github.ref == 'refs/heads/develop' ||
-#        github.ref == 'refs/heads/main'    ||
-#        startsWith(github.ref, 'refs/heads/release')
-#      run: |
-#        git config user.name "${GITHUB_ACTOR}"
-#        git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
-#        git tag -a "${{ env.software_version }}" -m "Version ${{ env.software_version }}"
-#        git push origin "${{ env.software_version }}"
-
-#    - name: Build Python Artifact
-#      run: |
-#        poetry build
-#
-#    - uses: actions/upload-artifact@v3
-#      with:
-#        name: python-artifact
-#        path: dist/*
-#
-#    - name: Publish to test.pypi.org
-#      id: pypi-test-publish
-#      if: |
-#        github.ref == 'refs/heads/develop' ||
-#        startsWith(github.ref, 'refs/heads/release')
-#      env:
-#        POETRY_PYPI_TOKEN_TESTPYPI: ${{secrets.POETRY_PYPI_TOKEN_TESTPYPI}}
-#      run: |
-#        poetry config repositories.testpypi https://test.pypi.org/legacy/
-#        poetry publish -r testpypi
-#
-#    - name: Publish to pypi.org
-#      if: ${{ github.ref == 'refs/heads/main' }}
-#      id: pypi-publish
-#      env:
-#        POETRY_PYPI_TOKEN_PYPI: ${{secrets.POETRY_PYPI_TOKEN_PYPI}}
-#      run: |
-#        poetry publish
+          echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
+          echo "venue=ops" >> $GITHUB_ENV
+
+      - name: Run Snyk as a blocking step
+        uses: snyk/actions/python-3.10@master
+        env:
+            SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          command: test
+          args: >
+              --org=${{ secrets.SNYK_ORG_ID }}
+              --project-name=${{ github.repository }}
+              --severity-threshold=high
+              --fail-on=all
+      - name: Run Snyk on Python
+        uses: snyk/actions/python-3.10@master
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          command: monitor
+          args: >
+            --org=${{ secrets.SNYK_ORG_ID }}
+            --project-name=${{ github.repository }}
+
+      - name: Commit Version Bump
+        # If building the `develop`, a `release` branch, or `main`,
+        # then we commit the version bump back to the repo.
+        if: |
+          github.ref == 'refs/heads/develop' ||
+          github.ref == 'refs/heads/main'    ||
+          startsWith(github.ref, 'refs/heads/release')
+        run: |
+          git config --global user.name 'batchee bot'
+          git config --global user.email 'batchee@noreply.github.com'
+          git commit -am "/version ${{ env.software_version }}"
+          git push
+
+      # Builds and pushes the package to the Python Package Index (PyPI)
+      - name: Build Python Artifact
+        run: |
+          poetry build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: python-artifact
+          path: dist/*
+      - name: Publish to test.pypi.org
+        id: pypi-test-publish
+        if: |
+          github.ref == 'refs/heads/develop' ||
+          startsWith(github.ref, 'refs/heads/release')
+        env:
+          POETRY_PYPI_TOKEN_TESTPYPI: ${{secrets.PYPI_TOKEN_TESTPYPI}}
+        run: |
+          poetry config repositories.testpypi https://test.pypi.org/legacy/
+          poetry publish -r testpypi
+      - name: Publish to pypi.org
+        if: ${{ github.ref == 'refs/heads/main' }}
+        id: pypi-publish
+        env:
+          POETRY_PYPI_TOKEN_PYPI: ${{secrets.PYPI_TOKEN_PYPI}}
+        run: |
+          poetry publish
+
+      # Builds and pushes a Docker image
+      - name: Log in to the Container registry
+        if: ${{ !startsWith(github.ref, 'refs/heads/main/') }}
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        if: ${{ !startsWith(github.ref, 'refs/heads/main/') }}
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=raw,pattern={{version}},value=${{ env.software_version }}
+            type=raw,value=${{ env.venue }}
+      - name: Build and push Docker image
+        if: ${{ !startsWith(github.ref, 'refs/heads/main/') }}
+        id: docker-push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: Dockerfile
+          build-args: |
+            SOURCE=${{env.pyproject_name}}[harmony]==${{ env.software_version }}
+          push: true
+          pull: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Push Tag
+        if: |
+          github.ref == 'refs/heads/develop' ||
+          github.ref == 'refs/heads/main'    ||
+          startsWith(github.ref, 'refs/heads/release')
+        run: |
+          git config user.name "${GITHUB_ACTOR}"
+          git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
+          git tag -a "${{ env.software_version }}" -m "Version ${{ env.software_version }}"
+          git push origin "${{ env.software_version }}"
diff --git a/.github/workflows/release-created.yml b/.github/workflows/release-created.yml
index 62a79d7..226b347 100644
--- a/.github/workflows/release-created.yml
+++ b/.github/workflows/release-created.yml
@@ -1,5 +1,3 @@
-# As soon as a release branch is pushed, the minor version on develop is bumped automatically.
-#   This allows further development to continue on develop for ‘the next’ release.
 name: Release Branch Created
 
 # Run whenever a ref is created https://docs.github.com/en/actions/reference/events-that-trigger-workflows#create
@@ -20,21 +18,21 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: 'refs/heads/develop'
-#      - uses: actions/setup-python@v4
-#        with:
-#          python-version: '3.10'
-#      - name: Install Poetry
-#        uses: abatilo/actions-poetry@v2.3.0
-#        with:
-#          poetry-version: 1.3.2
-##      - name: Bump minor version
-##        run: |
-##          poetry version ${GITHUB_REF#refs/heads/release/}
-##          poetry version preminor
-##          echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
-##      - name: Commit Version Bump
-##        run: |
-##          git config --global user.name 'batchee bot'
-##          git config --global user.email 'batchee@noreply.github.com'
-##          git commit -am "/version ${{ env.software_version }}"
-##          git push
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.10
+      - name: Install Poetry
+        uses: abatilo/actions-poetry@v3.0.0
+        with:
+          poetry-version: 1.3.2
+      - name: Bump minor version
+        run: |
+          poetry version ${GITHUB_REF#refs/heads/release/}
+          poetry version preminor
+          echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
+      - name: Commit Version Bump
+        run: |
+          git config --global user.name 'stitchee bot'
+          git config --global user.email 'stitchee@noreply.github.com'
+          git commit -am "/version ${{ env.software_version }}"
+          git push
diff --git a/.github/workflows/wait-for-pypi.py b/.github/workflows/wait-for-pypi.py
new file mode 100644
index 0000000..cb282be
--- /dev/null
+++ b/.github/workflows/wait-for-pypi.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+import logging
+import subprocess
+import sys
+import tempfile
+
+import tenacity
+
+"""
+Sometimes the package published to PyPi is not immediately available for download from the index.
+This script simply repeatedly tries to download a specific version of a package
+from PyPI (or test.pypi) until it succeeds or a limit is exceeded.
+"""
+
+
+@tenacity.retry(
+    wait=tenacity.wait_exponential(multiplier=1, min=4, max=10),
+    retry=tenacity.retry_if_exception_type(subprocess.CalledProcessError),
+    stop=tenacity.stop_after_delay(120),
+    before_sleep=tenacity.before_sleep_log(logging.getLogger(__name__), logging.DEBUG),
+)
+def download_package(package):
+    subprocess.check_call(
+        [
+            sys.executable,
+            "-m",
+            "pip",
+            "--isolated",
+            "--no-cache-dir",
+            "download",
+            "--no-deps",
+            "-d",
+            tempfile.gettempdir(),
+            "--index-url",
+            "https://pypi.org/simple/",
+            "--extra-index-url",
+            "https://test.pypi.org/simple/",
+            package,
+        ]
+    )
+
+
+if __name__ == "__main__":
+    logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
+    package_spec = sys.argv[1]
+    download_package(package_spec)

From efa9702ca89397eed31316f28307eea6883f153e Mon Sep 17 00:00:00 2001
From: danielfromearth <daniel.kaufman@nasa.gov>
Date: Mon, 22 Apr 2024 14:24:06 -0400
Subject: [PATCH 2/2] update CHANGELOG.md

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 46a255b..48a3876 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [PR #84](https://github.com/danielfromearth/batchee/pull/84): Add readme badges
 - [PR #100](https://github.com/danielfromearth/batchee/pull/100): Add license
 - [PR #111](https://github.com/nasa/batchee/pull/111): Add codecov to CI pipeline
+- [PR #112](https://github.com/nasa/batchee/pull/111): Add SNYK and PyPI to CI pipeline
 ### Changed
 - [issue/11](https://github.com/danielfromearth/batchee/issues/11): Rename from concat_batcher to batchee
 - [issue/21](https://github.com/danielfromearth/batchee/issues/21): Improve CICD workflows