feat: pull gateway_mfr build from upstream to reduce build time

- pull gateway_mfr build from upstream - update pypi and test-pypi actions - add action to check gateway-mfr-rs version weekly and auto-update if necessary - remove old unused backup action - bumping minor version as this is a fairly substantial change to the build process - update README to add more release info and update ROCK Pi hardware info Closes: #43 Closes: #36 Relates-to: #20
NebraLtd · Dec 7, 2021 · 16caade · 16caade
1 parent 354d3dc
commit 16caade
Show file tree

Hide file tree

Showing 6 changed files with 133 additions and 171 deletions.
diff --git a/.github/workflows/publish-gateway-mfr-rs.yml.bak b/.github/workflows/publish-gateway-mfr-rs.yml.bak
diff --git a/.github/workflows/publish-to-pypi-test.yml b/.github/workflows/publish-to-pypi-test.yml
@@ -3,47 +3,9 @@ name: Publish Python 🐍 distribution 📦 to TestPyPI
 on: [push]
 
 jobs:
-  rust-compile:
-    name: Build gateway-mfr-rs
-    runs-on: ubuntu-latest
-    steps:
-      - name: Install stable toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          target: arm-unknown-linux-gnueabihf
-          override: true
-
-      - name: Clone repo
-        run: |
-          git clone --branch v0.1.5 https://github.com/helium/gateway-mfr-rs.git .
-
-      - name: Run cargo check
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: true
-          command: check
-
-      - name: Build gateway-mfr-rs
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: true
-          command: build
-          args: --target arm-unknown-linux-gnueabihf --release
-
-      - name: Copy release file
-        run: |
-          cp ./target/arm-unknown-linux-gnueabihf/release/gateway_mfr gateway_mfr
-
-      - uses: actions/upload-artifact@v2
-        with:
-          name: gateway_mfr
-          path: ./gateway_mfr
-
   build-n-publish:
     name: Build and publish Python 🐍 distribution 📦 to TestPyPI
     runs-on: ubuntu-18.04
-    needs: [rust-compile]
 
     steps:
       - uses: actions/checkout@master
@@ -53,18 +15,32 @@ jobs:
         with:
           python-version: 3.7
 
-      - uses: actions/download-artifact@v2
-        with:
-          name: gateway_mfr
-          path: ./gateway_mfr
-
-      - name: Move gateway_mfr in place
+      - name: Fetch gateway-mfr-rs
+        env:
+          GATEWAY_MFR_RS_RELEASE: v0.1.6
         run: |
-          chmod +x gateway_mfr/gateway_mfr
+          wget "https://github.com/helium/gateway-mfr-rs/releases/download/${GATEWAY_MFR_RS_RELEASE}/gateway-mfr-${GATEWAY_MFR_RS_RELEASE}-arm-unknown-linux-gnueabihf.tar.gz"
+          wget "https://github.com/helium/gateway-mfr-rs/releases/download/${GATEWAY_MFR_RS_RELEASE}/gateway-mfr-${GATEWAY_MFR_RS_RELEASE}-arm-unknown-linux-gnueabihf.checksum"
+          SHA256=$( shasum -a 256 "gateway-mfr-${GATEWAY_MFR_RS_RELEASE}-arm-unknown-linux-gnueabihf.tar.gz" )
+          
+          # Verify the checksum 
+          if grep -q "${SHA256}" "gateway-mfr-${GATEWAY_MFR_RS_RELEASE}-arm-unknown-linux-gnueabihf.checksum"; then
+            echo "Checksum verified for gateway_mfr. Unpacking tarball..."
+            # Unpack the tarball
+            tar -xvf "gateway-mfr-${GATEWAY_MFR_RS_RELEASE}-arm-unknown-linux-gnueabihf.tar.gz"
+            exit 0
+          else
+            echo "Checksum does not match. Please check download manually and rerun!"
+            exit 1
+          fi
+
+      - name: Change permissions on gateway_mfr
+        run: |
+          chmod +x gateway_mfr
 
       - name: Move gateway_mfr in place
         run: |
-          mv gateway_mfr/gateway_mfr hm_pyhelper/
+          mv gateway_mfr hm_pyhelper/
 
       - name: Install pypa/build
         run: |
@@ -79,10 +55,6 @@ jobs:
           name: wheels
           path: ./dist/*
 
-      - name: Get changed files
-        id: changed-files
-        uses: tj-actions/[email protected]
-
       - name: Publish distribution 📦 to Test PyPI
         uses: pypa/gh-action-pypi-publish@master
         with:

diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml
@@ -6,47 +6,9 @@ on:
   workflow_dispatch:
 
 jobs:
-  rust-compile:
-    name: Build gateway-mfr-rs
-    runs-on: ubuntu-latest
-    steps:
-      - name: Install stable toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          target: arm-unknown-linux-gnueabihf
-          override: true
-
-      - name: Clone repo
-        run: |
-          git clone --branch v0.1.5 https://github.com/helium/gateway-mfr-rs.git .
-
-      - name: Run cargo check
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: true
-          command: check
-
-      - name: Build gateway-mfr-rs
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: true
-          command: build
-          args: --target arm-unknown-linux-gnueabihf --release
-
-      - name: Copy release file
-        run: |
-          cp ./target/arm-unknown-linux-gnueabihf/release/gateway_mfr gateway_mfr
-
-      - uses: actions/upload-artifact@v2
-        with:
-          name: gateway_mfr
-          path: ./gateway_mfr
-
   build-n-publish:
     name: Build and publish Python 🐍 distribution 📦 to PyPI
     runs-on: ubuntu-18.04
-    needs: [rust-compile]
 
     steps:
       - uses: actions/checkout@master
@@ -56,18 +18,32 @@ jobs:
         with:
           python-version: 3.7
 
-      - uses: actions/download-artifact@v2
-        with:
-          name: gateway_mfr
-          path: ./gateway_mfr
-
-      - name: Move gateway_mfr in place
+      - name: Fetch gateway-mfr-rs
+        env:
+          GATEWAY_MFR_RS_RELEASE: v0.1.6
+        run: |
+          wget "https://github.com/helium/gateway-mfr-rs/releases/download/${GATEWAY_MFR_RS_RELEASE}/gateway-mfr-${GATEWAY_MFR_RS_RELEASE}-arm-unknown-linux-gnueabihf.tar.gz"
+          wget "https://github.com/helium/gateway-mfr-rs/releases/download/${GATEWAY_MFR_RS_RELEASE}/gateway-mfr-${GATEWAY_MFR_RS_RELEASE}-arm-unknown-linux-gnueabihf.checksum"
+          SHA256=$( shasum -a 256 "gateway-mfr-${GATEWAY_MFR_RS_RELEASE}-arm-unknown-linux-gnueabihf.tar.gz" )
+          
+          # Verify the checksum 
+          if grep -q "${SHA256}" "gateway-mfr-${GATEWAY_MFR_RS_RELEASE}-arm-unknown-linux-gnueabihf.checksum"; then
+            echo "Checksum verified for gateway_mfr. Unpacking tarball..."
+            # Unpack the tarball
+            tar -xvf "gateway-mfr-${GATEWAY_MFR_RS_RELEASE}-arm-unknown-linux-gnueabihf.tar.gz"
+            exit 0
+          else
+            echo "Checksum does not match. Please check download manually and rerun!"
+            exit 1
+          fi
+
+      - name: Change permissions on gateway_mfr
         run: |
-          chmod +x gateway_mfr/gateway_mfr
+          chmod +x gateway_mfr
 
       - name: Move gateway_mfr in place
         run: |
-          mv gateway_mfr/gateway_mfr hm_pyhelper/
+          mv gateway_mfr hm_pyhelper/
 
       - name: Install pypa/build
         run: |
@@ -81,7 +57,7 @@ jobs:
         with:
           name: wheels
           path: ./dist/*
-          
+
       - name: Get Latest Release
         id: latest_version
         uses: abatilo/[email protected]

diff --git a/.github/workflows/update-gateway-mfr-rs.yml b/.github/workflows/update-gateway-mfr-rs.yml
@@ -0,0 +1,68 @@
+name: Check if we're using the latest Gateway MFR (gateway-mfr-rs) release
+
+on:
+  schedule:
+    - cron:  "0 0 * * 0" # Run weekly on sunday
+  workflow_dispatch:
+
+jobs:
+  latest-version:
+    name: Get Latest Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+      - name: Get Latest Release
+        id: latest_version
+        uses: abatilo/[email protected]
+        with:
+          owner: helium
+          repo: gateway-mfr-rs
+      - name: Perform check and update
+        env:
+          GATEWAY_MFR_RS_RELEASE: ${{ steps.latest_version.outputs.latest_tag }}
+        run: |
+          GITHUB_BRANCH=$( echo "${{ github.ref }}" | sed 's/refs\/heads\///g' )
+          CHECK_LATEST_RELEASE="GATEWAY_MFR_RS_RELEASE: ${GATEWAY_MFR_RS_RELEASE}"
+
+          echo "GATEWAY_MFR_RS_RELEASE=$GATEWAY_MFR_RS_RELEASE" >> $GITHUB_ENV
+          echo "GITHUB_BRANCH=$GITHUB_BRANCH" >> $GITHUB_ENV
+          
+          # Get the latest gateway-mfr-rs release
+          if grep -q "$CHECK_LATEST_RELEASE" .github/workflows/publish-to-pypi-test.yml; then
+            echo "We're on the latest Helium gateway-mfr-rs release $GATEWAY_MFR_RS_RELEASE."
+            exit 0
+          else
+            echo "We're not on the latest Helium gateway-mfr-rs release. Updating to $GATEWAY_MFR_RS_RELEASE."
+            sed -i -E 's/GATEWAY_MFR_RS_RELEASE: v.*/GATEWAY_MFR_RS_RELEASE: '$GATEWAY_MFR_RS_RELEASE'/g' .github/workflows/publish-to-pypi-test.yml
+            sed -i -E 's/GATEWAY_MFR_RS_RELEASE: v.*/GATEWAY_MFR_RS_RELEASE: '$GATEWAY_MFR_RS_RELEASE'/g' .github/workflows/publish-to-pypi.yml
+            UPDATED=true
+            echo "UPDATED=$UPDATED" >> $GITHUB_ENV
+            exit 0
+          fi
+      - name: Create Pull Request
+        if: env.UPDATED == 'true'
+        uses: peter-evans/create-pull-request@v3
+        with:
+          commit-message: Update gateway-mfr-rs to latest release ${{ env.GATEWAY_MFR_RS_RELEASE }}
+          branch: gateway-mfr-rs/${{ env.GATEWAY_MFR_RS_RELEASE }}
+          delete-branch: true
+          base: master
+          title: "gateway-mfr-rs: Update to latest release ${{ env.GATEWAY_MFR_RS_RELEASE }}"
+          body: |
+            Update gateway-mfr-rs to latest release ${{ env.GATEWAY_MFR_RS_RELEASE }}
+            Ref https://github.com/helium/gateway-mfr-rs/releases/tag/${{ env.GATEWAY_MFR_RS_RELEASE }}
+          draft: false
+      - name: Report Status
+        if: env.UPDATED == 'true'
+        uses: ravsamhq/notify-slack-action@master
+        with:
+          status: ${{ job.status }}
+          notification_title: 'Gateway-mfr-rs has been updated to ${{ env.GATEWAY_MFR_RS_RELEASE }}. Please review PR!'
+          message_format: '{emoji} *{workflow}* {status_message} in <{repo_url}|{repo}>'
+          footer: '<{run_url}|View Run> | Linked Repo <{repo_url}|{repo}> | <{workflow_url}|View Workflow>'
+          mention_groups: 'S02GCFWL27R'
+          notify_when: 'success'
+          token: ${{ secrets.GITHUB_TOKEN }}
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.MINER_MONITORING_SLACK }}
diff --git a/README.md b/README.md
@@ -6,6 +6,8 @@ This package is used in a number of Nebra software repos:
 - [hm-pktfwd](https://github.com/NebraLtd/hm-pktfwd/)
 - [hm-config](https://github.com/NebraLtd/hm-config/)
 - [hm-diag](https://github.com/NebraLtd/hm-diag/)
+- [hm-dashboard (private repo)](https://github.com/NebraLtd/hm-dashboard)
+- [Hotspot-Production-Tool (private repo)](https://github.com/NebraLtd/Hotspot-Production-Tool)
 
 The package is available on PyPI and PyPI test repos:
 - [PyPI hm-pyhelper](https://pypi.org/project/hm-pyhelper)
@@ -33,7 +35,8 @@ Note: Light hotspot software will also work on all models listed as type "full".
 | Nebra Pi 0 Light Hotspot X | NEBHNT-LGT-ZX | 1.2 | 22 | 24 | 23 | Light | False | USB Based Ethernet |
 | Nebra Beaglebone Light Hotspot | NEBHNT-BBB | 1.0 | 60 | 31  | 30  | Light | False | In Planning |
 | Nebra Pocket Beagle Light Hotspot | NEBHNT-PBB | 1.0 | 60 | 31 | 30 | Light | False | In Planning |
-| Nebra Hotspot HAT RockPi4 | NEBHNT-HHRK4 | 1.0 | 149 | 156 | 154 | Full | False | In Planning |
+| Nebra Hotspot HAT ROCK Pi 4 Indoor | NEBHNT-HHRK4 | 32766.0 | 149 | 156 (Physical pin 18) | 154 (Physical pin 16) | Full | False | In Planning |
+| Nebra Hotspot HAT ROCK Pi 4 Outdoor | NEBHNT-HHRK4-OUT | 32766.0 | 149 | 156 (Physical pin 18) | 154 (Physical pin 16)| Full | True | In Planning |
 | Nebra Hotspot HAT RPi | NEBHNT-HHRPI | 0.0 | 22 | 24 | 23 | Full | False | Should be compatible with 3+ & 4 |
 | Nebra Hotspot HAT RPi LIGHT | NEBHNT-HHRPL | 0.0 | 22 | 24 | 23 | Light | False | Light is compatible with all 40 pin headers |
 | Nebra Hotspot HAT Tinkerboard 2 | NEBHNT-HHTK | 2.0 | 167 | 163 | 162 | Full | False | Light would be compatible on TK1 |
@@ -189,3 +192,18 @@ To release, use the [Github new release flow](https://github.com/NebraLtd/hm-pyh
 
 **Full Changelog**: https://github.com/NebraLtd/hm-pyhelper/compare/v0.0.A...v0.0.Z
 ```
+
+### Release strategy
+
+The automated GitHub Actions in this repo do the following:
+- all pushes / PRs, regardless of branch, trigger a build of the wheels and python package which are released as build artifacts ([see below section](#test-release-artifacts))
+- pushes to master with an updated version number in `setup.py` are pushed to Test PyPI as well as being uploaded as build artifacts (note that if the version number isn't properly updated and is a duplicate of a previous one then the push to Test PyPI will fail)
+- any tagged releases on master branch ([see releasing process above](#releasing)) are built and published to PyPI as well as being uploaded as build artifacts
+
+### Test release artifacts
+
+Note that artifacts (wheels and source) are uploaded to the GitHub Actions artifacts even when the build fails or isn't pushed to PyPI/Test PyPI due to not being on the master branch.
+
+For example, [this failed build](https://github.com/NebraLtd/hm-pyhelper/actions/runs/1369814396), has artifacts uploaded [here](https://github.com/NebraLtd/hm-pyhelper/suites/4125934376/artifacts/105569066).
+
+These artifacts can be useful for testing releases without needing to bump version numbers.
diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 
 setup(
     name='hm_pyhelper',
-    version='0.11.14',
+    version='0.12.0',
     author="Nebra Ltd",
     author_email="[email protected]",
     description="Helium Python Helper",