ci: split off strategy tests in drive-abci

.github/workflows/tests-rs-drive-abci.yml

@@ -0,0 +1,299 @@
+on:
+  workflow_call:
+    inputs:
+      package:
+        description: The package name to run tests for
+        type: string
+        required: true
+      test-runner:
+        description: Runner for tests. Must be JSON valid string.
+        type: string
+        default: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]'
+      lint-runner:
+        description: Runner for linting. Must be JSON valid string.
+        type: string
+        default: '"ubuntu-22.04"'
+      check-each-feature:
+        description: If true, try to build each individual feature for this crate
+        type: boolean
+        default: false
+
+jobs:
+  lint:
+    name: Linting
+    runs-on: ${{ fromJSON(inputs.lint-runner) }}
+    permissions:
+      id-token: write
+      contents: read
+    timeout-minutes: 15
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials and bucket region
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Setup Rust
+        uses: ./.github/actions/rust
+        with:
+          components: clippy
+
+      - uses: clechasseur/rs-clippy-check@v3
+        with:
+          args: --package drive-abci --all-features --locked -- --no-deps
+        env:
+          RUSTC_WRAPPER: sccache
+          SCCACHE_BUCKET: multi-runner-cache-x1xibo9c
+          SCCACHE_REGION: ${{ secrets.AWS_REGION }}
+          SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
+
+  formatting:
+    name: Formatting
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+
+      - name: Setup Rust
+        uses: ./.github/actions/rust
+        with:
+          components: rustfmt
+          cache: false
+
+      - name: Check formatting
+        run: cargo fmt --check --package=drive-abci
+
+  unused_deps:
+    name: Unused dependencies
+    runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
+      contents: read
+    timeout-minutes: 15
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials and bucket region
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ secrets.AWS_REGION }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Setup Rust
+        uses: ./.github/actions/rust
+
+      - name: Get crate drive-abci info
+        id: crate_info
+        uses: ./.github/actions/crate_info
+        with:
+          package: drive-abci
+
+      - name: Find unused dependencies
+        uses: lklimek/cargo-machete@feat/workdir
+        env:
+          RUSTC_WRAPPER: sccache
+          SCCACHE_BUCKET: multi-runner-cache-x1xibo9c
+          SCCACHE_REGION: ${{ secrets.AWS_REGION }}
+          SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
+        with:
+          args: ${{ steps.crate_info.outputs.cargo_manifest_dir }}
+
+  detect_structure_changes:
+    name: Detect immutable structure changes
+    runs-on: ubuntu-22.04
+    # FIXME: as we use `gh pr view` below, this check can only
+    # run on pull requests. We should find a way to run it
+    # when manual triggers are used.
+    if: github.event_name == 'pull_request'
+    steps:
+      - name: Checkout base commit
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.ref }}
+
+      - name: Get all changed Rust files
+        id: files
+        run: |
+          gh pr view https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }} --json files --jq "[.files[].path] | map(select(test(\"packages/drive-abci/.*.rs$\")))" > files.json
+          echo "files=$(cat files.json)" >> $GITHUB_OUTPUT
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Save append_only and immutable structures
+        id: base_structures
+        run: |
+          mkdir base_structures
+          for file in $(jq -r '.[]' <<< "${{ steps.files.outputs.files }}"); do
+            if [[ $file == *.rs ]]; then
+              awk '/@append_only/,/}/' $file > "base_structures/$(basename $file).append_only.base"
+              awk '/@immutable/,/}/' $file > "base_structures/$(basename $file).immutable.base"
+            fi
+          done
+          echo "base_dir=$(pwd)/base_structures" >> $GITHUB_OUTPUT
+
+      - name: Checkout PR commit
+        uses: actions/checkout@v4
+
+      - name: Check structure changes
+        run: |
+          for file in $(jq -r '.[]' <<< "${{ steps.files.outputs.files }}"); do
+            if [[ $file == *.rs ]]; then
+              awk '/@append_only/,/}/' $file > "$(basename $file).append_only.pr"
+              awk '/@immutable/,/}/' $file > "$(basename $file).immutable.pr"
+
+              # Check append_only structures
+              if ! diff -u "${{ steps.base_structures.outputs.base_dir }}/$(basename $file).append_only.base" "$(basename $file).append_only.pr" | grep "^-" | grep -v "@append_only" | grep -v "^-" | grep -v "^///" | grep -v "^//" | grep -v "^-$"; then
+                echo "No deletions detected in @append_only structures in $file. Test passed."
+              else
+                echo "Deletions detected in @append_only structures in $file. Test failed."
+                exit 1
+              fi
+
+              # Check immutable structures
+              if ! diff -u "${{ steps.base_structures.outputs.base_dir }}/$(basename $file).immutable.base" "$(basename $file).immutable.pr" | grep -E "^\+[^/+]|^-[^/-]" ; then
+                echo "No code changes detected in @immutable structures in $file. Test passed."
+              else
+                echo "Code changes detected in @immutable structures in $file. Test failed."
+                exit 1
+              fi
+            fi
+          done
+
+  test:
+    name: Tests
+    runs-on: ${{ fromJSON(inputs.test-runner) }}
+    timeout-minutes: 15
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials and bucket region
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ secrets.AWS_REGION }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Setup Rust
+        uses: ./.github/actions/rust
+
+      - name: Run tests
+        run: |
+            cargo test --package=drive-abci --all-features --locked -- --skip strategy_tests
+        env:
+          RUSTC_WRAPPER: sccache
+          SCCACHE_BUCKET: multi-runner-cache-x1xibo9c
+          SCCACHE_REGION: ${{ secrets.AWS_REGION }}
+          SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
+
+  strategy_tests:
+    name: Strategy Tests (No Version Update)
+    runs-on: ${{ fromJSON(inputs.test-runner) }}
+    timeout-minutes: 15
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials and bucket region
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ secrets.AWS_REGION }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Setup Rust
+        uses: ./.github/actions/rust
+
+      - name: Run strategy tests
+        run: cargo test --package=drive-abci --test=strategy_tests --all-features --locked -- --skip upgrade_fork_tests
+        env:
+          RUSTC_WRAPPER: sccache
+          SCCACHE_BUCKET: multi-runner-cache-x1xibo9c
+          SCCACHE_REGION: ${{ secrets.AWS_REGION }}
+          SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
+
+  upgrade_fork_tests:
+    name: Upgrade Fork Tests
+    runs-on: ${{ fromJSON(inputs.test-runner) }}
+    timeout-minutes: 15
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials and bucket region
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ secrets.AWS_REGION }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Setup Rust
+        uses: ./.github/actions/rust
+
+      - name: Run upgrade fork tests
+        run: cargo test --package=drive-abci --test=strategy_tests --all-features --locked -- upgrade_fork_tests
+        env:
+          RUSTC_WRAPPER: sccache
+          SCCACHE_BUCKET: multi-runner-cache-x1xibo9c
+          SCCACHE_REGION: ${{ secrets.AWS_REGION }}
+          SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
+
+  check_each_feature:
+    name: Check each feature
+    runs-on: ${{ fromJSON(inputs.test-runner) }}
+    timeout-minutes: 10
+    if: ${{ inputs.check-each-feature }}
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v3
+
+      - name: Configure AWS credentials and bucket region
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ secrets.AWS_REGION }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Setup Rust
+        uses: ./.github/actions/rust
+
+      - name: Get crate ${{ runner.arch }} info
+        id: crate_info
+        uses: ./.github/actions/crate_info
+        with:
+          package: drive-abci
+
+      - name: Check each feature in drive-abci
+        env:
+          RUSTC_WRAPPER: sccache
+          SCCACHE_BUCKET: multi-runner-cache-x1xibo9c
+          SCCACHE_REGION: ${{ secrets.AWS_REGION }}
+          SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
+        run: |
+          echo Verify all features disabled
+          set -ex
+          features="${{ steps.crate_info.outputs.features }}"
+          fails=""
+          RUSTFLAGS="-D warnings"
+          cargo check --no-default-features --package "drive-abci" --locked
+          for feature in $features ;  do
+            echo " ============== Verify feature $feature =============="
+            cargo check \
+              --no-default-features \
+              --package "drive-abci" \
+              --features="${feature}" \
+              --locked || fails="${fails} ${feature}"
+          done
+
+          if [ -n "$fails" ] ; then
+            echo "Failed features: $fails"
+            exit 1
+          fi
+          echo "All features verified"

.github/workflows/tests-rs-package.yml

@@ -169,7 +169,7 @@ jobs:
   test:
     name: Tests
     runs-on: ${{ fromJSON(inputs.test-runner) }}
-    timeout-minutes: 25
+    timeout-minutes: 15
     steps:
       - name: Check out repo
         uses: actions/checkout@v4

.github/workflows/tests.yml

@@ -80,6 +80,8 @@ jobs:
       fail-fast: false
       matrix:
         rs-package: ${{ fromJson(needs.changes.outputs.rs-packages) }}
+        exclude:
+          - rs-package: 'drive-abci'
     uses: ./.github/workflows/tests-rs-package.yml
     with:
       package: ${{ matrix.rs-package }}
@@ -89,6 +91,22 @@ jobs:
       # Run drive tests on self-hosted 4x
       test-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]'
       check-each-feature: ${{ contains(fromJSON('["dash-sdk","rs-dapi-client","dapi-grpc","dpp","drive-abci"]'), matrix.rs-package) }}
+
+  rs-drive-abci:
+    name: Drive ABCI
+    needs:
+      - changes
+    secrets: inherit
+    if: contains(needs.changes.outputs.rs-packages, 'drive-abci')
+    uses: ./.github/workflows/tests-rs-drive-abci.yml
+    with:
+      package: ${{ matrix.rs-package }}
+      # lint-runner: ${{ contains(fromJSON('["drive-abci", "drive"]'), matrix.rs-package) && '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' || '"ubuntu-22.04"' }}
+      # FIXME: Clippy fails on github hosted runners, most likely due to RAM usage. Using self-hosted runners for now.
+      lint-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]'
+      # Run drive tests on self-hosted 4x
+      test-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]'
+      check-each-feature: true
 
   rs-crates-security:
     name: Rust crates security audit

packages/rs-drive-abci/src/error/execution.rs

@@ -90,7 +90,7 @@ pub enum ExecutionError {
 
     /// A protocol upgrade incoherence error occurred.
     #[error("protocol upgrade incoherence error: {0}")]
-    ProtocolUpgradeIncoherence(&'static str),
+    ProtocolUpgradeIncoherence(String),
 
     /// Data is missing from the drive.
     #[error("drive missing data error: {0}")]

packages/rs-drive-abci/src/execution/platform_events/protocol_upgrade/check_for_desired_protocol_upgrade/v0/mod.rs

@@ -54,7 +54,7 @@ impl<C> Platform<C> {
         if versions_passing_threshold.len() > 1 {
             return Err(Error::Execution(
                 ExecutionError::ProtocolUpgradeIncoherence(
-                    "only at most 1 version should be able to pass the threshold to upgrade",
+                    format!("only at most 1 version should be able to pass the threshold to upgrade, got versions [{}]", versions_passing_threshold.iter().map(|protocol_version| protocol_version.to_string()).collect::<Vec<_>>().join(", "))
                 ),
             ));
         }