Define parameters for Linux jobs in a matrix

Signed-off-by: Spencer Wilson <[email protected]>
open-quantum-safe · Aug 28, 2024 · 04f4a46 · 04f4a46
1 parent 84b84ec
commit 04f4a46
Showing 1 changed file with 123 additions and 112 deletions.
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -11,130 +11,141 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # total of 38 test runs here.
+        runner: [ubuntu-latest] # TODO add arm64
+        # Test all combinations of the following options.
+        # Generates 16 base test runs (4 on arm, 4 on x86_64).
+        compiler: [gcc, clang]
+        shared: [ON, OFF]
+        openssl: [ON, OFF]
+        # Default values to be overridden selectively
+        sha3-openssl: [OFF]
+        dlopen-openssl: [OFF]
+        dist-build: [ON]
+        opt-target: [auto]
+        stfl: [OFF]
+        stfl-keygen: [OFF]
+        libjade: [OFF]
+        build-options: ['']
         include:
-          - name: arm64
-            runner: oqs-arm64
-            container: openquantumsafe/ci-ubuntu-focal-arm64:latest
-            PYTEST_ARGS: --maxprocesses=10 --ignore=tests/test_kat_all.py
-            CMAKE_ARGS: -DOQS_ENABLE_SIG_STFL_LMS=ON -DOQS_ENABLE_SIG_STFL_XMSS=ON -DOQS_HAZARDOUS_EXPERIMENTAL_ENABLE_SIG_STFL_KEY_SIG_GEN=ON
-          - name: alpine
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-alpine-amd64:latest
-            CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DOQS_USE_OPENSSL=ON -DBUILD_SHARED_LIBS=ON -DOQS_HAZARDOUS_EXPERIMENTAL_ENABLE_SIG_STFL_KEY_SIG_GEN=ON -DOQS_ENABLE_SIG_STFL_XMSS=ON -DOQS_ENABLE_SIG_STFL_LMS=ON
-            PYTEST_ARGS: --ignore=tests/test_alg_info.py --ignore=tests/test_kat_all.py
-          - name: alpine
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-alpine-amd64:latest
-            CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DOQS_USE_OPENSSL=ON -DBUILD_SHARED_LIBS=ON -DOQS_LIBJADE_BUILD=ON -DOQS_MINIMAL_BUILD="${{ vars.LIBJADE_ALG_LIST }}"
-            PYTEST_ARGS: --ignore=tests/test_alg_info.py --ignore=tests/test_kat_all.py
-          - name: alpine-no-stfl-key-sig-gen
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-alpine-amd64:latest
-            CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DOQS_USE_OPENSSL=ON -DBUILD_SHARED_LIBS=ON -DOQS_HAZARDOUS_EXPERIMENTAL_ENABLE_SIG_STFL_KEY_SIG_GEN=OFF -DOQS_ENABLE_SIG_STFL_XMSS=ON -DOQS_ENABLE_SIG_STFL_LMS=ON
-            PYTEST_ARGS: --ignore=tests/test_alg_info.py --ignore=tests/test_kat_all.py
-          - name: alpine-openssl-all
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-alpine-amd64:latest
-            CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DOQS_USE_OPENSSL=ON -DBUILD_SHARED_LIBS=ON -DOQS_USE_AES_OPENSSL=ON -DOQS_USE_SHA2_OPENSSL=ON -DOQS_USE_SHA3_OPENSSL=ON -DOQS_HAZARDOUS_EXPERIMENTAL_ENABLE_SIG_STFL_KEY_SIG_GEN=ON -DOQS_ENABLE_SIG_STFL_XMSS=ON -DOQS_ENABLE_SIG_STFL_LMS=ON
-            PYTEST_ARGS: --ignore=tests/test_alg_info.py --ignore=tests/test_kat_all.py
-          - name: alpine-noopenssl
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-alpine-amd64:latest
-            CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DOQS_USE_OPENSSL=OFF -DOQS_HAZARDOUS_EXPERIMENTAL_ENABLE_SIG_STFL_KEY_SIG_GEN=ON -DOQS_ENABLE_SIG_STFL_XMSS=ON -DOQS_ENABLE_SIG_STFL_LMS=ON
-            PYTEST_ARGS: --ignore=tests/test_alg_info.py --ignore=tests/test_kat_all.py
-          - name: focal-nistr4-openssl
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-ubuntu-focal-x86_64:latest
-            CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DOQS_ALGS_ENABLED=NIST_R4
-            PYTEST_ARGS: --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py
-          - name: focal-nistonramp-openssl
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-ubuntu-focal-x86_64:latest
-            CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DOQS_ALGS_ENABLED=NIST_SIG_ONRAMP
-            PYTEST_ARGS: --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py
-          - name: focal-noopenssl
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-ubuntu-focal-x86_64:latest
-            CMAKE_ARGS: -DCMAKE_C_COMPILER=gcc-8 -DOQS_USE_OPENSSL=OFF
-            PYTEST_ARGS: --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py
-          - name: focal-noopenssl
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-ubuntu-focal-x86_64:latest
-            CMAKE_ARGS: -DCMAKE_C_COMPILER=gcc-8 -DOQS_USE_OPENSSL=OFF -DOQS_LIBJADE_BUILD=ON -DOQS_MINIMAL_BUILD="${{ vars.LIBJADE_ALG_LIST }}"
-            PYTEST_ARGS: --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py
-          - name: focal-shared-noopenssl
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-ubuntu-focal-x86_64:latest
-            CMAKE_ARGS: -DCMAKE_C_COMPILER=gcc-7 -DOQS_DIST_BUILD=OFF -DOQS_USE_OPENSSL=OFF -DBUILD_SHARED_LIBS=ON
-            PYTEST_ARGS: --ignore=tests/test_namespace.py --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py
-          - name: focal-clang15
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-ubuntu-focal-x86_64:latest
-            CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DCMAKE_C_COMPILER=clang-15
-            PYTEST_ARGS: --ignore=tests/test_kat_all.py
-          - name: jammy-std-openssl3
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-ubuntu-jammy:latest
-            CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DOQS_ALGS_ENABLED=STD -DBUILD_SHARED_LIBS=ON
-            PYTEST_ARGS: --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py
-          - name: jammy-std-openssl3-libjade
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-ubuntu-jammy:latest
-            CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DOQS_ALGS_ENABLED=STD -DBUILD_SHARED_LIBS=ON -DOQS_LIBJADE_BUILD=ON -DOQS_MINIMAL_BUILD="${{ vars.LIBJADE_ALG_LIST }}"
-            PYTEST_ARGS: --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py
-          - name: jammy-std-openssl3-dlopen
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-ubuntu-jammy:latest
-            CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DOQS_ALGS_ENABLED=STD -DBUILD_SHARED_LIBS=ON -DOQS_DLOPEN_OPENSSL=ON
-            PYTEST_ARGS: --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py
-          - name: jammy-std-openssl3-dlopen-libjade
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-ubuntu-jammy:latest
-            CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DOQS_ALGS_ENABLED=STD -DBUILD_SHARED_LIBS=ON -DOQS_DLOPEN_OPENSSL=ON -DOQS_LIBJADE_BUILD=ON -DOQS_MINIMAL_BUILD="${{ vars.LIBJADE_ALG_LIST }}"
-            PYTEST_ARGS: --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py
-          - name: address-sanitizer
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-ubuntu-focal-x86_64:latest
-            CMAKE_ARGS: -DCMAKE_C_COMPILER=clang-9 -DCMAKE_BUILD_TYPE=Debug -DUSE_SANITIZER=Address -DOQS_HAZARDOUS_EXPERIMENTAL_ENABLE_SIG_STFL_KEY_SIG_GEN=ON -DOQS_ENABLE_SIG_STFL_XMSS=ON -DOQS_ENABLE_SIG_STFL_LMS=ON
-            PYTEST_ARGS: --ignore=tests/test_distbuild.py --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py --maxprocesses=10
-          - name: address-sanitizer-no-stfl-key-sig-gen
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-ubuntu-focal-x86_64:latest
-            CMAKE_ARGS: -DCMAKE_C_COMPILER=clang-9 -DCMAKE_BUILD_TYPE=Debug -DUSE_SANITIZER=Address -DOQS_HAZARDOUS_EXPERIMENTAL_ENABLE_SIG_STFL_KEY_SIG_GEN=OFF -DOQS_ENABLE_SIG_STFL_XMSS=ON -DOQS_ENABLE_SIG_STFL_LMS=ON
-            PYTEST_ARGS: --ignore=tests/test_distbuild.py --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py --maxprocesses=10
-          - name: address-sanitizer-libjade
-            runner: ubuntu-latest
-            container: openquantumsafe/ci-ubuntu-focal-x86_64:latest
-            CMAKE_ARGS: -DCMAKE_C_COMPILER=clang-9 -DCMAKE_BUILD_TYPE=Debug -DUSE_SANITIZER=Address -DOQS_LIBJADE_BUILD=ON -DOQS_MINIMAL_BUILD="${{ vars.LIBJADE_ALG_LIST }}"
-            PYTEST_ARGS: --ignore=tests/test_distbuild.py --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py --maxprocesses=10
+          # Test the OpenSSL SHA3 implementation.
+          # It suffices to build only standards-track algs, as these rely heavily on SHA3.
+          # Generates 4 additional test runs (2 on arm, 2 on x86_64).
+          - shared: ON
+            openssl: ON
+            sha3-openssl: ON
+            build-options: -DOQS_ALGS_ENABLED=STD
+          # Test dynamic loading of libcrypto.
+          # Since we're primarily testing build-time behaviour, only build the on-ramp candidates.
+          # Generates 4 additional test runs (2 on arm, 2 on x86_64).
+          - shared: ON
+            openssl: ON
+            dlopen-ssl: OFF
+            build-options: -DOQS_ALGS_ENABLED=NIST_SIG_ONRAMP
+          # Test the "generic" code without DIST_BUILD.
+          # Do it without OpenSSL so that the generic common code gets tested too.
+          # Stateful sigs code is all generic, so additionally test it out here (without keygen).
+          # Generates 4 additional test runs (2 on arm, 2 on x86_64).
+          - shared: OFF
+            openssl: OFF
+            dist-build: OFF
+            opt-target: generic
+            stfl: ON
+            stfl-keygen: OFF
+          # Test the code without runtime feature detection.
+          # Do it without OpenSSL for the same reasons as above.
+          # Shared vs static doesn't really matter here, so just pick the opposite of the
+          # "generic" run to have more coverage.
+          # Test stateful sigs keygen here. We could also do it in the "generic" run, but the
+          # non-stateful tests will complete faster in this run, so total runtime will be lower
+          # if it's tested here.
+          # Generates 4 additional test runs (2 on arm, 2 on x86_64).
+          - shared: ON
+            openssl: OFF
+            dist-build: OFF
+            opt-target: auto
+            stfl: ON
+            stfl-keygen: ON
+          # Test the libjade implementation.
+          # libjade is not supported on arm.
+          # Static/shared/OpenSSL shouldn't matter here.
+          # Only build the algorithms supported by libjade.
+          # Generates 2 additional test runs (x86_64 only).
+          - runner: ubuntu-latest
+            shared: OFF
+            openssl: ON
+            libjade: ON
+            build-options: -DOQS_MINIMAL_BUILD="${{ vars.LIBJADE_ALG_LIST }}"
+          # Test the "generic" libjade implementation.
+          # Generates 2 additional test runs (x86_64 only).
+          - runner: ubuntu-latest
+            shared: ON
+            openssl: OFF
+            dist-build: OFF
+            opt-target: generic
+            libjade: ON
+            build-options: -DOQS_MINIMAL_BUILD="${{ vars.LIBJADE_ALG_LIST }}"
+          # Test the libjade implementation without runtime feature detection.
+          # Generates 2 additional test runs (x86_64 only).
+          - runner: ubuntu-latest
+            shared: OFF
+            openssl: OFF
+            dist-build: OFF
+            opt-target: auto
+            libjade: ON
+            build-options: -DOQS_MINIMAL_BUILD="${{ vars.LIBJADE_ALG_LIST }}"
+
     runs-on: ${{ matrix.runner }}
     container:
-      image: ${{ matrix.container }}
+      image: openquantumsafe/ci-ubuntu-latest:latest
     steps:
       - name: Checkout code
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # pin@v4
       - name: Configure
-        run: mkdir build && cd build && cmake -GNinja ${{ matrix.CMAKE_ARGS }} .. && cmake -LA -N ..
+        run: |
+          mkdir build && cd build && \
+          cmake -GNinja -DOQS_STRICT_WARNINGS=ON \
+            -DCMAKE_C_COMPILER=${{ matrix.compiler }} \
+            -DBUILD_SHARED_LIBS=${{ matrix.shared }} \
+            -DOQS_USE_OPENSSL=${{ matrix.openssl }} \
+            -DOQS_USE_SHA3_OPENSSL=${{ matrix.sha3-openssl }} \
+            -DOQS_DLOPEN_OPENSSL=${{ matrix.dlopen-openssl }} \
+            -DOQS_DIST_BUILD=${{ matrix.dist-build }} \
+            -DOQS_OPT_TARGET=${{ matrix.opt-target }} \
+            -DOQS_OPT_TARGET=${{ matrix.opt-target }} \
+            -DOQS_ENABLE_SIG_STFL_LMS=${{ matrix.stfl }} \
+            -DOQS_ENABLE_SIG_STFL_XMSS=${{ matrix.stfl }} \
+            -DOQS_HAZARDOUS_EXPERIMENTAL_ENABLE_SIG_STFL_KEY_SIG_GEN=${{ matrix.stfl-keygen }} \
+            -DOQS_LIBJADE_BUILD=${{ matrix.libjade }} \
+            ${{ matrix.build-options }} .. && \
+          cmake -LA -N ..
       - name: Build
         run: ninja
         working-directory: build
       - name: Run tests
         timeout-minutes: 60
-        run: mkdir -p tmp && python3 -m pytest --verbose --ignore=tests/test_code_conventions.py --numprocesses=auto ${{ matrix.PYTEST_ARGS }}
-      - name: Package .deb
-        if: matrix.name == 'jammy-std-openssl3'
-        run: cpack
-        working-directory: build
-      - name: Retain .deb file
-        if: matrix.name == 'jammy-std-openssl3'
-        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # pin@v3
-        with:
-          name: liboqs-openssl3-shared-x64
-          path: build/*.deb
-      - name: Check STD algorithm and alias
-        if: matrix.name == 'jammy-std-openssl3'
-        run: 'tests/dump_alg_info | grep -zoP "ML-DSA-44:\n    isnull: false" && tests/dump_alg_info | grep -zoP "ML-DSA-44-ipd:\n    isnull: true" && tests/dump_alg_info | grep -zoP "ML-KEM-512:\n    isnull: false"'
-        working-directory: build
+        run: |
+          mkdir -p tmp && \
+          python3 -m pytest --verbose \
+            --ignore=tests/test_code_conventions.py \
+            --ignore=tests/test_constant_time.py \
+            --ignore=tests/test_kat_all.py \
+            --numprocesses=auto
+# TODO: figure out where to put these
+#      - name: Package .deb
+#        if: matrix.name == 'jammy-std-openssl3'
+#        run: cpack
+#        working-directory: build
+#      - name: Retain .deb file
+#        if: matrix.name == 'jammy-std-openssl3'
+#        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # pin@v3
+#        with:
+#          name: liboqs-openssl3-shared-x64
+#          path: build/*.deb
+#      - name: Check STD algorithm and alias
+#        if: matrix.name == 'jammy-std-openssl3'
+#        run: 'tests/dump_alg_info | grep -zoP "ML-DSA-44:\n    isnull: false" && tests/dump_alg_info | grep -zoP "ML-DSA-44-ipd:\n    isnull: true" && tests/dump_alg_info | grep -zoP "ML-KEM-512:\n    isnull: false"'
+#        working-directory: build
 
   linux_arm_emulated:
     runs-on: ubuntu-latest