chore(ci): Take averages for compilation and execution report of smal…

…l programs (#6874)
noir-lang · Dec 19, 2024 · 47b6a8c · 47b6a8c
1 parent ee0754b
commit 47b6a8c
Show file tree

Hide file tree

Showing 3 changed files with 68 additions and 17 deletions.
diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml
@@ -252,13 +252,13 @@ jobs:
       - name: Generate Compilation report
         working-directory: ./test_programs
         run: |
-          ./compilation_report.sh
+          ./compilation_report.sh 0 1
           mv compilation_report.json ../compilation_report.json
 
       - name: Generate Execution report
         working-directory: ./test_programs
         run: |
-          ./execution_report.sh
+          ./execution_report.sh 0 1
           mv execution_report.json ../execution_report.json
           
       - name: Upload compilation report
@@ -286,10 +286,10 @@ jobs:
       matrix:
         include:
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts, is_library: true }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root, take_average: true }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner, take_average: true }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail, take_average: true  }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset, take_average: true }
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private }
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public }
 
@@ -330,20 +330,36 @@ jobs:
           path: test-repo
           ref: ${{ matrix.project.ref }}
 
-      - name: Generate compilation report
+      - name: Generate compilation report without averages
         working-directory: ./test-repo/${{ matrix.project.path }}
+        if: ${{ !matrix.project.take_average }}
         run: |
           mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh
           chmod +x ./compilation_report.sh
           ./compilation_report.sh 1
+
+      - name: Generate compilation report with averages
+        working-directory: ./test-repo/${{ matrix.project.path }}
+        if: ${{ matrix.project.take_average }}
+        run: |
+          mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh
+          chmod +x ./compilation_report.sh
+          ./compilation_report.sh 1 1
       
-      - name: Generate execution report
+      - name: Generate execution report without averages
         working-directory: ./test-repo/${{ matrix.project.path }}
-        if: ${{ !matrix.project.is_library }}
+        if: ${{ !matrix.project.is_library && !matrix.project.take_average }}
         run: |
           mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh
           ./execution_report.sh 1
 
+      - name: Generate execution report with averages
+        working-directory: ./test-repo/${{ matrix.project.path }}
+        if: ${{ !matrix.project.is_library && matrix.project.take_average }}
+        run: |
+          mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh
+          ./execution_report.sh 1 1
+
       - name: Move compilation report
         id: compilation_report
         shell: bash

diff --git a/test_programs/compilation_report.sh b/test_programs/compilation_report.sh
@@ -10,7 +10,7 @@ tests_to_profile=("sha256_regression" "regression_4709" "ram_blowup_regression")
 echo "{\"compilation_reports\": [ " > $current_dir/compilation_report.json
 
 # If there is an argument that means we want to generate a report for only the current directory
-if [ "$#" -ne 0 ]; then
+if [ "$1" == "1" ]; then
   base_path="$current_dir"
   tests_to_profile=(".")
 fi
@@ -32,12 +32,30 @@ for dir in ${tests_to_profile[@]}; do
     # The default package to run is the supplied list hardcoded at the top of the script
     PACKAGE_NAME=$dir
     # Otherwise default to the current directory as the package we want to run
-    if [ "$#" -ne 0 ]; then
+    if [ "$1" == "1" ]; then
       PACKAGE_NAME=$(basename $current_dir)
     fi
 
-    COMPILE_TIME=$((time nargo compile --force --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s')
-    echo -e " {\n    \"artifact_name\":\"$PACKAGE_NAME\",\n    \"time\":\"$COMPILE_TIME\"" >> $current_dir/compilation_report.json
+    NUM_RUNS=1
+    TOTAL_TIME=0
+
+    # Passing a second argument will take an average of five runs
+    # rather than 
+    if [ "$2" == "1" ]; then
+      NUM_RUNS=5
+    fi
+
+    for ((i = 1; i <= NUM_RUNS; i++)); do
+      COMPILE_TIME=$((time nargo compile --force --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s')
+      # Convert time to seconds and add to total time
+      TOTAL_TIME=$(echo "$TOTAL_TIME + $(echo $COMPILE_TIME | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc)" | bc)
+    done
+
+    AVG_TIME=$(echo "$TOTAL_TIME / $NUM_RUNS" | bc -l)
+    # Keep only last three decimal points
+    AVG_TIME=$(awk '{printf "%.3f\n", $1}' <<< "$AVG_TIME")
+
+    echo -e " {\n    \"artifact_name\":\"$PACKAGE_NAME\",\n    \"time\":\"0m"$AVG_TIME"s\"" >> $current_dir/compilation_report.json
     
     if (($ITER == $NUM_ARTIFACTS)); then
         echo "}" >> $current_dir/compilation_report.json

diff --git a/test_programs/execution_report.sh b/test_programs/execution_report.sh
@@ -10,7 +10,7 @@ tests_to_profile=("sha256_regression" "regression_4709" "ram_blowup_regression")
 echo "{\"execution_reports\": [ " > $current_dir/execution_report.json
 
 # If there is an argument that means we want to generate a report for only the current directory
-if [ "$#" -ne 0 ]; then
+if [ "$1" == "1" ]; then
   base_path="$current_dir"
   tests_to_profile=(".")
 fi
@@ -32,7 +32,7 @@ for dir in ${tests_to_profile[@]}; do
     # The default package to run is the supplied list hardcoded at the top of the script
     PACKAGE_NAME=$dir
     # Otherwise default to the current directory as the package we want to run
-    if [ "$#" -ne 0 ]; then
+    if [ "$1" == "1" ]; then
       PACKAGE_NAME=$(basename $current_dir)
     fi
 
@@ -44,8 +44,25 @@ for dir in ${tests_to_profile[@]}; do
       exit 1
     fi
 
-    COMPILE_TIME=$((time nargo execute --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s')
-    echo -e " {\n    \"artifact_name\":\"$PACKAGE_NAME\",\n    \"time\":\"$COMPILE_TIME\"" >> $current_dir/execution_report.json
+
+    NUM_RUNS=1
+    TOTAL_TIME=0
+
+    if [ "$2" == "1" ]; then
+      NUM_RUNS=5
+    fi
+
+    for ((i = 1; i <= NUM_RUNS; i++)); do
+      EXECUTION_TIME=$((time nargo execute --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s')
+      # Convert to seconds and add to total time
+      TOTAL_TIME=$(echo "$TOTAL_TIME + $(echo $EXECUTION_TIME | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc)" | bc)
+    done
+
+    AVG_TIME=$(echo "$TOTAL_TIME / $NUM_RUNS" | bc -l)
+    # Keep only last three decimal points
+    AVG_TIME=$(awk '{printf "%.3f\n", $1}' <<< "$AVG_TIME")
+
+    echo -e " {\n    \"artifact_name\":\"$PACKAGE_NAME\",\n    \"time\":\"0m"$AVG_TIME"s\"" >> $current_dir/execution_report.json
     
     if (($ITER == $NUM_ARTIFACTS)); then
         echo "}" >> $current_dir/execution_report.json