From 47b6a8c1c2220c58175e1b5bccb4a0a8a116d727 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 19 Dec 2024 12:58:14 -0500 Subject: [PATCH] chore(ci): Take averages for compilation and execution report of small programs (#6874) --- .github/workflows/reports.yml | 34 +++++++++++++++++++++-------- test_programs/compilation_report.sh | 26 ++++++++++++++++++---- test_programs/execution_report.sh | 25 +++++++++++++++++---- 3 files changed, 68 insertions(+), 17 deletions(-) diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index 37864dd9b68..f8750e5aaf5 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -252,13 +252,13 @@ jobs: - name: Generate Compilation report working-directory: ./test_programs run: | - ./compilation_report.sh + ./compilation_report.sh 0 1 mv compilation_report.json ../compilation_report.json - name: Generate Execution report working-directory: ./test_programs run: | - ./execution_report.sh + ./execution_report.sh 0 1 mv execution_report.json ../execution_report.json - name: Upload compilation report @@ -286,10 +286,10 @@ jobs: matrix: include: - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts, is_library: true } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset } + - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root, take_average: true } + - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner, take_average: true } + - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail, take_average: true } + - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset, take_average: true } - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private } - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public } @@ -330,20 +330,36 @@ jobs: path: test-repo ref: ${{ matrix.project.ref }} - - name: Generate compilation report + - name: Generate compilation report without averages working-directory: ./test-repo/${{ matrix.project.path }} + if: ${{ !matrix.project.take_average }} run: | mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh chmod +x ./compilation_report.sh ./compilation_report.sh 1 + + - name: Generate compilation report with averages + working-directory: ./test-repo/${{ matrix.project.path }} + if: ${{ matrix.project.take_average }} + run: | + mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh + chmod +x ./compilation_report.sh + ./compilation_report.sh 1 1 - - name: Generate execution report + - name: Generate execution report without averages working-directory: ./test-repo/${{ matrix.project.path }} - if: ${{ !matrix.project.is_library }} + if: ${{ !matrix.project.is_library && !matrix.project.take_average }} run: | mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh ./execution_report.sh 1 + - name: Generate execution report with averages + working-directory: ./test-repo/${{ matrix.project.path }} + if: ${{ !matrix.project.is_library && matrix.project.take_average }} + run: | + mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh + ./execution_report.sh 1 1 + - name: Move compilation report id: compilation_report shell: bash diff --git a/test_programs/compilation_report.sh b/test_programs/compilation_report.sh index d050e7c9c34..9f2f8b96147 100755 --- a/test_programs/compilation_report.sh +++ b/test_programs/compilation_report.sh @@ -10,7 +10,7 @@ tests_to_profile=("sha256_regression" "regression_4709" "ram_blowup_regression") echo "{\"compilation_reports\": [ " > $current_dir/compilation_report.json # If there is an argument that means we want to generate a report for only the current directory -if [ "$#" -ne 0 ]; then +if [ "$1" == "1" ]; then base_path="$current_dir" tests_to_profile=(".") fi @@ -32,12 +32,30 @@ for dir in ${tests_to_profile[@]}; do # The default package to run is the supplied list hardcoded at the top of the script PACKAGE_NAME=$dir # Otherwise default to the current directory as the package we want to run - if [ "$#" -ne 0 ]; then + if [ "$1" == "1" ]; then PACKAGE_NAME=$(basename $current_dir) fi - COMPILE_TIME=$((time nargo compile --force --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s') - echo -e " {\n \"artifact_name\":\"$PACKAGE_NAME\",\n \"time\":\"$COMPILE_TIME\"" >> $current_dir/compilation_report.json + NUM_RUNS=1 + TOTAL_TIME=0 + + # Passing a second argument will take an average of five runs + # rather than + if [ "$2" == "1" ]; then + NUM_RUNS=5 + fi + + for ((i = 1; i <= NUM_RUNS; i++)); do + COMPILE_TIME=$((time nargo compile --force --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s') + # Convert time to seconds and add to total time + TOTAL_TIME=$(echo "$TOTAL_TIME + $(echo $COMPILE_TIME | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc)" | bc) + done + + AVG_TIME=$(echo "$TOTAL_TIME / $NUM_RUNS" | bc -l) + # Keep only last three decimal points + AVG_TIME=$(awk '{printf "%.3f\n", $1}' <<< "$AVG_TIME") + + echo -e " {\n \"artifact_name\":\"$PACKAGE_NAME\",\n \"time\":\"0m"$AVG_TIME"s\"" >> $current_dir/compilation_report.json if (($ITER == $NUM_ARTIFACTS)); then echo "}" >> $current_dir/compilation_report.json diff --git a/test_programs/execution_report.sh b/test_programs/execution_report.sh index bdb1500d19e..bb3adf143f0 100755 --- a/test_programs/execution_report.sh +++ b/test_programs/execution_report.sh @@ -10,7 +10,7 @@ tests_to_profile=("sha256_regression" "regression_4709" "ram_blowup_regression") echo "{\"execution_reports\": [ " > $current_dir/execution_report.json # If there is an argument that means we want to generate a report for only the current directory -if [ "$#" -ne 0 ]; then +if [ "$1" == "1" ]; then base_path="$current_dir" tests_to_profile=(".") fi @@ -32,7 +32,7 @@ for dir in ${tests_to_profile[@]}; do # The default package to run is the supplied list hardcoded at the top of the script PACKAGE_NAME=$dir # Otherwise default to the current directory as the package we want to run - if [ "$#" -ne 0 ]; then + if [ "$1" == "1" ]; then PACKAGE_NAME=$(basename $current_dir) fi @@ -44,8 +44,25 @@ for dir in ${tests_to_profile[@]}; do exit 1 fi - COMPILE_TIME=$((time nargo execute --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s') - echo -e " {\n \"artifact_name\":\"$PACKAGE_NAME\",\n \"time\":\"$COMPILE_TIME\"" >> $current_dir/execution_report.json + + NUM_RUNS=1 + TOTAL_TIME=0 + + if [ "$2" == "1" ]; then + NUM_RUNS=5 + fi + + for ((i = 1; i <= NUM_RUNS; i++)); do + EXECUTION_TIME=$((time nargo execute --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s') + # Convert to seconds and add to total time + TOTAL_TIME=$(echo "$TOTAL_TIME + $(echo $EXECUTION_TIME | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc)" | bc) + done + + AVG_TIME=$(echo "$TOTAL_TIME / $NUM_RUNS" | bc -l) + # Keep only last three decimal points + AVG_TIME=$(awk '{printf "%.3f\n", $1}' <<< "$AVG_TIME") + + echo -e " {\n \"artifact_name\":\"$PACKAGE_NAME\",\n \"time\":\"0m"$AVG_TIME"s\"" >> $current_dir/execution_report.json if (($ITER == $NUM_ARTIFACTS)); then echo "}" >> $current_dir/execution_report.json