From 950339caae083850f05ec58d364d5e8b45867755 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 18 Oct 2024 10:56:04 +1100 Subject: [PATCH 1/5] Try some coverage stuff --- .github/workflows/gcc.yaml | 12 +++++-- .github/workflows/macos.yaml | 10 ++++-- .github/workflows/sonarcloud.yaml | 42 ++++++++++++++++------ .github/workflows/windows.yaml | 12 +++++-- cmake/TestRunner.cmake | 59 +++++++++++++++++++++++++++++++ sonar-project.properties | 9 ----- src/CMakeLists.txt | 9 +++++ 7 files changed, 125 insertions(+), 28 deletions(-) create mode 100644 cmake/TestRunner.cmake delete mode 100644 sonar-project.properties diff --git a/.github/workflows/gcc.yaml b/.github/workflows/gcc.yaml index d12005087..09d408a84 100644 --- a/.github/workflows/gcc.yaml +++ b/.github/workflows/gcc.yaml @@ -92,11 +92,17 @@ jobs: - name: Test timeout-minutes: 2 - working-directory: build/tests - run: ctest --output-on-failure -E "dsl/UDP" + working-directory: build + run: ninja run_all_tests -k 0 + + - name: Test Summary + if: ${{ !cancelled() }} + uses: test-summary/action@v2 + with: + paths: "build/reports/tests/*.junit.xml" - name: Upload Traces - if: always() + if: ${{ !cancelled() }} uses: actions/upload-artifact@v4 with: name: traces-gcc-${{ matrix.toolchain.version }} diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml index cc949da1d..eaf53f915 100644 --- a/.github/workflows/macos.yaml +++ b/.github/workflows/macos.yaml @@ -61,10 +61,16 @@ jobs: - name: Test timeout-minutes: 5 working-directory: build/tests - run: ctest --output-on-failure + run: ninja run_all_tests -k 0 + + - name: Test Summary + if: ${{ !cancelled() }} + uses: test-summary/action@v2 + with: + paths: "build/reports/tests/*.junit.xml" - name: Upload Traces - if: always() + if: ${{ !cancelled() }} uses: actions/upload-artifact@v4 with: name: traces-macos diff --git a/.github/workflows/sonarcloud.yaml b/.github/workflows/sonarcloud.yaml index 3a02f51e5..a7e407937 100644 --- a/.github/workflows/sonarcloud.yaml +++ b/.github/workflows/sonarcloud.yaml @@ -27,15 +27,15 @@ jobs: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - name: Install gcovr - run: pip install gcovr==7.2 + run: pip install gcovr==8.2 - name: Install sonar-scanner and build-wrapper uses: SonarSource/sonarcloud-github-c-cpp@v3 - name: Install CMake - uses: lukka/get-cmake@latest + uses: lukka/get-cmake@v3.30.5 with: - cmakeVersion: 3.27.1 + cmakeVersion: 3.30.5 ninjaVersion: 1.11.1 - name: Setup CCache @@ -49,12 +49,12 @@ jobs: cmake -E make_directory build cmake -S . -B build \ -GNinja \ - -DCMAKE_CXX_FLAGS="--coverage" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DBUILD_TESTS=ON \ -DCMAKE_BUILD_TYPE=Debug \ + -DBUILD_TESTS=ON \ -DCI_BUILD=ON \ + -DENABLE_COVERAGE=ON \ -DENABLE_CLANG_TIDY=OFF - name: Build the code in debug mode @@ -63,25 +63,45 @@ jobs: - name: Run tests to generate coverage statistics timeout-minutes: 10 - working-directory: build/tests - run: ctest --output-on-failure + working-directory: build + run: ninja run_all_tests -k 0 + + - name: Test Summary + if: ${{ !cancelled() }} + uses: test-summary/action@v2 + with: + paths: "build/reports/tests/*.junit.xml" - name: Collect coverage into one XML report - if: always() - run: gcovr --gcov-ignore-parse-errors=negative_hits.warn_once_per_file --exclude-unreachable-branches --exclude-noncode-lines --sonarqube > coverage.xml + if: ${{ !cancelled() }} + run: | + gcovr \ + --root . \ + --object-directory build \ + --force-color \ + --no-markers \ + --decisions \ + --calls \ + --exclude-noncode-lines \ + --gcov-ignore-parse-errors negative_hits.warn \ + --sonarqube "coverage.xml" - name: Run sonar-scanner - if: always() + if: ${{ !cancelled() }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} run: | sonar-scanner \ + --define sonar.projectKey=Fastcode_NUClear \ + --define sonar.organization=fastcode \ + --define sonar.sources=src \ + --define sonar.tests=tests \ --define sonar.cfamily.compile-commands=build/compile_commands.json \ --define sonar.coverageReportPaths=coverage.xml - name: Upload Traces - if: always() + if: ${{ !cancelled }} uses: actions/upload-artifact@v4 with: name: traces-sonar diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml index 2163cc35b..dd728120b 100644 --- a/.github/workflows/windows.yaml +++ b/.github/workflows/windows.yaml @@ -81,11 +81,17 @@ jobs: - name: Test timeout-minutes: 5 - working-directory: build/tests - run: ctest --output-on-failure + working-directory: build + run: ninja run_all_tests -k 0 + + - name: Test Summary + if: ${{ !cancelled() }} + uses: test-summary/action@v2 + with: + paths: "build/reports/tests/*.junit.xml" - name: Upload Traces - if: always() + if: ${{ !cancelled() }} uses: actions/upload-artifact@v4 with: name: traces-windows-${{ matrix.toolchain.name }} diff --git a/cmake/TestRunner.cmake b/cmake/TestRunner.cmake new file mode 100644 index 000000000..f2f2c617a --- /dev/null +++ b/cmake/TestRunner.cmake @@ -0,0 +1,59 @@ +# Collect all currently added targets in all subdirectories +# +# Parameters: +# - _result the list containing all found targets +# - _dir root directory to start looking from +function(get_all_catch_test_targets _result _dir) + get_property( + _subdirs + DIRECTORY "${_dir}" + PROPERTY SUBDIRECTORIES + ) + foreach(_subdir IN LISTS _subdirs) + get_all_catch_test_targets(${_result} "${_subdir}") + endforeach() + + unset(catch_targets) + get_directory_property(_sub_targets DIRECTORY "${_dir}" BUILDSYSTEM_TARGETS) + foreach(target ${_sub_targets}) + get_target_property(target_type ${target} TYPE) + if(target_type STREQUAL "EXECUTABLE") + get_target_property(target_link_libraries ${target} INTERFACE_LINK_LIBRARIES) + if(target_link_libraries MATCHES "Catch2::Catch2") + list(APPEND catch_targets ${target}) + endif() + endif() + endforeach() + + set(${_result} + ${${_result}} ${catch_targets} + PARENT_SCOPE + ) +endfunction() + +# Find all executable targets that link to Catch2::Catch2WithMain || Catch2::Catch2 +get_all_catch_test_targets(all_targets ${PROJECT_SOURCE_DIR}) + +# Create a custom command for each test target to run it +# Make sure that coverage data is written with paths relative to the source directory +unset(reports) +foreach(target ${all_targets}) + + set(sonarqube_report_file "${PROJECT_BINARY_DIR}/reports/tests/${target}.sonarqube.xml") + set(junit_report_file "${PROJECT_BINARY_DIR}/reports/tests/${target}.junit.xml") + list(APPEND reports ${sonarqube_report_file} ${junit_report_file}) + add_custom_command( + OUTPUT ${sonarqube_report_file} ${junit_report_file} + COMMAND $ --reporter console --reporter SonarQube::out=${sonarqube_report_file} --reporter + JUnit::out=${junit_report_file} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + COMMENT "Running test ${target}" + ) +endforeach() + +# Create a custom target that depends on all test targets +add_custom_target( + run_all_tests + DEPENDS ${reports} + COMMENT "Running all Catch2 tests" +) diff --git a/sonar-project.properties b/sonar-project.properties deleted file mode 100644 index 60dc88e40..000000000 --- a/sonar-project.properties +++ /dev/null @@ -1,9 +0,0 @@ -sonar.projectName=NUClear -sonar.projectKey=Fastcode_NUClear -sonar.organization=fastcode -sonar.projectVersion=1.0 - -sonar.sources=src -sonar.tests=tests - -sonar.sourceEncoding=UTF-8 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9ce7a65ac..65f180916 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -37,6 +37,15 @@ target_link_libraries(nuclear ${CMAKE_THREAD_LIBS_INIT}) set_target_properties(nuclear PROPERTIES POSITION_INDEPENDENT_CODE ON) target_compile_features(nuclear PUBLIC cxx_std_14) +option(ENABLE_COVERAGE "Compile with coverage support enabled.") +if(ENABLE_COVERAGE) + if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + message(WARNING "Coverage is enabled but not build in debug mode. Coverage results may be misleading.") + endif() + target_compile_options(nuclear PUBLIC -fprofile-arcs -ftest-coverage -fprofile-abs-path -fprofile-update=atomic) + target_link_options(nuclear PUBLIC -fprofile-arcs) +endif(ENABLE_COVERAGE) + # Enable warnings, and all warnings are errors if(MSVC) target_compile_options(nuclear PRIVATE /W4 /WX) From eeca6f2a5d51bac4372667f5b340a6d6904b361b Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 18 Oct 2024 11:11:36 +1100 Subject: [PATCH 2/5] add the test runner --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index b5064757d..4c70ae8a6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,6 +89,7 @@ option(BUILD_TESTS "Builds all of the NUClear unit tests." ON) if(BUILD_TESTS) enable_testing() add_subdirectory(tests) + include(TestRunner) endif() # Add the documentation subdirectory From 6f6ded8a020c73f17781b69f49cfce010d4335ad Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 18 Oct 2024 11:13:08 +1100 Subject: [PATCH 3/5] Fix mac --- .github/workflows/macos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml index eaf53f915..4483dd9f6 100644 --- a/.github/workflows/macos.yaml +++ b/.github/workflows/macos.yaml @@ -60,7 +60,7 @@ jobs: - name: Test timeout-minutes: 5 - working-directory: build/tests + working-directory: build run: ninja run_all_tests -k 0 - name: Test Summary From d84c30b9baa0872d603fd0f3eceee726d0e2c6c1 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 18 Oct 2024 11:41:53 +1100 Subject: [PATCH 4/5] . --- .github/workflows/sonarcloud.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sonarcloud.yaml b/.github/workflows/sonarcloud.yaml index a7e407937..75739d9f1 100644 --- a/.github/workflows/sonarcloud.yaml +++ b/.github/workflows/sonarcloud.yaml @@ -101,7 +101,7 @@ jobs: --define sonar.coverageReportPaths=coverage.xml - name: Upload Traces - if: ${{ !cancelled }} + if: ${{ !cancelled() }} uses: actions/upload-artifact@v4 with: name: traces-sonar From c08a7030a50d0922ac31eb44f5d0854a4c30dbf8 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Sat, 26 Oct 2024 14:52:32 +1100 Subject: [PATCH 5/5] 1 at a time --- .github/workflows/sonarcloud.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sonarcloud.yaml b/.github/workflows/sonarcloud.yaml index 75739d9f1..07bc629b2 100644 --- a/.github/workflows/sonarcloud.yaml +++ b/.github/workflows/sonarcloud.yaml @@ -64,7 +64,7 @@ jobs: - name: Run tests to generate coverage statistics timeout-minutes: 10 working-directory: build - run: ninja run_all_tests -k 0 + run: ninja run_all_tests -j1 -k 0 - name: Test Summary if: ${{ !cancelled() }}