From 86e64ff40b6ede72c5ea2a8de3a4a98b32a60c94 Mon Sep 17 00:00:00 2001 From: Thomas Cluzel Date: Tue, 22 Sep 2020 09:14:22 +0200 Subject: [PATCH 1/2] Add default filename for pylint report --- Dockerfile | 33 ++++++------ README.md | 98 +++++++++++++++++++++++------------ conf/sonar-scanner.properties | 1 + tests/functions.bash | 18 ++----- 4 files changed, 87 insertions(+), 63 deletions(-) diff --git a/Dockerfile b/Dockerfile index 18ab155..5dde0b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,25 +1,19 @@ # Builder image for analysis tools FROM debian:10.5-slim AS builder -# Get sonar-scanner and C/C++ tools sources -ADD https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-4.4.0.2170.zip \ - https://downloads.sourceforge.net/project/cppcheck/cppcheck/1.90/cppcheck-1.90.tar.gz \ - https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/rough-auditing-tool-for-security/rats-2.4.tgz \ - http://downloads.sourceforge.net/project/expat/expat/2.0.1/expat-2.0.1.tar.gz \ - https://frama-c.com/download/frama-c-20.0-Calcium.tar.gz \ - https://github.com/facebook/infer/releases/download/v0.17.0/infer-linux64-v0.17.0.tar.xz \ - / - -# Compile tools from source +# Install tools from sources RUN echo 'deb http://ftp.fr.debian.org/debian/ bullseye main contrib non-free' >> /etc/apt/sources.list \ && apt-get update \ && apt-get install -y --no-install-recommends \ + curl=7.72.0-* \ + # for C/C++ tools make=4.3-* \ g\+\+=4:10.1.0-* \ python3=3.8.2-* \ libpcre3-dev=2:8.39-* \ unzip=6.0-* \ xz-utils=5.2.4-* \ + # for Frama-C ocaml=4.08.1-* \ ocaml-findlib=1.8.1-* \ libfindlib-ocaml-dev=1.8.1-* \ @@ -27,17 +21,21 @@ RUN echo 'deb http://ftp.fr.debian.org/debian/ bullseye main contrib non-free' > libyojson-ocaml-dev=1.7.0-* \ libzarith-ocaml-dev=1.9.1-* \ menhir=20200624-* \ - # Unzip sonar-scanner + # sonar-scanner + && curl -ksSLO https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-4.4.0.2170.zip \ && unzip sonar-scanner-cli-4.4.0.2170.zip \ && mv /sonar-scanner-4.4.0.2170 /sonar-scanner \ - # Compile CppCheck + # CppCheck + && curl -ksSLO https://downloads.sourceforge.net/project/cppcheck/cppcheck/1.90/cppcheck-1.90.tar.gz \ && tar -zxvf cppcheck-1.90.tar.gz \ && make -C cppcheck-1.90/ install \ MATCHCOMPILER="yes" \ FILESDIR="/usr/share/cppcheck" \ HAVE_RULES="yes" \ CXXFLAGS="-O2 -DNDEBUG -Wall -Wno-sign-compare -Wno-unused-function -Wno-deprecated-declarations" \ - # Compile RATS (and expat) + # RATS (and expat) + && curl -ksSLO https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/rough-auditing-tool-for-security/rats-2.4.tgz \ + && curl -ksSLO http://downloads.sourceforge.net/project/expat/expat/2.0.1/expat-2.0.1.tar.gz \ && tar -xvzf expat-2.0.1.tar.gz \ && cd expat-2.0.1 \ && ./configure \ @@ -51,14 +49,16 @@ RUN echo 'deb http://ftp.fr.debian.org/debian/ bullseye main contrib non-free' > && make install \ && ./rats \ && cd .. \ - # Compile Frama-C + # Frama-C + && curl -ksSLO https://frama-c.com/download/frama-c-20.0-Calcium.tar.gz \ && tar -zxvf frama-c-20.0-Calcium.tar.gz \ && cd frama-c-20.0-Calcium \ && ./configure --disable-gui --disable-wp \ && make \ && make install \ && cd .. \ - # Decompress Infer + # Infer + && curl -ksSLO https://github.com/facebook/infer/releases/download/v0.17.0/infer-linux64-v0.17.0.tar.xz \ && tar -C /opt -Jxvf infer-linux64-v0.17.0.tar.xz ################################################################################ @@ -178,7 +178,8 @@ RUN echo 'deb http://ftp.fr.debian.org/debian/ bullseye main contrib non-free' > # Make sonar-scanner, CNES pylint and C/C++ tools executable ENV PYTHONPATH="$PYTHONPATH:/opt/python/cnes-pylint-extension-5.0.0/checkers" \ PATH="$SONAR_SCANNER_HOME/bin:/usr/local/bin:$PATH" \ - PYLINTHOME="$SONAR_SCANNER_HOME/.pylint.d" + PYLINTHOME="$SONAR_SCANNER_HOME/.pylint.d" \ + JAVA_HOME="/usr/lib/jvm/java-11-openjdk-amd64" # Switch to an unpriviledged user USER sonar-scanner diff --git a/README.md b/README.md index 96190a2..15a05e4 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ _This image is made to be used in conjunction with a pre-configured SonarQube se -v "$(pwd):/usr/src" \ lequal/sonar-scanner ``` + This docker command is equivalent to `sonar-scanner -Dsonar.host.url="url of your SonarQube instance"`. * If the SonarQube server is running in a container on the same computer, you will need to connect both containers (server and client) to the same bridge so that they can communicate. To do so: ```sh $ docker network create -d bridge sonarbridge @@ -68,17 +69,17 @@ $ docker run \ # where my-script.bash is a file in the current working directory ``` -For information on how to use these tools, refer to the official documentation of the tool. +For information on how to use these tools, refer to their official documentations. #### How to use embedded CNES pylintrc -There are 3 `pylintrc` embedded in the image under `/opt/python`: +There are 3 _pylintrc_ embedded in the image under `/opt/python`: * `pylintrc_RNC_sonar_2017_A_B` * `pylintrc_RNC_sonar_2017_C` * `pylintrc_RNC_sonar_2017_D` -To use one of these files when running pylint from within the container: +To use one of these files when running `pylint` from within the container: ```sh # pylint with a CNES pylintrc @@ -91,7 +92,25 @@ $ docker run \ # where my-script.py is a python module in the current working directory ``` -To import pylint results in SonarQube see the [official documentation](https://docs.sonarqube.org/7.9/analysis/languages/python/#header-3). (Summed up: Run pylint with the following template: `pylint --rcfile= -r n --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" > `. Activate at least one pylint rule in the Quality Profile the project uses for Python and set `sonar.python.pylint.reportPath` in `sonar-project.properties`.) +To import pylint results in SonarQube see the [official documentation](https://docs.sonarqube.org/7.9/analysis/languages/python/#header-3). (Summed up: Run pylint with the following template: `pylint --rcfile= -r n --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" > pylint-report.txt`. Activate at least one pylint rule in the Quality Profile the project uses for Python.) + +#### How to use other pylintrcs + +You may want to use the embedded `pylint` with a pylintrc of yours . In this case, the easiest way to do so is to put a _pylintrc_ file along with the sources. + +To then use it: + +```sh +# pylint with a custom pylintrc +$ docker run \ + --rm \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):/usr/src" \ + lequal/sonar-scanner \ + pylint --rcfile=/usr/src/custom_pylintrc my-script.py +# where my-script.py is a python module in the current working directory +# and custom_pylintrc is a pylintrc in the current working directory +``` ### Examples usage in CI @@ -101,37 +120,50 @@ _These examples still need to be tested._ #### Jenkins -Here is an example of a jenkins file that call this image to analyze a project. +Here are 2 examples of a declarative Jenkinsfile and a scripted Jenkinsfile that call this image in a stage to analyze a project. ```groovy +// Declarative pipeline +def sonarqubeURL = 'https://my-sonarqube.com' + pipeline { agent any + stages { - stage('Test') { + stage('Sonar scan') { steps { - sh ''' - mkdir -p .sonarcache + sh """ docker run --rm \ - -u "$(id -u):$(id -g)" \ - -e SONAR_HOST_URL="https://my-sonarqube.com" \ - -v "$(pwd):/usr/src" \ - -v ".sonarcache:/opt/sonar-scanner/.sonar/cache" \ - lequal/sonar-scanner - ''' - - cache { - caches { - path { - '.sonarcache' - } - } - } + -u "\$(id -u):\$(id -g)" \ + -e SONAR_HOST_URL="${sonarqubeURL}" \ + -v "\$(pwd):/usr/src" \ + lequal/sonar-scanner + """ } } } } ``` +```groovy +// Scripted pipeline +def sonarqubeURL = 'https://my-sonarqube.com' + +node { + checkout scm + + stage('Sonar scan') { + sh """ + docker run --rm \ + -u "\$(id -u):\$(id -g)" \ + -e SONAR_HOST_URL="${sonarqubeURL}" \ + -v "\$(pwd):/usr/src" \ + lequal/sonar-scanner + """ + } +} +``` + #### GitHub Actions Here is a GitHub Actions job of a GitHub Actions workflow that call this image to analyze a project. @@ -200,17 +232,17 @@ sonar-scanning: ## Analysis tools included -| Tool | Version | -|--------------------------------------------------------------------------------|----------------------| -| [sonar-scanner](https://docs.sonarqube.org/latest/analysis/scan/sonarscanner/) | 4.4.0.2170 | -| [ShellCheck](https://github.com/koalaman/shellcheck) | 0.7.1 | -| [pylint](http://pylint.pycqa.org/en/latest/user_guide/index.html) | 2.5.0 | -| [CNES pylint extension](https://github.com/cnescatlab/cnes-pylint-extension) | 5.0.0 | -| [CppCheck](https://github.com/danmar/cppcheck) | 1.90 | -| [Vera++](https://bitbucket.org/verateam/vera/wiki/Home) | 1.2.1 | -| [RATS](https://code.google.com/archive/p/rough-auditing-tool-for-security/) | 2.4 | -| [Frama-C](https://frama-c.com/index.html) | 20.0 | -| [Infer](https://fbinfer.com/) | 0.17.0 | +| Tool | Version | Default report file | +|--------------------------------------------------------------------------------|---------------|---------------------| +| [sonar-scanner](https://docs.sonarqube.org/latest/analysis/scan/sonarscanner/) | 4.4.0.2170 | | +| [ShellCheck](https://github.com/koalaman/shellcheck) | 0.7.1 | | +| [pylint](http://pylint.pycqa.org/en/latest/user_guide/index.html) | 2.5.0 | pylint-report.txt | +| [CNES pylint extension](https://github.com/cnescatlab/cnes-pylint-extension) | 5.0.0 | | +| [CppCheck](https://github.com/danmar/cppcheck) | 1.90 | cppcheck-report.xml | +| [Vera++](https://bitbucket.org/verateam/vera/wiki/Home) | 1.2.1 | vera-report.xml | +| [RATS](https://code.google.com/archive/p/rough-auditing-tool-for-security/) | 2.4 | rats-report.xml | +| [Frama-C](https://frama-c.com/index.html) | 20.0 | | +| [Infer](https://fbinfer.com/) | 0.17.0 | | ## Developer's guide diff --git a/conf/sonar-scanner.properties b/conf/sonar-scanner.properties index e7bbb4e..9d5f9f0 100644 --- a/conf/sonar-scanner.properties +++ b/conf/sonar-scanner.properties @@ -11,3 +11,4 @@ sonar.cxx.cppcheck.reportPath=cppcheck-report.xml sonar.cxx.vera.reportPath=vera-report.xml sonar.cxx.rats.reportPath=rats-report.xml +sonar.python.pylint.reportPath=pylint-report.txt diff --git a/tests/functions.bash b/tests/functions.bash index 03bdd3c..13fc881 100644 --- a/tests/functions.bash +++ b/tests/functions.bash @@ -243,7 +243,7 @@ test_language() # 5: (optional) store the standard output in the temporary result file, either "yes" or "no" (default: "yes") # # Example: -# $ cmd="pylint -f json --rcfile=/opt/python/pylintrc_RNC_sonar_2017_A_B tests//src/*.py" +# $ cmd="pylint -f json --rcfile=/opt/python/pylintrc_RNC_sonar_2017_A_B tests/python/src/*.py" # $ test_analysis_tool "pylint" "$cmd" "tests/python/reference-pylint-results.json" "tests/python/tmp-pylint-results.json" test_analysis_tool() { @@ -306,7 +306,7 @@ test_analysis_tool() # Example: # $ ruleViolated="cppcheck:arrayIndexOutOfBounds" # $ expected_sensor="INFO: Sensor C++ (Community) CppCheckSensor \[cxx\]" -# $ expected_import="INFO: Processing report '/usr/src/tests/c_cpp/cppcheck-report.xml'" +# $ expected_import="INFO: CXX-CPPCHECK processed = 1" # $ test_import_analysis_results "CppCheck" "CppCheck Dummy Project" "cppcheck-dummy-project" "CNES_C_A" "c++" \ # "tests/c_cpp" "cppcheck" "$ruleViolated" "$expected_sensor" "$expected_import" test_import_analysis_results() @@ -378,27 +378,17 @@ test_import_analysis_results() fi # Analyse the project and collect the analysis files (that match the default names) - usrSrcFolder="$PWD" - if [ "$projectKey" = "framac-dummy-project" ] - then - # Sonar Frama-C Plugin failed to find the result files when used with sonar.projectBaseDir - # Hence, we bind mount the project folder in /usr/src directly - # GitHub issue: https://github.com/cnescatlab/sonar-frama-c-plugin/issues/43 - usrSrcFolder="$PWD/$languageFolder" - languageFolder="" - fi analysis_output=$(docker run --rm -u "$(id -u):$(id -g)" \ -e SONAR_HOST_URL="$SONARQUBE_URL" \ --net "$SONARQUBE_NETWORK" \ - -v "$usrSrcFolder:/usr/src" \ + -v "$PWD:/usr/src" \ -v "$PWD/.sonarcache:/opt/sonar-scanner/.sonar/cache" \ + -w "/usr/src/$languageFolder"\ lequal/sonar-scanner \ - "-Dsonar.projectBaseDir=/usr/src/$languageFolder" \ "-Dsonar.projectKey=$projectKey" \ "-Dsonar.projectName=$projectName" \ "-Dsonar.projectVersion=1.0" \ "-Dsonar.sources=$sourceFolder" \ - "-Dsonar.python.pylint.reportPath=pylint-report.txt" \ 2>&1) for line in "$expected_sensor" "$expected_import" do From 70e7ef7994ca70242d3fbb1ae977de3faf545463 Mon Sep 17 00:00:00 2001 From: Thomas Cluzel Date: Wed, 23 Sep 2020 09:55:46 +0200 Subject: [PATCH 2/2] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 15a05e4..f9aeff8 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ $ docker run \ # where my-script.bash is a file in the current working directory ``` -For information on how to use these tools, refer to their official documentations. +For information on how to use these tools, refer to their official documentation. #### How to use embedded CNES pylintrc @@ -112,6 +112,8 @@ $ docker run \ # and custom_pylintrc is a pylintrc in the current working directory ``` +On the other hand, if you want to use a CNES _pylintrc_ for your project you can download it directly from github. They are stored on this repository under [pylintrc.d](https://github.com/cnescatlab/sonar-scanner/tree/master/pylintrc.d). + ### Examples usage in CI This image was made for CI, hence here are some examples. Make sur to use the right URL for your SonarQube instance instead of `my-sonarqube.com`.