diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f33c274b5..be223b8bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Setup Build Env + - name: Install Build Tools run: sudo ./scripts/install-build-tools.sh - name: Setup Local Dependencies run: ./scripts/setup-dependencies.sh @@ -38,7 +38,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Setup Build Env + - name: Install Build Tools run: sudo ./scripts/install-build-tools.sh - name: Setup Local Dependencies run: ./scripts/setup-dependencies.sh @@ -50,7 +50,7 @@ jobs: name: Pylint runs-on: ubuntu-22.04 continue-on-error: true - timeout-minutes: 10 + timeout-minutes: 5 strategy: matrix: python-version: ["3.10"] @@ -62,10 +62,25 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Setup Build Env + - name: Install Build Tools run: sudo ./scripts/install-build-tools.sh - name: Lint with Pylint run: ./scripts/pylint.sh + shellcheck: + name: Shellcheck + runs-on: ubuntu-22.04 + continue-on-error: true + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install shellcheck + run: | + sudo apt-get update + sudo apt-get install -y shellcheck + - name: Lint with Shellcheck + run: ./scripts/shellcheck.sh -S error -v unit-and-integration-test: name: Unit and Integration Tests runs-on: ubuntu-22.04 @@ -74,7 +89,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Setup Build Env + - name: Install Build Tools run: sudo ./scripts/install-build-tools.sh - name: Setup Local Dependencies run: ./scripts/setup-dependencies.sh @@ -84,7 +99,7 @@ jobs: run: ./scripts/test.sh - name: Shorten SHA id: vars - run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV - uses: actions/upload-artifact@v4 if: ${{ !env.ACT }} name: Archive Test Results @@ -114,4 +129,3 @@ jobs: name: OpenCBDC Transaction Processor docs for ${{ steps.vars.outputs.sha_short }} path: ./doxygen_generated/html/* retention-days: 7 - diff --git a/scripts/create-e2e-report.sh b/scripts/create-e2e-report.sh index 51214a4d0..771fb698c 100755 --- a/scripts/create-e2e-report.sh +++ b/scripts/create-e2e-report.sh @@ -11,9 +11,9 @@ function readAndFormatLogs() { return fi - for logfile in $(ls $logdir); do + for logfile in "$logdir"/*; do logfile_path="$logdir/$logfile" - logfile_content=$(cat $logfile_path) + logfile_content=$(<"$logfile_path") message+="\n
\n$logfile\n\n\`\`\`\n$logfile_content\n\`\`\`\n
\n" done echo "$message" diff --git a/scripts/install-build-tools.sh b/scripts/install-build-tools.sh index 523789d99..197ab904b 100755 --- a/scripts/install-build-tools.sh +++ b/scripts/install-build-tools.sh @@ -17,7 +17,7 @@ fi # Supporting these versions for buildflow PYTHON_VERSIONS=("3.10" "3.11" "3.12") -echo "Python3 versions supported: ${PYTHON_VERSIONS[@]}" +echo "Python3 versions supported: ${PYTHON_VERSIONS[*]}" # check if supported version of python3 is already installed, and save the version PY_INSTALLED='' diff --git a/scripts/lint.sh b/scripts/lint.sh index c6e4b0936..2197ab050 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -25,9 +25,12 @@ if [ -n "$whitespace_files" ] || [ -n "$newline_files" ] ; then exit 1 fi -check_format_files=$(git ls-files | grep -E "tools|tests|src|cmake-tests" \ - | grep -E "\..*pp") -clang-format --style=file --Werror --dry-run ${check_format_files[@]} +check_format_files=$(git ls-files | \ + grep -E "tools|tests|src|cmake-tests" | \ + grep -E "\..*pp") + +echo "${check_format_files}" | \ + xargs -n1 -I{} clang-format --style=file --Werror --dry-run {} if ! command -v clang-tidy &>/dev/null; then echo "clang-tidy does not appear to be installed" diff --git a/scripts/native-system-benchmark.sh b/scripts/native-system-benchmark.sh index 1b7923fa2..f295f2861 100755 --- a/scripts/native-system-benchmark.sh +++ b/scripts/native-system-benchmark.sh @@ -151,7 +151,7 @@ on_int() { printf 'Interrupting all components\n' trap '' SIGINT # avoid interrupting ourself for i in $PIDS; do # intentionally unquoted - if [[ -n "RECORD" ]]; then + if [[ -n "$RECORD" ]]; then kill -SIGINT -- "-$i" else kill -SIGINT -- "$i" @@ -194,7 +194,7 @@ on_int() { printf 'Terminating any remaining processes\n' for i in $PIDS; do # intentionally unquoted - if [[ -n "RECORD" ]]; then + if [[ -n "$RECORD" ]]; then kill -SIGTERM -- "-$i" else kill -SIGTERM -- "$i" @@ -253,7 +253,7 @@ run() { COMP= case "$RECORD" in perf) - $@ &> "$PROC_LOG" & + "$@" &> "$PROC_LOG" & COMP="$!" perf record -F 99 -a -g -o "$PNAME".perf -p "$COMP" &> "$PERF_LOG" & PERFS="$PERFS $!";; @@ -261,7 +261,7 @@ run() { ${DBG} "$@" &> "$PROC_LOG" & COMP="$!";; *) - $@ &> "$PROC_LOG" & + "$@" &> "$PROC_LOG" & COMP="$!";; esac @@ -324,7 +324,7 @@ launch() { "$RT"/scripts/wait-for-it.sh -q -t 5 -h localhost -p "$ep" done printf 'Launched logical %s %d, replica %d [PID: %d]\n' "$1" "$id" "$node" "$PID" - if [[ -n "RECORD" ]]; then + if [[ -n "$RECORD" ]]; then PIDS="$PIDS $(getpgid $PID)" else PIDS="$PIDS $PID" @@ -337,7 +337,7 @@ launch() { "$RT"/scripts/wait-for-it.sh -q -t 5 -h localhost -p "$ep" done printf 'Launched %s %d [PID: %d]\n' "$1" "$id" "$PID" - if [[ -n "RECORD" ]]; then + if [[ -n "$RECORD" ]]; then PIDS="$PIDS $(getpgid $PID)" else PIDS="$PIDS $PID" diff --git a/scripts/shellcheck.sh b/scripts/shellcheck.sh new file mode 100755 index 000000000..6ddc9032e --- /dev/null +++ b/scripts/shellcheck.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env bash + +CYAN="\e[36m" +YELLOW="\e[33m" +BLUE="\e[34m" +RED="\e[31m" +GREEN="\e[32m" +RST_COLOR="\e[0m" + +ROOT="$(cd "$(dirname "$0")"/.. && pwd)" +SHELLCHECK_REPORT="${ROOT}/shellcheck-report.txt" +START_TIME=$(date "+%Y-%m-%d %H:%M:%S") + +show_usage() { + cat << EOF +Usage: $0 [options] + +Options: + -h, --help print this help and exit + -e, --exclude-code exclude specific error code, can be repeated + -S, --severity=LEVEL set severity level (info, warning, error) + -v, --view view shellcheck report + +Usage: ./scripts/shellcheck.sh [-e|--exclude-code=CODE] [-S|--severity=LEVEL] [-v|--view] + +example: ./scripts/shellcheck.sh -e SC1091 -e SC1090 -S warning -v + +EOF +} + +parse_cli_args() { + echo; echo "Command line arguments: $0 $*"; echo + + SEVERITY= + EXCLUDE_CODES= + VIEW= + + _help= + _invalid_arg_count=0 + while [[ $# -gt 0 ]]; do + optarg= + _err=0 + shft_cnt=1 + if [[ "$1" = '--' ]]; then + break + elif [[ "$1" =~ ^-- && $# -gt 1 && ! "$2" =~ ^- ]]; then + optarg="$2" + shft_cnt=2 + elif [[ "$1" =~ ^-[^-] && $# -gt 1 && ! "$2" =~ ^- ]]; then + optarg="$2" + shft_cnt=2 + elif [[ "$1" =~ ^-[^-] ]]; then + optarg="${1/??/}" + fi + + case "$1" in + -S*|--severity*) + # don't let the user enter -S LEVEL more than once + if [[ -n "$SEVERITY" ]]; then + printf "${RED}[Error]${RST_COLOR} Severity level already set to: '${CYAN}%s${RST_COLOR}'\n" "${SEVERITY}" + _err=1 + # valid if -S has any of 'info', 'warning', 'error' + elif [[ "${optarg}" == "info" || "${optarg}" == "warning" || "${optarg}" == "error" ]]; then + SEVERITY="${optarg}" + else + # continue and disregard invalid severity level + printf "${RED}[Error]${RST_COLOR} invalid severity level: '${CYAN}%s${RST_COLOR}'\n" "${optarg}" + _err=1 + fi + shift "$shft_cnt" + ;; + -e*|--exclude-code*) + # strip whitespace from optarg + CODE=$(echo "${optarg}" | tr -d '[:space:]') + # valid if matching format SC1000-SC9999 + if [[ "${optarg}" =~ ^SC[0-9]{4}$ ]]; then + # if empty then populate with just error code, otherwise add pipe before new code for grep later + if [[ -z "${EXCLUDE_CODES}" ]]; then + EXCLUDE_CODES+="${CODE}" + else + EXCLUDE_CODES+=",${CODE}" + fi + else + # continue just don't save invalid error code + printf "${RED}[Error]${RST_COLOR} Invalid error code entered: %s\n" "${optarg}" + _err=1 + fi + shift "$shft_cnt" + ;; + -v|--view) + VIEW="True" + shift "$shft_cnt" + ;; + -h|--help) + _help=1 + ;; + *) + printf "${RED}[Error]${RST_COLOR} Unrecognized option: %s\n" "$1" + _err=1 + shift "$shft_cnt" + ;; + esac + + _invalid_arg_count=$(( "${_invalid_arg_count}" + 1 )) + + # exit on help message + if [[ $_help -eq 1 ]]; then + show_usage + exit 0 + fi + # continue on invalid arg, let user know but don't exit + if [[ "$_err" -eq 1 ]]; then + printf "${RED}[Error]${RST_COLOR} Invalid argument: %s\n" "${optarg}" + fi + + done + + if [[ "$_invalid_arg_count" -gt 0 ]]; then + # print the help message just once if one or more invalid arguments + echo; echo -e "${RED}[Error]${RST_COLOR} Some invalid arguments were detected."; echo + show_usage + fi +} + +check_shellcheck_install() { + if ! command -v shellcheck &>/dev/null; then + echo -e "${RED}[ERROR]${RST_COLOR} shellcheck is not installed." + echo "Run 'sudo ./scripts/install-build-tools.sh' to install shellcheck." + exit 1 + fi +} + +get_num_cores() { + NUM_CORES=1 + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + NUM_CORES=$(grep -c ^processor /proc/cpuinfo) + elif [[ "$OSTYPE" == "darwin"* ]]; then + NUM_CORES=$(sysctl -n hw.ncpu) + else + NUM_CORES=1 + fi +} + +run_shellcheck() { + # check if git is installed + if command -v git &>/dev/null; then + echo "Using git ls-files to find shell scripts..."; echo + if [[ -z "$EXCLUDE_CODES" ]]; then + git ls-files '*.sh' | xargs -n 1 -P "$NUM_CORES" shellcheck -s bash > "$SHELLCHECK_REPORT" + else + git ls-files '*.sh' | xargs -n 1 -P "$NUM_CORES" shellcheck -s bash --exclude="$EXCLUDE_CODES" > "$SHELLCHECK_REPORT" + fi + else + echo "git is not installed. Using find to compile list of shell scripts..."; echo + if [[ -z "$EXCLUDE_CODES" ]]; then + find "$ROOT" -name '*.sh' -print0 | xargs -0 -n 1 -P "$NUM_CORES" shellcheck -s bash > "$SHELLCHECK_REPORT" + else + find "$ROOT" -name '*.sh' -print0 | xargs -0 -n 1 -P "$NUM_CORES" shellcheck -s bash --exclude="$EXCLUDE_CODES" > "$SHELLCHECK_REPORT" + fi + fi +} + +check_report() { + # check if shellcheck file was created + if [[ ! -f "$SHELLCHECK_REPORT" ]]; then + echo -e "${RED}[ERROR]${RST_COLOR} Shellcheck report was not created: ${SHELLCHECK_REPORT}" + echo "Exiting..." + exit 1 + # if shellcheck report was generated within the last minute then ok + elif [[ -f "$SHELLCHECK_REPORT" ]]; then + FILE_MODIFIED=$(date -r "$SHELLCHECK_REPORT" "+%s") + TIME_DIFF=$(( CURRENT_TIME - FILE_MODIFIED )) + # in testing takes a few seconds at most + if [[ "$TIME_DIFF" -gt 60 ]]; then + echo "Shellcheck report exists but not created by this run. Exiting..." + exit 1 + fi + # check if shellcheck report is empty + elif [[ ! -s "$SHELLCHECK_REPORT" ]]; then + echo "Shellcheck report is empty: ${SHELLCHECK_REPORT}" + echo "Either there are no info/warning/error messages for all shell scripts" + echo "in the codebase or shellcheck failed to run successfully. Exiting..." + exit 0 + fi +} + +view_report() { + # view non-empty shellcheck report, includes info, warnings, errors + if [[ "$VIEW" == "True" ]]; then + echo; echo -e "----------------------- ${BLUE}Start of Shellcheck report${RST_COLOR} -----------------------"; echo + cat "$SHELLCHECK_REPORT" + echo; echo -e "------------------------ ${BLUE}End of Shellcheck report${RST_COLOR} ------------------------"; echo + fi +} + +determine_pass_fail() { + # detect if fatal errors are in shellcheck report + echo -e "Checking for errors in shellcheck report with severity level '${CYAN}${SEVERITY}${RST_COLOR}'"; echo + echo -e "Excluding error codes: '${YELLOW}${EXCLUDE_CODES}${RST_COLOR}'"; echo + + # if severity not set, set it to error as default + if [[ -z "$SEVERITY" ]]; then + SEVERITY="error" + fi + + # if any messages of severity level or more strict are present, use for grepping report + case "$SEVERITY" in + "info") REGEX_SEVERITY=("info" "warning" "error") ;; + "warning") REGEX_SEVERITY=("warning" "error") ;; + "error") REGEX_SEVERITY=("error") ;; + esac + + TOTAL_ERRORS= + for SEV_LEVEL in "${REGEX_SEVERITY[@]}"; do + # get count of errors of severity level + ERROR_COUNT=$(grep -Ec "\(${SEV_LEVEL}\):" "$SHELLCHECK_REPORT") + TOTAL_ERRORS=$(( TOTAL_ERRORS + ERROR_COUNT )) + if [[ "$ERROR_COUNT" -gt 0 ]]; then + echo -e "Found ${RED}${ERROR_COUNT}${RST_COLOR} violations of '${CYAN}${SEV_LEVEL}${RST_COLOR}' severity level"; echo + fi + done + + # if no errors found, then shellcheck passed + if [[ "$TOTAL_ERRORS" -eq 0 ]]; then + echo -e "${GREEN}[PASS]${RST_COLOR} Shellcheck did not detect violations scanning with severity level '${CYAN}${SEVERITY}${RST_COLOR}'" + echo; echo -e "${GREEN}Shellcheck passed.${RST_COLOR} See report: ${SHELLCHECK_REPORT}"; echo + exit 0 + else + echo -e "${RED}[FAIL]${RST_COLOR} Shellcheck found unexcused violations scanning with severity level '${CYAN}${SEVERITY}${RST_COLOR}'" + echo; echo -e "${RED}Shellcheck failed.${RST_COLOR} See report: ${SHELLCHECK_REPORT}"; echo + exit 1 + fi +} + +main() { + # exit if shellcheck is not installed + check_shellcheck_install + # parse command line arguments + parse_cli_args "$@" + # run shellcheck in parallel on all tracked shell scripts + get_num_cores + # checking status of this run will give failure if info/warning/error is found by default + run_shellcheck + # check if shellcheck report was created + check_report + # display report contents if view set in cli + view_report + # determine status by parsing shellcheck report to see if any messages + # of the severity level or more strict are present to determine failure (true errors) + determine_pass_fail +} + +main "$@" diff --git a/scripts/test-e2e-minikube.sh b/scripts/test-e2e-minikube.sh index d25900163..b4dc55759 100755 --- a/scripts/test-e2e-minikube.sh +++ b/scripts/test-e2e-minikube.sh @@ -7,7 +7,7 @@ BUILD_DOCKER=${TESTRUN_BUILD_DOCKER:-1} # Make sure we have the necessary tools installed required_executables=(minikube docker go helm kubectl) -for e in ${required_executables[@]}; do +for e in "${required_executables[@]}"; do if ! command -v $e &> /dev/null; then echo "'$e' command not be found! This is required to run. Please install it." exit 1 diff --git a/scripts/wait-for-it.sh b/scripts/wait-for-it.sh index d990e0d36..df6ad2646 100755 --- a/scripts/wait-for-it.sh +++ b/scripts/wait-for-it.sh @@ -150,7 +150,7 @@ if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then WAITFORIT_ISBUSY=1 # Check if busybox timeout uses -t flag # (recent Alpine versions don't support -t anymore) - if timeout &>/dev/stdout | grep -q -e '-t '; then + if timeout |& tee /dev/stdout | grep -q -e '-t '; then WAITFORIT_BUSYTIMEFLAG="-t" fi else