From 67a0cdb55f87b6536c2717389cf9024421684856 Mon Sep 17 00:00:00 2001 From: charles37 Date: Mon, 28 Oct 2024 13:21:09 -0400 Subject: [PATCH] tokc-hardware-ci: ported test script --- .github/workflows/treadmill-ci-test.yml | 24 +++++++++++++++ .github/workflows/treadmill-ci.yml | 30 ++++++++++++------ hwci/tests/__init__.py | 10 ++++++ hwci/tests/adc.py | 30 ++++++++++++++++++ hwci/tests/adc_continuous.py | 3 ++ hwci/tests/ble_advertising.py | 5 +++ hwci/tests/ble_env_sense.py | 6 ++++ hwci/tests/ble_passive_scanning.py | 26 ++++++++++++++++ hwci/tests/c_hello_and_printf_long.py | 28 +++++++++++++++++ hwci/tests/console_recv_long.py | 5 +++ hwci/tests/console_recv_short.py | 5 +++ hwci/tests/gpio_test.py | 21 +++++++++++++ hwci/tests/ipc_logic.py | 24 +++++++++++++++ hwci/tests/lua_hello.py | 3 ++ hwci/tests/malloc_test01.py | 3 ++ hwci/tests/malloc_test02.py | 3 ++ hwci/tests/mpu_stack_growth.py | 16 ++++++++++ hwci/tests/mpu_walk_region.py | 16 ++++++++++ hwci/tests/multi_alarm_simple_test.py | 41 +++++++++++++++++++++++++ hwci/tests/multi_alarm_test.py | 0 hwci/tests/printf_long.py | 6 ++++ hwci/tests/sensors.py | 23 ++++++++++++++ hwci/tests/stack_size_test01.py | 3 ++ hwci/tests/stack_size_test02.py | 3 ++ hwci/tests/whileone.py | 17 ++++++++++ 25 files changed, 341 insertions(+), 10 deletions(-) create mode 100644 hwci/tests/__init__.py create mode 100644 hwci/tests/adc.py create mode 100644 hwci/tests/adc_continuous.py create mode 100644 hwci/tests/ble_advertising.py create mode 100644 hwci/tests/ble_env_sense.py create mode 100644 hwci/tests/ble_passive_scanning.py create mode 100644 hwci/tests/c_hello_and_printf_long.py create mode 100644 hwci/tests/console_recv_long.py create mode 100644 hwci/tests/console_recv_short.py create mode 100644 hwci/tests/gpio_test.py create mode 100644 hwci/tests/ipc_logic.py create mode 100644 hwci/tests/lua_hello.py create mode 100644 hwci/tests/malloc_test01.py create mode 100644 hwci/tests/malloc_test02.py create mode 100644 hwci/tests/mpu_stack_growth.py create mode 100644 hwci/tests/mpu_walk_region.py create mode 100644 hwci/tests/multi_alarm_simple_test.py create mode 100644 hwci/tests/multi_alarm_test.py create mode 100644 hwci/tests/printf_long.py create mode 100644 hwci/tests/sensors.py create mode 100644 hwci/tests/stack_size_test01.py create mode 100644 hwci/tests/stack_size_test02.py create mode 100644 hwci/tests/whileone.py diff --git a/.github/workflows/treadmill-ci-test.yml b/.github/workflows/treadmill-ci-test.yml index 67a827a..1120ee3 100644 --- a/.github/workflows/treadmill-ci-test.yml +++ b/.github/workflows/treadmill-ci-test.yml @@ -11,6 +11,7 @@ on: push: branches: - main + - dev/test_ci_branch # Pull requests from forks will not have access to the required GitHub API # secrets below, even if they are using an appropriate deployment environment # and the workflow runs have been approved according to this environment's @@ -45,5 +46,28 @@ jobs: # Use the latest upstream Tock kernel / userspace components: tock-kernel-ref: 'master' libtock-c-ref: 'master' + tests: | + tests/ble_env_sense.py + tests/ble_advertising.py + tests/ble_passive_scanning.py + tests/c_hello.py + tests/printf_long.py + tests/c_hello_and_printf_long.py + tests/sensors.py + tests/console_recv_short.py + tests/console_recv_long.py + tests/lua_hello.py + tests/malloc_test01.py + tests/malloc_test02.py + tests/stack_size_test01.py + tests/stack_size_test02.py + tests/mpu_stack_growth.py + tests/mpu_walk_region.py + tests/adc.py + tests/adc_continuous.py + tests/ipc_logic.py + tests/gpio_interrupt.py + tests/whileone.py + tests/multi_alarm_simple_test.py secrets: inherit diff --git a/.github/workflows/treadmill-ci.yml b/.github/workflows/treadmill-ci.yml index e0885d5..956f750 100644 --- a/.github/workflows/treadmill-ci.yml +++ b/.github/workflows/treadmill-ci.yml @@ -39,6 +39,10 @@ on: libtock-c-ref: required: true type: string + tests: + required: false + type: string + default: "tests/c_hello.py" # Default to single test for backward compatibility jobs: test-prepare: @@ -116,7 +120,11 @@ jobs: # identically: HOST_TYPE: nbd-netboot HOST_ARCH: arm64 + TESTS: ${{ inputs.tests }} run: | + # Read the list of tests and convert to JSON array + TEST_LIST=$(echo "$TESTS" | jq -R -s -c 'split("\n") | map(select(length > 0))') + # When we eventually launch tests on multiple hardware platforms in # parallel, we need to supply different SUB_TEST_IDs here: SUB_TEST_ID="0" @@ -126,6 +134,7 @@ jobs: # runner (connected to the exact board we want to run tests on). RUNNER_ID="tml-gh-actions-runner-${GITHUB_REPOSITORY_ID}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${SUB_TEST_ID}" + # Obtain a new just-in-time runner registration token: RUNNER_CONFIG_JSON="$(gh api \ -H "Accept: application/vnd.github+json" \ @@ -162,18 +171,13 @@ jobs: TML_JOB_ID="$(echo "$TML_JOB_ID_JSON" | jq -r .job_id)" echo "Enqueued Treadmill job with ID $TML_JOB_ID" + # Pass the job IDs and other configuration data into the outputs of # this step, such that we can run test-execute job instances for each # Treadmill job we've started: - echo "tml-job-ids=[ \ - \"$TML_JOB_ID\" \ - ]" >> "$GITHUB_OUTPUT" + echo "tml-job-ids=[ \"$TML_JOB_ID\" ]" >> "$GITHUB_OUTPUT" + echo "tml-jobs={ \"$TML_JOB_ID\": { \"runner-id\": \"$RUNNER_ID\", \"tests\": $TEST_LIST } }" >> "$GITHUB_OUTPUT" - echo "tml-jobs={ \ - \"$TML_JOB_ID\": { \ - \"runner-id\": \"$RUNNER_ID\", \ - } \ - }" >> "$GITHUB_OUTPUT" test-execute: needs: test-prepare @@ -258,11 +262,17 @@ jobs: pip install -r hwci/requirements.txt -c hwci/requirements-frozen.txt - name: Run tests + env: + JSON_TEST_ARRAY: ${{ toJSON(fromJSON(needs.test-prepare.outputs.tml-jobs)[matrix.tml-job-id].tests) }} run: | source ./hwcienv/bin/activate - cd ./hwci + cd ./tools/hwci export PYTHONPATH="$PWD:$PYTHONPATH" - python3 core/main.py --board boards/nrf52dk.py --test tests/c_hello.py + + echo "$JSON_TEST_ARRAY" | jq -r -c '.[]' | while read TEST; do + echo "===== RUNNING TEST $TEST =====" + python3 core/main.py --board boards/nrf52dk.py --test "$TEST" || echo "===== Test failed! =====" + done - name: Request shutdown after successful job completion run: | diff --git a/hwci/tests/__init__.py b/hwci/tests/__init__.py new file mode 100644 index 0000000..b64be6c --- /dev/null +++ b/hwci/tests/__init__.py @@ -0,0 +1,10 @@ +# Licensed under the Apache License, Version 2.0 or the MIT License. +# SPDX-License-Identifier: Apache-2.0 OR MIT +# Copyright Tock Contributors 2024. + +from .console_hello_test import ( + OneshotTest, + AnalyzeConsoleTest, + WaitForConsoleMessageTest, +) +from .c_hello import test as c_hello_test diff --git a/hwci/tests/adc.py b/hwci/tests/adc.py new file mode 100644 index 0000000..db4ecc7 --- /dev/null +++ b/hwci/tests/adc.py @@ -0,0 +1,30 @@ +from utils.test_helpers import AnalyzeConsoleTest +import logging + + +class AdcTest(AnalyzeConsoleTest): + def __init__(self): + super().__init__(apps=["tests/adc/adc"]) + + def analyze(self, lines): + adc_driver_found = False + adc_readings_found = False + for line in lines: + if "ADC driver exists" in line: + adc_driver_found = True + if "ADC Reading:" in line: + adc_readings_found = True + break + if "No ADC driver!" in line: + logging.warning("No ADC driver available.") + return # Test passes if ADC is not available + if adc_driver_found and adc_readings_found: + logging.info("ADC Test passed with readings.") + elif adc_driver_found: + raise Exception("ADC Test failed: Driver found but no readings.") + else: + logging.warning("ADC Test skipped: No ADC driver.") + return # Test passes if ADC is not available + + +test = AdcTest() diff --git a/hwci/tests/adc_continuous.py b/hwci/tests/adc_continuous.py new file mode 100644 index 0000000..d7f1603 --- /dev/null +++ b/hwci/tests/adc_continuous.py @@ -0,0 +1,3 @@ +from utils.test_helpers import WaitForConsoleMessageTest + +test = WaitForConsoleMessageTest(["tests/adc/adc_continuous"], "No ADC driver!") diff --git a/hwci/tests/ble_advertising.py b/hwci/tests/ble_advertising.py new file mode 100644 index 0000000..44a669a --- /dev/null +++ b/hwci/tests/ble_advertising.py @@ -0,0 +1,5 @@ +from utils.test_helpers import WaitForConsoleMessageTest + +test = WaitForConsoleMessageTest( + ["ble_advertising"], "Now advertising every 300 ms as 'TockOS'" +) diff --git a/hwci/tests/ble_env_sense.py b/hwci/tests/ble_env_sense.py new file mode 100644 index 0000000..60a2673 --- /dev/null +++ b/hwci/tests/ble_env_sense.py @@ -0,0 +1,6 @@ +from utils.test_helpers import WaitForConsoleMessageTest + +test = WaitForConsoleMessageTest( + ["services/ble-env-sense", "services/ble-env-sense/test-with-sensors"], + "BLE ERROR: Code = 16385", +) diff --git a/hwci/tests/ble_passive_scanning.py b/hwci/tests/ble_passive_scanning.py new file mode 100644 index 0000000..f8bbed7 --- /dev/null +++ b/hwci/tests/ble_passive_scanning.py @@ -0,0 +1,26 @@ +from utils.test_helpers import AnalyzeConsoleTest +import logging + + +class BlePassiveScanningTest(AnalyzeConsoleTest): + def __init__(self): + super().__init__(apps=["ble_passive_scanning"]) + + def analyze(self, lines): + found_advertisements = False + for line in lines: + if "PDU Type:" in line: + found_advertisements = True + break + if found_advertisements: + logging.info("BLE Passive Scanning Test passed.") + else: + logging.warning( + "BLE Passive Scanning Test could not detect advertisements. Ensure BLE devices are nearby." + ) + raise Exception( + "BLE Passive Scanning Test failed: No advertisements detected." + ) + + +test = BlePassiveScanningTest() diff --git a/hwci/tests/c_hello_and_printf_long.py b/hwci/tests/c_hello_and_printf_long.py new file mode 100644 index 0000000..c80ef93 --- /dev/null +++ b/hwci/tests/c_hello_and_printf_long.py @@ -0,0 +1,28 @@ +from utils.test_helpers import AnalyzeConsoleTest +import logging + + +class CHelloAndPrintfLongTest(AnalyzeConsoleTest): + def __init__(self): + super().__init__(apps=["c_hello", "tests/printf_long"]) + + def analyze(self, lines): + expected_messages = [ + "Hello World!", + "Hi welcome to Tock. This test makes sure that a greater than 64 byte message can be printed.", + "And a short message.", + ] + messages_found = {msg: False for msg in expected_messages} + + for line in lines: + for msg in expected_messages: + if msg in line: + messages_found[msg] = True + + for msg, found in messages_found.items(): + if not found: + raise Exception(f"Did not find expected message: '{msg}'") + logging.info("C Hello and Printf Long Test passed.") + + +test = CHelloAndPrintfLongTest() diff --git a/hwci/tests/console_recv_long.py b/hwci/tests/console_recv_long.py new file mode 100644 index 0000000..4372533 --- /dev/null +++ b/hwci/tests/console_recv_long.py @@ -0,0 +1,5 @@ +from utils.test_helpers import WaitForConsoleMessageTest + +test = WaitForConsoleMessageTest( + ["tests/console/console_recv_long"], "[SHORT] Error doing UART receive: -2" +) diff --git a/hwci/tests/console_recv_short.py b/hwci/tests/console_recv_short.py new file mode 100644 index 0000000..85cc668 --- /dev/null +++ b/hwci/tests/console_recv_short.py @@ -0,0 +1,5 @@ +from utils.test_helpers import WaitForConsoleMessageTest + +test = WaitForConsoleMessageTest( + ["tests/console/console_recv_short"], "[SHORT] Error doing UART receive: -2" +) diff --git a/hwci/tests/gpio_test.py b/hwci/tests/gpio_test.py new file mode 100644 index 0000000..3b08391 --- /dev/null +++ b/hwci/tests/gpio_test.py @@ -0,0 +1,21 @@ +from utils.test_helpers import AnalyzeConsoleTest +import logging + + +class GpioInterruptTest(AnalyzeConsoleTest): + def __init__(self): + super().__init__(apps=["tests/gpio/gpio_interrupt"]) + + def analyze(self, lines): + interrupt_detected = False + for line in lines: + if "GPIO Interrupt!" in line: + interrupt_detected = True + break + if interrupt_detected: + logging.info("GPIO Interrupt Test passed.") + else: + raise Exception("GPIO Interrupt Test failed: No interrupt detected.") + + +test = GpioInterruptTest() diff --git a/hwci/tests/ipc_logic.py b/hwci/tests/ipc_logic.py new file mode 100644 index 0000000..13e9a7f --- /dev/null +++ b/hwci/tests/ipc_logic.py @@ -0,0 +1,24 @@ +from utils.test_helpers import AnalyzeConsoleTest +import logging + + +class IpcLogicTest(AnalyzeConsoleTest): + def __init__(self): + super().__init__( + apps=[ + "tutorials/05_ipc/led", + "tutorials/05_ipc/rng", + "tutorials/05_ipc/logic", + ] + ) + + def analyze(self, lines): + expected_output = "Number of LEDs:" + for line in lines: + if expected_output in line: + logging.info("IPC Logic Test passed.") + return + raise Exception("IPC Logic Test failed: Expected output not found.") + + +test = IpcLogicTest() diff --git a/hwci/tests/lua_hello.py b/hwci/tests/lua_hello.py new file mode 100644 index 0000000..02f3442 --- /dev/null +++ b/hwci/tests/lua_hello.py @@ -0,0 +1,3 @@ +from utils.test_helpers import WaitForConsoleMessageTest + +test = WaitForConsoleMessageTest(["lua-hello"], "Hello from Lua!") diff --git a/hwci/tests/malloc_test01.py b/hwci/tests/malloc_test01.py new file mode 100644 index 0000000..9a34da5 --- /dev/null +++ b/hwci/tests/malloc_test01.py @@ -0,0 +1,3 @@ +from utils.test_helpers import WaitForConsoleMessageTest + +test = WaitForConsoleMessageTest(["tests/malloc_test01"], "malloc01: success") diff --git a/hwci/tests/malloc_test02.py b/hwci/tests/malloc_test02.py new file mode 100644 index 0000000..8e921a6 --- /dev/null +++ b/hwci/tests/malloc_test02.py @@ -0,0 +1,3 @@ +from utils.test_helpers import WaitForConsoleMessageTest + +test = WaitForConsoleMessageTest(["tests/malloc_test02"], "malloc02: success") diff --git a/hwci/tests/mpu_stack_growth.py b/hwci/tests/mpu_stack_growth.py new file mode 100644 index 0000000..399afde --- /dev/null +++ b/hwci/tests/mpu_stack_growth.py @@ -0,0 +1,16 @@ +from utils.test_helpers import AnalyzeConsoleTest + + +class MpuStackGrowthTest(AnalyzeConsoleTest): + def __init__(self): + super().__init__(apps=["tests/mpu/mpu_stack_growth"]) + + def analyze(self, lines): + for line in lines: + if "[TEST] MPU Stack Growth" in line: + # Test started successfully + return + raise Exception("MPU Stack Growth test did not start") + + +test = MpuStackGrowthTest() diff --git a/hwci/tests/mpu_walk_region.py b/hwci/tests/mpu_walk_region.py new file mode 100644 index 0000000..0e03be6 --- /dev/null +++ b/hwci/tests/mpu_walk_region.py @@ -0,0 +1,16 @@ +from utils.test_helpers import AnalyzeConsoleTest + + +class MpuWalkRegionTest(AnalyzeConsoleTest): + def __init__(self): + super().__init__(apps=["tests/mpu/mpu_walk_region"]) + + def analyze(self, lines): + for line in lines: + if "[TEST] MPU Walk Regions" in line: + # Test started successfully + return + raise Exception("MPU Walk Region test did not start") + + +test = MpuWalkRegionTest() diff --git a/hwci/tests/multi_alarm_simple_test.py b/hwci/tests/multi_alarm_simple_test.py new file mode 100644 index 0000000..0af582a --- /dev/null +++ b/hwci/tests/multi_alarm_simple_test.py @@ -0,0 +1,41 @@ +from utils.test_helpers import AnalyzeConsoleTest +import logging + + +class MultiAlarmSimpleTest(AnalyzeConsoleTest): + def __init__(self): + super().__init__(apps=["tests/multi_alarm_simple_test"]) + + def analyze(self, lines): + # Initialize counts for each alarm + alarm_counts = {1: 0, 2: 0} + + for line in lines: + tokens = line.strip().split() + if len(tokens) >= 3: + try: + alarm_index = int(tokens[0]) + # Optional: parse timestamps if needed + # now = int(tokens[1]) + # expiration = int(tokens[2]) + + # Record counts + if alarm_index in [1, 2]: + alarm_counts[alarm_index] += 1 + except ValueError: + continue # Skip lines that don't parse correctly + + logging.info(f"Alarm counts: {alarm_counts}") + count1 = alarm_counts.get(1, 0) + count2 = alarm_counts.get(2, 0) + if count1 < 2 or count2 < 1: + raise Exception("MultiAlarmSimpleTest failed: Not enough alarms fired") + if count1 < 2 * count2: + raise Exception( + "MultiAlarmSimpleTest failed: Alarm 1 did not fire at least twice as often as Alarm 2" + ) + + logging.info("MultiAlarmSimpleTest passed") + + +test = MultiAlarmSimpleTest() diff --git a/hwci/tests/multi_alarm_test.py b/hwci/tests/multi_alarm_test.py new file mode 100644 index 0000000..e69de29 diff --git a/hwci/tests/printf_long.py b/hwci/tests/printf_long.py new file mode 100644 index 0000000..7926f09 --- /dev/null +++ b/hwci/tests/printf_long.py @@ -0,0 +1,6 @@ +from utils.test_helpers import WaitForConsoleMessageTest + +test = WaitForConsoleMessageTest( + ["tests/printf_long"], + "Hi welcome to Tock. This test makes sure that a greater than 64 byte message can be printed.", +) diff --git a/hwci/tests/sensors.py b/hwci/tests/sensors.py new file mode 100644 index 0000000..243d686 --- /dev/null +++ b/hwci/tests/sensors.py @@ -0,0 +1,23 @@ +from utils.test_helpers import AnalyzeConsoleTest +import logging + + +class SensorsTest(AnalyzeConsoleTest): + def __init__(self): + super().__init__(apps=["sensors"]) + + def analyze(self, lines): + sensors_detected = False + for line in lines: + if "Sampling" in line: + sensors_detected = True + if "deg C" in line or "Light Intensity" in line or "Humidity" in line: + logging.info(f"Sensor reading: {line.strip()}") + return + if sensors_detected: + raise Exception("Sensors Test failed: No sensor readings found.") + else: + logging.warning("Sensors Test skipped: No sensors detected.") + + +test = SensorsTest() diff --git a/hwci/tests/stack_size_test01.py b/hwci/tests/stack_size_test01.py new file mode 100644 index 0000000..c5cec35 --- /dev/null +++ b/hwci/tests/stack_size_test01.py @@ -0,0 +1,3 @@ +from utils.test_helpers import WaitForConsoleMessageTest + +test = WaitForConsoleMessageTest(["tests/stack_size_test01"], "Stack Test App") diff --git a/hwci/tests/stack_size_test02.py b/hwci/tests/stack_size_test02.py new file mode 100644 index 0000000..b344355 --- /dev/null +++ b/hwci/tests/stack_size_test02.py @@ -0,0 +1,3 @@ +from utils.test_helpers import WaitForConsoleMessageTest + +test = WaitForConsoleMessageTest(["tests/stack_size_test02"], "Stack Test App") diff --git a/hwci/tests/whileone.py b/hwci/tests/whileone.py new file mode 100644 index 0000000..68b3ce6 --- /dev/null +++ b/hwci/tests/whileone.py @@ -0,0 +1,17 @@ +from utils.test_helpers import AnalyzeConsoleTest + + +class WhileOneTest(AnalyzeConsoleTest): + def __init__(self): + super().__init__(apps=["tests/whileone"]) + + def analyze(self, lines): + # This app does not produce output, but we can check for the absence of crashes + for line in lines: + if "Kernel panic" in line or "Fault" in line: + raise Exception("App crashed with a fault") + # If no faults, the test passes + return + + +test = WhileOneTest()