From 96f8f5e3efe9d4ae1bb92b585059f1b4dfbcee98 Mon Sep 17 00:00:00 2001 From: Abdelatif Guettouche Date: Thu, 10 Mar 2022 13:45:26 +0100 Subject: [PATCH] Add initial hardware testing support (#6313) - Added workflow triggered by cron or label "hil_test" - Added examples with both pytest and unity --- .github/scripts/on-push.sh | 4 +- .github/scripts/sketch_utils.sh | 10 +-- .github/scripts/tests_build.sh | 56 ++++++++++++ .github/scripts/tests_run.sh | 71 +++++++++++++++ .github/workflows/hil.yml | 120 ++++++++++++++++++++++++++ .github/workflows/publish.yml | 38 ++++++++ .gitignore | 3 + tests/.gitignore | 2 + tests/hello_world/hello_world.ino | 12 +++ tests/hello_world/test_hello_world.py | 2 + tests/pytest.ini | 13 +++ tests/requirements.txt | 13 +++ tests/unity/test_unity.py | 2 + tests/unity/unity.ino | 33 +++++++ 14 files changed, 372 insertions(+), 7 deletions(-) create mode 100755 .github/scripts/tests_build.sh create mode 100755 .github/scripts/tests_run.sh create mode 100644 .github/workflows/hil.yml create mode 100644 .github/workflows/publish.yml create mode 100644 tests/.gitignore create mode 100644 tests/hello_world/hello_world.ino create mode 100644 tests/hello_world/test_hello_world.py create mode 100644 tests/pytest.ini create mode 100644 tests/requirements.txt create mode 100644 tests/unity/test_unity.py create mode 100644 tests/unity/unity.ino diff --git a/.github/scripts/on-push.sh b/.github/scripts/on-push.sh index 5d8ffe9cd..6e92b9be8 100755 --- a/.github/scripts/on-push.sh +++ b/.github/scripts/on-push.sh @@ -58,7 +58,7 @@ fi SCRIPTS_DIR="./.github/scripts" if [ "$BUILD_PIO" -eq 0 ]; then - source ./.github/scripts/install-arduino-ide.sh + source ${SCRIPTS_DIR}/install-arduino-ide.sh source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh FQBN_ESP32="espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app" @@ -80,7 +80,7 @@ if [ "$BUILD_PIO" -eq 0 ]; then build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX else - source ./${SCRIPTS_DIR}/install-platformio-esp32.sh + source ${SCRIPTS_DIR}/install-platformio-esp32.sh # PlatformIO ESP32 Test BOARD="esp32dev" OPTIONS="board_build.partitions = huge_app.csv" diff --git a/.github/scripts/sketch_utils.sh b/.github/scripts/sketch_utils.sh index abc6a0476..d8e4aead5 100755 --- a/.github/scripts/sketch_utils.sh +++ b/.github/scripts/sketch_utils.sh @@ -32,13 +32,13 @@ function build_sketch(){ # build_sketch +function count_sketches(){ # count_sketches [target] local path=$1 local target=$2 - if [ $# -lt 2 ]; then + if [ $# -lt 1 ]; then echo "ERROR: Illegal number of parameters" - echo "USAGE: ${0} count " + echo "USAGE: ${0} count [target]" fi rm -rf sketches.txt @@ -47,7 +47,7 @@ function count_sketches(){ # count_sketches return 0 fi - local sketches=$(find $path -name *.ino) + local sketches=$(find $path -name *.ino | sort) local sketchnum=0 for sketch in $sketches; do local sketchdir=$(dirname $sketch) @@ -55,7 +55,7 @@ function count_sketches(){ # count_sketches local sketchname=$(basename $sketch) if [[ "$sketchdirname.ino" != "$sketchname" ]]; then continue - elif [[ -f "$sketchdir/.skip.$target" ]]; then + elif [[ -n $target ]] && [[ -f "$sketchdir/.skip.$target" ]]; then continue else echo $sketch >> sketches.txt diff --git a/.github/scripts/tests_build.sh b/.github/scripts/tests_build.sh new file mode 100755 index 000000000..6de547d37 --- /dev/null +++ b/.github/scripts/tests_build.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +SCRIPTS_DIR="./.github/scripts" +BUILD_CMD="" + +if [ $# -eq 3 ]; then + chunk_build=1 +elif [ $# -eq 2 ]; then + chunk_build=0 +else + echo "ERROR: Illegal number of parameters" + echo "USAGE: + ${0} + ${0} + " + exit 0 +fi + +target=$1 + +case "$target" in + "esp32") fqbn="espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app" + ;; + "esp32s2") fqbn="espressif:esp32:esp32s2:PSRAM=enabled,PartitionScheme=huge_app" + ;; + "esp32c3") fqbn="espressif:esp32:esp32c3:PartitionScheme=huge_app" + ;; +esac + +if [ -z $fqbn ]; then + echo "Unvalid chip $1" + exit 0 +fi + +source ${SCRIPTS_DIR}/install-arduino-ide.sh +source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh + +args="$ARDUINO_IDE_PATH $ARDUINO_USR_PATH \"$fqbn\"" + +if [ $chunk_build -eq 1 ]; then + chunk_index=$2 + chunk_max=$3 + + if [ "$chunk_index" -gt "$chunk_max" ] && [ "$chunk_max" -ge 2 ]; then + chunk_index=$chunk_max + fi + BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh chunk_build" + args+=" $target $PWD/tests $chunk_index $chunk_max" +else + sketchdir=$2 + BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh build" + args+=" $PWD/tests/$sketchdir/$sketchdir.ino" +fi + +${BUILD_CMD} ${args} + diff --git a/.github/scripts/tests_run.sh b/.github/scripts/tests_run.sh new file mode 100755 index 000000000..a13f3c00c --- /dev/null +++ b/.github/scripts/tests_run.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +target=$1 +chunk_idex=$2 +chunks_num=$3 + +SCRIPTS_DIR="./.github/scripts" +COUNT_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh count" + +source ${SCRIPTS_DIR}/install-arduino-ide.sh + +if [ "$chunks_num" -le 0 ]; then + echo "ERROR: Chunks count must be positive number" + return 1 +fi +if [ "$chunk_idex" -ge "$chunks_num" ] && [ "$chunks_num" -ge 2 ]; then + echo "ERROR: Chunk index must be less than chunks count" + return 1 +fi + +set +e +${COUNT_SKETCHES} $PWD/tests $target +sketchcount=$? +set -e +sketches=$(cat sketches.txt) +rm -rf sketches.txt + +chunk_size=$(( $sketchcount / $chunks_num )) +all_chunks=$(( $chunks_num * $chunk_size )) +if [ "$all_chunks" -lt "$sketchcount" ]; then + chunk_size=$(( $chunk_size + 1 )) +fi + +start_index=0 +end_index=0 +if [ "$chunk_idex" -ge "$chunks_num" ]; then + start_index=$chunk_idex + end_index=$sketchcount +else + start_index=$(( $chunk_idex * $chunk_size )) + if [ "$sketchcount" -le "$start_index" ]; then + echo "Skipping job" + return 0 + fi + + end_index=$(( $(( $chunk_idex + 1 )) * $chunk_size )) + if [ "$end_index" -gt "$sketchcount" ]; then + end_index=$sketchcount + fi +fi + +start_num=$(( $start_index + 1 )) +sketchnum=0 + +for sketch in $sketches; do + sketchdir=$(dirname $sketch) + sketchdirname=$(basename $sketchdir) + sketchname=$(basename $sketch) + sketchnum=$(($sketchnum + 1)) + if [ "$sketchnum" -le "$start_index" ] \ + || [ "$sketchnum" -gt "$end_index" ]; then + continue + fi + echo "" + echo "Test for Sketch Index $(($sketchnum - 1)) - $sketchdirname" + pytest tests -k test_$sketchdirname --junit-xml=tests/$sketchdirname/$sketchdirname.xml + result=$? + if [ $result -ne 0 ]; then + return $result + fi +done diff --git a/.github/workflows/hil.yml b/.github/workflows/hil.yml new file mode 100644 index 000000000..b55e686c5 --- /dev/null +++ b/.github/workflows/hil.yml @@ -0,0 +1,120 @@ +name: Run tests in hardware + +on: + pull_request: + types: [opened, reopened, synchronize, labeled] + + schedule: + - cron: '0 2 * * *' + +env: + MAX_CHUNKS: 15 + +concurrency: + group: hil-${{github.event.pull_request.number || github.ref}} + cancel-in-progress: true + +jobs: + gen_chunks: + if: | + contains(github.event.pull_request.labels.*.name, 'hil_test') || + github.event_name == 'schedule' + name: Generate Chunks matrix + runs-on: ubuntu-latest + outputs: + chunks: ${{ steps.gen-chunks.outputs.chunks }} + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + + - name: Generate Chunks matrix + id: gen-chunks + run: | + set +e + bash .github/scripts/sketch_utils.sh count tests + sketches=$((? - 1)) + if [[ $sketches -gt ${{env.MAX_CHUNKS}} ]]; then + $sketches=${{env.MAX_CHUNKS}} + fi + set -e + rm sketches.txt + CHUNKS=$(jq -c -n '$ARGS.positional' --args `seq 0 1 $sketches`) + echo "::set-output name=chunks::${CHUNKS}" + + Build: + needs: gen_chunks + name: ${{matrix.chip}}-Build#${{matrix.chunks}} + runs-on: ubuntu-latest + strategy: + matrix: + chip: ['esp32', 'esp32s2', 'esp32c3'] + chunks: ${{fromJson(needs.gen_chunks.outputs.chunks)}} + + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + + - name: Build sketches + run: | + bash .github/scripts/tests_build.sh ${{matrix.chip}} ${{matrix.chunks}} ${{env.MAX_CHUNKS}} + - name: Upload ${{matrix.chip}}-${{matrix.chunks}} artifacts + uses: actions/upload-artifact@v2 + with: + name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts + path: | + tests/*/build/*.bin + tests/*/build/*.json + Test: + needs: [gen_chunks, Build] + name: ${{matrix.chip}}-Test#${{matrix.chunks}} + runs-on: ESP32 + strategy: + fail-fast: false + matrix: + chip: ['esp32', 'esp32s2', 'esp32c3'] + chunks: ${{fromJson(needs.gen_chunks.outputs.chunks)}} + container: + image: python:3.10.1-bullseye + options: --privileged + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Download ${{matrix.chip}}-${{matrix.chunks}} artifacts + uses: actions/download-artifact@v2 + with: + name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts + path: tests/ + + - name: Check Artifacts + run: | + ls -R tests + cat tests/*/build/build.options.json + + - name: Install dependencies + run: | + pip install -U pip + pip install -r tests/requirements.txt + + - name: Run Tests + run: | + bash .github/scripts/tests_run.sh ${{matrix.chip}} ${{matrix.chunks}} ${{env.MAX_CHUNKS}} + + - name: Upload test result artifacts + uses: actions/upload-artifact@v2 + if: always() + with: + name: test_results-${{matrix.chip}}-${{matrix.chunks}} + path: tests/*/*.xml + + event_file: + name: "Event File" + needs: Test + runs-on: ubuntu-latest + steps: + - name: Upload + uses: actions/upload-artifact@v2 + with: + name: Event File + path: ${{github.event_path}} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..686ce2f23 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,38 @@ +name: Unit Test Results + +on: + workflow_run: + workflows: [Run tests in hardware] + + types: + - completed + +jobs: + debug: + name: Debug + runs-on: ubuntu-latest + + steps: + - name: Debug Action + uses: hmarr/debug-action@v2.0.0 + + unit-test-results: + name: Unit Test Results + runs-on: ubuntu-latest + steps: + - name: Download and Extract Artifacts + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + run: | + mkdir -p artifacts && cd artifacts + artifacts_url=${{ github.event.workflow_run.artifacts_url }} + gh api "$artifacts_url" -q '.artifacts[] | [.name, .archive_download_url] | @tsv' | while read artifact + do + IFS=$'\t' read name url <<< "$artifact" + gh api $url > "$name.zip" + unzip -d "$name" "$name.zip" + done + - name: Publish Unit Test Results + uses: EnricoMi/publish-unit-test-result-action@v1 + with: + commit: ${{ github.event.workflow diff --git a/.gitignore b/.gitignore index 5114d19da..df9c98da7 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ boards.sloeber.txt # Ignore docs build (Sphinx) docs/build docs/source/_build + +# Test log files +*.log diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 000000000..3d4333929 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,2 @@ +build/ +__pycache__/ diff --git a/tests/hello_world/hello_world.ino b/tests/hello_world/hello_world.ino new file mode 100644 index 000000000..0cc110d42 --- /dev/null +++ b/tests/hello_world/hello_world.ino @@ -0,0 +1,12 @@ +void setup(){ + // Open serial communications and wait for port to open: + Serial.begin(115200); + while (!Serial) { + ; + } + + Serial.println("Hello Arduino!"); +} + +void loop(){ +} diff --git a/tests/hello_world/test_hello_world.py b/tests/hello_world/test_hello_world.py new file mode 100644 index 000000000..f7ed50cde --- /dev/null +++ b/tests/hello_world/test_hello_world.py @@ -0,0 +1,2 @@ +def test_hello_world(dut): + dut.expect('Hello Arduino!') diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 000000000..ef7e6d7c5 --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,13 @@ +[pytest] +addopts = --embedded-services esp,arduino + +# log related +log_cli = True +log_cli_level = INFO +log_cli_format = %(asctime)s %(levelname)s %(message)s +log_cli_date_format = %Y-%m-%d %H:%M:%S + +log_file = test.log +log_file_level = INFO +log_file_format = %(asctime)s %(levelname)s %(message)s +log_file_date_format = %Y-%m-%d %H:%M:%S diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 000000000..c4095ef6c --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,13 @@ +pyserial>=3.0 +esptool>=3.1 +pytest-cov +cryptography<3.4; platform_machine == "armv7l" + +pytest>=6.2.0 +pexpect>=4.4 + +pytest-embedded>=0.5.1 +pytest-embedded-serial>=0.5.1 +pytest-embedded-serial-esp>=0.5.1 +pytest-embedded-arduino>=0.5.1 + diff --git a/tests/unity/test_unity.py b/tests/unity/test_unity.py new file mode 100644 index 000000000..a5a391cc5 --- /dev/null +++ b/tests/unity/test_unity.py @@ -0,0 +1,2 @@ +def test_unity(dut): + dut.expect_unity_test_output(timeout=240) diff --git a/tests/unity/unity.ino b/tests/unity/unity.ino new file mode 100644 index 000000000..2a1c131ca --- /dev/null +++ b/tests/unity/unity.ino @@ -0,0 +1,33 @@ +#include + + +/* These functions are intended to be called before and after each test. */ +void setUp(void) { +} + +void tearDown(void){ +} + + +void test_pass(void){ + TEST_ASSERT_EQUAL(1, 1); +} + +void test_fail(void){ + TEST_ASSERT_EQUAL(1, 1); +} + +void setup() { + Serial.begin(115200); + while (!Serial) { + ; + } + + UNITY_BEGIN(); + RUN_TEST(test_pass); + RUN_TEST(test_fail); + UNITY_END(); +} + +void loop() { +} -- GitLab