Local testing via Makefile & Docker (#19981)

.github/workflows/test-builds.yml

 
     - name: Run ${{ matrix.test-platform }} Tests
       run: |
-        # Inline tests script
-        chmod +x buildroot/bin/*
-        chmod +x buildroot/tests/*
-        export PATH=./buildroot/bin/:./buildroot/tests/:${PATH}
-        run_tests . ${{ matrix.test-platform }}
+        make tests-single-ci TEST_TARGET=${{ matrix.test-platform }}

Makefile

+help:
+	@echo "Tasks for local development:"
+	@echo "* tests-single-ci:             Run a single test from inside the CI"
+	@echo "* tests-single-local:          Run a single test locally"
+	@echo "* tests-single-local-docker:   Run a single test locally, using docker-compose"
+	@echo "* tests-all-local:             Run all tests locally"
+	@echo "* tests-all-local-docker:      Run all tests locally, using docker-compose"
+	@echo "* setup-local-docker:          Setup local docker-compose"
+	@echo ""
+	@echo "Options for testing:"
+	@echo "  TEST_TARGET          Set when running tests-single-*, to select the"
+	@echo "                       test. If you set it to ALL it will run all "
+	@echo "                       tests, but some of them are broken: use "
+	@echo "                       tests-all-* instead to run only the ones that "
+	@echo "                       run on GitHub CI"
+	@echo "  ONLY_TEST            Limit tests to only those that contain this, or"
+	@echo "                       the index of the test (1-based)"
+	@echo "  VERBOSE_PLATFORMIO   If you want the full PIO output, set any value"
+	@echo "  GIT_RESET_HARD       Used by CI: reset all local changes. WARNING:"
+	@echo "                       THIS WILL UNDO ANY CHANGES YOU'VE MADE!"
+.PHONY: help
+
+tests-single-ci:
+	export GIT_RESET_HARD=true
+	$(MAKE) tests-single-local TEST_TARGET=$(TEST_TARGET)
+.PHONY: tests-single-ci
+
+tests-single-local:
+	@if ! test -n "$(TEST_TARGET)" ; then echo "***ERROR*** Set TEST_TARGET=<your-module> or use make tests-all-local" ; return 1; fi
+	chmod +x buildroot/bin/*
+	chmod +x buildroot/tests/*
+	export PATH=./buildroot/bin/:./buildroot/tests/:${PATH} \
+	  && export VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) \
+	  && run_tests . $(TEST_TARGET) "$(ONLY_TEST)"
+.PHONY: tests-single-local
+
+tests-single-local-docker:
+	@if ! test -n "$(TEST_TARGET)" ; then echo "***ERROR*** Set TEST_TARGET=<your-module> or use make tests-all-local-docker" ; return 1; fi
+	docker-compose run --rm marlin $(MAKE) tests-single-local TEST_TARGET=$(TEST_TARGET) VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) GIT_RESET_HARD=$(GIT_RESET_HARD) ONLY_TEST="$(ONLY_TEST)"
+.PHONY: tests-single-local-docker
+
+tests-all-local:
+	chmod +x buildroot/bin/*
+	chmod +x buildroot/tests/*
+	export PATH=./buildroot/bin/:./buildroot/tests/:${PATH} \
+	  && export VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) \
+	  && for TEST_TARGET in $$(./get_test_targets.py) ; do echo "Running tests for $$TEST_TARGET" ; run_tests . $$TEST_TARGET ; done
+.PHONY: tests-all-local
+
+tests-all-local-docker:
+	docker-compose run --rm marlin $(MAKE) tests-all-local VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) GIT_RESET_HARD=$(GIT_RESET_HARD)
+.PHONY: tests-all-local-docker
+
+setup-local-docker:
+	docker-compose build
+.PHONY: setup-local-docker

README.md

 - This branch is for fixing bugs and integrating any new features for the duration of the Marlin 2.0.x life-cycle.
 - Follow the [Coding Standards](https://marlinfw.org/docs/development/coding_standards.html) to gain points with the maintainers.
 - Please submit Feature Requests and Bug Reports to the [Issue Queue](https://github.com/MarlinFirmware/Marlin/issues/new/choose). Support resources are also listed there.
+- Whenever you add new features, be sure to add tests to `buildroot/tests` and then run your tests locally, if possible.
+  - It's optional: Running all the tests on Windows might take a long time, and they will run anyway on GitHub.
+  - If you're running the tests on Linux (or on WSL with the code on a Linux volume) the speed is much faster.
+  - You can use `make tests-all-local` or `make tests-single-local TEST_TARGET=...`.
+  - If you prefer Docker you can use `make tests-all-local-docker` or `make tests-all-local-docker TEST_TARGET=...`.
 
 ### [RepRap.org Wiki Page](https://reprap.org/wiki/Marlin)
 

buildroot/share/git/mftest

 #
 
 MFINFO=$(mfinfo) || exit 1
-[[ -d Marlin/src ]] || { echo "Please 'cd' up to repo root." ; exit 1 ; }
+[[ -d Marlin/src ]] || { echo "Please 'cd' to the Marlin repo root." ; exit 1 ; }
+
+perror() { echo -e "$0: \033[0;31m$1 -- $2\033[0m" ; }
+errout() { echo -e "\033[0;31m$1\033[0m" ; }
+bugout() { ((DEBUG)) && echo -e "\033[0;32m$1\033[0m" ; }
+
+usage() {
+  echo "
+Usage: mftest [-t|--env=<env>] [-n|--num=<num>] [-m|--make] [-y|--build=<Y|n>]
+       mftest [-a|--autobuild]
+       mftest [-r|--rebuild]
+       mftest [-u|--autoupload] [-n|--num=<num>]
+
+OPTIONS
+  -t --env         The environment of the test to apply / run. (As named in platformio.ini.)
+  -n --num         The index of the test to run. (In *-tests file order.)
+  -m --make        Use the make / Docker method for the build.
+  -y --build       Skip 'Do you want to build this test?' and assume YES.
+  -h --help        Print this help.
+  -a --autobuild   PIO Build using the MOTHERBOARD environment.
+  -u --autoupload  PIO Upload using the MOTHERBOARD environment.
+  -v --verbose     Extra output for debugging.
+
+env shortcuts: tree due esp lin lpc|lpc8 lpc9 m128 m256|mega stm|f1 f4 f7 s6 teensy|t31|t32 t35|t36 t40|t41
+
+"
+}
 
 TESTPATH=buildroot/tests
 
 ISEXEC='^exec_'
 ISCONT='\\ *$'
 
-# Get the environment and test number from the command
-TESTENV=${1:-'-'}
-CHOICE=${2:-0}
-AUTOENV=0
+# Get environment, test number, etc. from the command
+TESTENV='-'
+CHOICE=0
+DEBUG=0
+
+while getopts 'abhmruvyn:t:-:' OFLAG; do
+  case "${OFLAG}" in
+    a) AUTO_BUILD=1 ; bugout "Auto-Build target..." ;;
+    h) EXIT_USAGE=1 ;;
+    m) USE_MAKE=1 ; bugout "Using make with Docker..." ;;
+    n) case "$OPTARG" in
+         *[!0-9]*) perror "option requires a number" $OFLAG ; EXIT_USAGE=1 ;;
+                *) CHOICE="$OPTARG" ; bugout "Got a number: $CHOICE" ;;
+       esac
+       ;;
+    r) REBUILD=1         ; bugout "Rebuilding previous..."    ;;
+    t) TESTENV="$OPTARG" ; bugout "Got a target: $TESTENV" ;;
+    u) AUTO_BUILD=2      ; bugout "Auto-Upload target..."  ;;
+    v) DEBUG=1           ; bugout "Debug ON" ;;
+    y) BUILD_YES='Y'     ; bugout "Build will initiate..."    ;;
+    -) IFS="=" read -r ONAM OVAL <<< "$OPTARG"
+       case "$ONAM" in
+         help) [[ -z "$OVAL" ]] || perror "option can't take value $OVAL" $ONAM ; EXIT_USAGE=1 ;;
+    autobuild) AUTO_BUILD=1 ; bugout "Auto-Build target..."  ;;
+   autoupload) AUTO_BUILD=2 ; bugout "Auto-Upload target..." ;;
+          env) case "$OVAL" in
+                 '') perror "option requires a value" $ONAM ; EXIT_USAGE=1 ;;
+                  *) TESTENV="$OVAL" ; bugout "Got a target: $TESTENV" ;;
+               esac
+               ;;
+          num) case "$OVAL" in
+                 [0-9]+) CHOICE="$OVAL" ; bugout "Got a number: $CHOICE" ;;
+                      *) perror "option requires a value" $ONAM ; EXIT_USAGE=1 ;;
+               esac
+               ;;
+      rebuild) REBUILD=1      ; bugout "Rebuilding previous..."    ;;
+         make) USE_MAKE=1     ; bugout "Using make with Docker..." ;;
+debug|verbose) DEBUG=1        ; bugout "Debug ON" ;;
+        build) case "$OVAL" in
+                 ''|y|yes) BUILD_YES='Y' ;;
+                     n|no) BUILD_YES='N' ;;
+                        *) perror "option value must be y, n, yes, or no" $ONAM ; EXIT_USAGE=1 ;;
+               esac
+               bugout "Build will initiate? ($BUILD_YES)"
+               ;;
+            *) perror "Unknown flag" "$OPTARG" ; EXIT_USAGE=1 ;;
+       esac
+       ;;
+  esac
+done
+
+((EXIT_USAGE)) && { usage ; exit 1 ; }
+
+if ((REBUILD)); then
+  bugout "Rebuilding previous..."
+  # Build with the last-built env
+  [[ -f "$STATE_FILE" ]] || { errout "No previous (-r) build state found." ; exit 1 ; }
+  read TESTENV <"$STATE_FILE"
+  pio run -d . -e $TESTENV
+  exit
+fi
 
-# Allow shorthand for test name
 case $TESTENV in
-    tree) pio run -d . -e include_tree ; exit 1 ;;
-     due) TESTENV='DUE' ;;
-     esp) TESTENV='esp32' ;;
-    lin*) TESTENV='linux_native' ;;
- lpc?(8)) TESTENV='LPC1768' ;;
-    lpc9) TESTENV='LPC1769' ;;
-    m128) TESTENV='mega1280' ;;
-    m256) TESTENV='mega2560' ;;
-    mega) TESTENV='mega2560' ;;
-     stm) TESTENV='STM32F103RE' ;;
-      f1) TESTENV='STM32F103RE' ;;
-      f4) TESTENV='STM32F4' ;;
-      f7) TESTENV='STM32F7' ;;
-      s6) TESTENV='FYSETC_S6' ;;
-  teensy) TESTENV='teensy31' ;;
-     t31) TESTENV='teensy31' ;;
-     t32) TESTENV='teensy31' ;;
-     t35) TESTENV='teensy35' ;;
-     t36) TESTENV='teensy35' ;;
-     t40) TESTENV='teensy41' ;;
-     t41) TESTENV='teensy41' ;;
-
--h|--help)  echo -e "$(basename $0) : Marlin Firmware test, build, and upload\n"
-            echo "Usage: $(basename $0) ................. Select env and test to apply / run"
-            echo "       $(basename $0) [-y] env ........ Select a test for env to apply / run"
-            echo "       $(basename $0) [-y] env test ... Apply / run the specified env test"
-            echo "       $(basename $0) -b [variant] .... Auto-build the specified variant"
-            echo "       $(basename $0) -u [variant] .... Auto-build and upload the specified variant"
-            echo
-            echo "env shortcuts: tree due esp lin lpc|lpc8 lpc9 m128 m256|mega stm|f1 f4 f7 s6 teensy|t31|t32 t35|t36 t40|t41"
-            exit
-            ;;
-
-          # Build with the last-built env
-      -r) [[ -f "$STATE_FILE" ]] || { echo "No previous (-r) build state found." ; exit 1 ; }
-          read TESTENV <"$STATE_FILE"
-          pio run -d . -e $TESTENV
-          exit
-          ;;
-
-   -[bu]) MB=$( grep -E "^\s*#define MOTHERBOARD" Marlin/Configuration.h | awk '{ print $3 }' | $SED 's/BOARD_//' )
-          [[ -z $MB ]] && { echo "Error - Can't read MOTHERBOARD setting." ; exit 1 ; }
-          BLINE=$( grep -E "define\s+BOARD_$MB\b" Marlin/src/core/boards.h )
-          BNUM=$( $SED -E 's/^.+BOARD_[^ ]+ +([0-9]+).+$/\1/' <<<"$BLINE" )
-          BDESC=$( $SED -E 's/^.+\/\/ *(.+)$/\1/' <<<"$BLINE" )
-          [[ -z $BNUM ]] && { echo "Error - Can't find $MB in boards list." ; exit 1 ; }
-          readarray -t ENVS <<< $( grep -EA1 "MB\(.*\b$MB\b.*\)" Marlin/src/pins/pins.h | grep -E '#include.+//.+env:.+' | grep -oE 'env:[^ ]+' | $SED -E 's/env://' )
-          [[ -z $ENVS ]] && { echo "Error - Can't find target(s) for $MB ($BNUM)." ; exit 1 ; }
-          ECOUNT=${#ENVS[*]}
-
-          if [[ $ECOUNT == 1 ]]; then
-            TARGET=$ENVS
-          else
-            if [[ $CHOICE == 0 ]]; then
-              #
-              # List env names and numbers. Get selection.
-              #
-              echo "Available targets for \"$BDESC\" | $MB ($BNUM):"
-
-              IND=0 ; for ENV in "${ENVS[@]}"; do let IND++ ; echo " $IND) $ENV" ; done
-
-              if [[ $ECOUNT > 1 ]]; then
-                for (( ; ; ))
-                do
-                  read -p "Select a target for '$MB' (1-$ECOUNT) : " CHOICE
-                  [[ -z "$CHOICE" ]] && { echo '(canceled)' ; exit 1 ; }
-                  [[ $CHOICE =~ $ISNUM ]] && ((CHOICE >= 1 && CHOICE <= ECOUNT)) && break
-                  echo ">>> Invalid environment choice '$CHOICE'."
-                done
-                echo
-              fi
-            else
-              echo "Detected \"$BDESC\" | $MB ($BNUM)."
-              [[ $CHOICE > $ECOUNT ]] && { echo "Environment selection out of range." ; exit 1 ; }
-            fi
-            TARGET="${ENVS[$CHOICE-1]}"
-            echo "Selected $TARGET"
-          fi
-
-          echo "$TARGET" >"$STATE_FILE"
-
-          if [[ $TESTENV == "-u" ]]; then
-            echo "Build/Uploading environment $TARGET for board $MB ($BNUM)..." ; echo
-            pio run -t upload -e $TARGET
-          else
-            echo "Building environment $TARGET for board $MB ($BNUM)..." ; echo
-            pio run -e $TARGET
-          fi
-          exit
-          ;;
-
-          # The -y flag may come first
-      -y) TESTENV=${2:-'-'} ; CHOICE=${3:-0} ;;
-
-  -[a-z]) echo "Unknown flag $TESTENV" ; exit 1 ;;
-       -) ;;
+   tree) pio run -d . -e include_tree ; exit 1 ;;
+    due) TESTENV='DUE' ;;
+    esp) TESTENV='esp32' ;;
+   lin*) TESTENV='linux_native' ;;
+lpc?(8)) TESTENV='LPC1768' ;;
+   lpc9) TESTENV='LPC1769' ;;
+   m128) TESTENV='mega1280' ;;
+   m256) TESTENV='mega2560' ;;
+   mega) TESTENV='mega2560' ;;
+    stm) TESTENV='STM32F103RE' ;;
+     f1) TESTENV='STM32F103RE' ;;
+     f4) TESTENV='STM32F4' ;;
+     f7) TESTENV='STM32F7' ;;
+     s6) TESTENV='FYSETC_S6' ;;
+ teensy) TESTENV='teensy31' ;;
+    t31) TESTENV='teensy31' ;;
+    t32) TESTENV='teensy31' ;;
+    t35) TESTENV='teensy35' ;;
+    t36) TESTENV='teensy35' ;;
+    t40) TESTENV='teensy41' ;;
+    t41) TESTENV='teensy41' ;;
 esac
 
+if ((AUTO_BUILD)); then
+  #
+  # List environments that apply to the current MOTHERBOARD.
+  #
+  echo ; echo -n "Auto " ; ((AUTO_BUILD == 2)) && echo "Upload..." || echo "Build..."
+  MB=$( grep -E "^\s*#define MOTHERBOARD" Marlin/Configuration.h | awk '{ print $3 }' | $SED 's/BOARD_//' )
+  [[ -z $MB ]] && { echo "Error - Can't read MOTHERBOARD setting." ; exit 1 ; }
+  BLINE=$( grep -E "define\s+BOARD_$MB\b" Marlin/src/core/boards.h )
+  BNUM=$( $SED -E 's/^.+BOARD_[^ ]+ +([0-9]+).+$/\1/' <<<"$BLINE" )
+  BDESC=$( $SED -E 's/^.+\/\/ *(.+)$/\1/' <<<"$BLINE" )
+  [[ -z $BNUM ]] && { echo "Error - Can't find $MB in boards list." ; exit 1 ; }
+  ENVS=( $( grep -EA1 "MB\(.*\b$MB\b.*\)" Marlin/src/pins/pins.h | grep -E '#include.+//.+env:[^ ]+' | grep -oE 'env:[^ ]+' | $SED -E 's/env://' ) )
+  [[ -z $ENVS ]] && { errout "Error - Can't find target(s) for $MB ($BNUM)." ; exit 1 ; }
+  ECOUNT=${#ENVS[*]}
+
+  if [[ $ECOUNT == 1 ]]; then
+    TARGET=$ENVS
+  else
+    if [[ $CHOICE == 0 ]]; then
+      # List env names and numbers. Get selection.
+      echo "Available targets for \"$BDESC\" | $MB ($BNUM):"
+
+      IND=0 ; for ENV in "${ENVS[@]}"; do let IND++ ; echo " $IND) $ENV" ; done
+
+      if [[ $ECOUNT > 1 ]]; then
+        for (( ; ; ))
+        do
+          read -p "Select a target for '$MB' (1-$ECOUNT) : " CHOICE
+          [[ -z "$CHOICE" ]] && { echo '(canceled)' ; exit 1 ; }
+          [[ $CHOICE =~ $ISNUM ]] && ((CHOICE >= 1 && CHOICE <= ECOUNT)) && break
+          errout ">>> Invalid environment choice '$CHOICE'."
+        done
+        echo
+      fi
+    else
+      echo "Detected \"$BDESC\" | $MB ($BNUM)."
+      [[ $CHOICE > $ECOUNT ]] && { echo "Environment selection out of range." ; exit 1 ; }
+    fi
+    TARGET="${ENVS[$CHOICE-1]}"
+    echo "Selected $TARGET"
+  fi
+
+  echo "$TARGET" >"$STATE_FILE"
+
+  if ((AUTO_BUILD == 2)); then
+    echo "Uploading environment $TARGET for board $MB ($BNUM)..." ; echo
+    pio run -t upload -e $TARGET
+  else
+    echo "Building environment $TARGET for board $MB ($BNUM)..." ; echo
+    pio run -e $TARGET
+  fi
+  exit
+fi
+
 #
 # List available tests and ask for selection
 #
   for (( ; ; ))
   do
     read -p "Select a test to apply (1-$IND) : " NAMEIND
-    [[ -z "$NAMEIND" ]] && { echo '(canceled)' ; exit 1 ; }
+    [[ -z "$NAMEIND" ]] && { errout "(canceled)" ; exit 1 ; }
     [[ $NAMEIND =~ $ISNUM ]] && ((NAMEIND >= 1 && NAMEIND <= IND)) && { TESTENV=${NAMES[$NAMEIND-1]} ; echo ; break ; }
-    echo "Invalid selection."
+    errout "Invalid selection."
   done
 fi
 
 # Get the contents of the test file
-OUT=$( cat $TESTPATH/$TESTENV-tests 2>/dev/null ) || { echo "Can't find test '$TESTENV'." ; exit 1 ; }
+OUT=$( cat $TESTPATH/$TESTENV-tests 2>/dev/null ) || { errout "Can't find test '$TESTENV'." ; exit 1 ; }
 
 # Count up the number of tests
 TESTCOUNT=$( awk "/$ISEXEC/{a++}END{print a}" <<<"$OUT" )
 
 # User entered a number?
-(( CHOICE && CHOICE > TESTCOUNT )) && { echo "Invalid test selection '$CHOICE' (1-$TESTCOUNT)." ; exit 1 ; }
+(( CHOICE && CHOICE > TESTCOUNT )) && { errout "Invalid test selection '$CHOICE' (1-$TESTCOUNT)." ; exit 1 ; }
 
 if [[ $CHOICE == 0 ]]; then
   #
     while IFS= read -r LINE
     do
       if [[ $LINE =~ $ISEXEC ]]; then
-        DESC=$( "$SED" -E 's/^.+"(.*)".*$/\1/g' <<<"$LINE" )
+        DESC=$( "$SED" -E 's/^exec_test \$1 \$2 "([^"]+)".*$/\1/g' <<<"$LINE" )
         (( ++IND < 10 )) && echo -n " "
         echo " $IND) $DESC"
       fi
     for (( ; ; ))
     do
       read -p "Select a '$TESTENV' test (1-$TESTCOUNT) : " CHOICE
-      [[ -z "$CHOICE" ]] && { echo '(canceled)' ; exit 1 ; }
+      [[ -z "$CHOICE" ]] && { errout "(canceled)" ; exit 1 ; }
       [[ $CHOICE =~ $ISNUM ]] && ((CHOICE >= 1 && CHOICE <= TESTCOUNT)) && break
-      echo ">>> Invalid test selection '$CHOICE'."
+      errout ">>> Invalid test selection '$CHOICE'."
     done
   fi
 fi
 #
 # Run the specified test lines
 #
+echo -ne "\033[0;33m"
 echo "$OUT" | {
   IND=0
   GOTX=0
         }
         ((IND == CHOICE)) && {
           GOTX=1
-          [[ $CMD == "" ]] && CMD="$LINE" || CMD=$( echo -e "$CMD$LINE" | $SED -e 's/\\//g' )
-          [[ $LINE =~ $ISCONT ]] || { echo $CMD ; eval "$CMD" ; CMD="" ; }
+          [[ $CMD == "" ]] && CMD="$LINE" || CMD=$( echo -e "$CMD$LINE" | $SED -e 's/\\//g' | $SED -E 's/ +/ /g' )
+          [[ $LINE =~ $ISCONT ]] || { echo "$CMD" ; eval "$CMD" ; CMD="" ; }
         }
       fi
     fi
   done
 }
+echo -ne "\033[0m"
 
 # Make clear it's a TEST
 opt_set CUSTOM_MACHINE_NAME "\"$TESTENV-tests ($CHOICE)\""
 
-# Get a -y parameter the lazy way
-[[ "$2" == "-y" || "$3" == "-y" ]] && BUILD_YES='Y'
-
 # Build the test too?
-if [[ $BUILD_YES != 'Y' ]]; then
+if [[ -z "$BUILD_YES" ]]; then
   echo
   read -p "Build $TESTENV test #$CHOICE (y/N) ? " BUILD_YES
 fi
 
 [[ $BUILD_YES == 'Y' || $BUILD_YES == 'Yes' ]] && {
-  pio run -d . -e $TESTENV
+  ((USE_MAKE)) && make tests-single-local TEST_TARGET=$TESTENV ONLY_TEST=$CHOICE
+  ((USE_MAKE)) || pio run -d . -e $TESTENV
   echo "$TESTENV" >"$STATE_FILE"
 }

buildroot/tests/ARMED-tests

 use_example_configs ArmEd
 opt_set X_DRIVER_TYPE TMC2130
 opt_set Y_DRIVER_TYPE TMC2208
-exec_test $1 $2 "ArmEd Example Configuration with mixed TMC Drivers"
+exec_test $1 $2 "ArmEd Example Configuration with mixed TMC Drivers" "$3"
 
 # clean up
 restore_configs

buildroot/tests/BIGTREE_BTT002-tests

 opt_set SERIAL_PORT 1
 opt_set X_DRIVER_TYPE TMC2209
 opt_set Y_DRIVER_TYPE TMC2130
-exec_test $1 $2 "BigTreeTech BTT002 Default Configuration plus TMC steppers"
+exec_test $1 $2 "BigTreeTech BTT002 Default Configuration plus TMC steppers" "$3"
 
 # clean up
 restore_configs

buildroot/tests/BIGTREE_GTR_V1_0-tests

 opt_enable FIL_RUNOUT4_PULL
 opt_set FIL_RUNOUT8_STATE HIGH
 opt_enable FIL_RUNOUT8_PULL
-exec_test $1 $2 "BigTreeTech GTR 8 Extruders with Auto-Fan, Mixed TMC Drivers, and Runout Sensors with distinct states"
+exec_test $1 $2 "BigTreeTech GTR 8 Extruders with Auto-Fan, Mixed TMC Drivers, and Runout Sensors with distinct states" "$3"
 
 restore_configs
 opt_set MOTHERBOARD BOARD_BTT_GTR_V1_0
 opt_set DEFAULT_Ki_LIST "{ 1.08 }"
 opt_set DEFAULT_Kd_LIST "{ 114.0, 112.0, 110.0, 108.0 }"
 opt_enable TOOLCHANGE_FILAMENT_SWAP TOOLCHANGE_MIGRATION_FEATURE TOOLCHANGE_FS_INIT_BEFORE_SWAP TOOLCHANGE_FS_PRIME_FIRST_USED PID_PARAMS_PER_HOTEND
-exec_test $1 $2 "BigTreeTech GTR 6 Extruders Triple Z"
+exec_test $1 $2 "BigTreeTech GTR 6 Extruders Triple Z" "$3"
 
 # clean up
 restore_configs

buildroot/tests/BIGTREE_SKR_PRO-tests

 restore_configs
 opt_set MOTHERBOARD BOARD_BTT_SKR_PRO_V1_1
 opt_set SERIAL_PORT 1
-exec_test $1 $2 "BigTreeTech SKR Pro Default Configuration"
+exec_test $1 $2 "BigTreeTech SKR Pro Default Configuration" "$3"
 
 restore_configs
 opt_set MOTHERBOARD BOARD_BTT_SKR_PRO_V1_1
 opt_set X_DRIVER_TYPE TMC2209
 opt_set Y_DRIVER_TYPE TMC2130
 opt_enable BLTOUCH EEPROM_SETTINGS AUTO_BED_LEVELING_3POINT Z_SAFE_HOMING
-exec_test $1 $2 "BigTreeTech SKR Pro 3 Extruders, Auto-Fan, BLTOUCH, mixed TMC drivers"
+exec_test $1 $2 "BigTreeTech SKR Pro 3 Extruders, Auto-Fan, BLTOUCH, mixed TMC drivers" "$3"
 
 # clean up
 restore_configs

buildroot/tests/DUE-tests

 opt_set TEMP_SENSOR_CHAMBER 3
 opt_add TEMP_CHAMBER_PIN 6
 opt_set HEATER_CHAMBER_PIN 45
-exec_test $1 $2 "RAMPS4DUE_EFB with ABL (Bilinear), ExtUI, S-Curve, many options."
+exec_test $1 $2 "RAMPS4DUE_EFB with ABL (Bilinear), ExtUI, S-Curve, many options." "$3"
 
 #
 # RADDS with BLTouch, ABL(B), 3 x Z auto-align
 opt_add Z3_MAX_PIN 3
 pins_set ramps/RAMPS X_MAX_PIN -1
 pins_set ramps/RAMPS Y_MAX_PIN -1
-exec_test $1 $2 "RADDS with ABL (Bilinear), Triple Z Axis, Z_STEPPER_AUTO_ALIGN"
+exec_test $1 $2 "RADDS with ABL (Bilinear), Triple Z Axis, Z_STEPPER_AUTO_ALIGN" "$3"
 
 #
 # Test SWITCHING_EXTRUDER
 opt_set EXTRUDERS 2
 opt_set NUM_SERVOS 1
 opt_enable SWITCHING_EXTRUDER ULTIMAKERCONTROLLER BEEP_ON_FEEDRATE_CHANGE POWER_LOSS_RECOVERY
-exec_test $1 $2 "RAMPS4DUE_EEF with SWITCHING_EXTRUDER, POWER_LOSS_RECOVERY"
+exec_test $1 $2 "RAMPS4DUE_EEF with SWITCHING_EXTRUDER, POWER_LOSS_RECOVERY" "$3"

buildroot/tests/DUE_archim-tests

 # Test Archim 1
 #
 use_example_configs UltiMachine/Archim1
-exec_test $1 $2 "Archim 1 base configuration"
+exec_test $1 $2 "Archim 1 base configuration" "$3"
 
 #
 # Test Archim 2
 #
 use_example_configs UltiMachine/Archim2
-exec_test $1 $2 "Archim 2 base configuration"
+exec_test $1 $2 "Archim 2 base configuration" "$3"
 
 restore_configs

buildroot/tests/FLYF407ZG-tests

 opt_set SERIAL_PORT -1
 opt_set X_DRIVER_TYPE TMC2208
 opt_set Y_DRIVER_TYPE TMC2130
-exec_test $1 $2 "FLYF407ZG Default Config with mixed TMC Drivers"
+exec_test $1 $2 "FLYF407ZG Default Config with mixed TMC Drivers" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/FYSETC_F6_13-tests

 restore_configs
 opt_set MOTHERBOARD BOARD_FYSETC_F6_13
 opt_enable DGUS_LCD_UI_FYSETC
-exec_test $1 $2 "FYSETC F6 1.3 with DGUS"
+exec_test $1 $2 "FYSETC F6 1.3 with DGUS" "$3"
 
 # clean up
 restore_configs

buildroot/tests/FYSETC_S6-tests

 use_example_configs FYSETC/S6
 opt_set Y_DRIVER_TYPE TMC2209
 opt_set Z_DRIVER_TYPE TMC2130
-exec_test $1 $2 "FYSETC S6 Example"
+exec_test $1 $2 "FYSETC S6 Example" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/LERDGEX-tests

 restore_configs
 opt_set MOTHERBOARD BOARD_LERDGE_X
 opt_set SERIAL_PORT 1
-exec_test $1 $2 "LERDGE X with Default Configuration"
+exec_test $1 $2 "LERDGE X with Default Configuration" "$3"
 
 # clean up
 restore_configs

buildroot/tests/LPC1768-tests

 opt_set MOTHERBOARD BOARD_RAMPS_14_RE_ARM_EFB
 opt_enable VIKI2 SDSUPPORT SDCARD_READONLY SERIAL_PORT_2 NEOPIXEL_LED
 opt_set NEOPIXEL_PIN P1_16
-exec_test $1 $2 "ReARM EFB VIKI2, SDSUPPORT, 2 Serial ports (USB CDC + UART0), NeoPixel"
+exec_test $1 $2 "ReARM EFB VIKI2, SDSUPPORT, 2 Serial ports (USB CDC + UART0), NeoPixel" "$3"
 
 #restore_configs
 #use_example_configs Mks/Sbase
 opt_set NUM_SERVOS 2
 opt_set SERVO_DELAY "{ 300, 300 }"
 opt_enable SWITCHING_NOZZLE SWITCHING_NOZZLE_E1_SERVO_NR ULTIMAKERCONTROLLER
-exec_test $1 $2 "MKS SBASE with SWITCHING_NOZZLE"
+exec_test $1 $2 "MKS SBASE with SWITCHING_NOZZLE" "$3"
 
 restore_configs
 opt_set MOTHERBOARD BOARD_RAMPS_14_RE_ARM_EEB
            LCD_INFO_MENU ARC_SUPPORT BEZIER_CURVE_SUPPORT EXTENDED_CAPABILITIES_REPORT AUTO_REPORT_TEMPERATURES SDCARD_SORT_ALPHA EMERGENCY_PARSER
 opt_set GRID_MAX_POINTS_X 16
 opt_set NOZZLE_TO_PROBE_OFFSET "{ 0, 0, 0 }"
-exec_test $1 $2 "Re-ARM with NOZZLE_AS_PROBE and many features."
+exec_test $1 $2 "Re-ARM with NOZZLE_AS_PROBE and many features." "$3"
 
 # clean up
 restore_configs

buildroot/tests/LPC1769-tests

 # Build with the default configurations
 #
 use_example_configs Azteeg/X5GT
-exec_test $1 $2 "Azteeg X5GT Example Configuration"
+exec_test $1 $2 "Azteeg X5GT Example Configuration" "$3"
 
 restore_configs
 opt_set MOTHERBOARD BOARD_SMOOTHIEBOARD
            LCD_INFO_MENU ARC_SUPPORT BEZIER_CURVE_SUPPORT EXTENDED_CAPABILITIES_REPORT AUTO_REPORT_TEMPERATURES SDCARD_SORT_ALPHA EMERGENCY_PARSER
 opt_disable SD_PROCEDURE_DEPTH
 opt_set GRID_MAX_POINTS_X 16
-exec_test $1 $2 "Smoothieboard with many features"
+exec_test $1 $2 "Smoothieboard with many features" "$3"
 
 restore_configs
 opt_set MOTHERBOARD BOARD_SMOOTHIEBOARD
            Z_SAFE_HOMING ADVANCED_PAUSE_FEATURE PARK_HEAD_ON_PAUSE \
            LCD_INFO_MENU ARC_SUPPORT BEZIER_CURVE_SUPPORT EXTENDED_CAPABILITIES_REPORT AUTO_REPORT_TEMPERATURES SDCARD_SORT_ALPHA EMERGENCY_PARSER
 opt_set GRID_MAX_POINTS_X 16
-exec_test $1 $2 "Smoothieboard with TFTGLCD_PANEL_SPI"
+exec_test $1 $2 "Smoothieboard with TFTGLCD_PANEL_SPI" "$3"
 
 #restore_configs
 #opt_set MOTHERBOARD BOARD_AZTEEG_X5_MINI_WIFI
            SENSORLESS_PROBING Z_SAFE_HOMING X_STALL_SENSITIVITY Y_STALL_SENSITIVITY Z_STALL_SENSITIVITY TMC_DEBUG \
            EXPERIMENTAL_I2CBUS
 opt_disable PSU_CONTROL
-exec_test $1 $2 "Cohesion3D Remix DELTA + ABL Bilinear + EEPROM + SENSORLESS_PROBING"
+exec_test $1 $2 "Cohesion3D Remix DELTA + ABL Bilinear + EEPROM + SENSORLESS_PROBING" "$3"
 
 # clean up
 restore_configs

buildroot/tests/NUCLEO_F767ZI-tests

 opt_enable BLTOUCH Z_SAFE_HOMING SPEAKER
 opt_set X_DRIVER_TYPE TMC2209
 opt_set Y_DRIVER_TYPE TMC2208
-exec_test $1 $2 "Mixed timer usage"
+exec_test $1 $2 "Mixed timer usage" "$3"
 
 # clean up
 restore_configs

buildroot/tests/SAMD51_grandcentral_m4-tests

            MOVE_Z_WHEN_IDLE BABYSTEP_ZPROBE_OFFSET BABYSTEP_ZPROBE_GFX_OVERLAY \
            LIN_ADVANCE ADVANCED_PAUSE_FEATURE PARK_HEAD_ON_PAUSE MONITOR_DRIVER_STATUS SENSORLESS_HOMING \
            SQUARE_WAVE_STEPPING TMC_DEBUG EXPERIMENTAL_SCURVE
-exec_test $1 $2 "Build Grand Central M4 Default Configuration"
+exec_test $1 $2 "Build Grand Central M4 Default Configuration" "$3"
 
 # clean up
 restore_configs

buildroot/tests/STM32F070CB_malyan-tests

 restore_configs
 opt_set MOTHERBOARD BOARD_MALYAN_M200_V2
 opt_set SERIAL_PORT -1
-exec_test $1 $2 "Malyan M200 v2 Default Config"
+exec_test $1 $2 "Malyan M200 v2 Default Config" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/STM32F070RB_malyan-tests

 restore_configs
 opt_set MOTHERBOARD BOARD_MALYAN_M200_V2
 opt_set SERIAL_PORT -1
-exec_test $1 $2 "Malyan M200 v2 Default Config"
+exec_test $1 $2 "Malyan M200 v2 Default Config" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/STM32F103CB_malyan-tests

 set -e
 
 use_example_configs Malyan/M200
-exec_test $1 $2 "Malyan M200"
+exec_test $1 $2 "Malyan M200" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/STM32F103RC_btt-tests

 opt_set Z_SLAVE_ADDRESS 2
 opt_set E0_SLAVE_ADDRESS 3
 
-exec_test $1 $2 "BigTreeTech SKR Mini E3 1.0 - Basic Config with TMC2209 HW Serial"
+exec_test $1 $2 "BigTreeTech SKR Mini E3 1.0 - Basic Config with TMC2209 HW Serial" "$3"
 
 # clean up
 restore_configs

buildroot/tests/STM32F103RC_btt_USB-tests

 opt_set MOTHERBOARD BOARD_BTT_SKR_MINI_V1_1
 opt_set SERIAL_PORT 1
 opt_set SERIAL_PORT_2 -1
-exec_test $1 $2 "BigTreeTech SKR Mini v1.1 - Basic Configuration"
+exec_test $1 $2 "BigTreeTech SKR Mini v1.1 - Basic Configuration" "$3"
 
 # clean up
 restore_configs

buildroot/tests/STM32F103RC_fysetc-tests

 # Build with the default configurations
 #
 use_example_configs "Creality/Ender-3/FYSETC Cheetah 1.2/base"
-exec_test $1 $2 "Cheetah 1.2 Configuration"
+exec_test $1 $2 "Cheetah 1.2 Configuration" "$3"
 
 # clean up
 restore_configs

buildroot/tests/STM32F103RC_meeb-tests

 opt_set Y_DRIVER_TYPE TMC2208
 opt_set Z_DRIVER_TYPE TMC2208
 opt_set E0_DRIVER_TYPE TMC2208
-exec_test $1 $2 "MEEB_3DP - Basic Config with TMC2208 SW Serial"
+exec_test $1 $2 "MEEB_3DP - Basic Config with TMC2208 SW Serial" "$3"
 
 # clean up
 restore_configs

buildroot/tests/STM32F103RE-tests

 opt_enable EEPROM_SETTINGS EEPROM_CHITCHAT REPRAP_DISCOUNT_SMART_CONTROLLER SDSUPPORT \
            PAREN_COMMENTS GCODE_MOTION_MODES SINGLENOZZLE TOOLCHANGE_FILAMENT_SWAP TOOLCHANGE_PARK \
            BAUD_RATE_GCODE GCODE_MACROS NOZZLE_PARK_FEATURE NOZZLE_CLEAN_FEATURE
-exec_test $1 $2 "STM32F1R EEPROM_SETTINGS EEPROM_CHITCHAT REPRAP_DISCOUNT_SMART_CONTROLLER SDSUPPORT PAREN_COMMENTS GCODE_MOTION_MODES"
+exec_test $1 $2 "STM32F1R EEPROM_SETTINGS EEPROM_CHITCHAT REPRAP_DISCOUNT_SMART_CONTROLLER SDSUPPORT PAREN_COMMENTS GCODE_MOTION_MODES" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/STM32F103RET6_creality-tests

 #
 use_example_configs "Creality/Ender-3 V2"
 opt_enable MARLIN_DEV_MODE
-exec_test $1 $2 "Ender 3 v2"
+exec_test $1 $2 "Ender 3 v2" "$3"
 
 restore_configs

buildroot/tests/STM32F103RE_btt-tests

 opt_set SERIAL_PORT_2 -1
 opt_set X_DRIVER_TYPE TMC2209
 opt_set Y_DRIVER_TYPE TMC2130
-exec_test $1 $2 "BigTreeTech SKR E3 DIP v1.0 - Basic Config with mixed TMC Drivers"
+exec_test $1 $2 "BigTreeTech SKR E3 DIP v1.0 - Basic Config with mixed TMC Drivers" "$3"
 
 # clean up
 restore_configs

buildroot/tests/STM32F103RE_btt_USB-tests

 opt_set MOTHERBOARD BOARD_BTT_SKR_E3_DIP
 opt_set SERIAL_PORT 1
 opt_set SERIAL_PORT_2 -1
-exec_test $1 $2 "BigTreeTech SKR E3 DIP v1.0 - Basic Configuration"
+exec_test $1 $2 "BigTreeTech SKR E3 DIP v1.0 - Basic Configuration" "$3"
 
 # clean up
 restore_configs

buildroot/tests/STM32F103VE_longer-tests

 
 use_example_configs Alfawise/U20
 opt_enable BAUD_RATE_GCODE
-exec_test $1 $2 "Full-featured U20 config"
+exec_test $1 $2 "Full-featured U20 config" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/STM32F4-tests

 # Build with the default configurations
 #
 use_example_configs STM32/STM32F4
-exec_test $1 $2 "STM32F4 Default Configuration"
+exec_test $1 $2 "STM32F4 Default Configuration" "$3"
 
 # clean up
 restore_configs

buildroot/tests/STM32F401VE_STEVAL-tests

 restore_configs
 opt_set MOTHERBOARD BOARD_STEVAL_3DP001V1
 opt_set SERIAL_PORT -1
-exec_test $1 $2 "STM32F401VE_STEVAL Default Config"
+exec_test $1 $2 "STM32F401VE_STEVAL Default Config" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/STM32F407VE_black-tests

 restore_configs
 use_example_configs STM32/Black_STM32F407VET6
 opt_enable BAUD_RATE_GCODE
-exec_test $1 $2 "Full-featured Sample Black STM32F407VET6 config"
+exec_test $1 $2 "Full-featured Sample Black STM32F407VET6 config" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/STM32F7-tests

 restore_configs
 opt_set MOTHERBOARD BOARD_REMRAM_V1
 opt_set SERIAL_PORT 1
-exec_test $1 $2 "Default Configuration"
+exec_test $1 $2 "Default Configuration" "$3"
 
 # clean up
 restore_configs

buildroot/tests/at90usb1286_cdc-tests

 #
 restore_configs
 opt_set MOTHERBOARD BOARD_BRAINWAVE_PRO
-exec_test $1 $2 "Default Configuration"
+exec_test $1 $2 "Default Configuration" "$3"
 
 # clean up
 restore_configs

buildroot/tests/at90usb1286_dfu-tests

 #
 restore_configs
 opt_set MOTHERBOARD BOARD_PRINTRBOARD
-exec_test $1 $2 "Printrboard Configuration"
+exec_test $1 $2 "Printrboard Configuration" "$3"
 
 restore_configs
 opt_set MOTHERBOARD BOARD_PRINTRBOARD_REVF
 opt_enable MINIPANEL
-exec_test $1 $2 "Printrboard RevF with MiniPanel and Stepper DAC (in pins file)"
+exec_test $1 $2 "Printrboard RevF with MiniPanel and Stepper DAC (in pins file)" "$3"
 
 # clean up
 restore_configs

buildroot/tests/esp32-tests

 opt_add WIFI_SSID "\"ssid\""
 opt_add WIFI_PWD "\"password\""
 opt_set TX_BUFFER_SIZE 64
-exec_test $1 $2 "ESP32 with WIFISUPPORT and WEBSUPPORT"
+exec_test $1 $2 "ESP32 with WIFISUPPORT and WEBSUPPORT" "$3"
 
 #
 # Build with TMC drivers using hardware serial
 opt_set Z_SLAVE_ADDRESS 2
 opt_set E0_SLAVE_ADDRESS 3
 opt_enable HOTEND_IDLE_TIMEOUT
-exec_test $1 $2 "ESP32, TMC HW Serial, Hotend Idle"
+exec_test $1 $2 "ESP32, TMC HW Serial, Hotend Idle" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/jgaurora_a5s_a1-tests

 # Build with the default configurations
 #
 use_example_configs JGAurora/A5S
-exec_test $1 $2 "JGAurora/A5S Configuration"
+exec_test $1 $2 "JGAurora/A5S Configuration" "$3"
 
 # clean up
 restore_configs

buildroot/tests/linux_native-tests

 opt_set MOTHERBOARD BOARD_LINUX_RAMPS
 opt_set TEMP_SENSOR_BED 1
 opt_enable PIDTEMPBED EEPROM_SETTINGS BAUD_RATE_GCODE
-exec_test $1 $2 "Linux with EEPROM"
+exec_test $1 $2 "Linux with EEPROM" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/malyan_M300-tests

 use_example_configs "delta/Malyan M300"
 opt_disable AUTO_BED_LEVELING_3POINT
 opt_set LCD_SERIAL_PORT 1
-exec_test $1 $2 "Malyan M300 (delta)"
+exec_test $1 $2 "Malyan M300 (delta)" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/mega1280-tests

 opt_set POWER_MONITOR_VOLTAGE_PIN 15
 opt_set CLOSED_LOOP_ENABLE_PIN 44
 opt_set CLOSED_LOOP_MOVE_COMPLETE_PIN 45
-exec_test $1 $2 "Spindle, MESH_BED_LEVELING, closed loop, Power Monitor, and LCD"
+exec_test $1 $2 "Spindle, MESH_BED_LEVELING, closed loop, Power Monitor, and LCD" "$3"
 
 #
 # Test DUAL_X_CARRIAGE
 opt_set TEMP_SENSOR_1 1
 opt_enable USE_XMAX_PLUG DUAL_X_CARRIAGE REPRAPWORLD_KEYPAD
 opt_set REPRAPWORLD_KEYPAD_MOVE_STEP 10.0
-exec_test $1 $2 "TT Oscar | DUAL_X_CARRIAGE"
+exec_test $1 $2 "TT Oscar | DUAL_X_CARRIAGE" "$3"
 
 #
 # Delta Config (generic) + Probeless
 #
 use_example_configs delta/generic
 opt_enable REPRAP_DISCOUNT_SMART_CONTROLLER DELTA_AUTO_CALIBRATION DELTA_CALIBRATION_MENU
-exec_test $1 $2 "RAMPS | DELTA | RRD LCD | DELTA_AUTO_CALIBRATION | DELTA_CALIBRATION_MENU"
+exec_test $1 $2 "RAMPS | DELTA | RRD LCD | DELTA_AUTO_CALIBRATION | DELTA_CALIBRATION_MENU" "$3"
 
 #
 # Delta Config (generic) + ABL bilinear + BLTOUCH
 use_example_configs delta/generic
 opt_set LCD_LANGUAGE cz
 opt_enable REPRAP_DISCOUNT_SMART_CONTROLLER DELTA_CALIBRATION_MENU AUTO_BED_LEVELING_BILINEAR BLTOUCH
-exec_test $1 $2 "DELTA | RRD LCD | ABL Bilinear | BLTOUCH"
+exec_test $1 $2 "DELTA | RRD LCD | ABL Bilinear | BLTOUCH" "$3"
 
 # clean up
 restore_configs

buildroot/tests/mega2560-tests

            MULTI_NOZZLE_DUPLICATION CLASSIC_JERK LIN_ADVANCE EXTRA_LIN_ADVANCE_K QUICK_HOME \
            LCD_SET_PROGRESS_MANUALLY PRINT_PROGRESS_SHOW_DECIMALS SHOW_REMAINING_TIME \
            BABYSTEPPING BABYSTEP_XY NANODLP_Z_SYNC I2C_POSITION_ENCODERS M114_DETAIL
-exec_test $1 $2 "Azteeg X3 Pro | EXTRUDERS 5 | RRDFGSC | UBL | LIN_ADVANCE ..."
+exec_test $1 $2 "Azteeg X3 Pro | EXTRUDERS 5 | RRDFGSC | UBL | LIN_ADVANCE ..." "$3"
 
 #
 # Add a Sled Z Probe, use UBL Cartesian moves, use Japanese language
 opt_set LCD_LANGUAGE jp_kana
 opt_disable SEGMENT_LEVELED_MOVES
 opt_enable BABYSTEPPING BABYSTEP_XY BABYSTEP_ZPROBE_OFFSET DOUBLECLICK_FOR_Z_BABYSTEPPING BABYSTEP_HOTEND_Z_OFFSET BABYSTEP_DISPLAY_TOTAL M114_DETAIL
-exec_test $1 $2 "Azteeg X3 Pro | EXTRUDERS 5 | RRDFGSC | UBL | LIN_ADVANCE | Sled Probe | Skew | JP-Kana | Babystep offsets ..."
+exec_test $1 $2 "Azteeg X3 Pro | EXTRUDERS 5 | RRDFGSC | UBL | LIN_ADVANCE | Sled Probe | Skew | JP-Kana | Babystep offsets ..." "$3"
 
 #
 # Test a Servo Probe
            NO_VOLUMETRICS EXTENDED_CAPABILITIES_REPORT AUTO_REPORT_TEMPERATURES AUTOTEMP G38_PROBE_TARGET JOYSTICK \
            PRUSA_MMU2 MMU2_MENUS PRUSA_MMU2_S_MODE DIRECT_STEPPING DETECT_BROKEN_ENDSTOP \
            FILAMENT_RUNOUT_SENSOR NOZZLE_PARK_FEATURE ADVANCED_PAUSE_FEATURE Z_SAFE_HOMING
-exec_test $1 $2 "RAMPS | ZONESTAR + Chinese | MMU2 | Servo | 3-Point + Debug | G38 ..."
+exec_test $1 $2 "RAMPS | ZONESTAR + Chinese | MMU2 | Servo | 3-Point + Debug | G38 ..." "$3"
 
 #
 # 5 runout sensors with distinct states
 opt_add M100_FREE_MEMORY_CORRUPTOR
 opt_set PWM_MOTOR_CURRENT "{ 1300, 1300, 1250 }"
 opt_set I2C_SLAVE_ADDRESS 63
-exec_test $1 $2 "MEGACONTROLLER | Minipanel | M100 | PWM_MOTOR_CURRENT | PRINTCOUNTER | Advanced Pause ..."
+exec_test $1 $2 "MEGACONTROLLER | Minipanel | M100 | PWM_MOTOR_CURRENT | PRINTCOUNTER | Advanced Pause ..." "$3"
 
 #
 # Mixing Extruder with 5 steppers, Greek
            USE_CONTROLLER_FAN CONTROLLER_FAN_EDITABLE CONTROLLER_FAN_IGNORE_Z
 opt_set MIXING_STEPPERS 5
 opt_set LCD_LANGUAGE ru
-exec_test $1 $2 "Azteeg X3 | Mixing Extruder (x5) | Gradient Mix | Greek"
+exec_test $1 $2 "Azteeg X3 | Mixing Extruder (x5) | Gradient Mix | Greek" "$3"
 
 #
 # Test SPEAKER with BOARD_BQ_ZUM_MEGA_3D and BQ_LCD_SMART_CONTROLLER
 opt_set FAN_KICKSTART_TIME 100
 opt_set XY_FREQUENCY_LIMIT 15
 opt_add FILWIDTH_PIN 5
-exec_test $1 $2 "Mightyboard Rev. E | CoreXY, Gradient Mix | Endstop Int. | Home Y > X | FW Retract ..."
+exec_test $1 $2 "Mightyboard Rev. E | CoreXY, Gradient Mix | Endstop Int. | Home Y > X | FW Retract ..." "$3"
 
 ######## Other Standard LCD/Panels ##############
 #
 opt_add L6470_CHAIN_MOSI_PIN 40
 opt_add L6470_CHAIN_SS_PIN   42
 opt_add "ENABLE_RESET_L64XX_CHIPS(V) NOOP"
-exec_test $1 $2 "DELTA, RAMPS, L6470, UBL, Allen Key, EEPROM, OLED_PANEL_TINYBOY2..."
+exec_test $1 $2 "DELTA, RAMPS, L6470, UBL, Allen Key, EEPROM, OLED_PANEL_TINYBOY2..." "$3"
 
 #
 # Delta Config (FLSUN AC because it's complex)
 #
 use_example_configs delta/FLSUN/auto_calibrate
-exec_test $1 $2 "RAMPS 1.3 | DELTA | FLSUN AC Config"
+exec_test $1 $2 "RAMPS 1.3 | DELTA | FLSUN AC Config" "$3"
 
 #
 # Makibox Config  need to check board type for Teensy++ 2.0
            MARLIN_BRICKOUT MARLIN_INVADERS MARLIN_SNAKE \
            MONITOR_DRIVER_STATUS STEALTHCHOP_XY STEALTHCHOP_Z STEALTHCHOP_E HYBRID_THRESHOLD \
            USE_ZMIN_PLUG SENSORLESS_HOMING TMC_DEBUG M114_DETAIL
-exec_test $1 $2 "RAMPS | Mixed TMC | Sensorless | RRDFGSC | Games"
+exec_test $1 $2 "RAMPS | Mixed TMC | Sensorless | RRDFGSC | Games" "$3"
 
 #
 # SCARA with Mixed TMC
 opt_set Y_DRIVER_TYPE TMC2130
 opt_set Z_DRIVER_TYPE TMC2130_STANDALONE
 opt_set E0_DRIVER_TYPE TMC2660
-exec_test $1 $2 "RAMPS | SCARA | Mixed TMC | EEPROM"
+exec_test $1 $2 "RAMPS | SCARA | Mixed TMC | EEPROM" "$3"
 
 #
 # tvrrug Config need to check board type for sanguino atmega644p

buildroot/tests/mks_robin-tests

 set -e
 
 use_example_configs Mks/Robin
-exec_test $1 $2 "MKS Robin config (FSMC Color UI)"
+exec_test $1 $2 "MKS Robin config (FSMC Color UI)" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/mks_robin_lite-tests

 opt_set SERIAL_PORT 1
 opt_enable EEPROM_SETTINGS
 opt_enable SDSUPPORT
-exec_test $1 $2 "Default Configuration with Fallback SD EEPROM"
+exec_test $1 $2 "Default Configuration with Fallback SD EEPROM" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/mks_robin_mini-tests

 opt_set MOTHERBOARD BOARD_MKS_ROBIN_MINI
 opt_set EXTRUDERS 1
 opt_set TEMP_SENSOR_1 0
-exec_test $1 $2 "MKS Robin mini"
+exec_test $1 $2 "MKS Robin mini" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/mks_robin_nano35-tests

 #
 use_example_configs Mks/Robin
 opt_set MOTHERBOARD BOARD_MKS_ROBIN_NANO
-exec_test $1 $2 "MKS Robin nano v1.2 Emulated DOGM FSMC"
+exec_test $1 $2 "MKS Robin nano v1.2 Emulated DOGM FSMC" "$3"
 
 #
 # MKS Robin v2 nano Emulated DOGM SPI
 opt_set MOTHERBOARD BOARD_MKS_ROBIN_NANO_V2
 opt_disable TFT_INTERFACE_FSMC
 opt_enable TFT_INTERFACE_SPI
-exec_test $1 $2 "MKS Robin v2 nano Emulated DOGM SPI"
+exec_test $1 $2 "MKS Robin v2 nano Emulated DOGM SPI" "$3"
 
 #
 # MKS Robin nano v1.2 LVGL FSMC
 opt_set MOTHERBOARD BOARD_MKS_ROBIN_NANO
 opt_disable TFT_CLASSIC_UI TFT_COLOR_UI TOUCH_SCREEN TFT_RES_320x240
 opt_enable TFT_LVGL_UI TFT_RES_480x320
-exec_test $1 $2 "MKS Robin nano v1.2 LVGL FSMC"
+exec_test $1 $2 "MKS Robin nano v1.2 LVGL FSMC" "$3"
 
 #
 # MKS Robin v2 nano LVGL SPI
 opt_set MOTHERBOARD BOARD_MKS_ROBIN_NANO_V2
 opt_disable TFT_INTERFACE_FSMC TFT_COLOR_UI TOUCH_SCREEN TFT_RES_320x240
 opt_enable TFT_INTERFACE_SPI TFT_LVGL_UI TFT_RES_480x320
-exec_test $1 $2 "MKS Robin v2 nano LVGL SPI"
+exec_test $1 $2 "MKS Robin v2 nano LVGL SPI" "$3"
 
 #
 # MKS Robin v2 nano New Color UI 480x320 SPI
 opt_set MOTHERBOARD BOARD_MKS_ROBIN_NANO_V2
 opt_disable TFT_INTERFACE_FSMC TFT_RES_320x240
 opt_enable TFT_INTERFACE_SPI TFT_RES_480x320
-exec_test $1 $2 "MKS Robin v2 nano New Color UI 480x320 SPI"
+exec_test $1 $2 "MKS Robin v2 nano New Color UI 480x320 SPI" "$3"
 
 #
 # MKS Robin v2 nano LVGL SPI + TMC
 opt_enable TFT_INTERFACE_SPI TFT_LVGL_UI TFT_RES_480x320
 opt_set X_DRIVER_TYPE TMC2209
 opt_set Y_DRIVER_TYPE TMC2209
-exec_test $1 $2 "MKS Robin v2 nano LVGL SPI + TMC"
+exec_test $1 $2 "MKS Robin v2 nano LVGL SPI + TMC" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/mks_robin_pro-tests

 opt_set Y_DRIVER_TYPE TMC2130
 opt_set TEMP_SENSOR_BED 1
 opt_disable THERMAL_PROTECTION_HOTENDS
-exec_test $1 $2 "MKS Robin Pro, TMC Drivers, no thermal protection"
+exec_test $1 $2 "MKS Robin Pro, TMC Drivers, no thermal protection" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/mks_robin_stm32-tests

 set -e
 
 use_example_configs Mks/Robin
-exec_test $1 $2 "MKS Robin base configuration"
+exec_test $1 $2 "MKS Robin base configuration" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/rambo-tests

            SLOW_PWM_HEATERS THERMAL_PROTECTION_CHAMBER LIN_ADVANCE EXTRA_LIN_ADVANCE_K \
            HOST_ACTION_COMMANDS HOST_PROMPT_SUPPORT PINS_DEBUGGING MAX7219_DEBUG M114_DETAIL
 opt_add DEBUG_POWER_LOSS_RECOVERY
-exec_test $1 $2 "RAMBO | EXTRUDERS 2 | CHAR LCD + SD | FIX Probe | ABL-Linear | Advanced Pause | PLR | LEDs ..."
+exec_test $1 $2 "RAMBO | EXTRUDERS 2 | CHAR LCD + SD | FIX Probe | ABL-Linear | Advanced Pause | PLR | LEDs ..." "$3"
 
 #
 # Full size Rambo Dual Endstop CNC
            S_CURVE_ACCELERATION X_DUAL_STEPPER_DRIVERS X_DUAL_ENDSTOPS Y_DUAL_STEPPER_DRIVERS Y_DUAL_ENDSTOPS \
            ADAPTIVE_STEP_SMOOTHING CNC_COORDINATE_SYSTEMS GCODE_MOTION_MODES
 opt_disable MIN_SOFTWARE_ENDSTOP_Z MAX_SOFTWARE_ENDSTOPS
-exec_test $1 $2 "Rambo CNC Configuration"
+exec_test $1 $2 "Rambo CNC Configuration" "$3"
 
 #
 # Rambo heated bed only
 opt_set Y_DRIVER_TYPE TMC2130
 opt_set Z_DRIVER_TYPE TMC2130
 opt_set E0_DRIVER_TYPE TMC2130
-exec_test $1 $2 "Einsy RAMBo with TMC2130"
+exec_test $1 $2 "Einsy RAMBo with TMC2130" "$3"
 
 # clean up
 restore_configs

buildroot/tests/rumba32-tests

 opt_set TEMP_SENSOR_BED 1
 opt_disable THERMAL_PROTECTION_BED
 opt_set X_DRIVER_TYPE TMC2130
-exec_test $1 $2 "RUMBA32 V1.0 with TMC2130, PID Bed, and bed thermal protection disabled"
+exec_test $1 $2 "RUMBA32 V1.0 with TMC2130, PID Bed, and bed thermal protection disabled" "$3"
 
 # Build examples
 restore_configs
 opt_set TEMP_SENSOR_BED 1
 opt_set X_DRIVER_TYPE TMC2130
 opt_set Y_DRIVER_TYPE TMC2208
-exec_test $1 $2 "RUMBA32 V1.1 with TMC2130, TMC2208, PID Bed, EEPROM settings, and graphic LCD controller"
+exec_test $1 $2 "RUMBA32 V1.1 with TMC2130, TMC2208, PID Bed, EEPROM settings, and graphic LCD controller" "$3"
 
 # Build examples
 restore_configs
 opt_set SERIAL_PORT -1
 opt_set X_DRIVER_TYPE TMC2130
 opt_set Y_DRIVER_TYPE TMC2208
-exec_test $1 $2 "RUMBA32 MKS Default Config with Mixed TMC Drivers"
+exec_test $1 $2 "RUMBA32 MKS Default Config with Mixed TMC Drivers" "$3"
 
 # cleanup
 restore_configs

buildroot/tests/run_tests

 
 exec_test () {
   printf "\n\033[0;32m[Test $2] \033[0m$3...\n"
-  if platformio run --project-dir $1 -e $2 --silent; then
+  # Check to see if we should skip tests
+  if [[ -n "$4" ]] ; then
+    if [[ ! "$3" =~ $4 ]] ; then
+      printf "\033[1;33mSkipped\033[0m\n"
+      return 0
+    fi
+  fi
+  if [[ -z "$VERBOSE_PLATFORMIO" ]] ; then
+    silent="--silent"
+  else
+    silent=""
+  fi
+  if platformio run --project-dir $1 -e $2 $silent; then
     printf "\033[0;32mPassed\033[0m\n"
     return 0
   else
-    git reset --hard HEAD
+    if [[ -n $GIT_RESET_HARD ]]; then
+      git reset --hard HEAD
+    else
+      restore_configs
+    fi
     printf "\033[0;31mFailed!\033[0m\n"
     return 1
   fi
     testenv=$(basename $f | cut -d"-" -f1)
     printf "Running \033[0;32m$f\033[0m Tests\n"
     exec_test $1 "$testenv --target clean" "Setup Build Environment"
-    $f $1 $testenv
-    git reset --hard HEAD
+    if [[ $GIT_RESET_HARD == "true" ]]; then
+      git reset --hard HEAD
+    else
+      restore_configs
+    fi
   done
 else
   exec_test $1 "$2 --target clean" "Setup Build Environment"
-  $2-tests $1 $2
-  git reset --hard HEAD
+  test_name="$3"
+  # If the test name is 1 or 2 digits, treat it as an index
+  if [[ "$test_name" =~ ^[0-9][0-9]?$ ]] ; then
+    # Find the test name that corresponds to that index
+    test_name="$(cat buildroot/tests/$2-tests | grep -e '^exec_test' | sed -n "$3p" | sed "s/.*\$1 \$2 \"\([^\"]*\).*/\1/g")"
+    if [[ -z "$test_name" ]] ; then
+      # Fail if none matches
+      printf "\033[0;31mCould not find test \033[0m#$3\033[0;31m in \033[0mbuildroot/tests/$2-tests\n"
+      exit 1
+    else
+      printf "\033[0;32mMatching test \033[0m#$3\033[0;32m: '\033[0m$test_name\033[0;32m'\n"
+    fi
+  fi
+  $2-tests $1 $2 "$test_name"
+  if [[ $GIT_RESET_HARD == "true" ]]; then
+    git reset --hard HEAD
+  else
+    restore_configs
+  fi
 fi
 printf "\033[0;32mAll tests completed successfully\033[0m\n"

buildroot/tests/sanguino1284p-tests

 #
 restore_configs
 opt_set MOTHERBOARD BOARD_SANGUINOLOLU_12
-exec_test $1 $2 "Default Configuration"
+exec_test $1 $2 "Default Configuration" "$3"
 
 # clean up
 restore_configs

buildroot/tests/sanguino644p-tests

 #
 restore_configs
 opt_set MOTHERBOARD BOARD_SANGUINOLOLU_12
-exec_test $1 $2 "Default Configuration"
+exec_test $1 $2 "Default Configuration" "$3"
 
 # clean up
 restore_configs

buildroot/tests/teensy31-tests

 
 restore_configs
 opt_set MOTHERBOARD BOARD_TEENSY31_32
-exec_test $1 $2 "Teensy3.1 with default config"
+exec_test $1 $2 "Teensy3.1 with default config" "$3"
 
 #
 # Test many features together
            HOST_ACTION_COMMANDS HOST_PROMPT_SUPPORT
 opt_set I2C_SLAVE_ADDRESS 63
 opt_set GRID_MAX_POINTS_X 16
-exec_test $1 $2 "Teensy3.1 with many features"
+exec_test $1 $2 "Teensy3.1 with many features" "$3"

buildroot/tests/teensy35-tests

 
 restore_configs
 opt_set MOTHERBOARD BOARD_TEENSY35_36
-exec_test $1 $2 "Teensy3.5 with default config"
+exec_test $1 $2 "Teensy3.5 with default config" "$3"
 
 #
 # Test as many features together as possible
            HOST_ACTION_COMMANDS HOST_PROMPT_SUPPORT
 opt_set I2C_SLAVE_ADDRESS 63
 opt_set GRID_MAX_POINTS_X 16
-exec_test $1 $2 "Teensy3.5 with many features"
+exec_test $1 $2 "Teensy3.5 with many features" "$3"
 
 #
 # Test a Sled Z Probe with Linear leveling
 restore_configs
 opt_set MOTHERBOARD BOARD_TEENSY35_36
 opt_enable EEPROM_SETTINGS Z_PROBE_SLED Z_SAFE_HOMING AUTO_BED_LEVELING_LINEAR DEBUG_LEVELING_FEATURE GCODE_MACROS
-exec_test $1 $2 "Sled Z Probe with Linear leveling"
+exec_test $1 $2 "Sled Z Probe with Linear leveling" "$3"
 
 #
 # Test a Servo Probe
 opt_set EXTRUDERS 2
 opt_set TEMP_SENSOR_1 1
 opt_enable MAGNETIC_PARKING_EXTRUDER ULTIMAKERCONTROLLER
-exec_test $1 $2 "MAGNETIC_PARKING_EXTRUDER with LCD"
+exec_test $1 $2 "MAGNETIC_PARKING_EXTRUDER with LCD" "$3"
 
 #
 # Mixing Extruder
 opt_set MOTHERBOARD BOARD_TEENSY35_36
 opt_enable MIXING_EXTRUDER DIRECT_MIXING_IN_G1 GRADIENT_MIX GRADIENT_VTOOL REPRAP_DISCOUNT_FULL_GRAPHIC_SMART_CONTROLLER
 opt_set MIXING_STEPPERS 2
-exec_test $1 $2 "Mixing Extruder"
+exec_test $1 $2 "Mixing Extruder" "$3"
 
 #
 # Test SWITCHING_EXTRUDER
 opt_add X_CS_PIN 46
 opt_add Y_CS_PIN 47
 opt_enable USE_ZMIN_PLUG MONITOR_DRIVER_STATUS SENSORLESS_HOMING
-exec_test $1 $2 "Teensy 3.5/3.6 COREXY"
+exec_test $1 $2 "Teensy 3.5/3.6 COREXY" "$3"
 
 #
 # Enable COREXZ
 restore_configs
 opt_set MOTHERBOARD BOARD_TEENSY35_36
 opt_enable COREXZ
-exec_test $1 $2 "Teensy 3.5/3.6 COREXZ"
+exec_test $1 $2 "Teensy 3.5/3.6 COREXZ" "$3"
 
 #
 # Enable Dual Z with Dual Z endstops
 opt_enable Z_MULTI_ENDSTOPS USE_XMAX_PLUG
 pins_set ramps/RAMPS X_MAX_PIN -1
 opt_add Z2_MAX_PIN 2
-exec_test $1 $2 "Dual Z with Dual Z endstops"
+exec_test $1 $2 "Dual Z with Dual Z endstops" "$3"
 
 # Clean up
 restore_configs

buildroot/tests/teensy41-tests

 
 restore_configs
 opt_set MOTHERBOARD BOARD_TEENSY41
-exec_test $1 $2 "Teensy4.1 with default config"
+exec_test $1 $2 "Teensy4.1 with default config" "$3"
 
 #
 # Test as many features together as possible
 opt_add EXTUI_EXAMPLE
 opt_set I2C_SLAVE_ADDRESS 63
 opt_set GRID_MAX_POINTS_X 16
-exec_test $1 $2 "Teensy4.1 with many features"
+exec_test $1 $2 "Teensy4.1 with many features" "$3"
 
 #
 # Test a Sled Z Probe with Linear leveling
 restore_configs
 opt_set MOTHERBOARD BOARD_TEENSY41
 opt_enable EEPROM_SETTINGS Z_PROBE_SLED Z_SAFE_HOMING AUTO_BED_LEVELING_LINEAR DEBUG_LEVELING_FEATURE GCODE_MACROS
-exec_test $1 $2 "Sled Z Probe with Linear leveling"
+exec_test $1 $2 "Sled Z Probe with Linear leveling" "$3"
 
 #
 # Test a Servo Probe
 opt_set TEMP_SENSOR_1 1
 opt_set SERIAL_PORT_2 -2
 opt_enable EEPROM_SETTINGS MAGNETIC_PARKING_EXTRUDER
-exec_test $1 $2 "Ethernet, EEPROM, Magnetic Parking Extruder, No LCD"
+exec_test $1 $2 "Ethernet, EEPROM, Magnetic Parking Extruder, No LCD" "$3"
 
 #
 # Mixing Extruder
 opt_set MOTHERBOARD BOARD_TEENSY41
 opt_enable MIXING_EXTRUDER DIRECT_MIXING_IN_G1 GRADIENT_MIX GRADIENT_VTOOL
 opt_set MIXING_STEPPERS 2
-exec_test $1 $2 "Mixing Extruder"
+exec_test $1 $2 "Mixing Extruder" "$3"
 
 #
 # Test SWITCHING_EXTRUDER
 opt_add X_CS_PIN 46
 opt_add Y_CS_PIN 47
 opt_enable USE_ZMIN_PLUG MONITOR_DRIVER_STATUS SENSORLESS_HOMING
-exec_test $1 $2 "Teensy 4.0/4.1 COREXY"
+exec_test $1 $2 "Teensy 4.0/4.1 COREXY" "$3"
 
 #
 # Enable COREXZ
 restore_configs
 opt_set MOTHERBOARD BOARD_TEENSY41
 opt_enable COREXZ
-exec_test $1 $2 "Teensy 4.0/4.1 COREXZ"
+exec_test $1 $2 "Teensy 4.0/4.1 COREXZ" "$3"
 
 #
 # Enable Dual Z with Dual Z endstops
 opt_set NUM_Z_STEPPER_DRIVERS 2
 pins_set ramps/RAMPS X_MAX_PIN -1
 opt_add Z2_MAX_PIN 2
-exec_test $1 $2 "Dual Z with Dual Z endstops"
+exec_test $1 $2 "Dual Z with Dual Z endstops" "$3"
 
 # Clean up
 restore_configs

docker-compose.yml

+version: "3.8"
+services:
+  # The main image: this doesn't run any particular command, but is mainly used
+  # for running tests locally
+  marlin:
+    image: marlin-dev
+    build:
+      dockerfile: Dockerfile
+      context: docker
+    working_dir: /code
+    volumes:
+    - .:/code
+    - platformio-cache:/root/.platformio
+
+volumes:
+  # This volume holds installed libraries for PlatformIO. If this is deleted you
+  # will have to download all the dependencies again, which can be a very slow
+  # process
+  platformio-cache:

docker/Dockerfile

+FROM python:3.9.0-buster
+
+RUN pip install -U https://github.com/platformio/platformio-core/archive/develop.zip
+RUN platformio update
+# To get the test platforms
+RUN pip install PyYaml
+#ENV PATH /code/buildroot/bin/:/code/buildroot/tests/:${PATH}

get_test_targets.py

+#!/usr/bin/env python
+"""
+Extract the builds used in Github CI, so that we can run them locally
+"""
+import yaml
+
+
+with open('.github/workflows/test-builds.yml') as f:
+	github_configuration = yaml.safe_load(f)
+test_platforms = github_configuration\
+	['jobs']['test_builds']['strategy']['matrix']['test-platform']
+print(' '.join(test_platforms))