At present test_pinmux_status() assumes that test_pinmux_dev() has run beforehand. Drop this assumption so we can run the tests in parallel.

Signed-off-by: Simon Glass sjg@chromium.org ---

test/py/tests/test_pinmux.py | 1 + 1 file changed, 1 insertion(+)

diff --git a/test/py/tests/test_pinmux.py b/test/py/tests/test_pinmux.py index b3ae2ab0240..794994e12d1 100644 --- a/test/py/tests/test_pinmux.py +++ b/test/py/tests/test_pinmux.py @@ -68,6 +68,7 @@ def test_pinmux_dev(u_boot_console): def test_pinmux_status(u_boot_console): """Test that 'pinmux status' displays selected pincontroller's pin muxing descriptions.""" + u_boot_console.run_command('pinmux dev pinctrl') output = u_boot_console.run_command('pinmux status')

assert (not 'pinctrl-gpio:' in output)

Add a new 'singlethread' marker to allow tests to be skipped when running in parallel.

Signed-off-by: Simon Glass sjg@chromium.org ---

test/py/conftest.py | 17 +++++++++++++++++ test/py/pytest.ini | 1 + 2 files changed, 18 insertions(+)

diff --git a/test/py/conftest.py b/test/py/conftest.py index 2ba34479e03..dbe1acd9e37 100644 --- a/test/py/conftest.py +++ b/test/py/conftest.py @@ -521,6 +521,22 @@ def setup_requiredtool(item): if not tool_is_in_path(tool): pytest.skip('tool "%s" not in $PATH' % tool)

+def setup_singlethread(item): + """Process any 'singlethread' marker for a test. + + Skip this test if running in parallel. + + Args: + item: The pytest test item. + + Returns: + Nothing. + """ + for single in item.iter_markers('singlethread'): + worker_id = os.environ.get("PYTEST_XDIST_WORKER") + if worker_id and worker_id != 'master': + pytest.skip('must run single-threaded') + def start_test_section(item): anchors[item.name] = log.start_section(item.name)

@@ -541,6 +557,7 @@ def pytest_runtest_setup(item): setup_boardspec(item) setup_buildconfigspec(item) setup_requiredtool(item) + setup_singlethread(item)

def pytest_runtest_protocol(item, nextitem): """pytest hook: Called to execute a test. diff --git a/test/py/pytest.ini b/test/py/pytest.ini index e93d010f1fa..26d83f83e00 100644 --- a/test/py/pytest.ini +++ b/test/py/pytest.ini @@ -11,3 +11,4 @@ markers = notbuildconfigspec: U-Boot: Describes required disabled Kconfig options. requiredtool: U-Boot: Required host tools for a test. slow: U-Boot: Specific test will run slowly. + singlethread: Cannot run in parallel

Use a different temporary dir for each test, to allow them to run in parallel.

Signed-off-by: Simon Glass sjg@chromium.org ---

test/py/tests/test_fit_ecdsa.py | 4 +++- test/py/tests/test_fit_hashes.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/test/py/tests/test_fit_ecdsa.py b/test/py/tests/test_fit_ecdsa.py index 87b60812229..cc6c0c4dc42 100644 --- a/test/py/tests/test_fit_ecdsa.py +++ b/test/py/tests/test_fit_ecdsa.py @@ -10,6 +10,7 @@ signature is then extracted, and verified against pyCryptodome. This test doesn't run the sandbox. It only checks the host tool 'mkimage' """

+import os import pytest import u_boot_utils as util from Cryptodome.Hash import SHA256 @@ -84,7 +85,8 @@ def test_fit_ecdsa(u_boot_console): cons = u_boot_console mkimage = cons.config.build_dir + '/tools/mkimage' datadir = cons.config.source_dir + '/test/py/tests/vboot/' - tempdir = cons.config.result_dir + tempdir = os.path.join(cons.config.result_dir, 'ecdsa') + os.makedirs(tempdir, exist_ok=True) key_file = f'{tempdir}/ecdsa-test-key.pem' fit_file = f'{tempdir}/test.fit' dtc('sandbox-kernel.dts') diff --git a/test/py/tests/test_fit_hashes.py b/test/py/tests/test_fit_hashes.py index e228ea96d31..4891e77ca2d 100644 --- a/test/py/tests/test_fit_hashes.py +++ b/test/py/tests/test_fit_hashes.py @@ -10,6 +10,7 @@ output of a fixed data block with known good hashes. This test doesn't run the sandbox. It only checks the host tool 'mkimage' """

+import os import pytest import u_boot_utils as util

@@ -93,7 +94,9 @@ def test_mkimage_hashes(u_boot_console): cons = u_boot_console mkimage = cons.config.build_dir + '/tools/mkimage' datadir = cons.config.source_dir + '/test/py/tests/vboot/' - tempdir = cons.config.result_dir + tempdir = os.path.join(cons.config.result_dir, 'hashes') + os.makedirs(tempdir, exist_ok=True) + fit_file = f'{tempdir}/test.fit' dtc('sandbox-kernel.dts')

This test seems to fail when run in parallel. Mark it single-threaded to avoid any problems.

Signed-off-by: Simon Glass sjg@chromium.org ---

test/py/tests/test_efi_bootmgr/test_efi_bootmgr.py | 1 + 1 file changed, 1 insertion(+)

diff --git a/test/py/tests/test_efi_bootmgr/test_efi_bootmgr.py b/test/py/tests/test_efi_bootmgr/test_efi_bootmgr.py index 75a6e7c9629..1bb59d8fcf8 100644 --- a/test/py/tests/test_efi_bootmgr/test_efi_bootmgr.py +++ b/test/py/tests/test_efi_bootmgr/test_efi_bootmgr.py @@ -7,6 +7,7 @@ import pytest @pytest.mark.boardspec('sandbox') @pytest.mark.buildconfigspec('cmd_efidebug') @pytest.mark.buildconfigspec('cmd_bootefi_bootmgr') +@pytest.mark.singlethread def test_efi_bootmgr(u_boot_console, efi_bootmgr_data): """ Unit test for UEFI bootmanager The efidebug command is used to set up UEFI load options.

Mark all the tests in this file as slow, since they take a while.

Signed-off-by: Simon Glass sjg@chromium.org ---

test/py/tests/test_gpt.py | 3 +++ 1 file changed, 3 insertions(+)

diff --git a/test/py/tests/test_gpt.py b/test/py/tests/test_gpt.py index f707d9f2534..cb44e1d7896 100644 --- a/test/py/tests/test_gpt.py +++ b/test/py/tests/test_gpt.py @@ -13,6 +13,9 @@ These tests rely on a 4 MB disk image, which is automatically created by the test. """

+# Mark all tests here as slow +pytestmark = pytest.mark.slow + class GptTestDiskImage(object): """Disk Image used by the GPT tests."""

Tidy up this code a little. Also use '-k' consistently, since -m is more limited in what it can accept.

Signed-off-by: Simon Glass sjg@chromium.org ---

test/run | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/test/run b/test/run index 869406cd8d2..0c8aeb5b0af 100755 --- a/test/run +++ b/test/run @@ -13,9 +13,12 @@ run_test() { [ $? -ne 0 ] && failures=$((failures+1)) }

-# SKip slow tests if requested -[ "$1" == "quick" ] && mark_expr="not slow" -[ "$1" == "quick" ] && skip=--skip-net-tests +# Select test attributes +if [ "$1" = "quick" ]; then + mark_expr="not slow" + skip=--skip-net-tests +fi + [ "$1" == "tools" ] && tools_only=y

failures=0 @@ -23,7 +26,7 @@ failures=0 if [ -z "$tools_only" ]; then # Run all tests that the standard sandbox build can support run_test "sandbox" ./test/py/test.py --bd sandbox --build \ - -m "${mark_expr}" + -k "${mark_expr}" fi

# Run tests which require sandbox_spl

At present when -n is used, all workers try to build U-Boot at once. Add a lock to ensure that only one of them builds, with the others using the build that is produced.

The lock file is removed on startup.

Signed-off-by: Simon Glass sjg@chromium.org ---

test/py/conftest.py | 29 +++++++++++++++++++++++++---- test/py/requirements.txt | 2 ++ 2 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/test/py/conftest.py b/test/py/conftest.py index 906387db1c8..304e93164aa 100644 --- a/test/py/conftest.py +++ b/test/py/conftest.py @@ -15,9 +15,11 @@ import atexit import configparser import errno +import filelock import io import os import os.path +from pathlib import Path import pytest import re from _pytest.runner import runtestprotocol @@ -27,6 +29,8 @@ import sys log = None console = None

+TEST_PY_DIR = os.path.dirname(os.path.abspath(__file__)) + def mkdir_p(path): """Create a directory path.

@@ -111,6 +115,18 @@ def run_build(config, source_dir, build_dir, board_type, log): runner.close() log.status_pass('OK')

+def pytest_xdist_setupnodes(config, specs): + """Clear out any 'done' file from a previous build""" + global build_done_file + build_dir = config.getoption('build_dir') + board_type = config.getoption('board_type') + source_dir = os.path.dirname(os.path.dirname(TEST_PY_DIR)) + if not build_dir: + build_dir = source_dir + '/build-' + board_type + build_done_file = Path(build_dir) / 'build.done' + if build_done_file.exists(): + os.remove(build_done_file) + def pytest_configure(config): """pytest hook: Perform custom initialization at startup time.

@@ -145,8 +161,7 @@ def pytest_configure(config): global console global ubconfig

- test_py_dir = os.path.dirname(os.path.abspath(__file__)) - source_dir = os.path.dirname(os.path.dirname(test_py_dir)) + source_dir = os.path.dirname(os.path.dirname(TEST_PY_DIR))

board_type = config.getoption('board_type') board_type_filename = board_type.replace('-', '_') @@ -177,7 +192,13 @@ def pytest_configure(config): log = multiplexed_log.Logfile(result_dir + '/test-log.html')

if config.getoption('build'): - run_build(config, source_dir, build_dir, board_type, log) + worker_id = os.environ.get("PYTEST_XDIST_WORKER") + with filelock.FileLock(os.path.join(build_dir, 'build.lock')): + build_done_file = Path(build_dir) / 'build.done' + if (not worker_id or worker_id == 'master' or + not build_done_file.exists()): + run_build(config, source_dir, build_dir, board_type, log) + build_done_file.touch()

class ArbitraryAttributeContainer(object): pass @@ -209,7 +230,7 @@ def pytest_configure(config): else: parse_config('include/autoconf.mk')

- ubconfig.test_py_dir = test_py_dir + ubconfig.test_py_dir = TEST_PY_DIR ubconfig.source_dir = source_dir ubconfig.build_dir = build_dir ubconfig.result_dir = result_dir diff --git a/test/py/requirements.txt b/test/py/requirements.txt index 2d0fcc965cf..a02f014b7e8 100644 --- a/test/py/requirements.txt +++ b/test/py/requirements.txt @@ -2,6 +2,7 @@ atomicwrites==1.4.1 attrs==19.3.0 coverage==4.5.4 extras==1.0.0 +filelock==3.0.12 fixtures==3.0.0 importlib-metadata==0.23 linecache2==1.0.0 @@ -15,6 +16,7 @@ pyelftools==0.27 pygit2==0.28.2 pyparsing==2.4.2 pytest==5.2.1 +pytest-xdist==2.5.0 python-mimeparse==1.6.0 python-subunit==1.3.0 requests==2.25.1

Running tests in parallel is much faster, e.g. 15 seconds to run the tests on sandbox (only), instead of 100 seconds (on a 16-core machine). Add a 'make pcheck' option to access this feature.

Note that the tools/ tests still run each tool's tests once after the other, although within that, they do run in parallel. So for example, the buildman tests run in parallel, then the binman tests run in parallel. There would be a signiificant advantage to running them all in parallel together, but that would require a large amount of refactoring, e.g. with more use of pytest fixtures.

Update the documentation to represent the current state.

Signed-off-by: Simon Glass sjg@chromium.org ---

Makefile | 8 ++++++-- doc/develop/py_testing.rst | 37 ++++++++++++++++++++--------------- doc/develop/testing.rst | 6 +++++- test/run | 40 ++++++++++++++++++++++++++++++-------- 4 files changed, 64 insertions(+), 27 deletions(-)

diff --git a/Makefile b/Makefile index ff25f929748..ff0e295d9f1 100644 --- a/Makefile +++ b/Makefile @@ -521,8 +521,8 @@ env_h := include/generated/environment.h

no-dot-config-targets := clean clobber mrproper distclean \ help %docs check% coccicheck \ - ubootversion backup tests check qcheck tcheck pylint \ - pylint_err + ubootversion backup tests check pcheck qcheck tcheck \ + pylint pylint_err

config-targets := 0 mixed-targets := 0 @@ -2352,6 +2352,7 @@ help: @echo 'Test targets:' @echo '' @echo ' check - Run all automated tests that use sandbox' + @echo ' pcheck - Run quick automated tests in parallel' @echo ' qcheck - Run quick automated tests that use sandbox' @echo ' tcheck - Run quick automated tests on tools' @echo ' pylint - Run pylint on all Python files' @@ -2397,6 +2398,9 @@ help: tests check: $(srctree)/test/run

+pcheck: + $(srctree)/test/run parallel + qcheck: $(srctree)/test/run quick

diff --git a/doc/develop/py_testing.rst b/doc/develop/py_testing.rst index 06f919609b5..92fbd227219 100644 --- a/doc/develop/py_testing.rst +++ b/doc/develop/py_testing.rst @@ -121,31 +121,36 @@ more options. Running tests in parallel ~~~~~~~~~~~~~~~~~~~~~~~~~

-Note: This does not fully work yet and is documented only so you can try to -fix the problems. +Note: Not all tests can run in parallel at present, so the usual approach is +to just run those that can.

First install support for parallel tests::

+ sudo apt install python3-pytest-xdist + +or::: + pip3 install pytest-xdist

-Then build sandbox in a suitable build directory. It is not possible to use -the --build flag with xdist. +Then run the tests in parallel using the -n flag::

-Finally, run the tests in parallel using the -n flag:: + test/py/test.py -B sandbox --build --build-dir /tmp/b/sandbox -q -k \ + 'not slow and not bootstd and not spi_flash' -n16

- # build sandbox first, in a suitable build directory. It is not possible - # to use the --build flag with -n - test/py/test.py -B sandbox --build-dir /tmp/b/sandbox -q -k 'not slow' -n32 +You can also use `make pcheck` to run all tests in parallel. This uses a maximum +of 16 threads, since the setup time is significant and there are under 1000 +tests.

-At least the following non-slow tests are known to fail: +Note that the `test-log.html` output does not work correctly at present with +parallel testing. All the threads write to it at once, so it is garbled.

-- test_fit_ecdsa -- test_bind_unbind_with_uclass -- ut_dm_spi_flash -- test_gpt_rename_partition -- test_gpt_swap_partitions -- test_pinmux_status -- test_sqfs_load +Note that the `tools/` tests still run each tool's tests once after the other, +although within that, they do run in parallel. So for example, the buildman +tests run in parallel, then the binman tests run in parallel. There would be a +significant advantage to running them all in parallel together, but that would +require a large amount of refactoring, e.g. with more use of pytest fixtures. +The code-coverage tests are omitted since they cannot run in parallel due to a +Python limitation.

Testing under a debugger diff --git a/doc/develop/testing.rst b/doc/develop/testing.rst index 1abe4d7f0f0..5afeb42f696 100644 --- a/doc/develop/testing.rst +++ b/doc/develop/testing.rst @@ -28,8 +28,12 @@ run. Type this::

make tcheck

+You can also run a selection tests in parallel with:: + + make pcheck + All of the above use the test/run script with a paremeter to select which tests -are run. +are run. See :doc:`py_testing` for more information.

Sandbox diff --git a/test/run b/test/run index 0c8aeb5b0af..810b47e08d0 100755 --- a/test/run +++ b/test/run @@ -14,27 +14,46 @@ run_test() { }

# Select test attributes +ut_mark_expr=test_ut if [ "$1" = "quick" ]; then mark_expr="not slow" + ut_mark_expr="test_ut and not slow" skip=--skip-net-tests fi

[ "$1" == "tools" ] && tools_only=y

+if [ "$1" = "parallel" ]; then + if ! echo 'import xdist' | python3 2>/dev/null; then + echo "Please install python3-pytest-xdist - see doc/develop/py_testing.rst" + exit 1 + fi + jobs="$(($(nproc) > 16 ? 16 : $(nproc)))" + para="-n${jobs} -q" + prompt="Building and..." + skip=--skip-net-tests + mark_expr="not slow and not bootstd and not spi_flash" + ut_mark_expr="test_ut and not slow and not bootstd and not spi_flash" + echo "Note: test log is garbled with parallel tests" +fi + failures=0

if [ -z "$tools_only" ]; then # Run all tests that the standard sandbox build can support - run_test "sandbox" ./test/py/test.py --bd sandbox --build \ + echo "${prompt}" + run_test "sandbox" ./test/py/test.py --bd sandbox --build ${para} \ -k "${mark_expr}" fi

# Run tests which require sandbox_spl -run_test "sandbox_spl" ./test/py/test.py --bd sandbox_spl --build \ +echo "${prompt}" +run_test "sandbox_spl" ./test/py/test.py --bd sandbox_spl --build ${para} \ -k 'test_ofplatdata or test_handoff or test_spl'

# Run the sane tests with sandbox_noinst (i.e. without OF_PLATDATA_INST) -run_test "sandbox_spl" ./test/py/test.py --bd sandbox_noinst --build \ +echo "${prompt}" +run_test "sandbox_spl" ./test/py/test.py --bd sandbox_noinst --build ${para} \ -k 'test_ofplatdata or test_handoff or test_spl'

if [ -z "$tools_only" ]; then @@ -42,8 +61,9 @@ if [ -z "$tools_only" ]; then # build which does not enable CONFIG_OF_LIVE for the live device tree, so we can # check that functionality is the same. The standard sandbox build (above) uses # CONFIG_OF_LIVE. + echo "${prompt}" run_test "sandbox_flattree" ./test/py/test.py --bd sandbox_flattree \ - --build -k test_ut + ${para} --build -k "${ut_mark_expr}" fi

# Set up a path to dtc (device-tree compiler) and libfdt.py, a library it @@ -64,10 +84,14 @@ run_test "dtoc" ./tools/dtoc/dtoc -t # This needs you to set up Python test coverage tools. # To enable Python test coverage on Debian-type distributions (e.g. Ubuntu): # $ sudo apt-get install python-pytest python-coverage -export PATH=$PATH:${TOOLS_DIR} -run_test "binman code coverage" ./tools/binman/binman test -T -run_test "dtoc code coverage" ./tools/dtoc/dtoc -T -run_test "fdt code coverage" ./tools/dtoc/test_fdt -T + +# Code-coverage tests cannot run in parallel, so skip them in that case +if [ -z "${para}" ]; then + export PATH=$PATH:${TOOLS_DIR} + run_test "binman code coverage" ./tools/binman/binman test -T + run_test "dtoc code coverage" ./tools/dtoc/dtoc -T + run_test "fdt code coverage" ./tools/dtoc/test_fdt -T +fi

if [ $failures == 0 ]; then echo "Tests passed!"