Drop some the prototypes for functions which were removed in earlier series.

Signed-off-by: Simon Glass sjg@chromium.org ---

include/test/suites.h | 7 ------- 1 file changed, 7 deletions(-)

diff --git a/include/test/suites.h b/include/test/suites.h index 2ceef577f7f..e40ad634702 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -36,8 +36,6 @@ int do_ut_bloblist(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_cmd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_common(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); -int do_ut_compression(struct cmd_tbl *cmdtp, int flag, int argc, - char *const argv[]); int do_ut_dm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_env(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_exit(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); @@ -55,14 +53,9 @@ int do_ut_overlay(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_pci_mps(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); -int do_ut_print(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_seama(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_setexpr(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); -int do_ut_str(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); -int do_ut_time(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); -int do_ut_unicode(struct cmd_tbl *cmdtp, int flag, int argc, - char *const argv[]); int do_ut_upl(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);

#endif /* __TEST_SUITES_H__ */

Move these operations into separate functions so that it is clearer what is needed. These functions can also be called from somewhere other than ut_run_list().

Signed-off-by: Simon Glass sjg@chromium.org ---

test/test-main.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/test/test-main.c b/test/test-main.c index cfb3504941d..871fc1f22b4 100644 --- a/test/test-main.c +++ b/test/test-main.c @@ -73,6 +73,19 @@ void ut_set_state(struct unit_test_state *uts) cur_test_state = uts; }

+void ut_init_state(struct unit_test_state *uts) +{ + memset(uts, '\0', sizeof(*uts)); +} + +void ut_uninit_state(struct unit_test_state *uts) +{ + if (IS_ENABLED(CONFIG_SANDBOX)) { + os_free(uts->fdt_copy); + os_free(uts->other_fdt); + } +} + /** * dm_test_pre_run() - Get ready to run a driver model test * @@ -664,10 +677,11 @@ int ut_run_list(const char *category, const char *prefix, struct unit_test *tests, int count, const char *select_name, int runs_per_test, bool force_run, const char *test_insert) { - struct unit_test_state uts = { .fail_count = 0 }; + struct unit_test_state uts; bool has_dm_tests = false; int ret;

+ ut_init_state(&uts); if (!CONFIG_IS_ENABLED(OF_PLATDATA) && ut_list_has_dm_tests(tests, count, prefix, select_name)) { has_dm_tests = true; @@ -703,10 +717,6 @@ int ut_run_list(const char *category, const char *prefix, /* Best efforts only...ignore errors */ if (has_dm_tests) dm_test_restore(uts.of_root); - if (IS_ENABLED(CONFIG_SANDBOX)) { - os_free(uts.fdt_copy); - os_free(uts.other_fdt); - }

if (uts.skip_count) printf("Skipped: %d, ", uts.skip_count); @@ -714,6 +724,7 @@ int ut_run_list(const char *category, const char *prefix, printf("Test '%s' not found\n", select_name); else printf("Failures: %d\n", uts.fail_count); + ut_uninit_state(&uts);

return ret; }

A limitation of most linker_list macros is that they cannot easily be used in data structures. This is because they include code inside their expressions.

Provide a way to support this, with new ll_start_decl() and ll_end_decl() macros.

Signed-off-by: Simon Glass sjg@chromium.org ---

include/linker_lists.h | 51 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+)

diff --git a/include/linker_lists.h b/include/linker_lists.h index f9a2ee0c762..c05b99c886f 100644 --- a/include/linker_lists.h +++ b/include/linker_lists.h @@ -186,6 +186,57 @@ _ll_result; \ })

+/** + * Declares a symbol that points to the start/end of the list. The name of the + * (new) symbol is arbitrary and can be anything that is not already declared in + * the file where it appears. It is provided in _sym and can then be used (later + * in the same file) within a data structure. + * + * The _type and _list arguments must match those passed to ll_entry_start/end() + * + * Example: + * + * :: + * Here we want to record the start of each sub-command in a list. We have two + * sub-commands, 'bob' and 'mary'. + * + * In bob.c: + * ll_entry_declare(struct my_sub_cmd, bob_cmd, cmd_sub) = {...} + * + * In mary.c: + * ll_entry_declare(struct my_sub_cmd, mary_cmd, cmd_sub) = {...} + * + * In a different file where we want a list the start of all sub-commands. + * It is not possible to use ll_entry_start() in a data structure, due to its + * use of code inside expressions - ({ ... }) - so this fails to compile: + * + * In sub_cmds.c: + * struct cmd_sub *my_list[] = { + * ll_entry_start(cmd_sub, bob), + * ll_entry_start(cmd_sub, bob), + * } + * + * Instead, we can use: + * + * ll_start_decl(bob, struct my_sub_cmd, cmd_sub); + * ll_start_decl(mary, struct my_sub_cmd, cmd_sub); + * + * struct cmd_sub *my_list[] = { + * bob, + * mary, + * } + * + * So 'bob' is declared as symbol, a struct my_list * which points to the + * start of the bob sub-commands. It is then used in my_list[] + */ +#define ll_start_decl(_sym, _type, _list) \ + static _type _sym[0] __aligned(CONFIG_LINKER_LIST_ALIGN) \ + __maybe_unused __section("__u_boot_list_2_" #_list "_1") + +#define ll_end_decl(_sym, _type, _list) \ + static _type _sym[0] __aligned(CONFIG_LINKER_LIST_ALIGN) \ + __maybe_unused __section("__u_boot_list_2_" #_list "_3") + /** * ll_entry_get() - Retrieve entry from linker-generated array by name * @_type: Data type of the entry

Some suites have a different name from that used in the linker list. That makes it hard to programmatically match the name printed when the suite runs to the linker-list name it has.

Update the names so they are the same.

Signed-off-by: Simon Glass sjg@chromium.org ---

include/test/cmd.h | 2 +- test/boot/upl.c | 3 +-- test/cmd/addrmap.c | 2 +- test/cmd/cmd_ut_cmd.c | 4 ++-- test/cmd/exit.c | 2 +- test/cmd/mem.c | 2 +- test/cmd/pci_mps.c | 2 +- test/cmd/setexpr.c | 2 +- test/dm/test-dm.c | 3 +-- test/env/cmd_ut_env.c | 3 +-- 10 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/include/test/cmd.h b/include/test/cmd.h index c200570e423..3d1e3e3bddb 100644 --- a/include/test/cmd.h +++ b/include/test/cmd.h @@ -10,6 +10,6 @@ #include <test/test.h>

/* Declare a new command test */ -#define CMD_TEST(_name, _flags) UNIT_TEST(_name, _flags, cmd_test) +#define CMD_TEST(_name, _flags) UNIT_TEST(_name, _flags, cmd)

#endif /* __TEST_CMD_H__ */ diff --git a/test/boot/upl.c b/test/boot/upl.c index 99f02b7951b..ef65cf34055 100644 --- a/test/boot/upl.c +++ b/test/boot/upl.c @@ -432,6 +432,5 @@ int do_ut_upl(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) struct unit_test *tests = UNIT_TEST_SUITE_START(upl_test); const int n_ents = UNIT_TEST_SUITE_COUNT(upl_test);

- return cmd_ut_category("cmd_upl", "cmd_upl_", tests, n_ents, argc, - argv); + return cmd_ut_category("upl", "cmd_upl_", tests, n_ents, argc, argv); } diff --git a/test/cmd/addrmap.c b/test/cmd/addrmap.c index b34be895f5d..43897615aa9 100644 --- a/test/cmd/addrmap.c +++ b/test/cmd/addrmap.c @@ -30,6 +30,6 @@ int do_ut_addrmap(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) struct unit_test *tests = UNIT_TEST_SUITE_START(addrmap_test); const int n_ents = UNIT_TEST_SUITE_COUNT(addrmap_test);

- return cmd_ut_category("cmd_addrmap", "cmd_addrmap_", tests, n_ents, + return cmd_ut_category("addrmap", "cmd_addrmap_", tests, n_ents, argc, argv); } diff --git a/test/cmd/cmd_ut_cmd.c b/test/cmd/cmd_ut_cmd.c index e77fa1c7f01..ad192563c5a 100644 --- a/test/cmd/cmd_ut_cmd.c +++ b/test/cmd/cmd_ut_cmd.c @@ -13,8 +13,8 @@

int do_ut_cmd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) { - struct unit_test *tests = UNIT_TEST_SUITE_START(cmd_test); - const int n_ents = UNIT_TEST_SUITE_COUNT(cmd_test); + struct unit_test *tests = UNIT_TEST_SUITE_START(cmd); + const int n_ents = UNIT_TEST_SUITE_COUNT(cmd);

return cmd_ut_category("cmd", "cmd_test_", tests, n_ents, argc, argv); } diff --git a/test/cmd/exit.c b/test/cmd/exit.c index af58a57fca7..cf65c05e283 100644 --- a/test/cmd/exit.c +++ b/test/cmd/exit.c @@ -116,6 +116,6 @@ int do_ut_exit(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) struct unit_test *tests = UNIT_TEST_SUITE_START(exit_test); const int n_ents = UNIT_TEST_SUITE_COUNT(exit_test);

- return cmd_ut_category("cmd_exit", "exit_test_", tests, n_ents, + return cmd_ut_category("exit", "exit_test_", tests, n_ents, argc, argv); } diff --git a/test/cmd/mem.c b/test/cmd/mem.c index f1bbab6055b..d6f2544a4e9 100644 --- a/test/cmd/mem.c +++ b/test/cmd/mem.c @@ -14,6 +14,6 @@ int do_ut_mem(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) struct unit_test *tests = UNIT_TEST_SUITE_START(mem_test); const int n_ents = UNIT_TEST_SUITE_COUNT(mem_test);

- return cmd_ut_category("cmd_mem", "mem_test_", tests, n_ents, argc, + return cmd_ut_category("mem", "mem_test_", tests, n_ents, argc, argv); } diff --git a/test/cmd/pci_mps.c b/test/cmd/pci_mps.c index a265105600c..0c57d15d636 100644 --- a/test/cmd/pci_mps.c +++ b/test/cmd/pci_mps.c @@ -35,6 +35,6 @@ int do_ut_pci_mps(struct cmd_tbl *cmdtp, int flag, int argc, struct unit_test *tests = UNIT_TEST_SUITE_START(pci_mps_test); const int n = UNIT_TEST_SUITE_COUNT(pci_mps_test);

- return cmd_ut_category("cmd_pci_mps", "pci_mps_test_", tests, n, + return cmd_ut_category("pci_mps", "pci_mps_test_", tests, n, argc, argv); } diff --git a/test/cmd/setexpr.c b/test/cmd/setexpr.c index 21a3268bd81..9b3277f2ad9 100644 --- a/test/cmd/setexpr.c +++ b/test/cmd/setexpr.c @@ -485,6 +485,6 @@ int do_ut_setexpr(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) struct unit_test *tests = UNIT_TEST_SUITE_START(setexpr_test); const int n_ents = UNIT_TEST_SUITE_COUNT(setexpr_test);

- return cmd_ut_category("cmd_setexpr", "setexpr_test_", tests, n_ents, + return cmd_ut_category("setexpr", "setexpr_test_", tests, n_ents, argc, argv); } diff --git a/test/dm/test-dm.c b/test/dm/test-dm.c index 4bc2c45db61..7cfbefe02da 100644 --- a/test/dm/test-dm.c +++ b/test/dm/test-dm.c @@ -11,6 +11,5 @@ int do_ut_dm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) struct unit_test *tests = UNIT_TEST_SUITE_START(dm_test); const int n_ents = UNIT_TEST_SUITE_COUNT(dm_test);

- return cmd_ut_category("driver model", "dm_test_", tests, n_ents, argc, - argv); + return cmd_ut_category("dm", "dm_test_", tests, n_ents, argc, argv); } diff --git a/test/env/cmd_ut_env.c b/test/env/cmd_ut_env.c index 9f16a978f2a..494b5db395b 100644 --- a/test/env/cmd_ut_env.c +++ b/test/env/cmd_ut_env.c @@ -80,6 +80,5 @@ int do_ut_env(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) struct unit_test *tests = UNIT_TEST_SUITE_START(env_test); const int n_ents = UNIT_TEST_SUITE_COUNT(env_test);

- return cmd_ut_category("environment", "env_test_", - tests, n_ents, argc, argv); + return cmd_ut_category("env", "env_test_", tests, n_ents, argc, argv); }

Add a Python test which runs 'ut all' and then checks that the expected suites are present and all tests in each suite are run.

This can help to check that nothing is missing.

Update 'ut info' to ignore the 'all' suite when counting the number of suites, since that is really just a combination of all the other suites.

Adjust the message for skipped tests so that appears even if no particular test was selected. This helps the new 'test_suite' test see what is going on.

Signed-off-by: Simon Glass sjg@chromium.org ---

test/cmd_ut.c | 2 +- test/py/tests/test_suite.py | 175 ++++++++++++++++++++++++++++++++++++ test/test-main.c | 6 +- 3 files changed, 178 insertions(+), 5 deletions(-) create mode 100644 test/py/tests/test_suite.py

diff --git a/test/cmd_ut.c b/test/cmd_ut.c index b6ceeb94a30..b81708b3e57 100644 --- a/test/cmd_ut.c +++ b/test/cmd_ut.c @@ -153,7 +153,7 @@ static int do_ut_info(struct cmd_tbl *cmdtp, int flag, int argc, { const char *flags;

- printf("Test suites: %d\n", (int)ARRAY_SIZE(cmd_ut_sub)); + printf("Test suites: %d\n", (int)ARRAY_SIZE(cmd_ut_sub) - 1); printf("Total tests: %d\n", (int)UNIT_TEST_ALL_COUNT());

flags = cmd_arg1(argc, argv); diff --git a/test/py/tests/test_suite.py b/test/py/tests/test_suite.py new file mode 100644 index 00000000000..ed636bbde74 --- /dev/null +++ b/test/py/tests/test_suite.py @@ -0,0 +1,175 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright 2024 Google LLC + +import pytest +import re + +# List of test suites we expect to find with 'ut info' and 'ut all' +EXPECTED_SUITES = [ + 'addrmap', 'bdinfo', 'bloblist', 'bootm', 'bootstd', + 'cmd', 'common', 'dm', 'env', 'exit', + 'fdt', 'font', 'hush', 'info', 'lib', + 'loadm', 'log', 'mbr', 'measurement', 'mem', + 'overlay', 'pci_mps', 'setexpr', 'upl', + ] + + +# Set this to True to aid debugging of tests +DEBUG_ME = True + + +def collect_info(cons, output): + """Run all unit tests and check the resulting output + + Args: + cons: U-Boot console object + output: Output from running 'ut all' + + Returns: + tuple: + set: suite names that were found in output + set: test names that were found in output + dict: test count for each suite: + key: suite name + value: number of tests for the suite found in output + set: missing suites (compared to EXPECTED_SUITES) + set: extra suites (compared to EXPECTED_SUITES) + """ + suites = set() + tests = set() + cur_suite = None + test_count = None + exp_test_count = {} + + # Collect suites{} + for line in output.splitlines(): + m = re.search('----Running ([^ ]*) tests----', line) + if m: + if DEBUG_ME and cur_suite and cur_suite != 'info': + cons.log.info(f'suite: {cur_suite} expected {exp_test_count[cur_suite]} found {test_count}') + + cur_suite = m.group(1) + if DEBUG_ME: + cons.log.info(f'cur_suite: {cur_suite}') + suites.add(cur_suite) + + test_count = 0 + m = re.match(rf'Running (\d+) {cur_suite} tests', line) + if m: + exp_test_count[cur_suite] = int(m.group(1)) + m = re.search(r'Test: (\w*): ([-a-z0-9_]*.c)?( .*)?', line) + if m: + test_name = m.group(1) + msg = m.group(3) + if DEBUG_ME: + cons.log.info(f"test_name {test_name} msg '{msg}'") + if msg == ' (flat tree)' and test_name not in tests: + tests.add(test_name) + test_count += 1 + if not msg or 'skipped as it is manual' in msg: + tests.add(test_name) + test_count += 1 + if DEBUG_ME: + cons.log.info(f'suite: {cur_suite} expected {exp_test_count[cur_suite]} found {test_count}') + cons.log.info(f"Tests: {' '.join(sorted(list(tests)))}") + + # Figure out what is missing, or extra + missing = set() + extra = set(suites) + for suite in EXPECTED_SUITES: + if suite in extra: + extra.remove(suite) + else: + missing.add(suite) + + return suites, tests, exp_test_count, missing, extra + + +def process_ut_info(cons, output): + """Process the output of the 'ut info' command + + Args: + cons: U-Boot console object + output: Output from running 'ut all' + + Returns: + tuple: + int: Number of suites reported + int: Number of tests reported + dict: test count for each suite: + key: suite name + value: number of tests reported for the suite + + """ + suite_count = None + total_test_count = None + test_count = {} + for line in output.splitlines(): + if DEBUG_ME: + cons.log.info(f'line: {line}') + m = re.match(r'Test suites: (.*)', line) + if m: + suite_count = int(m.group(1)) + m = re.match(r'Total tests: (.*)', line) + if m: + total_test_count = int(m.group(1)) + m = re.match(r' *([0-9?]*) (\w*)', line) + if m: + test_count[m.group(2)] = m.group(1) + return suite_count, total_test_count, test_count + + +@pytest.mark.buildconfigspec('sandbox') +@pytest.mark.notbuildconfigspec('sandbox_spl') +@pytest.mark.notbuildconfigspec('sandbox64') +def test_suite(u_boot_console): + """Perform various checks on the unit tests, including: + + - The number of suites matches that reported by the 'ut info' + - Where available, the number of tests is each suite matches that + reported by 'ut info -s' + - The total number of tests adds up to the total that are actually run + with 'ut all' + - All suites are run with 'ut all' + - The expected set of suites is run (the list is hard-coded in this test) + + """ + cons = u_boot_console + with cons.log.section('Run all unit tests'): + # ut hush hush_test_simple_dollar prints "Unknown command" on purpose. + with u_boot_console.disable_check('unknown_command'): + output = cons.run_command('ut all') + + # Process the output from the run + with cons.log.section('Check output'): + suites, all_tests, exp_test_count, missing, extra = collect_info(cons, + output) + + # Make sure we got a test count for each suite (ignore 'info' since it isn't + # a real suite + assert suites - exp_test_count.keys() == {'info'} + + # Run 'ut info' and compare with the log results + with cons.log.section('Check suite test-counts'): + output = cons.run_command('ut info -s') + + suite_count, total_test_count, test_count = process_ut_info(cons, + output) + + if missing or extra: + cons.log.info(f"suites: {' '.join(sorted(list(suites)))}") + cons.log.error(f'missing: {sorted(list(missing))}') + cons.log.error(f'extra: {sorted(list(extra))}') + + assert not missing + assert not extra + + cons.log.info(str(exp_test_count)) + for suite in EXPECTED_SUITES: + # 'info' is not really a suite, just a subcommand of 'ut' + if suite != 'info': + assert test_count[suite] in ['?', str(exp_test_count[suite])],\ + f'suite {suite} expected {exp_test_count[suite]}' + + assert suite_count == len(EXPECTED_SUITES) + assert total_test_count == len(all_tests) diff --git a/test/test-main.c b/test/test-main.c index e12c5a95e45..391d4e7bb0e 100644 --- a/test/test-main.c +++ b/test/test-main.c @@ -636,10 +636,8 @@ static int ut_run_tests(struct unit_test_state *uts, const char *prefix, return -EBADF; } if (!uts->force_run) { - if (select_name) { - printf("Test '%s' skipped as it is manual (use -f to run it)\n", - test_name); - } + printf("Test: %s: skipped as it is manual (use -f to run it)\n", + test_name); continue; } }

The current cmd_ut_sub[] array was fine when there were only a few test suites. But is quite unwieldy now:

- it requires a separate do_ut_xxx for each suite, even though the code for most is almost identical - running more than one suite requires running multiple commands, and there is no record of which suites passed or failed - 'ut all' runs all suites but reports their results individually - we need lots of #ifdefs in the array, mirroring those in the makefile but maintained in a separate place

In fact the tests are all in the same linker list. The suites are grouped, so it is possible to access the information without a command.

Introduce a 'suite' array, which holds the cmd_ut_...() function to call, but can also support running a suite without that function. This means that the array of struct cmd_tbl is transformed into an array of 'struct suite'.

This will allow removal of many of the functions, particularly those without test-specific init.

Signed-off-by: Simon Glass sjg@chromium.org ---

include/test/suites.h | 4 + test/cmd_ut.c | 205 +++++++++++++++++++++++++++++++++--------- 2 files changed, 166 insertions(+), 43 deletions(-)

diff --git a/include/test/suites.h b/include/test/suites.h index e40ad634702..c1119e44999 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -10,6 +10,10 @@ struct cmd_tbl; struct unit_test;

+/* 'command' functions normally called do_xxx where xxx is the command name */ +typedef int (*ut_cmd_func)(struct cmd_tbl *cmd, int flags, int argc, + char *const argv[]); + /** * cmd_ut_category() - Run a category of unit tests * diff --git a/test/cmd_ut.c b/test/cmd_ut.c index b81708b3e57..10557be6d6d 100644 --- a/test/cmd_ut.c +++ b/test/cmd_ut.c @@ -11,6 +11,24 @@ #include <test/test.h> #include <test/ut.h>

+/** + * struct suite - A set of tests for a certain topic + * + * All tests end up in a single 'struct unit_test' linker-list array, in order + * of the suite they are in + * + * @name: Name of suite + * @start: First test in suite + * end: End test in suite (points to the first test in the next suite + * cmd: Command to use to run the suite + */ +struct suite { + const char *name; + struct unit_test *start; + struct unit_test *end; + ut_cmd_func cmd; +}; + static int do_ut_all(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);

@@ -54,83 +72,147 @@ int cmd_ut_category(const char *name, const char *prefix, return ret ? CMD_RET_FAILURE : 0; }

-static struct cmd_tbl cmd_ut_sub[] = { - U_BOOT_CMD_MKENT(all, CONFIG_SYS_MAXARGS, 1, do_ut_all, "", ""), - U_BOOT_CMD_MKENT(info, 1, 1, do_ut_info, "", ""), +/* declare linker-list symbols for the start and end of a suite */ +#define SUITE_DECL(_name) \ + ll_start_decl(suite_start_ ## _name, struct unit_test, ut_ ## _name); \ + ll_end_decl(suite_end_ ## _name, struct unit_test, ut_ ## _name) + +/* declare a test suite which uses a subcommand to run */ +#define SUITE_CMD(_name, _cmd_func) { \ + #_name, \ + suite_start_ ## _name, \ + suite_end_ ## _name, \ + _cmd_func, \ + } + +/* declare a test suite which can be run directly without a subcommand */ +#define SUITE(_name) { \ + #_name, \ + suite_start_ ## _name, \ + suite_end_ ## _name, \ + NULL, \ + } + +SUITE_DECL(info); +SUITE_DECL(bdinfo); +SUITE_DECL(bootstd); +SUITE_DECL(cmd); +SUITE_DECL(common); +SUITE_DECL(dm); +SUITE_DECL(env); +SUITE_DECL(exit); +SUITE_DECL(fdt); +SUITE_DECL(font); +SUITE_DECL(optee); +SUITE_DECL(overlay); +SUITE_DECL(lib); +SUITE_DECL(log); +SUITE_DECL(mbr); +SUITE_DECL(mem); +SUITE_DECL(setexpr); +SUITE_DECL(measurement); +SUITE_DECL(bloblist); +SUITE_DECL(bootm); +SUITE_DECL(addrmap); +SUITE_DECL(hush); +SUITE_DECL(loadm); +SUITE_DECL(pci_mps); +SUITE_DECL(seama); +SUITE_DECL(upl); + +static struct suite suites[] = { + SUITE_CMD(info, do_ut_info), #ifdef CONFIG_CMD_BDI - U_BOOT_CMD_MKENT(bdinfo, CONFIG_SYS_MAXARGS, 1, do_ut_bdinfo, "", ""), + SUITE_CMD(bdinfo, do_ut_bdinfo), #endif #ifdef CONFIG_UT_BOOTSTD - U_BOOT_CMD_MKENT(bootstd, CONFIG_SYS_MAXARGS, 1, do_ut_bootstd, - "", ""), + SUITE_CMD(bootstd, do_ut_bootstd), #endif #ifdef CONFIG_CMDLINE - U_BOOT_CMD_MKENT(cmd, CONFIG_SYS_MAXARGS, 1, do_ut_cmd, "", ""), + SUITE_CMD(cmd, do_ut_cmd), #endif - U_BOOT_CMD_MKENT(common, CONFIG_SYS_MAXARGS, 1, do_ut_common, "", ""), + SUITE_CMD(common, do_ut_common), #if defined(CONFIG_UT_DM) - U_BOOT_CMD_MKENT(dm, CONFIG_SYS_MAXARGS, 1, do_ut_dm, "", ""), + SUITE_CMD(dm, do_ut_dm), #endif #if defined(CONFIG_UT_ENV) - U_BOOT_CMD_MKENT(env, CONFIG_SYS_MAXARGS, 1, do_ut_env, "", ""), + SUITE_CMD(env, do_ut_env), #endif - U_BOOT_CMD_MKENT(exit, CONFIG_SYS_MAXARGS, 1, do_ut_exit, "", ""), + SUITE_CMD(exit, do_ut_exit), #ifdef CONFIG_CMD_FDT - U_BOOT_CMD_MKENT(fdt, CONFIG_SYS_MAXARGS, 1, do_ut_fdt, "", ""), + SUITE_CMD(fdt, do_ut_fdt), #endif #ifdef CONFIG_CONSOLE_TRUETYPE - U_BOOT_CMD_MKENT(font, CONFIG_SYS_MAXARGS, 1, do_ut_font, "", ""), + SUITE_CMD(font, do_ut_font), #endif #ifdef CONFIG_UT_OPTEE - U_BOOT_CMD_MKENT(optee, CONFIG_SYS_MAXARGS, 1, do_ut_optee, "", ""), + SUITE_CMD(optee, do_ut_optee), #endif #ifdef CONFIG_UT_OVERLAY - U_BOOT_CMD_MKENT(overlay, CONFIG_SYS_MAXARGS, 1, do_ut_overlay, "", ""), + SUITE_CMD(overlay, do_ut_overlay), #endif #ifdef CONFIG_UT_LIB - U_BOOT_CMD_MKENT(lib, CONFIG_SYS_MAXARGS, 1, do_ut_lib, "", ""), + SUITE_CMD(lib, do_ut_lib), #endif #ifdef CONFIG_UT_LOG - U_BOOT_CMD_MKENT(log, CONFIG_SYS_MAXARGS, 1, do_ut_log, "", ""), + SUITE_CMD(log, do_ut_log), #endif #if defined(CONFIG_SANDBOX) && defined(CONFIG_CMD_MBR) && defined(CONFIG_CMD_MMC) \ && defined(CONFIG_MMC_SANDBOX) && defined(CONFIG_MMC_WRITE) - U_BOOT_CMD_MKENT(mbr, CONFIG_SYS_MAXARGS, 1, do_ut_mbr, "", ""), + SUITE_CMD(mbr, do_ut_mbr), #endif - U_BOOT_CMD_MKENT(mem, CONFIG_SYS_MAXARGS, 1, do_ut_mem, "", ""), + SUITE_CMD(mem, do_ut_mem), #if defined(CONFIG_SANDBOX) && defined(CONFIG_CMD_SETEXPR) - U_BOOT_CMD_MKENT(setexpr, CONFIG_SYS_MAXARGS, 1, do_ut_setexpr, "", - ""), + SUITE_CMD(setexpr, do_ut_setexpr), #endif #ifdef CONFIG_MEASURED_BOOT - U_BOOT_CMD_MKENT(measurement, CONFIG_SYS_MAXARGS, 1, do_ut_measurement, - "", ""), + SUITE_CMD(measurement, do_ut_measurement), #endif #ifdef CONFIG_SANDBOX - U_BOOT_CMD_MKENT(bloblist, CONFIG_SYS_MAXARGS, 1, do_ut_bloblist, - "", ""), - U_BOOT_CMD_MKENT(bootm, CONFIG_SYS_MAXARGS, 1, do_ut_bootm, "", ""), + SUITE_CMD(bloblist, do_ut_bloblist), + SUITE_CMD(bootm, do_ut_bootm), #endif #ifdef CONFIG_CMD_ADDRMAP - U_BOOT_CMD_MKENT(addrmap, CONFIG_SYS_MAXARGS, 1, do_ut_addrmap, "", ""), + SUITE_CMD(addrmap, do_ut_addrmap), #endif #if CONFIG_IS_ENABLED(HUSH_PARSER) - U_BOOT_CMD_MKENT(hush, CONFIG_SYS_MAXARGS, 1, do_ut_hush, "", ""), + SUITE_CMD(hush, do_ut_hush), #endif #ifdef CONFIG_CMD_LOADM - U_BOOT_CMD_MKENT(loadm, CONFIG_SYS_MAXARGS, 1, do_ut_loadm, "", ""), + SUITE_CMD(loadm, do_ut_loadm), #endif #ifdef CONFIG_CMD_PCI_MPS - U_BOOT_CMD_MKENT(pci_mps, CONFIG_SYS_MAXARGS, 1, do_ut_pci_mps, "", ""), + SUITE_CMD(pci_mps, do_ut_pci_mps), #endif #ifdef CONFIG_CMD_SEAMA - U_BOOT_CMD_MKENT(seama, CONFIG_SYS_MAXARGS, 1, do_ut_seama, "", ""), + SUITE_CMD(seama, do_ut_seama), #endif #ifdef CONFIG_CMD_UPL - U_BOOT_CMD_MKENT(upl, CONFIG_SYS_MAXARGS, 1, do_ut_upl, "", ""), + SUITE_CMD(upl, do_ut_upl), #endif };

+/** run_suite() - Run a suite of tests */ +static int run_suite(struct suite *ste, struct cmd_tbl *cmdtp, int flag, + int argc, char *const argv[]) +{ + int ret; + + if (ste->cmd) { + ret = ste->cmd(cmdtp, flag, argc, argv); + } else { + int n_ents = ste->end - ste->start; + char prefix[30]; + + /* use a standard prefix */ + snprintf(prefix, sizeof(prefix), "%s_test", ste->name); + ret = cmd_ut_category(ste->name, prefix, ste->start, n_ents, + argc, argv); + } + + return ret; +} + static int do_ut_all(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) { @@ -138,9 +220,12 @@ static int do_ut_all(struct cmd_tbl *cmdtp, int flag, int argc, int retval; int any_fail = 0;

- for (i = 1; i < ARRAY_SIZE(cmd_ut_sub); i++) { - printf("----Running %s tests----\n", cmd_ut_sub[i].name); - retval = cmd_ut_sub[i].cmd(cmdtp, flag, 1, &cmd_ut_sub[i].name); + for (i = 0; i < ARRAY_SIZE(suites); i++) { + struct suite *ste = &suites[i]; + char *const argv[] = {(char *)ste->name, NULL}; + + printf("----Running %s tests----\n", ste->name); + retval = run_suite(ste, cmdtp, flag, 1, argv); if (!any_fail) any_fail = retval; } @@ -153,7 +238,7 @@ static int do_ut_info(struct cmd_tbl *cmdtp, int flag, int argc, { const char *flags;

- printf("Test suites: %d\n", (int)ARRAY_SIZE(cmd_ut_sub) - 1); + printf("Test suites: %d\n", (int)ARRAY_SIZE(suites)); printf("Total tests: %d\n", (int)UNIT_TEST_ALL_COUNT());

flags = cmd_arg1(argc, argv); @@ -162,16 +247,38 @@ static int do_ut_info(struct cmd_tbl *cmdtp, int flag, int argc,

puts("\nTests Suite\n"); puts("----- -----\n"); - for (i = 1; i < ARRAY_SIZE(cmd_ut_sub); i++) - printf("%5s %s\n", "?", cmd_ut_sub[i].name); + for (i = 1; i < ARRAY_SIZE(suites); i++) { + struct suite *ste = &suites[i]; + long n_ent = ste->end - ste->start; + + if (n_ent) + printf("%5ld %s\n", n_ent, ste->name); + else + printf("%5s %s\n", "?", ste->name); + } }

return 0; }

+static struct suite *find_suite(const char *name) +{ + struct suite *ste; + int i; + + for (i = 0, ste = suites; i < ARRAY_SIZE(suites); i++, ste++) { + if (!strcmp(ste->name, name)) + return ste; + } + + return NULL; +} + static int do_ut(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) { - struct cmd_tbl *cp; + struct suite *ste; + const char *name; + int ret;

if (argc < 2) return CMD_RET_USAGE; @@ -180,12 +287,24 @@ static int do_ut(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) argc--; argv++;

- cp = find_cmd_tbl(argv[0], cmd_ut_sub, ARRAY_SIZE(cmd_ut_sub)); + name = argv[0]; + if (!strcmp(name, "all")) { + ret = do_ut_all(cmdtp, flag, argc, argv); + } else if (!strcmp(name, "info")) { + ret = do_ut_info(cmdtp, flag, argc, argv); + } else { + ste = find_suite(argv[0]); + if (!ste) { + printf("Suite '%s' not found\n", argv[0]); + return CMD_RET_FAILURE; + }

- if (cp) - return cp->cmd(cmdtp, flag, argc, argv); + ret = run_suite(ste, cmdtp, flag, argc, argv); + } + if (ret) + return ret;

- return CMD_RET_USAGE; + return 0; }

U_BOOT_LONGHELP(ut,

Use the new suite-runner to run these tests instead.

Signed-off-by: Simon Glass sjg@chromium.org ---

include/test/suites.h | 1 - test/cmd/Makefile | 2 -- test/cmd/cmd_ut_cmd.c | 20 -------------------- test/cmd_ut.c | 2 +- 4 files changed, 1 insertion(+), 24 deletions(-) delete mode 100644 test/cmd/cmd_ut_cmd.c

diff --git a/include/test/suites.h b/include/test/suites.h index b6af9295424..21158692e9b 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -37,7 +37,6 @@ int do_ut_bootstd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_bloblist(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); -int do_ut_cmd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_common(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_dm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_env(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); diff --git a/test/cmd/Makefile b/test/cmd/Makefile index 583e7c2eec4..f83c53696d0 100644 --- a/test/cmd/Makefile +++ b/test/cmd/Makefile @@ -3,8 +3,6 @@ # Copyright (c) 2013 Google, Inc # Copyright 2022-2023 Arm Limited and/or its affiliates open-source-office@arm.com

-obj-y += cmd_ut_cmd.o - obj-$(CONFIG_$(XPL_)CMDLINE) += command.o ifdef CONFIG_HUSH_PARSER obj-$(CONFIG_CONSOLE_RECORD) += test_echo.o diff --git a/test/cmd/cmd_ut_cmd.c b/test/cmd/cmd_ut_cmd.c deleted file mode 100644 index ad192563c5a..00000000000 --- a/test/cmd/cmd_ut_cmd.c +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright 2023 Google LLC - * Written by Simon Glass sjg@chromium.org - * - * Unit tests for command functions - */ - -#include <command.h> -#include <test/cmd.h> -#include <test/suites.h> -#include <test/ut.h> - -int do_ut_cmd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) -{ - struct unit_test *tests = UNIT_TEST_SUITE_START(cmd); - const int n_ents = UNIT_TEST_SUITE_COUNT(cmd); - - return cmd_ut_category("cmd", "cmd_test_", tests, n_ents, argc, argv); -} diff --git a/test/cmd_ut.c b/test/cmd_ut.c index 2ee9b589b52..761556cc763 100644 --- a/test/cmd_ut.c +++ b/test/cmd_ut.c @@ -129,7 +129,7 @@ static struct suite suites[] = { SUITE_CMD(bootstd, do_ut_bootstd), #endif #ifdef CONFIG_CMDLINE - SUITE_CMD(cmd, do_ut_cmd), + SUITE(cmd), #endif SUITE_CMD(common, do_ut_common), #if defined(CONFIG_UT_DM)

Use the new suite-runner to run these tests instead.

Signed-off-by: Simon Glass sjg@chromium.org ---

include/test/suites.h | 1 - test/cmd_ut.c | 2 +- test/dm/Makefile | 2 -- test/dm/test-dm.c | 15 --------------- 4 files changed, 1 insertion(+), 19 deletions(-) delete mode 100644 test/dm/test-dm.c

diff --git a/include/test/suites.h b/include/test/suites.h index d6c0061ff87..414613c593e 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -37,7 +37,6 @@ int do_ut_bootstd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_bloblist(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); -int do_ut_dm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_env(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_exit(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_fdt(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); diff --git a/test/cmd_ut.c b/test/cmd_ut.c index 4b529cbd699..a3dcbef1740 100644 --- a/test/cmd_ut.c +++ b/test/cmd_ut.c @@ -133,7 +133,7 @@ static struct suite suites[] = { #endif SUITE(common), #if defined(CONFIG_UT_DM) - SUITE_CMD(dm, do_ut_dm), + SUITE(dm), #endif #if defined(CONFIG_UT_ENV) SUITE_CMD(env, do_ut_env), diff --git a/test/dm/Makefile b/test/dm/Makefile index bcb52ef1067..e44f3d89e77 100644 --- a/test/dm/Makefile +++ b/test/dm/Makefile @@ -3,8 +3,6 @@ # Copyright (c) 2013 Google, Inc # Copyright 2022-2023 Arm Limited and/or its affiliates open-source-office@arm.com

-obj-$(CONFIG_UT_DM) += test-dm.o - # Tests for particular subsystems - when enabling driver model for a new # subsystem you must add sandbox tests here. ifeq ($(CONFIG_XPL_BUILD),y) diff --git a/test/dm/test-dm.c b/test/dm/test-dm.c deleted file mode 100644 index 8f83110163a..00000000000 --- a/test/dm/test-dm.c +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Copyright (c) 2013 Google, Inc - */ - -#include <test/suites.h> -#include <test/test.h> - -int do_ut_dm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) -{ - struct unit_test *tests = UNIT_TEST_SUITE_START(dm); - const int n_ents = UNIT_TEST_SUITE_COUNT(dm); - - return cmd_ut_category("dm", "dm_test_", tests, n_ents, argc, argv); -}

Use the new suite-runner to run these tests instead.

Signed-off-by: Simon Glass sjg@chromium.org ---

include/test/suites.h | 1 - test/cmd/exit.c | 9 --------- test/cmd_ut.c | 2 +- 3 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/include/test/suites.h b/include/test/suites.h index b78436fe2b2..70b9af1fd05 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -37,7 +37,6 @@ int do_ut_bootstd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_bloblist(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); -int do_ut_exit(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_fdt(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_font(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_hush(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); diff --git a/test/cmd/exit.c b/test/cmd/exit.c index d1eacdc36c2..71c37edcdf6 100644 --- a/test/cmd/exit.c +++ b/test/cmd/exit.c @@ -110,12 +110,3 @@ static int cmd_exit_test(struct unit_test_state *uts) return 0; } EXIT_TEST(cmd_exit_test, UTF_CONSOLE); - -int do_ut_exit(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) -{ - struct unit_test *tests = UNIT_TEST_SUITE_START(exit); - const int n_ents = UNIT_TEST_SUITE_COUNT(exit); - - return cmd_ut_category("exit", "exit_test_", tests, n_ents, - argc, argv); -} diff --git a/test/cmd_ut.c b/test/cmd_ut.c index a34998605cc..88a32129f0a 100644 --- a/test/cmd_ut.c +++ b/test/cmd_ut.c @@ -138,7 +138,7 @@ static struct suite suites[] = { #if defined(CONFIG_UT_ENV) SUITE(env), #endif - SUITE_CMD(exit, do_ut_exit), + SUITE(exit), #ifdef CONFIG_CMD_FDT SUITE_CMD(fdt, do_ut_fdt), #endif

Use the new suite-runner to run these tests instead.

Signed-off-by: Simon Glass sjg@chromium.org ---

include/test/suites.h | 1 - test/cmd/font.c | 8 -------- test/cmd_ut.c | 2 +- 3 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/include/test/suites.h b/include/test/suites.h index bf3a9851322..587d6a5b473 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -37,7 +37,6 @@ int do_ut_bootstd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_bloblist(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); -int do_ut_font(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_hush(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_lib(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_loadm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); diff --git a/test/cmd/font.c b/test/cmd/font.c index d860efe8ce5..af88d1b5459 100644 --- a/test/cmd/font.c +++ b/test/cmd/font.c @@ -85,11 +85,3 @@ static int font_test_base(struct unit_test_state *uts) } FONT_TEST(font_test_base, UTF_SCAN_PDATA | UTF_SCAN_FDT | UTF_CONSOLE | UTF_DM); - -int do_ut_font(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) -{ - struct unit_test *tests = UNIT_TEST_SUITE_START(font); - const int n_ents = UNIT_TEST_SUITE_COUNT(font); - - return cmd_ut_category("font", "font_test_", tests, n_ents, argc, argv); -} diff --git a/test/cmd_ut.c b/test/cmd_ut.c index f6c28c73981..f9cf7b93412 100644 --- a/test/cmd_ut.c +++ b/test/cmd_ut.c @@ -143,7 +143,7 @@ static struct suite suites[] = { SUITE(fdt), #endif #ifdef CONFIG_CONSOLE_TRUETYPE - SUITE_CMD(font, do_ut_font), + SUITE(font), #endif #ifdef CONFIG_UT_OPTEE SUITE_CMD(optee, do_ut_optee),

Use the new suite-runner to run these tests instead.

Signed-off-by: Simon Glass sjg@chromium.org ---

include/test/suites.h | 1 - test/cmd_ut.c | 2 +- test/log/Makefile | 2 -- test/log/log_ut.c | 20 -------------------- 4 files changed, 1 insertion(+), 24 deletions(-) delete mode 100644 test/log/log_ut.c

diff --git a/include/test/suites.h b/include/test/suites.h index 08b35b276be..72f98eb3558 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -39,7 +39,6 @@ int do_ut_bloblist(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_hush(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_loadm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); -int do_ut_log(struct cmd_tbl *cmdtp, int flag, int argc, char * const argv[]); int do_ut_mbr(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_measurement(struct cmd_tbl *cmdtp, int flag, int argc, char * const argv[]); int do_ut_mem(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); diff --git a/test/cmd_ut.c b/test/cmd_ut.c index ba101a48b98..c490498b4c5 100644 --- a/test/cmd_ut.c +++ b/test/cmd_ut.c @@ -155,7 +155,7 @@ static struct suite suites[] = { SUITE(lib), #endif #ifdef CONFIG_UT_LOG - SUITE_CMD(log, do_ut_log), + SUITE(log), #endif #if defined(CONFIG_SANDBOX) && defined(CONFIG_CMD_MBR) && defined(CONFIG_CMD_MMC) \ && defined(CONFIG_MMC_SANDBOX) && defined(CONFIG_MMC_WRITE) diff --git a/test/log/Makefile b/test/log/Makefile index 08eea70e344..24b7c46786d 100644 --- a/test/log/Makefile +++ b/test/log/Makefile @@ -7,8 +7,6 @@ obj-$(CONFIG_CMD_LOG) += log_filter.o

ifdef CONFIG_UT_LOG

-obj-y += log_ut.o - ifdef CONFIG_SANDBOX obj-$(CONFIG_LOG_SYSLOG) += syslog_test.o obj-$(CONFIG_LOG_SYSLOG) += syslog_test_ndebug.o diff --git a/test/log/log_ut.c b/test/log/log_ut.c deleted file mode 100644 index 2aa3afe286a..00000000000 --- a/test/log/log_ut.c +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Copyright (c) 2020, Heinrich Schuchardt xypron.glpk@gmx.de - * - * Logging function tests. - */ - -#include <console.h> -#include <log.h> -#include <test/log.h> -#include <test/suites.h> - -int do_ut_log(struct cmd_tbl *cmdtp, int flag, int argc, char * const argv[]) -{ - struct unit_test *tests = UNIT_TEST_SUITE_START(log); - const int n_ents = UNIT_TEST_SUITE_COUNT(log); - - return cmd_ut_category("log", "log_test_", - tests, n_ents, argc, argv); -}

Use the new suite-runner to run these tests instead.

Signed-off-by: Simon Glass sjg@chromium.org ---

include/test/suites.h | 1 - test/cmd/Makefile | 2 +- test/cmd/mem.c | 19 ------------------- test/cmd_ut.c | 2 +- 4 files changed, 2 insertions(+), 22 deletions(-) delete mode 100644 test/cmd/mem.c

diff --git a/include/test/suites.h b/include/test/suites.h index 612167fe053..cb6b6fba240 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -40,7 +40,6 @@ int do_ut_bloblist(struct cmd_tbl *cmdtp, int flag, int argc, int do_ut_hush(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_loadm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_measurement(struct cmd_tbl *cmdtp, int flag, int argc, char * const argv[]); -int do_ut_mem(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_optee(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_overlay(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); diff --git a/test/cmd/Makefile b/test/cmd/Makefile index f83c53696d0..d8a5e77402d 100644 --- a/test/cmd/Makefile +++ b/test/cmd/Makefile @@ -10,7 +10,7 @@ endif ifdef CONFIG_CONSOLE_RECORD obj-$(CONFIG_CMD_PAUSE) += test_pause.o endif -obj-y += exit.o mem.o +obj-y += exit.o obj-$(CONFIG_X86) += cpuid.o msr.o obj-$(CONFIG_CMD_ADDRMAP) += addrmap.o obj-$(CONFIG_CMD_BDI) += bdinfo.o diff --git a/test/cmd/mem.c b/test/cmd/mem.c deleted file mode 100644 index 2b78e421a9c..00000000000 --- a/test/cmd/mem.c +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Executes tests for memory-related commands - * - * Copyright 2020 Google LLC - */ - -#include <command.h> -#include <test/suites.h> -#include <test/test.h> - -int do_ut_mem(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) -{ - struct unit_test *tests = UNIT_TEST_SUITE_START(mem); - const int n_ents = UNIT_TEST_SUITE_COUNT(mem); - - return cmd_ut_category("mem", "mem_test_", tests, n_ents, argc, - argv); -} diff --git a/test/cmd_ut.c b/test/cmd_ut.c index f5fbcf0d686..4e57fcf0005 100644 --- a/test/cmd_ut.c +++ b/test/cmd_ut.c @@ -161,7 +161,7 @@ static struct suite suites[] = { && defined(CONFIG_MMC_SANDBOX) && defined(CONFIG_MMC_WRITE) SUITE(mbr), #endif - SUITE_CMD(mem, do_ut_mem), + SUITE(mem), #if defined(CONFIG_SANDBOX) && defined(CONFIG_CMD_SETEXPR) SUITE_CMD(setexpr, do_ut_setexpr), #endif

Use the new suite-runner to run these tests instead.

Signed-off-by: Simon Glass sjg@chromium.org ---

include/test/suites.h | 1 - test/boot/measurement.c | 10 ---------- test/cmd_ut.c | 2 +- 3 files changed, 1 insertion(+), 12 deletions(-)

diff --git a/include/test/suites.h b/include/test/suites.h index 775b0a5a131..1a40af6599a 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -39,7 +39,6 @@ int do_ut_bloblist(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_hush(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_loadm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); -int do_ut_measurement(struct cmd_tbl *cmdtp, int flag, int argc, char * const argv[]); int do_ut_optee(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_overlay(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); diff --git a/test/boot/measurement.c b/test/boot/measurement.c index 018e8af45f0..5a49c7a6b23 100644 --- a/test/boot/measurement.c +++ b/test/boot/measurement.c @@ -53,13 +53,3 @@ static int measure(struct unit_test_state *uts) return 0; } MEASUREMENT_TEST(measure, 0); - -int do_ut_measurement(struct cmd_tbl *cmdtp, int flag, int argc, - char *const argv[]) -{ - struct unit_test *tests = UNIT_TEST_SUITE_START(measurement); - const int n_ents = UNIT_TEST_SUITE_COUNT(measurement); - - return cmd_ut_category("measurement", "measurement_test_", tests, - n_ents, argc, argv); -} diff --git a/test/cmd_ut.c b/test/cmd_ut.c index c53b24bfde9..6f5aa7e4924 100644 --- a/test/cmd_ut.c +++ b/test/cmd_ut.c @@ -166,7 +166,7 @@ static struct suite suites[] = { SUITE(setexpr), #endif #ifdef CONFIG_MEASURED_BOOT - SUITE_CMD(measurement, do_ut_measurement), + SUITE(measurement), #endif #ifdef CONFIG_SANDBOX SUITE_CMD(bloblist, do_ut_bloblist),

Use the new suite-runner to run these tests instead.

Signed-off-by: Simon Glass sjg@chromium.org ---

include/test/suites.h | 1 - test/boot/bootm.c | 9 --------- test/cmd_ut.c | 2 +- 3 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/include/test/suites.h b/include/test/suites.h index 32b7e8e3d5a..77af692be3c 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -32,7 +32,6 @@ int cmd_ut_category(const char *name, const char *prefix,

int do_ut_addrmap(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); -int do_ut_bootm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_bootstd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_hush(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); diff --git a/test/boot/bootm.c b/test/boot/bootm.c index c8e32023acb..1ce5c3cd6bf 100644 --- a/test/boot/bootm.c +++ b/test/boot/bootm.c @@ -256,12 +256,3 @@ static int bootm_test_subst_both(struct unit_test_state *uts) return 0; } BOOTM_TEST(bootm_test_subst_both, 0); - -int do_ut_bootm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) -{ - struct unit_test *tests = UNIT_TEST_SUITE_START(bootm); - const int n_ents = UNIT_TEST_SUITE_COUNT(bootm); - - return cmd_ut_category("bootm", "bootm_test_", tests, n_ents, - argc, argv); -} diff --git a/test/cmd_ut.c b/test/cmd_ut.c index e0e01510016..eb5ef099cb8 100644 --- a/test/cmd_ut.c +++ b/test/cmd_ut.c @@ -170,7 +170,7 @@ static struct suite suites[] = { #endif #ifdef CONFIG_SANDBOX SUITE(bloblist), - SUITE_CMD(bootm, do_ut_bootm), + SUITE(bootm), #endif #ifdef CONFIG_CMD_ADDRMAP SUITE_CMD(addrmap, do_ut_addrmap),

Use the new suite-runner to run these tests instead.

Signed-off-by: Simon Glass sjg@chromium.org ---

include/test/suites.h | 1 - test/cmd_ut.c | 2 +- test/hush/Makefile | 1 - test/hush/cmd_ut_hush.c | 19 ------------------- 4 files changed, 1 insertion(+), 22 deletions(-) delete mode 100644 test/hush/cmd_ut_hush.c

diff --git a/include/test/suites.h b/include/test/suites.h index 54fddaa6bac..b96a78f7129 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -32,7 +32,6 @@ int cmd_ut_category(const char *name, const char *prefix,

int do_ut_bootstd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); -int do_ut_hush(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_loadm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_optee(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_overlay(struct cmd_tbl *cmdtp, int flag, int argc, diff --git a/test/cmd_ut.c b/test/cmd_ut.c index f68da4dcf76..39489ba0811 100644 --- a/test/cmd_ut.c +++ b/test/cmd_ut.c @@ -176,7 +176,7 @@ static struct suite suites[] = { SUITE(addrmap), #endif #if CONFIG_IS_ENABLED(HUSH_PARSER) - SUITE_CMD(hush, do_ut_hush), + SUITE(hush), #endif #ifdef CONFIG_CMD_LOADM SUITE_CMD(loadm, do_ut_loadm), diff --git a/test/hush/Makefile b/test/hush/Makefile index 4c3a0be857a..febdc82e8aa 100644 --- a/test/hush/Makefile +++ b/test/hush/Makefile @@ -3,7 +3,6 @@ # (C) Copyright 2021 # Francis Laniel, Amarula Solutions, francis.laniel@amarulasolutions.com

-obj-y += cmd_ut_hush.o obj-y += if.o ifdef CONFIG_CONSOLE_RECORD obj-y += dollar.o diff --git a/test/hush/cmd_ut_hush.c b/test/hush/cmd_ut_hush.c deleted file mode 100644 index 141203d766b..00000000000 --- a/test/hush/cmd_ut_hush.c +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * (C) Copyright 2021 - * Francis Laniel, Amarula Solutions, francis.laniel@amarulasolutions.com - */ - -#include <command.h> -#include <test/hush.h> -#include <test/suites.h> -#include <test/ut.h> - -int do_ut_hush(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) -{ - struct unit_test *tests = UNIT_TEST_SUITE_START(hush); - const int n_ents = UNIT_TEST_SUITE_COUNT(hush); - - return cmd_ut_category("hush", "hush_test_", - tests, n_ents, argc, argv); -}

Use the new suite-runner to run these tests instead.

Signed-off-by: Simon Glass sjg@chromium.org ---

include/test/suites.h | 2 -- test/cmd/pci_mps.c | 10 ---------- test/cmd_ut.c | 2 +- 3 files changed, 1 insertion(+), 13 deletions(-)

diff --git a/include/test/suites.h b/include/test/suites.h index f6b33da5a28..49c18ebf456 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -35,8 +35,6 @@ int do_ut_bootstd(struct cmd_tbl *cmdtp, int flag, int argc, int do_ut_optee(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_overlay(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); -int do_ut_pci_mps(struct cmd_tbl *cmdtp, int flag, int argc, - char *const argv[]); int do_ut_seama(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_upl(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);

diff --git a/test/cmd/pci_mps.c b/test/cmd/pci_mps.c index 2595ea103ff..8b3ea4a6134 100644 --- a/test/cmd/pci_mps.c +++ b/test/cmd/pci_mps.c @@ -28,13 +28,3 @@ static int test_pci_mps_safe(struct unit_test_state *uts) return 0; } PCI_MPS_TEST(test_pci_mps_safe, UTF_CONSOLE); - -int do_ut_pci_mps(struct cmd_tbl *cmdtp, int flag, int argc, - char * const argv[]) -{ - struct unit_test *tests = UNIT_TEST_SUITE_START(pci_mps); - const int n = UNIT_TEST_SUITE_COUNT(pci_mps); - - return cmd_ut_category("pci_mps", "pci_mps_test_", tests, n, - argc, argv); -} diff --git a/test/cmd_ut.c b/test/cmd_ut.c index 0a6284e4962..7caf41da096 100644 --- a/test/cmd_ut.c +++ b/test/cmd_ut.c @@ -182,7 +182,7 @@ static struct suite suites[] = { SUITE(loadm), #endif #ifdef CONFIG_CMD_PCI_MPS - SUITE_CMD(pci_mps, do_ut_pci_mps), + SUITE(pci_mps), #endif #ifdef CONFIG_CMD_SEAMA SUITE_CMD(seama, do_ut_seama),

Use the new suite-runner to run these tests instead.

It is not clear that these actually work, since they are not enabled on sandbox for some reason.

Signed-off-by: Simon Glass sjg@chromium.org ---

include/test/suites.h | 1 - test/boot/upl.c | 8 -------- test/cmd_ut.c | 2 +- 3 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/include/test/suites.h b/include/test/suites.h index cd758b8c464..20e6a2a113b 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -35,6 +35,5 @@ int do_ut_bootstd(struct cmd_tbl *cmdtp, int flag, int argc, int do_ut_optee(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); int do_ut_overlay(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); -int do_ut_upl(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);

#endif /* __TEST_SUITES_H__ */ diff --git a/test/boot/upl.c b/test/boot/upl.c index 2c9b8bc8bc0..aa58cdf083b 100644 --- a/test/boot/upl.c +++ b/test/boot/upl.c @@ -426,11 +426,3 @@ static int upl_test_info_norun(struct unit_test_state *uts) return 0; } UPL_TEST(upl_test_info_norun, UTF_CONSOLE | UTF_MANUAL); - -int do_ut_upl(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) -{ - struct unit_test *tests = UNIT_TEST_SUITE_START(upl); - const int n_ents = UNIT_TEST_SUITE_COUNT(upl); - - return cmd_ut_category("upl", "cmd_upl_", tests, n_ents, argc, argv); -} diff --git a/test/cmd_ut.c b/test/cmd_ut.c index c6b7ccdedbd..0d055119ce3 100644 --- a/test/cmd_ut.c +++ b/test/cmd_ut.c @@ -188,7 +188,7 @@ static struct suite suites[] = { SUITE(seama), #endif #ifdef CONFIG_CMD_UPL - SUITE_CMD(upl, do_ut_upl), + SUITE(upl), #endif };

This is not needed anymore. If a test suite is not built, then it will have no linker-list entries. So we can just check for that and know that the suite is not present.

This allows removal of the #ifdefs and the need to keep them in sync with the associated Makefile rules, which has actually failed, since the help does not match what commands are actually present.

Signed-off-by: Simon Glass sjg@chromium.org ---

test/cmd_ut.c | 75 ++++++++++++++++--------------------- test/py/tests/test_suite.py | 19 +++++++--- 2 files changed, 46 insertions(+), 48 deletions(-)

diff --git a/test/cmd_ut.c b/test/cmd_ut.c index 798710972e5..2ba99a26c63 100644 --- a/test/cmd_ut.c +++ b/test/cmd_ut.c @@ -120,76 +120,55 @@ SUITE_DECL(seama); SUITE_DECL(upl);

static struct suite suites[] = { -#ifdef CONFIG_CMD_BDI SUITE(bdinfo), -#endif #ifdef CONFIG_UT_BOOTSTD SUITE_CMD(bootstd, do_ut_bootstd), #endif -#ifdef CONFIG_CMDLINE SUITE(cmd), -#endif SUITE(common), -#if defined(CONFIG_UT_DM) SUITE(dm), -#endif -#if defined(CONFIG_UT_ENV) SUITE(env), -#endif SUITE(exit), -#ifdef CONFIG_CMD_FDT SUITE(fdt), -#endif -#ifdef CONFIG_CONSOLE_TRUETYPE SUITE(font), -#endif #ifdef CONFIG_UT_OPTEE SUITE_CMD(optee, do_ut_optee), #endif #ifdef CONFIG_UT_OVERLAY SUITE_CMD(overlay, do_ut_overlay), #endif -#ifdef CONFIG_UT_LIB SUITE(lib), -#endif -#ifdef CONFIG_UT_LOG SUITE(log), -#endif -#if defined(CONFIG_SANDBOX) && defined(CONFIG_CMD_MBR) && defined(CONFIG_CMD_MMC) \ - && defined(CONFIG_MMC_SANDBOX) && defined(CONFIG_MMC_WRITE) SUITE(mbr), -#endif SUITE(mem), -#if defined(CONFIG_SANDBOX) && defined(CONFIG_CMD_SETEXPR) SUITE(setexpr), -#endif -#ifdef CONFIG_MEASURED_BOOT SUITE(measurement), -#endif -#ifdef CONFIG_SANDBOX SUITE(bloblist), SUITE(bootm), -#endif -#ifdef CONFIG_CMD_ADDRMAP SUITE(addrmap), -#endif -#if CONFIG_IS_ENABLED(HUSH_PARSER) SUITE(hush), -#endif -#ifdef CONFIG_CMD_LOADM SUITE(loadm), -#endif -#ifdef CONFIG_CMD_PCI_MPS SUITE(pci_mps), -#endif -#ifdef CONFIG_CMD_SEAMA SUITE(seama), -#endif -#ifdef CONFIG_CMD_UPL SUITE(upl), -#endif };

+/** + * has_tests() - Check if a suite has tests, i.e. is supported in this build + * + * If the suite is run using a command, we have to assume that tests may be + * present, since we have no visibility + * + * @ste: Suite to check + * Return: true if supported, false if not + */ +static bool has_tests(struct suite *ste) +{ + int n_ents = ste->end - ste->start; + + return n_ents || ste->cmd; +} + /** run_suite() - Run a suite of tests */ static int run_suite(struct suite *ste, struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) @@ -222,10 +201,12 @@ static int do_ut_all(struct cmd_tbl *cmdtp, int flag, int argc, struct suite *ste = &suites[i]; char *const argv[] = {(char *)ste->name, NULL};

- printf("----Running %s tests----\n", ste->name); - retval = run_suite(ste, cmdtp, flag, 1, argv); - if (!any_fail) - any_fail = retval; + if (has_tests(ste)) { + printf("----Running %s tests----\n", ste->name); + retval = run_suite(ste, cmdtp, flag, 1, argv); + if (!any_fail) + any_fail = retval; + } }

return any_fail; @@ -234,9 +215,17 @@ static int do_ut_all(struct cmd_tbl *cmdtp, int flag, int argc, static int do_ut_info(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) { + int suite_count, i; const char *flags;

- printf("Test suites: %d\n", (int)ARRAY_SIZE(suites)); + for (suite_count = 0, i = 0; i < ARRAY_SIZE(suites); i++) { + struct suite *ste = &suites[i]; + + if (has_tests(ste)) + suite_count++; + } + + printf("Test suites: %d\n", suite_count); printf("Total tests: %d\n", (int)UNIT_TEST_ALL_COUNT());

flags = cmd_arg1(argc, argv); @@ -251,7 +240,7 @@ static int do_ut_info(struct cmd_tbl *cmdtp, int flag, int argc,

if (n_ent) printf("%5ld %s\n", n_ent, ste->name); - else + else if (ste->cmd) printf("%5s %s\n", "?", ste->name); } } diff --git a/test/py/tests/test_suite.py b/test/py/tests/test_suite.py index 938dfb468f6..1c248602199 100644 --- a/test/py/tests/test_suite.py +++ b/test/py/tests/test_suite.py @@ -106,7 +106,7 @@ def process_ut_info(cons, output): test_count = {} for line in output.splitlines(): if DEBUG_ME: - cons.log.info(f'line: {line}') + cons.log.info(f'line: {line.rstrip()}') m = re.match(r'Test suites: (.*)', line) if m: suite_count = int(m.group(1)) @@ -122,7 +122,7 @@ def process_ut_info(cons, output): @pytest.mark.buildconfigspec('sandbox') @pytest.mark.notbuildconfigspec('sandbox_spl') @pytest.mark.notbuildconfigspec('sandbox64') -def test_suite(u_boot_console): +def test_suite(u_boot_console, u_boot_config): """Perform various checks on the unit tests, including:

- The number of suites matches that reported by the 'ut info' @@ -135,6 +135,7 @@ def test_suite(u_boot_console):

""" cons = u_boot_console + buildconfig = u_boot_config.buildconfig with cons.log.section('Run all unit tests'): # ut hush hush_test_simple_dollar prints "Unknown command" on purpose. with u_boot_console.disable_check('unknown_command'): @@ -144,9 +145,17 @@ def test_suite(u_boot_console): with cons.log.section('Check output'): suites, all_tests, exp_test_count, missing, extra = collect_info(cons, output) + cons.log.info(f'missing {missing}') + cons.log.info(f'extra {extra}')

# Make sure we got a test count for each suite - assert suites - exp_test_count.keys() == set() + assert not (suites - exp_test_count.keys()) + + # Deal with missing suites + with cons.log.section('Check missing suites'): + if 'config_cmd_seama' not in buildconfig: + cons.log.info("CMD_SEAMA not enabled: Ignoring suite 'seama'") + missing.discard('seama')

# Run 'ut info' and compare with the log results with cons.log.section('Check suite test-counts'): @@ -160,8 +169,8 @@ def test_suite(u_boot_console): cons.log.error(f'missing: {sorted(list(missing))}') cons.log.error(f'extra: {sorted(list(extra))}')

- assert not missing - assert not extra + assert not missing, f'Missing suites {missing}' + assert not extra, f'Extra suites {extra}'

cons.log.info(str(exp_test_count)) for suite in EXPECTED_SUITES:

Use a struct to hold the stats, since we also want to have the same stats for all runs as we have for each suite.

Signed-off-by: Simon Glass sjg@chromium.org ---

include/test/test.h | 17 +++++++++++++---- test/test-main.c | 29 +++++++++++++++-------------- test/ut.c | 4 ++-- 3 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/include/test/test.h b/include/test/test.h index 21c0478befe..bc8f0bbe501 100644 --- a/include/test/test.h +++ b/include/test/test.h @@ -9,11 +9,21 @@ #include <malloc.h> #include <linux/bitops.h>

-/* - * struct unit_test_state - Entire state of test system +/** + * struct ut_stats - Statistics about tests run * * @fail_count: Number of tests that failed * @skip_count: Number of tests that were skipped + */ +struct ut_stats { + int fail_count; + int skip_count; +}; + +/* + * struct unit_test_state - Entire state of test system + * + * @cur: Statistics for the current run * @start: Store the starting mallinfo when doing leak test * @of_live: true to use livetree if available, false to use flattree * @of_root: Record of the livetree root node (used for setting up tests) @@ -34,8 +44,7 @@ * @actual_str: Temporary string used to hold actual string value */ struct unit_test_state { - int fail_count; - int skip_count; + struct ut_stats cur; struct mallinfo start; struct device_node *of_root; bool of_live; diff --git a/test/test-main.c b/test/test-main.c index 391d4e7bb0e..e8aecd267c7 100644 --- a/test/test-main.c +++ b/test/test-main.c @@ -448,7 +448,7 @@ static int test_post_run(struct unit_test_state *uts, struct unit_test *test) */ static int skip_test(struct unit_test_state *uts) { - uts->skip_count++; + uts->cur.skip_count++;

return -EAGAIN; } @@ -460,7 +460,7 @@ static int skip_test(struct unit_test_state *uts) * the name of each test before running it. * * @uts: Test state to update. The caller should ensure that this is zeroed for - * the first call to this function. On exit, @uts->fail_count is + * the first call to this function. On exit, @uts->cur.fail_count is * incremented by the number of failures (0, one hopes) * @test_name: Test to run * @name: Name of test, possibly skipping a prefix that should not be displayed @@ -510,7 +510,7 @@ static int ut_run_test(struct unit_test_state *uts, struct unit_test *test, * SPL. * * @uts: Test state to update. The caller should ensure that this is zeroed for - * the first call to this function. On exit, @uts->fail_count is + * the first call to this function. On exit, @uts->cur.fail_count is * incremented by the number of failures (0, one hopes) * @test: Test to run * Return: 0 if all tests passed, -EAGAIN if the test should be skipped, -1 if @@ -574,7 +574,7 @@ static int ut_run_test_live_flat(struct unit_test_state *uts, * the name of each test before running it. * * @uts: Test state to update. The caller should ensure that this is zeroed for - * the first call to this function. On exit, @uts->fail_count is + * the first call to this function. On exit, @uts->cur.fail_count is * incremented by the number of failures (0, one hopes) * @prefix: String prefix for the tests. Any tests that have this prefix will be * printed without the prefix, so that it is easier to see the unique part @@ -632,7 +632,7 @@ static int ut_run_tests(struct unit_test_state *uts, const char *prefix, if (len < 6 || strcmp(test_name + len - 6, "_norun")) { printf("Test '%s' is manual so must have a name ending in _norun\n", test_name); - uts->fail_count++; + uts->cur.fail_count++; return -EBADF; } if (!uts->force_run) { @@ -641,23 +641,24 @@ static int ut_run_tests(struct unit_test_state *uts, const char *prefix, continue; } } - old_fail_count = uts->fail_count; + old_fail_count = uts->cur.fail_count;

if (one && upto == pos) { ret = ut_run_test_live_flat(uts, one); - if (uts->fail_count != old_fail_count) { + if (uts->cur.fail_count != old_fail_count) { printf("Test '%s' failed %d times (position %d)\n", one->name, - uts->fail_count - old_fail_count, pos); + uts->cur.fail_count - old_fail_count, + pos); } return -EBADF; }

for (i = 0; i < uts->runs_per_test; i++) ret = ut_run_test_live_flat(uts, test); - if (uts->fail_count != old_fail_count) { + if (uts->cur.fail_count != old_fail_count) { printf("Test '%s' failed %d times\n", test_name, - uts->fail_count - old_fail_count); + uts->cur.fail_count - old_fail_count); } found++; if (ret == -EAGAIN) @@ -668,7 +669,7 @@ static int ut_run_tests(struct unit_test_state *uts, const char *prefix, if (select_name && !found) return -ENOENT;

- return uts->fail_count ? -EBADF : 0; + return uts->cur.fail_count ? -EBADF : 0; }

int ut_run_list(struct unit_test_state *uts, const char *category, @@ -716,12 +717,12 @@ int ut_run_list(struct unit_test_state *uts, const char *category, if (has_dm_tests) dm_test_restore(uts->of_root);

- if (uts->skip_count) - printf("Skipped: %d, ", uts->skip_count); + if (uts->cur.skip_count) + printf("Skipped: %d, ", uts->cur.skip_count); if (ret == -ENOENT) printf("Test '%s' not found\n", select_name); else - printf("Failures: %d\n", uts->fail_count); + printf("Failures: %d\n", uts->cur.fail_count);

return ret; } diff --git a/test/ut.c b/test/ut.c index 7454da3e001..a16fdfb3a93 100644 --- a/test/ut.c +++ b/test/ut.c @@ -21,7 +21,7 @@ void ut_fail(struct unit_test_state *uts, const char *fname, int line, { gd->flags &= ~(GD_FLG_SILENT | GD_FLG_RECORD); printf("%s:%d, %s(): %s\n", fname, line, func, cond); - uts->fail_count++; + uts->cur.fail_count++; }

void ut_failf(struct unit_test_state *uts, const char *fname, int line, @@ -35,7 +35,7 @@ void ut_failf(struct unit_test_state *uts, const char *fname, int line, vprintf(fmt, args); va_end(args); putc('\n'); - uts->fail_count++; + uts->cur.fail_count++; }

ulong ut_check_free(void)

Add a function to show the stats, so we can decide when to print it.

This slightly adjusts the output, so that any 'test not found' message appears on its own line after all other output.

The 'failures' message now appears in lower case so update pytest accordingly.

Signed-off-by: Simon Glass sjg@chromium.org ---

arch/sandbox/cpu/spl.c | 1 + include/test/ut.h | 8 ++++++++ test/py/tests/test_spl.py | 2 +- test/py/tests/test_upl.py | 2 +- test/py/tests/test_ut.py | 2 +- test/py/tests/test_vbe.py | 2 +- test/py/tests/test_vpl.py | 2 +- test/test-main.c | 14 +++++++++----- 8 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/arch/sandbox/cpu/spl.c b/arch/sandbox/cpu/spl.c index 0ad23e4ba95..544ea8717eb 100644 --- a/arch/sandbox/cpu/spl.c +++ b/arch/sandbox/cpu/spl.c @@ -153,6 +153,7 @@ void spl_board_init(void) ut_init_state(&uts); ret = ut_run_list(&uts, "spl", NULL, tests, count, state->select_unittests, 1, false, NULL); + ut_report(&uts.cur, "Tests"); ut_uninit_state(&uts); /* continue execution into U-Boot */ } diff --git a/include/test/ut.h b/include/test/ut.h index a51defc3b90..55bb5aa2092 100644 --- a/include/test/ut.h +++ b/include/test/ut.h @@ -530,4 +530,12 @@ int ut_run_list(struct unit_test_state *uts, const char *category, const char *select_name, int runs_per_test, bool force_run, const char *test_insert);

+/** + * ut_report() - Report stats on a test run + * + * @stats: Stats to show + * @prefix: Name of this run type, e.g. "Tests" or "Total tests" + */ +void ut_report(struct ut_stats *stats, const char *prefix); + #endif diff --git a/test/py/tests/test_spl.py b/test/py/tests/test_spl.py index 42e4c4342b2..474f430a344 100644 --- a/test/py/tests/test_spl.py +++ b/test/py/tests/test_spl.py @@ -36,7 +36,7 @@ def test_spl(u_boot_console, ut_spl_subtest): cons = u_boot_console cons.restart_uboot_with_flags(['-u', '-k', ut_spl_subtest.split()[1]]) output = cons.get_spawn_output().replace('\r', '') - assert 'Failures: 0' in output + assert 'failures: 0' in output finally: # Restart afterward in case a non-SPL test is run next. This should not # happen since SPL tests are run in their own invocation of test.py, but diff --git a/test/py/tests/test_upl.py b/test/py/tests/test_upl.py index 3164bda6b71..d94359d8b9b 100644 --- a/test/py/tests/test_upl.py +++ b/test/py/tests/test_upl.py @@ -35,4 +35,4 @@ def test_upl_handoff(u_boot_console):

# Check the FIT offsets look correct output = cons.run_command('ut upl -f upl_test_info_norun') - assert 'Failures: 0' in output + assert 'failures: 0' in output diff --git a/test/py/tests/test_ut.py b/test/py/tests/test_ut.py index cacf11f7c0a..d2d8ce10755 100644 --- a/test/py/tests/test_ut.py +++ b/test/py/tests/test_ut.py @@ -607,4 +607,4 @@ def test_ut(u_boot_console, ut_subtest): assert 'Unknown command 'quux' - try 'help'' in output else: output = u_boot_console.run_command('ut ' + ut_subtest) - assert output.endswith('Failures: 0') + assert output.endswith('failures: 0') diff --git a/test/py/tests/test_vbe.py b/test/py/tests/test_vbe.py index 50b6c1cd911..861df3f8266 100644 --- a/test/py/tests/test_vbe.py +++ b/test/py/tests/test_vbe.py @@ -117,4 +117,4 @@ def test_vbe(u_boot_console): with cons.log.section('Kernel load'): output = cons.run_command_list(cmd.splitlines())

- assert 'Failures: 0' in output[-1] + assert 'failures: 0' in output[-1] diff --git a/test/py/tests/test_vpl.py b/test/py/tests/test_vpl.py index 4af578b9173..8c472ca7a92 100644 --- a/test/py/tests/test_vpl.py +++ b/test/py/tests/test_vpl.py @@ -26,7 +26,7 @@ def test_vpl(u_boot_console, ut_vpl_subtest): cons = u_boot_console cons.restart_uboot_with_flags(['-u', '-k', ut_vpl_subtest.split()[1]]) output = cons.get_spawn_output().replace('\r', '') - assert 'Failures: 0' in output + assert 'failures: 0' in output finally: # Restart afterward in case a non-VPL test is run next. This should not # happen since VPL tests are run in their own invocation of test.py, but diff --git a/test/test-main.c b/test/test-main.c index e36bc37d29e..a0a3f6086ef 100644 --- a/test/test-main.c +++ b/test/test-main.c @@ -673,6 +673,14 @@ static int ut_run_tests(struct unit_test_state *uts, const char *prefix, return uts->cur.fail_count ? -EBADF : 0; }

+void ut_report(struct ut_stats *stats, const char *prefix) +{ + printf("%s run: %d, ", prefix, stats->test_count); + if (stats->skip_count) + printf("skipped: %d, ", stats->skip_count); + printf("failures: %d\n", stats->fail_count); +} + int ut_run_list(struct unit_test_state *uts, const char *category, const char *prefix, struct unit_test *tests, int count, const char *select_name, int runs_per_test, bool force_run, @@ -718,13 +726,9 @@ int ut_run_list(struct unit_test_state *uts, const char *category, if (has_dm_tests) dm_test_restore(uts->of_root);

- printf("Tests run: %d, ", uts->cur.test_count); - if (uts->cur.skip_count) - printf("Skipped: %d, ", uts->cur.skip_count); + ut_report(&uts->cur, "Tests"); if (ret == -ENOENT) printf("Test '%s' not found\n", select_name); - else - printf("Failures: %d\n", uts->cur.fail_count);

return ret; }

Put the suites in order by name, for easier code-maintenance. This also helps find test results for a particular swuit in the 'ut all' output.

Signed-off-by: Simon Glass sjg@chromium.org ---

test/cmd_ut.c | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/test/cmd_ut.c b/test/cmd_ut.c index f33918ddd00..3afab422e4a 100644 --- a/test/cmd_ut.c +++ b/test/cmd_ut.c @@ -90,7 +90,10 @@ int cmd_ut_category(struct unit_test_state *uts, const char *name, NULL, \ }

+SUITE_DECL(addrmap); SUITE_DECL(bdinfo); +SUITE_DECL(bloblist); +SUITE_DECL(bootm); SUITE_DECL(bootstd); SUITE_DECL(cmd); SUITE_DECL(common); @@ -99,25 +102,25 @@ SUITE_DECL(env); SUITE_DECL(exit); SUITE_DECL(fdt); SUITE_DECL(font); -SUITE_DECL(optee); -SUITE_DECL(overlay); +SUITE_DECL(hush); SUITE_DECL(lib); +SUITE_DECL(loadm); SUITE_DECL(log); SUITE_DECL(mbr); -SUITE_DECL(mem); -SUITE_DECL(setexpr); SUITE_DECL(measurement); -SUITE_DECL(bloblist); -SUITE_DECL(bootm); -SUITE_DECL(addrmap); -SUITE_DECL(hush); -SUITE_DECL(loadm); +SUITE_DECL(mem); +SUITE_DECL(optee); +SUITE_DECL(overlay); SUITE_DECL(pci_mps); SUITE_DECL(seama); +SUITE_DECL(setexpr); SUITE_DECL(upl);

static struct suite suites[] = { + SUITE(addrmap), SUITE(bdinfo), + SUITE(bloblist), + SUITE(bootm), #ifdef CONFIG_UT_BOOTSTD SUITE_CMD(bootstd, do_ut_bootstd), #endif @@ -128,25 +131,22 @@ static struct suite suites[] = { SUITE(exit), SUITE(fdt), SUITE(font), + SUITE(hush), + SUITE(lib), + SUITE(loadm), + SUITE(log), + SUITE(mbr), + SUITE(measurement), + SUITE(mem), #ifdef CONFIG_UT_OPTEE SUITE_CMD(optee, do_ut_optee), #endif #ifdef CONFIG_UT_OVERLAY SUITE_CMD(overlay, do_ut_overlay), #endif - SUITE(lib), - SUITE(log), - SUITE(mbr), - SUITE(mem), - SUITE(setexpr), - SUITE(measurement), - SUITE(bloblist), - SUITE(bootm), - SUITE(addrmap), - SUITE(hush), - SUITE(loadm), SUITE(pci_mps), SUITE(seama), + SUITE(setexpr), SUITE(upl), };

Rather than having the help in the longhelp, put it in the suite info so 'ut info -s' can show it. This is tidier, particular due to the removal of #ifdefs

This means that the help text is present in the image (although not displayed with 'ut info -s') so the image-size increases. But with UNIT_TEST enabled, we expect large images so this doesn't seem important.

Signed-off-by: Simon Glass sjg@chromium.org ---

test/cmd_ut.c | 127 ++++++++++++++++---------------------------------- 1 file changed, 40 insertions(+), 87 deletions(-)

diff --git a/test/cmd_ut.c b/test/cmd_ut.c index 3afab422e4a..76f481f509b 100644 --- a/test/cmd_ut.c +++ b/test/cmd_ut.c @@ -19,14 +19,16 @@ * * @name: Name of suite * @start: First test in suite - * end: End test in suite (points to the first test in the next suite - * cmd: Command to use to run the suite + * @end: End test in suite (points to the first test in the next suite + * @cmd: Command to use to run the suite + * @help: Help-string to show for this suite */ struct suite { const char *name; struct unit_test *start; struct unit_test *end; ut_cmd_func cmd; + const char *help; };

static int do_ut_all(struct unit_test_state *uts, struct cmd_tbl *cmdtp, @@ -75,19 +77,21 @@ int cmd_ut_category(struct unit_test_state *uts, const char *name, ll_end_decl(suite_end_ ## _name, struct unit_test, ut_ ## _name)

/* declare a test suite which uses a subcommand to run */ -#define SUITE_CMD(_name, _cmd_func) { \ +#define SUITE_CMD(_name, _cmd_func, _help) { \ #_name, \ suite_start_ ## _name, \ suite_end_ ## _name, \ _cmd_func, \ + _help, \ }

/* declare a test suite which can be run directly without a subcommand */ -#define SUITE(_name) { \ +#define SUITE(_name, _help) { \ #_name, \ suite_start_ ## _name, \ suite_end_ ## _name, \ NULL, \ + _help, \ }

SUITE_DECL(addrmap); @@ -117,37 +121,37 @@ SUITE_DECL(setexpr); SUITE_DECL(upl);

static struct suite suites[] = { - SUITE(addrmap), - SUITE(bdinfo), - SUITE(bloblist), - SUITE(bootm), + SUITE(addrmap, "very basic test of addrmap command"), + SUITE(bdinfo, "bdinfo (board info) command"), + SUITE(bloblist, "bloblist implementation"), + SUITE(bootm, "bootm command"), #ifdef CONFIG_UT_BOOTSTD - SUITE_CMD(bootstd, do_ut_bootstd), + SUITE_CMD(bootstd, do_ut_bootstd, "standard boot implementation"), #endif - SUITE(cmd), - SUITE(common), - SUITE(dm), - SUITE(env), - SUITE(exit), - SUITE(fdt), - SUITE(font), - SUITE(hush), - SUITE(lib), - SUITE(loadm), - SUITE(log), - SUITE(mbr), - SUITE(measurement), - SUITE(mem), + SUITE(cmd, "various commands"), + SUITE(common, "tests for common/ directory"), + SUITE(dm, "driver model"), + SUITE(env, "environment"), + SUITE(exit, "shell exit and variables"), + SUITE(fdt, "fdt command"), + SUITE(font, "font command"), + SUITE(hush, "hush behaviour"), + SUITE(lib, "library functions"), + SUITE(loadm, "loadm command parameters and loading memory blob"), + SUITE(log, "logging functions"), + SUITE(mbr, "mbr command"), + SUITE(measurement, "TPM-based measured boot"), + SUITE(mem, "memory-related commands"), #ifdef CONFIG_UT_OPTEE - SUITE_CMD(optee, do_ut_optee), + SUITE_CMD(optee, do_ut_optee, "OP-TEE"), #endif #ifdef CONFIG_UT_OVERLAY - SUITE_CMD(overlay, do_ut_overlay), + SUITE_CMD(overlay, do_ut_overlay, "device tree overlays"), #endif - SUITE(pci_mps), - SUITE(seama), - SUITE(setexpr), - SUITE(upl), + SUITE(pci_mps, "PCI Express Maximum Payload Size"), + SUITE(seama, "seama command parameters loading and decoding"), + SUITE(setexpr, "setexpr command"), + SUITE(upl, "Universal payload support"), };

/** @@ -232,16 +236,19 @@ static int do_ut_info(struct cmd_tbl *cmdtp, int flag, int argc, if (flags && !strcmp("-s", flags)) { int i;

- puts("\nTests Suite\n"); - puts("----- -----\n"); + puts("\nTests Suite Purpose"); + puts("\n----- ------------ -------------------------\n"); for (i = 0; i < ARRAY_SIZE(suites); i++) { struct suite *ste = &suites[i]; long n_ent = ste->end - ste->start;

if (n_ent) - printf("%5ld %s\n", n_ent, ste->name); + printf("%5ld", n_ent); else if (ste->cmd) - printf("%5s %s\n", "?", ste->name); + printf("%5s", "?"); + else /* suite is not present */ + continue; + printf(" %-13.13s %s\n", ste->name, ste->help); } }

@@ -306,60 +313,6 @@ U_BOOT_LONGHELP(ut, "\nOptions for <suite>:" "\nall - execute all enabled tests" "\ninfo [-s] - show info about tests [and suites]" -#ifdef CONFIG_CMD_ADDRMAP - "\naddrmap - very basic test of addrmap command" -#endif -#ifdef CONFIG_CMD_BDI - "\nbdinfo - bdinfo command" -#endif -#ifdef CONFIG_SANDBOX - "\nbloblist - bloblist implementation" -#endif -#ifdef CONFIG_BOOTSTD - "\nbootstd - standard boot implementation" -#endif -#ifdef CONFIG_CMDLINE - "\ncmd - test various commands" -#endif - "\ncommon - tests for common/ directory" -#ifdef CONFIG_UT_DM - "\ndm - driver model" -#endif -#ifdef CONFIG_UT_ENV - "\nenv - environment" -#endif -#ifdef CONFIG_CMD_FDT - "\nfdt - fdt command" -#endif -#ifdef CONFIG_CONSOLE_TRUETYPE - "\nfont - font command" -#endif -#if CONFIG_IS_ENABLED(HUSH_PARSER) - "\nhush - Test hush behavior" -#endif -#ifdef CONFIG_CMD_LOADM - "\nloadm - loadm command parameters and loading memory blob" -#endif -#ifdef CONFIG_UT_LIB - "\nlib - library functions" -#endif -#ifdef CONFIG_UT_LOG - "\nlog - logging functions" -#endif - "\nmem - memory-related commands" -#ifdef CONFIG_UT_OPTEE - "\noptee - test OP-TEE" -#endif -#ifdef CONFIG_UT_OVERLAY - "\noverlay - device tree overlays" -#endif -#ifdef CONFIG_CMD_PCI_MPS - "\npci_mps - PCI Express Maximum Payload Size" -#endif - "\nsetexpr - setexpr command" -#ifdef CONFIG_CMD_SEAMA - "\nseama - seama command parameters loading and decoding" -#endif );

U_BOOT_CMD(