Buildman currently lacks testing in many areas, including its use of git, make and many command-line flags.
Add a functional test which covers some of these areas. So far it does a fake 'build' of all boards for the current source tree.
This version reads the real ~/.buildman and boards.cfg files. Future work will improve this.
Signed-off-by: Simon Glass sjg@chromium.org ---
Changes in v2: - Add a comment to _HandleCommandGit - Make sure the test temporary directory is removed
tools/buildman/buildman.py | 17 ++--- tools/buildman/control.py | 21 +++-- tools/buildman/func_test.py | 182 ++++++++++++++++++++++++++++++++++++++++++++ tools/buildman/toolchain.py | 4 +- 4 files changed, 206 insertions(+), 18 deletions(-) create mode 100644 tools/buildman/func_test.py
diff --git a/tools/buildman/buildman.py b/tools/buildman/buildman.py index 70c2142..6771c86 100755 --- a/tools/buildman/buildman.py +++ b/tools/buildman/buildman.py @@ -30,27 +30,20 @@ import terminal import toolchain
def RunTests(): + import func_test import test import doctest
result = unittest.TestResult() - for module in ['toolchain']: + for module in ['toolchain', 'gitutil']: suite = doctest.DocTestSuite(module) suite.run(result)
- # TODO: Surely we can just 'print' result? - print result - for test, err in result.errors: - print err - for test, err in result.failures: - print err - sys.argv = [sys.argv[0]] - suite = unittest.TestLoader().loadTestsFromTestCase(test.TestBuild) - result = unittest.TestResult() - suite.run(result) + for module in (test.TestBuild, func_test.TestFunctional): + suite = unittest.TestLoader().loadTestsFromTestCase(module) + suite.run(result)
- # TODO: Surely we can just 'print' result? print result for test, err in result.errors: print err diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 245d370..b35eb41 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -13,6 +13,7 @@ from builder import Builder import gitutil import patchstream import terminal +from terminal import Print import toolchain import command import subprocess @@ -77,12 +78,18 @@ def ShowActions(series, why_selected, boards_selected, builder, options): print ('Total boards to build for each commit: %d\n' % why_selected['all'])
-def DoBuildman(options, args): +def DoBuildman(options, args, toolchains=None, make_func=None): """The main control code for buildman
Args: options: Command line options object args: Command line arguments (list of strings) + toolchains: Toolchains to use - this should be a Toolchains() + object. If None, then it will be created and scanned + make_func: Make function to use for the builder. This is called + to execute 'make'. If this is None, the normal function + will be used, which calls the 'make' tool with suitable + arguments. This setting is useful for tests. """ if options.full_help: pager = os.getenv('PAGER') @@ -97,8 +104,10 @@ def DoBuildman(options, args): bsettings.Setup(options.config_file) options.git_dir = os.path.join(options.git, '.git')
- toolchains = toolchain.Toolchains() - toolchains.Scan(options.list_tool_chains) + if not toolchains: + toolchains = toolchain.Toolchains() + toolchains.GetSettings() + toolchains.Scan(options.list_tool_chains) if options.list_tool_chains: toolchains.List() print @@ -204,6 +213,8 @@ def DoBuildman(options, args): options.threads, options.jobs, gnu_make=gnu_make, checkout=True, show_unknown=options.show_unknown, step=options.step) builder.force_config_on_failure = not options.quick + if make_func: + builder.do_make = make_func
# For a dry run, just show our actions as a sanity check if options.dry_run: @@ -222,8 +233,8 @@ def DoBuildman(options, args): else: commits = None
- print GetActionSummary(options.summary, commits, board_selected, - options) + Print(GetActionSummary(options.summary, commits, board_selected, + options))
builder.SetDisplayOptions(options.show_errors, options.show_sizes, options.show_detail, options.show_bloat, diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py new file mode 100644 index 0000000..8711f9c --- /dev/null +++ b/tools/buildman/func_test.py @@ -0,0 +1,182 @@ +# +# Copyright (c) 2014 Google, Inc +# +# SPDX-License-Identifier: GPL-2.0+ +# + +import os +import shutil +import sys +import tempfile +import unittest + +import cmdline +import command +import control +import gitutil +import terminal +import toolchain + +class TestFunctional(unittest.TestCase): + """Functional test for buildman. + + This aims to test from just below the invocation of buildman (parsing + of arguments) to 'make' and 'git' invocation. It is not a true + emd-to-end test, as it mocks git, make and the tool chain. But this + makes it easier to detect when the builder is doing the wrong thing, + since in many cases this test code will fail. For example, only a + very limited subset of 'git' arguments is supported - anything + unexpected will fail. + """ + def setUp(self): + self._base_dir = tempfile.mkdtemp() + self._git_dir = os.path.join(self._base_dir, 'src') + self._buildman_pathname = sys.argv[0] + self._buildman_dir = os.path.dirname(sys.argv[0]) + command.test_result = self._HandleCommand + self._toolchains = toolchain.Toolchains() + self._toolchains.Add('gcc', test=False) + + def tearDown(self): + shutil.rmtree(self._base_dir) + + def _RunBuildman(self, *args): + return command.RunPipe([[self._buildman_pathname] + list(args)], + capture=True, capture_stderr=True) + + def _RunControl(self, *args): + sys.argv = [sys.argv[0]] + list(args) + options, args = cmdline.ParseArgs() + return control.DoBuildman(options, args, toolchains=self._toolchains, + make_func=self._HandleMake) + + def testFullHelp(self): + command.test_result = None + result = self._RunBuildman('-H') + help_file = os.path.join(self._buildman_dir, 'README') + self.assertEqual(len(result.stdout), os.path.getsize(help_file)) + self.assertEqual(0, len(result.stderr)) + self.assertEqual(0, result.return_code) + + def testHelp(self): + command.test_result = None + result = self._RunBuildman('-h') + help_file = os.path.join(self._buildman_dir, 'README') + self.assertTrue(len(result.stdout) > 1000) + self.assertEqual(0, len(result.stderr)) + self.assertEqual(0, result.return_code) + + def testGitSetup(self): + """Test gitutils.Setup(), from outside the module itself""" + command.test_result = command.CommandResult(return_code=1) + gitutil.Setup() + self.assertEqual(gitutil.use_no_decorate, False) + + command.test_result = command.CommandResult(return_code=0) + gitutil.Setup() + self.assertEqual(gitutil.use_no_decorate, True) + + def _HandleCommandGitLog(self, args): + if '-n0' in args: + return command.CommandResult(return_code=0) + + # Not handled, so abort + print 'git log', args + sys.exit(1) + + def _HandleCommandGit(self, in_args): + """Handle execution of a git command + + This uses a hacked-up parser. + + Args: + in_args: Arguments after 'git' from the command line + """ + git_args = [] # Top-level arguments to git itself + sub_cmd = None # Git sub-command selected + args = [] # Arguments to the git sub-command + for arg in in_args: + if sub_cmd: + args.append(arg) + elif arg[0] == '-': + git_args.append(arg) + else: + sub_cmd = arg + if sub_cmd == 'config': + return command.CommandResult(return_code=0) + elif sub_cmd == 'log': + return self._HandleCommandGitLog(args) + + # Not handled, so abort + print 'git', git_args, sub_cmd, args + sys.exit(1) + + def _HandleCommandNm(self, args): + return command.CommandResult(return_code=0) + + def _HandleCommandObjdump(self, args): + return command.CommandResult(return_code=0) + + def _HandleCommandSize(self, args): + return command.CommandResult(return_code=0) + + def _HandleCommand(self, **kwargs): + """Handle a command execution. + + The command is in kwargs['pipe-list'], as a list of pipes, each a + list of commands. The command should be emulated as required for + testing purposes. + + Returns: + A CommandResult object + """ + pipe_list = kwargs['pipe_list'] + if len(pipe_list) != 1: + print 'invalid pipe', kwargs + sys.exit(1) + cmd = pipe_list[0][0] + args = pipe_list[0][1:] + if cmd == 'git': + return self._HandleCommandGit(args) + elif cmd == './scripts/show-gnu-make': + return command.CommandResult(return_code=0, stdout='make') + elif cmd == 'nm': + return self._HandleCommandNm(args) + elif cmd == 'objdump': + return self._HandleCommandObjdump(args) + elif cmd == 'size': + return self._HandleCommandSize(args) + + # Not handled, so abort + print 'unknown command', kwargs + sys.exit(1) + return command.CommandResult(return_code=0) + + def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs): + """Handle execution of 'make' + + Args: + commit: Commit object that is being built + brd: Board object that is being built + stage: Stage that we are at (mrproper, config, build) + cwd: Directory where make should be run + args: Arguments to pass to make + kwargs: Arguments to pass to command.RunPipe() + """ + if stage == 'mrproper': + return command.CommandResult(return_code=0) + elif stage == 'config': + return command.CommandResult(return_code=0, + combined='Test configuration complete') + elif stage == 'build': + return command.CommandResult(return_code=0) + + # Not handled, so abort + print 'make', stage + sys.exit(1) + + def testCurrentSource(self): + """Very simple test to invoke buildman on the current source""" + self._RunControl() + lines = terminal.GetPrintTestLines() + self.assertTrue(lines[0].text.startswith('Building current source')) diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index 0e91294..27dc318 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -99,6 +99,9 @@ class Toolchains: def __init__(self): self.toolchains = {} self.paths = [] + self._make_flags = dict(bsettings.GetItems('make-flags')) + + def GetSettings(self): toolchains = bsettings.GetItems('toolchain') if not toolchains: print ("Warning: No tool chains - please add a [toolchain] section" @@ -110,7 +113,6 @@ class Toolchains: self.paths += glob.glob(value) else: self.paths.append(value) - self._make_flags = dict(bsettings.GetItems('make-flags'))
def Add(self, fname, test=True, verbose=False): """Add a toolchain to our list