It is sometimes useful to see the exact 'make' command used by buildman for a commit. Add an output file for this.

Signed-off-by: Simon Glass sjg@chromium.org ---

(no changes since v1)

tools/buildman/builderthread.py | 13 +++++++++++++ tools/buildman/buildman.rst | 8 ++++++++ tools/buildman/func_test.py | 13 +++++++++++++ 3 files changed, 34 insertions(+)

diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index 680efae02d7..7ba9a856dd5 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -273,14 +273,19 @@ class BuilderThread(threading.Thread):

# If we need to reconfigure, do that now cfg_file = os.path.join(out_dir, '.config') + cmd_list = [] if do_config or adjust_cfg: config_out = '' if self.mrproper: result = self.Make(commit, brd, 'mrproper', cwd, 'mrproper', *args, env=env) config_out += result.combined + cmd_list.append([self.builder.gnu_make, 'mrproper', + *args]) result = self.Make(commit, brd, 'config', cwd, *(args + config_args), env=env) + cmd_list.append([self.builder.gnu_make] + args + + config_args) config_out += result.combined do_config = False # No need to configure next time if adjust_cfg: @@ -290,6 +295,7 @@ class BuilderThread(threading.Thread): args.append('cfg') result = self.Make(commit, brd, 'build', cwd, *args, env=env) + cmd_list.append([self.builder.gnu_make] + args) if (result.return_code == 2 and ('Some images are invalid' in result.stderr)): # This is handled later by the check for output in @@ -303,6 +309,7 @@ class BuilderThread(threading.Thread): result.stderr = result.stderr.replace(src_dir + '/', '') if self.builder.verbose_build: result.stdout = config_out + result.stdout + result.cmd_list = cmd_list else: result.return_code = 1 result.stderr = 'No tool chain for %s\n' % brd.arch @@ -378,6 +385,12 @@ class BuilderThread(threading.Thread): with open(os.path.join(build_dir, 'out-env'), 'wb') as fd: for var in sorted(env.keys()): fd.write(b'%s="%s"' % (var, env[var])) + + with open(os.path.join(build_dir, 'out-cmd'), 'w', + encoding='utf-8') as fd: + for cmd in result.cmd_list: + print(' '.join(cmd), file=fd) + lines = [] for fname in BASE_ELF_FILENAMES: cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst index 9a2d913c785..11c72141791 100644 --- a/tools/buildman/buildman.rst +++ b/tools/buildman/buildman.rst @@ -1300,6 +1300,14 @@ You should use 'buildman -nv <criteria>' instead of greoing the boards.cfg file, since it may be dropped altogether in future.

+Checking the command +-------------------- + +Buildman writes out the toolchain information to a `toolchain` file within the +output directory. It also writes the commands used to build U-Boot in an +`out-cmd` file. You can check these if you suspect something strange is +happening. + TODO ----

diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 559e4edf74b..799c609446e 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -723,3 +723,16 @@ Some images are invalid''' control.get_allow_missing(False, False, 2, True)) self.assertEqual(False, control.get_allow_missing(False, True, 2, True)) + + def testCmdFile(self): + """Test that the -cmd-out file is produced""" + self._RunControl('-o', self._output_dir) + board0_dir = os.path.join(self._output_dir, 'current', 'board0') + self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done'))) + cmd_fname = os.path.join(board0_dir, 'out-cmd') + self.assertTrue(os.path.exists(cmd_fname)) + data = tools.read_file(cmd_fname) + lines = data.splitlines() + self.assertEqual(2, len(lines)) + self.assertRegex(lines[0], b'make O=/.*board0_defconfig') + self.assertRegex(lines[0], b'make O=/.*-s.*')

This cuts down build performance considerably and is not always needed, when checking for build errors, etc.

Add a flag to disable it.

Signed-off-by: Simon Glass sjg@chromium.org ---

(no changes since v1)

tools/buildman/builder.py | 5 ++++- tools/buildman/builderthread.py | 2 ++ tools/buildman/buildman.rst | 14 ++++++++++++++ tools/buildman/cmdline.py | 2 ++ tools/buildman/control.py | 2 +- tools/buildman/func_test.py | 25 +++++++++++++++++++++---- 6 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index c2a69027f88..107086cc0e5 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -194,6 +194,7 @@ class Builder: work_in_output: Use the output directory as the work directory and don't write to a separate output directory. thread_exceptions: List of exceptions raised by thread jobs + no_lto (bool): True to set the NO_LTO flag when building

Private members: _base_board_dict: Last-summarised Dict of boards @@ -253,7 +254,7 @@ class Builder: config_only=False, squash_config_y=False, warnings_as_errors=False, work_in_output=False, test_thread_exceptions=False, adjust_cfg=None, - allow_missing=False): + allow_missing=False, no_lto=False): """Create a new Builder object

Args: @@ -292,6 +293,7 @@ class Builder: C=val to set the value of C (val must have quotes if C is a string Kconfig allow_missing: Run build with BINMAN_ALLOW_MISSING=1 + no_lto (bool): True to set the NO_LTO flag when building

""" self.toolchains = toolchains @@ -331,6 +333,7 @@ class Builder: self.adjust_cfg = adjust_cfg self.allow_missing = allow_missing self._ide = False + self.no_lto = no_lto

if not self.squash_config_y: self.config_filenames += EXTRA_CONFIG_FILENAMES diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index 7ba9a856dd5..dae3d4ab9ff 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -255,6 +255,8 @@ class BuilderThread(threading.Thread): args.append('KCFLAGS=-Werror') if self.builder.allow_missing: args.append('BINMAN_ALLOW_MISSING=1') + if self.builder.no_lto: + args.append('NO_LTO=1') config_args = ['%s_defconfig' % brd.target] config_out = '' args.extend(self.builder.toolchains.GetMakeArguments(brd)) diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst index 11c72141791..800b83a89de 100644 --- a/tools/buildman/buildman.rst +++ b/tools/buildman/buildman.rst @@ -1123,6 +1123,20 @@ toolchain. For example: buildman -O clang-7 --board sandbox

+Building without LTO +-------------------- + +Link-time optimisation (LTO) is designed to reduce code size by globally +optimising the U-Boot build. Unfortunately this can dramatically slow down +builds. This is particularly noticeable when running a lot of builds. + +Use the -L (--no-lto) flag to disable LTO. + +.. code-block:: bash + + buildman -L --board sandbox + + Doing a simple build --------------------

diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index c485994e9fe..409013671be 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -71,6 +71,8 @@ def ParseArgs(): default=False, help="Don't convert y to 1 in configs") parser.add_option('-l', '--list-error-boards', action='store_true', default=False, help='Show a list of boards next to each error/warning') + parser.add_option('-L', '--no-lto', action='store_true', + default=False, help='Disable Link-time Optimisation (LTO) for builds') parser.add_option('--list-tool-chains', action='store_true', default=False, help='List available tool chains (use -v to see probing detail)') parser.add_option('-m', '--mrproper', action='store_true', diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 87e7d0e2012..13a462ae595 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -351,7 +351,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, work_in_output=options.work_in_output, test_thread_exceptions=test_thread_exceptions, adjust_cfg=adjust_cfg, - allow_missing=allow_missing) + allow_missing=allow_missing, no_lto=options.no_lto) builder.force_config_on_failure = not options.quick if make_func: builder.do_make = make_func diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 799c609446e..6d1557293f0 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -724,15 +724,32 @@ Some images are invalid''' self.assertEqual(False, control.get_allow_missing(False, True, 2, True))

- def testCmdFile(self): - """Test that the -cmd-out file is produced""" - self._RunControl('-o', self._output_dir) + def check_command(self, *extra_args): + """Run a command with the extra arguments and return the commands used + + Args: + extra_args (list of str): List of extra arguments + + Returns: + list of str: Lines returned in the out-cmd file + """ + self._RunControl('-o', self._output_dir, *extra_args) board0_dir = os.path.join(self._output_dir, 'current', 'board0') self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done'))) cmd_fname = os.path.join(board0_dir, 'out-cmd') self.assertTrue(os.path.exists(cmd_fname)) data = tools.read_file(cmd_fname) - lines = data.splitlines() + return data.splitlines() + + def testCmdFile(self): + """Test that the -cmd-out file is produced""" + lines = self.check_command() self.assertEqual(2, len(lines)) self.assertRegex(lines[0], b'make O=/.*board0_defconfig') self.assertRegex(lines[0], b'make O=/.*-s.*') + + def testNoLto(self): + """Test that the --no-lto flag works""" + lines = self.check_command('-L') + self.assertIn(b'NO_LTO=1', lines[0]) +

This is quite a useful thing to use when building since it avoids small size changes between commits. Add a -r flag for it.

Also undefine CONFIG_LOCALVERSION_AUTO since this appends the git hash to the version string, causing every build to be slightly different.

Signed-off-by: Simon Glass sjg@chromium.org ---

Changes in v2: - Drop CONFIG_LOCALVERSION_AUTO also

doc/build/reproducible.rst | 2 ++ tools/buildman/builder.py | 4 +++- tools/buildman/builderthread.py | 2 ++ tools/buildman/buildman.rst | 7 +++--- tools/buildman/cmdline.py | 2 ++ tools/buildman/control.py | 11 ++++++++- tools/buildman/func_test.py | 40 +++++++++++++++++++++++++++------ 7 files changed, 56 insertions(+), 12 deletions(-)

diff --git a/doc/build/reproducible.rst b/doc/build/reproducible.rst index 5423080633e..8b030f469d7 100644 --- a/doc/build/reproducible.rst +++ b/doc/build/reproducible.rst @@ -23,3 +23,5 @@ This date is shown when we launch U-Boot:

./u-boot -T U-Boot 2023.01 (Jan 01 2023 - 00:00:00 +0000) + +The same effect can be obtained with buildman using the `-r` flag. diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 107086cc0e5..7b9be887e55 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -195,6 +195,7 @@ class Builder: don't write to a separate output directory. thread_exceptions: List of exceptions raised by thread jobs no_lto (bool): True to set the NO_LTO flag when building + reproducible_builds (bool): True to set SOURCE_DATE_EPOCH=0 for builds

Private members: _base_board_dict: Last-summarised Dict of boards @@ -254,7 +255,7 @@ class Builder: config_only=False, squash_config_y=False, warnings_as_errors=False, work_in_output=False, test_thread_exceptions=False, adjust_cfg=None, - allow_missing=False, no_lto=False): + allow_missing=False, no_lto=False, reproducible_builds=False): """Create a new Builder object

Args: @@ -334,6 +335,7 @@ class Builder: self.allow_missing = allow_missing self._ide = False self.no_lto = no_lto + self.reproducible_builds = reproducible_builds

if not self.squash_config_y: self.config_filenames += EXTRA_CONFIG_FILENAMES diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index dae3d4ab9ff..8b88c68e5d2 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -257,6 +257,8 @@ class BuilderThread(threading.Thread): args.append('BINMAN_ALLOW_MISSING=1') if self.builder.no_lto: args.append('NO_LTO=1') + if self.builder.reproducible_builds: + args.append('SOURCE_DATE_EPOCH=0') config_args = ['%s_defconfig' % brd.target] config_out = '' args.extend(self.builder.toolchains.GetMakeArguments(brd)) diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst index 800b83a89de..c8b0db3d8b9 100644 --- a/tools/buildman/buildman.rst +++ b/tools/buildman/buildman.rst @@ -1023,14 +1023,15 @@ U-Boot's build system embeds information such as a build timestamp into the final binary. This information varies each time U-Boot is built. This causes various files to be rebuilt even if no source changes are made, which in turn requires that the final U-Boot binary be re-linked. This unnecessary work can -be avoided by turning off the timestamp feature. This can be achieved by -setting the SOURCE_DATE_EPOCH environment variable to 0. +be avoided by turning off the timestamp feature. This can be achieved using +the `-r` flag, which enables reproducible builds by setting +`SOURCE_DATE_EPOCH=0` when building.

Combining all of these options together yields the command-line shown below. This will provide the quickest possible feedback regarding the current content of the source tree, thus allowing rapid tested evolution of the code::

- SOURCE_DATE_EPOCH=0 ./tools/buildman/buildman -P tegra + ./tools/buildman/buildman -Pr tegra

Checking configuration diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index 409013671be..da7f1a99f6b 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -97,6 +97,8 @@ def ParseArgs(): default=False, help="Use full toolchain path in CROSS_COMPILE") parser.add_option('-P', '--per-board-out-dir', action='store_true', default=False, help="Use an O= (output) directory per board rather than per thread") + parser.add_option('-r', '--reproducible-builds', action='store_true', + help='Set SOURCE_DATE_EPOCH=0 to suuport a reproducible build') parser.add_option('-R', '--regen-board-list', action='store_true', help='Force regeneration of the list of boards, like the old boards.cfg file') parser.add_option('-s', '--summary', action='store_true', diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 13a462ae595..c3c53881998 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -338,6 +338,14 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, shutil.rmtree(output_dir) adjust_cfg = cfgutil.convert_list_to_dict(options.adjust_cfg)

+ # Drop LOCALVERSION_AUTO since it changes the version string on every commit + if options.reproducible_builds: + # If these are mentioned, leave the local version alone + if 'LOCALVERSION' in adjust_cfg or 'LOCALVERSION_AUTO' in adjust_cfg: + print('Not dropping LOCALVERSION_AUTO for reproducible build') + else: + adjust_cfg['LOCALVERSION_AUTO'] = '~' + builder = Builder(toolchains, output_dir, options.git_dir, options.threads, options.jobs, gnu_make=gnu_make, checkout=True, show_unknown=options.show_unknown, step=options.step, @@ -351,7 +359,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, work_in_output=options.work_in_output, test_thread_exceptions=test_thread_exceptions, adjust_cfg=adjust_cfg, - allow_missing=allow_missing, no_lto=options.no_lto) + allow_missing=allow_missing, no_lto=options.no_lto, + reproducible_builds=options.reproducible_builds) builder.force_config_on_failure = not options.quick if make_func: builder.do_make = make_func diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 6d1557293f0..cf91c339134 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -415,17 +415,19 @@ class TestFunctional(unittest.TestCase): kwargs: Arguments to pass to command.run_pipe() """ self._make_calls += 1 + out_dir = '' + for arg in args: + if arg.startswith('O='): + out_dir = arg[2:] if stage == 'mrproper': return command.CommandResult(return_code=0) elif stage == 'config': + fname = os.path.join(cwd or '', out_dir, '.config') + tools.write_file(fname, b'CONFIG_SOMETHING=1') return command.CommandResult(return_code=0, combined='Test configuration complete') elif stage == 'build': stderr = '' - out_dir = '' - for arg in args: - if arg.startswith('O='): - out_dir = arg[2:] fname = os.path.join(cwd or '', out_dir, 'u-boot') tools.write_file(fname, b'U-Boot')

@@ -739,17 +741,41 @@ Some images are invalid''' cmd_fname = os.path.join(board0_dir, 'out-cmd') self.assertTrue(os.path.exists(cmd_fname)) data = tools.read_file(cmd_fname) - return data.splitlines() + + config_fname = os.path.join(board0_dir, '.config') + self.assertTrue(os.path.exists(config_fname)) + cfg_data = tools.read_file(config_fname) + + return data.splitlines(), cfg_data

def testCmdFile(self): """Test that the -cmd-out file is produced""" - lines = self.check_command() + lines = self.check_command()[0] self.assertEqual(2, len(lines)) self.assertRegex(lines[0], b'make O=/.*board0_defconfig') self.assertRegex(lines[0], b'make O=/.*-s.*')

def testNoLto(self): """Test that the --no-lto flag works""" - lines = self.check_command('-L') + lines = self.check_command('-L')[0] self.assertIn(b'NO_LTO=1', lines[0])

+ def testReproducible(self): + """Test that the -r flag works""" + lines, cfg_data = self.check_command('-r') + self.assertIn(b'SOURCE_DATE_EPOCH=0', lines[0]) + + # We should see CONFIG_LOCALVERSION_AUTO unset + self.assertEqual(b'''CONFIG_SOMETHING=1 +# CONFIG_LOCALVERSION_AUTO is not set +''', cfg_data) + + with test_util.capture_sys_output() as (stdout, stderr): + lines, cfg_data = self.check_command('-r', '-a', 'LOCALVERSION') + self.assertIn(b'SOURCE_DATE_EPOCH=0', lines[0]) + + # We should see CONFIG_LOCALVERSION_AUTO unset + self.assertEqual(b'''CONFIG_SOMETHING=1 +CONFIG_LOCALVERSION=y +''', cfg_data) + self.assertIn('Not dropping LOCALVERSION_AUTO', stdout.getvalue())