This is not needed since we always do a full (non-incremental) build. Also it might be dangerous since it will try to delete everything below the base directory.

Fix this potentially nasty bug.

Signed-off-by: Simon Glass sjg@chromium.org ---

tools/buildman/builder.py | 2 ++ 1 file changed, 2 insertions(+)

diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 7002034..05ebfd2 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -1115,6 +1115,8 @@ class Builder: create. Having left over directories is confusing when the user wants to check the output manually. """ + if not self.commits: + return dir_list = [] for commit_upto in range(self.commit_count): dir_list.append(self._GetOutputDir(commit_upto))

When building current source for a single board, buildman puts the output in <output_dir>/current/current/<board>. Add an option to make it use <output_dir>/<board> instead. This removes the unnecessary directories in that case, controlled by the --no-subdirs/-N option.

Suggested-by: Tom Rini trini@ti.com Signed-off-by: Simon Glass sjg@chromium.org ---

tools/buildman/builder.py | 12 ++++++++---- tools/buildman/cmdline.py | 2 ++ tools/buildman/control.py | 8 ++++++-- tools/buildman/test.py | 8 ++++++++ 4 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 05ebfd2..ca74c36 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -174,7 +174,8 @@ class Builder: self.func_sizes = func_sizes

def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, - gnu_make='make', checkout=True, show_unknown=True, step=1): + gnu_make='make', checkout=True, show_unknown=True, step=1, + no_subdirs=False): """Create a new Builder object

Args: @@ -213,6 +214,7 @@ class Builder: self._step = step self.in_tree = False self._error_lines = 0 + self.no_subdirs = no_subdirs

self.col = terminal.Color()

@@ -392,15 +394,17 @@ class Builder: Args: commit_upto: Commit number to use (0..self.count-1) """ + commit_dir = None if self.commits: commit = self.commits[commit_upto] subject = commit.subject.translate(trans_valid_chars) commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, self.commit_count, commit.hash, subject[:20])) - else: + elif not self.no_subdirs: commit_dir = 'current' - output_dir = os.path.join(self.base_dir, commit_dir) - return output_dir + if not commit_dir: + return self.base_dir + return os.path.join(self.base_dir, commit_dir)

def GetBuildDir(self, commit_upto, target): """Get the name of the build directory for a commit number diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index 27d3c70..2b75653 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -55,6 +55,8 @@ def ParseArgs(): help='List available tool chains') parser.add_option('-n', '--dry-run', action='store_true', dest='dry_run', default=False, help="Do a dry run (describe actions, but do nothing)") + parser.add_option('-N', '--no-subdirs', action='store_true', dest='no_subdirs', + default=False, help="Don't create subdirectories when building current source for a single board") parser.add_option('-o', '--output-dir', type='string', dest='output_dir', default='..', help='Directory where all builds happen and buildman has its workspace (default is ../)') diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 663c4ce..df24509 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -209,12 +209,16 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, output_dir = options.output_dir if options.branch: dirname = options.branch.replace('/', '_') - output_dir = os.path.join(options.output_dir, dirname) + # As a special case allow the board directory to be placed in the + # output directory itself rather than any subdirectory. + if not options.no_subdirs: + output_dir = os.path.join(options.output_dir, dirname) if clean_dir and os.path.exists(output_dir): shutil.rmtree(output_dir) builder = Builder(toolchains, output_dir, options.git_dir, options.threads, options.jobs, gnu_make=gnu_make, checkout=True, - show_unknown=options.show_unknown, step=options.step) + show_unknown=options.show_unknown, step=options.step, + no_subdirs=options.no_subdirs) builder.force_config_on_failure = not options.quick if make_func: builder.do_make = make_func diff --git a/tools/buildman/test.py b/tools/buildman/test.py index f16d2fd..c085d2f 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -373,5 +373,13 @@ class TestBuild(unittest.TestCase): build.commit_count = 0 self.CheckDirs(build, '/current')

+ def testOutputDirNoSubdirs(self): + build = builder.Builder(self.toolchains, BASE_DIR, None, 1, 2, + checkout=False, show_unknown=False, + no_subdirs=True) + build.commits = None + build.commit_count = 0 + self.CheckDirs(build, '') + if __name__ == "__main__": unittest.main()

Adjust the -b flag to permit a range expression as well as a branch.

Signed-off-by: Simon Glass sjg@chromium.org Suggested-by: Daniel Schwierzeck daniel.schwierzeck@gmail.com ---

tools/buildman/README | 11 +++++++++++ tools/buildman/control.py | 19 +++++++++++++++---- tools/patman/gitutil.py | 24 +++++++++++++++++++----- 3 files changed, 45 insertions(+), 9 deletions(-)

diff --git a/tools/buildman/README b/tools/buildman/README index db1ec68..0500ce5 100644 --- a/tools/buildman/README +++ b/tools/buildman/README @@ -699,6 +699,17 @@ build the selected boards and display build status as it runs (i.e. -v is enabled automatically). Use -e to see errors/warnings as well.

+Building Ranges +=============== + +You can build a range of commits by specifying a range instead of a branch +when using the -b flag. For example: + + upstream/master..us-buildman + +will build commits in us-buildman that are not in upstream/master. + + Other options =============

diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 331b4f9..6a6743e 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -123,14 +123,22 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, # problems introduced by the first commit on the branch. col = terminal.Color() count = options.count + has_range = options.branch and '..' in options.branch if count == -1: if not options.branch: count = 1 else: - count, msg = gitutil.CountCommitsInBranch(options.git_dir, - options.branch) + if has_range: + count, msg = gitutil.CountCommitsInRange(options.git_dir, + options.branch) + else: + count, msg = gitutil.CountCommitsInBranch(options.git_dir, + options.branch) if count is None: sys.exit(col.Color(col.RED, msg)) + elif count == 0: + sys.exit(col.Color(col.RED, "Range '%s' has no commits" % + options.branch)) if msg: print col.Color(col.YELLOW, msg) count += 1 # Build upstream commit also @@ -172,8 +180,11 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, # to overwrite earlier ones by setting allow_overwrite=True if options.branch: if count == -1: - range_expr = gitutil.GetRangeInBranch(options.git_dir, - options.branch) + if has_range: + range_expr = options.branch + else: + range_expr = gitutil.GetRangeInBranch(options.git_dir, + options.branch) upstream_commit = gitutil.GetUpstream(options.git_dir, options.branch) series = patchstream.GetMetaDataForList(upstream_commit, diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py index 34c6b04..cc5a55a 100644 --- a/tools/patman/gitutil.py +++ b/tools/patman/gitutil.py @@ -154,6 +154,24 @@ def GetRangeInBranch(git_dir, branch, include_upstream=False): rstr = '%s%s..%s' % (upstream, '~' if include_upstream else '', branch) return rstr, msg

+def CountCommitsInRange(git_dir, range_expr): + """Returns the number of commits in the given range. + + Args: + git_dir: Directory containing git repo + range_expr: Range to check + Return: + Number of patches that exist in the supplied rangem or None if none + were found + """ + pipe = [LogCmd(range_expr, git_dir=git_dir, oneline=True)] + result = command.RunPipe(pipe, capture=True, capture_stderr=True, + raise_on_error=False) + if result.return_code: + return None, "Range '%s' not found or is invalid" % range_expr + patch_count = len(result.stdout.splitlines()) + return patch_count, None + def CountCommitsInBranch(git_dir, branch, include_upstream=False): """Returns the number of commits in the given branch.

@@ -167,11 +185,7 @@ def CountCommitsInBranch(git_dir, branch, include_upstream=False): range_expr, msg = GetRangeInBranch(git_dir, branch, include_upstream) if not range_expr: return None, msg - pipe = [LogCmd(range_expr, git_dir=git_dir, oneline=True), - ['wc', '-l']] - result = command.RunPipe(pipe, capture=True, oneline=True) - patch_count = int(result.stdout) - return patch_count, msg + return CountCommitsInRange(git_dir, range_expr)

def CountCommits(commit_range): """Returns the number of commits in the given range.

The assumption that the compiler name will always end in gcc is incorrect for clang and apparently on BSD.

Signed-off-by: Simon Glass sjg@chromium.org ---

tools/buildman/toolchain.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index 27dc318..e2a851e 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -30,7 +30,14 @@ class Toolchain: """ self.gcc = fname self.path = os.path.dirname(fname) - self.cross = os.path.basename(fname)[:-3] + + # Find the CROSS_COMPILE prefix to use for U-Boot. For example, + # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. + basename = os.path.basename(fname) + pos = basename.rfind('-') + self.cross = basename[:pos + 1] if pos != -1 else '' + + # The architecture is the first part of the name pos = self.cross.find('-') self.arch = self.cross[:pos] if pos != -1 else 'sandbox'

In some cases there may be multiple toolchains with the same name in the path. Provide an option to use the full path in the CROSS_COMPILE environment variable.

Note: Wolfgang mentioned that this is dangerous since in some cases there may be other tools on the path that are needed. So this is set up as an option, not the default. I will need test confirmation (i.e. that this commit fixes a real problem) before merging it.

Signed-off-by: Simon Glass sjg@chromium.org Suggested-by: Steve Rae srae@broadcom.com ---

tools/buildman/builder.py | 7 ++++++- tools/buildman/builderthread.py | 4 ++-- tools/buildman/cmdline.py | 2 ++ tools/buildman/control.py | 2 +- tools/buildman/toolchain.py | 20 ++++++++++++++------ 5 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index ca74c36..93d048b 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -175,7 +175,7 @@ class Builder:

def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, gnu_make='make', checkout=True, show_unknown=True, step=1, - no_subdirs=False): + no_subdirs=False, full_path=False): """Create a new Builder object

Args: @@ -189,6 +189,10 @@ class Builder: This is used for testing. show_unknown: Show unknown boards (those not built) in summary step: 1 to process every commit, n to process every nth commit + no_subdirs: Don't create subdirectories when building current + source for a single board + full_path: Return the full path in CROSS_COMPILE and don't set + PATH """ self.toolchains = toolchains self.base_dir = base_dir @@ -215,6 +219,7 @@ class Builder: self.in_tree = False self._error_lines = 0 self.no_subdirs = no_subdirs + self.full_path = full_path

self.col = terminal.Color()

diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index bc4541c..a803481 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -177,7 +177,7 @@ class BuilderThread(threading.Thread): commit = 'current'

# Set up the environment and command line - env = self.toolchain.MakeEnvironment() + env = self.toolchain.MakeEnvironment(self.builder.full_path) Mkdir(out_dir) args = [] cwd = work_dir @@ -284,7 +284,7 @@ class BuilderThread(threading.Thread): print >>fd, 'path', result.toolchain.path

# Write out the image and function size information and an objdump - env = result.toolchain.MakeEnvironment() + env = result.toolchain.MakeEnvironment(self.builder.full_path) lines = [] for fname in ['u-boot', 'spl/u-boot-spl']: cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index 2b75653..6ad376d 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -62,6 +62,8 @@ def ParseArgs(): help='Directory where all builds happen and buildman has its workspace (default is ../)') parser.add_option('-Q', '--quick', action='store_true', default=False, help='Do a rough build, with limited warning resolution') + parser.add_option('-p', '--full-path', action='store_true', + default=False, help="Use full toolchain path in CROSS_COMPILE") parser.add_option('-s', '--summary', action='store_true', default=False, help='Show a build summary') parser.add_option('-S', '--show-sizes', action='store_true', diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 6a6743e..803e1d0 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -230,7 +230,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, builder = Builder(toolchains, output_dir, options.git_dir, options.threads, options.jobs, gnu_make=gnu_make, checkout=True, show_unknown=options.show_unknown, step=options.step, - no_subdirs=options.no_subdirs) + no_subdirs=options.no_subdirs, full_path=options.full_path) builder.force_config_on_failure = not options.quick if make_func: builder.do_make = make_func diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index ab08193..cb693f4 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -41,7 +41,7 @@ class Toolchain: pos = self.cross.find('-') self.arch = self.cross[:pos] if pos != -1 else 'sandbox'

- env = self.MakeEnvironment() + env = self.MakeEnvironment(False)

# As a basic sanity check, run the C compiler with --version cmd = [fname, '--version'] @@ -81,15 +81,23 @@ class Toolchain: return prio return prio

- def MakeEnvironment(self): + def MakeEnvironment(self, full_path): """Returns an environment for using the toolchain.

- Thie takes the current environment, adds CROSS_COMPILE and - augments PATH so that the toolchain will operate correctly. + Thie takes the current environment and adds CROSS_COMPILE so that + the tool chain will operate correctly. + + Args: + full_path: Return the full path in CROSS_COMPILE and don't set + PATH """ env = dict(os.environ) - env['CROSS_COMPILE'] = self.cross - env['PATH'] = self.path + ':' + env['PATH'] + if full_path: + env['CROSS_COMPILE'] = os.path.join(self.path, self.cross) + else: + env['CROSS_COMPILE'] = self.cross + env['PATH'] = self.path + ':' + env['PATH'] + return env

This file is only partially documented. Add some more details.

Signed-off-by: Simon Glass sjg@chromium.org Suggested-by: Wolfgang Denk wd@denx.de ---

tools/buildman/README | 73 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 20 deletions(-)

diff --git a/tools/buildman/README b/tools/buildman/README index f3acf46..affcb6c 100644 --- a/tools/buildman/README +++ b/tools/buildman/README @@ -141,8 +141,8 @@ $ git clone git://git.denx.de/u-boot.git . $ git checkout -b my-branch origin/master $ # Add some commits to the branch, reading for testing

-2. Create ~/.buildman to tell buildman where to find tool chains. As an -example: +2. Create ~/.buildman to tell buildman where to find tool chains (see 'The +.buildman file' later for details). As an example:

# Buildman settings file

@@ -675,28 +675,61 @@ It is common when refactoring code for the rodata to decrease as the text size increases, and vice versa.

-Providing 'make' flags -====================== +The .buildman file +================== + +The .buildman file provides information about the available toolchains and +also allows build flags to be passed to 'make'. It consists of several +sections, with the section name in square brackets. Within each section are +a set of (tag, value) pairs. + +'[toolchain]' section + + This lists the available toolchains. The tag here doesn't matter, but + make sure it is unique. The value is the path to the toolchain. Buildman + will look in that path for a file ending in 'gcc'. It will then execute + it to check that it is a C compiler, passing only the --version flag to + it. If the return code is 0, buildman assumes that it is a valid C + compiler. It uses the first part of the name as the architecture and + strips off the last part when setting the CROSS_COMPILE environment + variable (parts are delimited with a hyphen). + + For example powerpc-linux-gcc will be noted as a toolchain for 'powerpc' + and CROSS_COMPILE will be set to powerpc-linux- when using it. + +'[toolchain-alias]' section + + This converts toolchain architecture names to U-Boot names. For example, + if an x86 toolchains is called i386-linux-gcc it will not normally be + used for architecture 'x86'. Adding 'x86: i386' to this section will + tell buildman that the i386 toolchain can be used for x86. + +'[make-flags]' section + + U-Boot's build system supports a few flags (such as BUILD_TAG) which + affect the build product. These flags can be specified in the buildman + settings file. They can also be useful when building U-Boot against other + open source software. + + [make-flags] + at91-boards=ENABLE_AT91_TEST=1 + snapper9260=${at91-boards} BUILD_TAG=442 + snapper9g45=${at91-boards} BUILD_TAG=443

-U-Boot's build system supports a few flags (such as BUILD_TAG) which affect -the build product. These flags can be specified in the buildman settings -file. They can also be useful when building U-Boot against other open source -software. + This will use 'make ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 + and 'make ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. A special + variable ${target} is available to access the target name (snapper9260 + and snapper9g20 in this case). Variables are resolved recursively. Note + that variables can only contain the characters A-Z, a-z, 0-9, hyphen (-) + and underscore (_).

-[make-flags] -at91-boards=ENABLE_AT91_TEST=1 -snapper9260=${at91-boards} BUILD_TAG=442 -snapper9g45=${at91-boards} BUILD_TAG=443 + It is expected that any variables added are dealt with in U-Boot's + config.mk file and documented in the README.

-This will use 'make ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 -and 'make ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. A special -variable ${target} is available to access the target name (snapper9260 and -snapper9g20 in this case). Variables are resolved recursively. Note that -variables can only contain the characters A-Z, a-z, 0-9, hyphen (-) and -underscore (_). + Note that you can pass ad-hoc options to the build using environment + variables, for example:

-It is expected that any variables added are dealt with in U-Boot's -config.mk file and documented in the README. + SOME_OPTION=1234 ./tools/buildman/buildman my_board

Quick Sanity Check

We should create a test setting file when running testes, not use whatever happens to be on the local machine.

Signed-off-by: Simon Glass sjg@chromium.org ---

tools/buildman/test.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/tools/buildman/test.py b/tools/buildman/test.py index c085d2f..d19f6ea 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -24,6 +24,16 @@ import commit import terminal import toolchain

+settings_data = ''' +# Buildman settings file + +[toolchain] +main: /usr/sbin + +[toolchain-alias] +x86: i386 x86_64 +''' + errors = [ '''main.c: In function 'main_loop': main.c:260:6: warning: unused variable 'joe' [-Wunused-variable] @@ -113,8 +123,11 @@ class TestBuild(unittest.TestCase): self.boards.AddBoard(board.Board(*brd)) self.boards.SelectBoards([])

+ # Add some test settings + bsettings.Setup(None) + bsettings.AddFile(settings_data) + # Set up the toolchains - bsettings.Setup() self.toolchains = toolchain.Toolchains() self.toolchains.Add('arm-linux-gcc', test=False) self.toolchains.Add('sparc-linux-gcc', test=False)

The site at https://www.kernel.org/pub/tools/crosstool/ is a convenient repository of toolchains which can be used for U-Boot. Add a feature to download and install a toolchain for a selected architecture automatically.

It isn't clear how long this site will stay in the current place and format, but we should be able to rely on bug reports if it changes.

Suggested-by: Marek Vašut marex@denx.de Suggested-by: Fabio Estevam festevam@gmail.com Signed-off-by: Simon Glass sjg@chromium.org ---

tools/buildman/README | 47 ++++++++- tools/buildman/bsettings.py | 10 ++ tools/buildman/cmdline.py | 4 + tools/buildman/control.py | 16 +++ tools/buildman/test.py | 6 ++ tools/buildman/toolchain.py | 233 ++++++++++++++++++++++++++++++++++++++++++-- 6 files changed, 303 insertions(+), 13 deletions(-)

diff --git a/tools/buildman/README b/tools/buildman/README index 093c177..d4e8ce4 100644 --- a/tools/buildman/README +++ b/tools/buildman/README @@ -173,9 +173,9 @@ to build x86 commits.

3. Make sure you have the require Python pre-requisites

-Buildman uses multiprocessing, Queue, shutil, StringIO and ConfigParser. -These should normally be available, but if you get an error like this then -you will need to obtain those modules: +Buildman uses multiprocessing, Queue, shutil, StringIO, ConfigParser and +urllib2. These should normally be available, but if you get an error like +this then you will need to obtain those modules:

ImportError: No module named multiprocessing

@@ -310,6 +310,47 @@ You can see that everything is covered, even some strange ones that won't be used (c88 and c99). This is a feature.

+5. Install new toolchains if needed + +You can download toolchains and update the [toolchain] section of the +settings file to find them. + +To make this easier, buildman can automatically download and install +toolchains from kernel.org. First list the available architectures: + +$ ./tools/buildman/buildman sandbox --fetch-arch list +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.3/ +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.2/ +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.5.1/ +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.2.4/ +Available architectures: alpha am33_2.0 arm avr32 bfin cris crisv32 frv h8300 +hppa hppa64 i386 ia64 m32r m68k mips mips64 or32 powerpc powerpc64 s390x sh4 +sparc sparc64 tilegx x86_64 xtensa + +Then pick one and download it: + +$ ./tools/buildman/buildman sandbox --fetch-arch or32 +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.3/ +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.2/ +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.5.1/ +Downloading: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.5.1//x86_64-gc... +Unpacking to: /home/sjg/.buildman-toolchains +Testing + - looking in '/home/sjg/.buildman-toolchains/gcc-4.5.1-nolibc/or32-linux/.' + - looking in '/home/sjg/.buildman-toolchains/gcc-4.5.1-nolibc/or32-linux/bin' + - found '/home/sjg/.buildman-toolchains/gcc-4.5.1-nolibc/or32-linux/bin/or32-linux-gcc' +Tool chain test: OK + +Buildman should now be set up to use your new toolchain. + +At the time of writing, U-Boot has these architectures: + + arc, arm, avr32, blackfin, m68k, microblaze, mips, nds32, nios2, openrisc + powerpc, sandbox, sh, sparc, x86 + +Of these, only arc, microblaze and nds32 are not available at kernel.org.. + + How to run it =============

diff --git a/tools/buildman/bsettings.py b/tools/buildman/bsettings.py index 9eb9b2b..b361469 100644 --- a/tools/buildman/bsettings.py +++ b/tools/buildman/bsettings.py @@ -43,3 +43,13 @@ def GetItems(section): return [] except: raise + +def SetItem(section, tag, value): + """Set an item and write it back to the settings file""" + global settings + global config_fname + + settings.set(section, tag, value) + if config_fname is not None: + with open(config_fname, 'w') as fd: + settings.write(fd) diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index 6ad376d..e884e19 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -36,6 +36,10 @@ def ParseArgs(): parser.add_option('-F', '--force-build-failures', dest='force_build_failures', action='store_true', default=False, help='Force build of previously-failed build') + parser.add_option('--fetch-arch', type='string', + help="Fetch a toolchain for architecture FETCH_ARCH ('list' to list)." + ' You can also fetch several toolchains separate by comma, or' + " 'all' to download all") parser.add_option('-g', '--git', type='string', help='Git repo containing branch to build', default='.') parser.add_option('-G', '--config-file', type='string', diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 803e1d0..6fe53cb 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -118,6 +118,22 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, print return 0

+ if options.fetch_arch: + if options.fetch_arch == 'list': + sorted_list = toolchains.ListArchs() + print 'Available architectures: %s\n' % ' '.join(sorted_list) + return 0 + else: + fetch_arch = options.fetch_arch + if fetch_arch == 'all': + fetch_arch = ','.join(toolchains.ListArchs()) + print 'Downloading toolchains: %s\n' % fetch_arch + for arch in fetch_arch.split(','): + ret = toolchains.FetchAndInstall(arch) + if ret: + return ret + return 0 + # Work out how many commits to build. We want to build everything on the # branch. We also build the upstream commit as a control so we can see # problems introduced by the first commit on the branch. diff --git a/tools/buildman/test.py b/tools/buildman/test.py index 25be43f..c0ad5d0 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -409,5 +409,11 @@ class TestBuild(unittest.TestCase): self.toolchains.Add('i386-linux-gcc', test=False) self.assertTrue(self.toolchains.Select('x86') != None)

+ def testToolchainDownload(self): + """Test that we can download toolchains""" + self.assertEqual('https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.3/x86_64-gcc...', + self.toolchains.LocateArchUrl('arm')) + + if __name__ == "__main__": unittest.main() diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index ad4df8c..d4c5d4a 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -5,11 +5,42 @@

import re import glob +from HTMLParser import HTMLParser import os +import sys +import tempfile +import urllib2

import bsettings import command

+# Simple class to collect links from a page +class MyHTMLParser(HTMLParser): + def __init__(self, arch): + """Create a new parser + + After the parser runs, self.links will be set to a list of the links + to .xz archives found in the page, and self.arch_link will be set to + the one for the given architecture (or None if not found). + + Args: + arch: Architecture to search for + """ + HTMLParser.__init__(self) + self.arch_link = None + self.links = [] + self._match = '_%s-' % arch + + def handle_starttag(self, tag, attrs): + if tag == 'a': + for tag, value in attrs: + if tag == 'href': + if value and value.endswith('.xz'): + self.links.append(value) + if self._match in value: + self.arch_link = value + + class Toolchain: """A single toolchain

@@ -20,7 +51,6 @@ class Toolchain: arch: Architecture of toolchain as determined from the first component of the filename. E.g. arm-linux-gcc becomes arm """ - def __init__(self, fname, test, verbose=False): """Create a new toolchain object.

@@ -116,18 +146,29 @@ class Toolchains: self.paths = [] self._make_flags = dict(bsettings.GetItems('make-flags'))

- def GetSettings(self): + def GetPathList(self): + """Get a list of available toolchain paths + + Returns: + List of strings, each a path to a toolchain mentioned in the + [toolchain] section of the settings file. + """ toolchains = bsettings.GetItems('toolchain') if not toolchains: print ("Warning: No tool chains - please add a [toolchain] section" " to your buildman config file %s. See README for details" % bsettings.config_fname)

+ paths = [] for name, value in toolchains: if '*' in value: - self.paths += glob.glob(value) + paths += glob.glob(value) else: - self.paths.append(value) + paths.append(value) + return paths + + def GetSettings(self): + self.paths += self.GetPathList()

def Add(self, fname, test=True, verbose=False): """Add a toolchain to our list @@ -147,6 +188,24 @@ class Toolchains: if add_it: self.toolchains[toolchain.arch] = toolchain

+ def ScanPath(self, path, verbose): + """Scan a path for a valid toolchain + + Args: + path: Path to scan + verbose: True to print out progress information + Returns: + Filename of C compiler if found, else None + """ + for subdir in ['.', 'bin', 'usr/bin']: + dirname = os.path.join(path, subdir) + if verbose: print " - looking in '%s'" % dirname + for fname in glob.glob(dirname + '/*gcc'): + if verbose: print " - found '%s'" % fname + return fname + return None + + def Scan(self, verbose): """Scan for available toolchains and select the best for each arch.

@@ -160,12 +219,9 @@ class Toolchains: if verbose: print 'Scanning for tool chains' for path in self.paths: if verbose: print " - scanning path '%s'" % path - for subdir in ['.', 'bin', 'usr/bin']: - dirname = os.path.join(path, subdir) - if verbose: print " - looking in '%s'" % dirname - for fname in glob.glob(dirname + '/*gcc'): - if verbose: print " - found '%s'" % fname - self.Add(fname, True, verbose) + fname = self.ScanPath(path, verbose) + if fname: + self.Add(fname, True, verbose)

def List(self): """List out the selected toolchains for each architecture""" @@ -264,3 +320,160 @@ class Toolchains: else: i += 1 return args + + def LocateArchUrl(self, fetch_arch): + """Find a toolchain available online + + Look in standard places for available toolchains. At present the + only standard place is at kernel.org. + + Args: + arch: Architecture to look for, or 'list' for all + Returns: + If fetch_arch is 'list', a tuple: + Machine architecture (e.g. x86_64) + List of toolchains + else + URL containing this toolchain, if avaialble, else None + """ + arch = command.OutputOneLine('uname', '-m') + base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' + versions = ['4.6.3', '4.6.2', '4.5.1', '4.2.4'] + links = [] + for version in versions: + url = '%s/%s/%s/' % (base, arch, version) + print 'Checking: %s' % url + response = urllib2.urlopen(url) + html = response.read() + parser = MyHTMLParser(fetch_arch) + parser.feed(html) + if fetch_arch == 'list': + links += parser.links + elif parser.arch_link: + return url + parser.arch_link + if fetch_arch == 'list': + return arch, links + return None + + def Download(self, url): + """Download a file to a temporary directory + + Args: + url: URL to download + Returns: + Tuple: + Temporary directory name + Full path to the downloaded archive file in that directory, + or None if there was an error while downloading + """ + print "Downloading: %s" % url + leaf = url.split('/')[-1] + tmpdir = tempfile.mkdtemp('.buildman') + response = urllib2.urlopen(url) + fname = os.path.join(tmpdir, leaf) + fd = open(fname, 'wb') + meta = response.info() + size = int(meta.getheaders("Content-Length")[0]) + done = 0 + block_size = 1 << 16 + status = '' + + # Read the file in chunks and show progress as we go + while True: + buffer = response.read(block_size) + if not buffer: + print chr(8) * (len(status) + 1), '\r', + break + + done += len(buffer) + fd.write(buffer) + status = r"%10d MiB [%3d%%]" % (done / 1024 / 1024, + done * 100 / size) + status = status + chr(8) * (len(status) + 1) + print status, + sys.stdout.flush() + fd.close() + if done != size: + print 'Error, failed to download' + os.remove(fname) + fname = None + return tmpdir, fname + + def Unpack(self, fname, dest): + """Unpack a tar file + + Args: + fname: Filename to unpack + dest: Destination directory + Returns: + Directory name of the first entry in the archive, without the + trailing / + """ + stdout = command.Output('tar', 'xvfJ', fname, '-C', dest) + return stdout.splitlines()[0][:-1] + + def TestSettingsHasPath(self, path): + """Check if builmand will find this toolchain + + Returns: + True if the path is in settings, False if not + """ + paths = self.GetPathList() + return path in paths + + def ListArchs(self): + """List architectures with available toolchains to download""" + host_arch, archives = self.LocateArchUrl('list') + re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*') + arch_set = set() + for archive in archives: + # Remove the host architecture from the start + arch = re_arch.match(archive[len(host_arch):]) + if arch: + arch_set.add(arch.group(1)) + return sorted(arch_set) + + def FetchAndInstall(self, arch): + """Fetch and install a new toolchain + + arch: + Architecture to fetch, or 'list' to list + """ + # Fist get the URL for this architecture + url = self.LocateArchUrl(arch) + if not url: + print ("Cannot find toolchain for arch '%s' - use 'list' to list" % + arch) + return 2 + home = os.environ['HOME'] + dest = os.path.join(home, '.buildman-toolchains') + if not os.path.exists(dest): + os.mkdir(dest) + + # Download the tar file for this toolchain and unpack it + tmpdir, tarfile = self.Download(url) + if not tarfile: + return 1 + print 'Unpacking to: %s' % dest, + sys.stdout.flush() + path = self.Unpack(tarfile, dest) + os.remove(tarfile) + os.rmdir(tmpdir) + print + + # Check that the toolchain works + print 'Testing' + dirpath = os.path.join(dest, path) + compiler_fname = self.ScanPath(dirpath, True) + if not compiler_fname: + print 'Could not locate C compiler - fetch failed.' + return 1 + toolchain = Toolchain(compiler_fname, True, True) + + # Make sure that it will be found by buildman + if not self.TestSettingsHasPath(dirpath): + print ("Adding 'download' to config file '%s'" % + bsettings.config_fname) + tools_dir = os.path.dirname(dirpath) + bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir) + return 0