On 15/05/2017 11:47 PM, "Simon Glass" sjg@chromium.org wrote:

Add a -b option which scans all the defconfigs and builds a database of all the CONFIG options used by each. This is useful for querying later.

At present this only works with the separate -b option, which does not move any configs. It would be possible to adjust the script to build the database automatically when moving configs, but this might not be useful as the database does not change that often.

Signed-off-by: Simon Glass sjg@chromium.org ---

tools/moveconfig.py | 81 ++++++++++++++++++++++++++++++ ++++++++++++++++++----- 1 file changed, 74 insertions(+), 7 deletions(-)

diff --git a/tools/moveconfig.py b/tools/moveconfig.py index ba40c33521..ed576f4b83 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -199,11 +199,13 @@ import glob import multiprocessing import optparse import os +import Queue import re import shutil import subprocess import sys import tempfile +import threading import time

SHOW_GNU_MAKE = 'scripts/show-gnu-make' @@ -260,6 +262,7 @@ COLOR_LIGHT_CYAN = '1;36' COLOR_WHITE = '1;37'

AUTO_CONF_PATH = 'include/config/auto.conf' +CONFIG_DATABASE = 'moveconfig.db'

### helper functions ### @@ -937,6 +940,34 @@ class KconfigParser:

return log

+ +class DatabaseThread(threading.Thread): + """This thread processes results from Slot threads. + + It collects the data in the master config directary. There is only one + result thread, and this helps to serialise the build output. + """ + def __init__(self, config_db, db_queue): + """Set up a new result thread + + Args: + builder: Builder which will be sent each result + """ + threading.Thread.__init__(self) + self.config_db = config_db + self.db_queue= db_queue + + def run(self): + """Called to start up the result thread. + + We collect the next result job and pass it on to the build. + """ + while True: + defconfig, configs = self.db_queue.get() + self.config_db[defconfig] = configs + self.db_queue.task_done() + + class Slot:

"""A slot to store a subprocess. @@ -946,7 +977,8 @@ class Slot: for faster processing. """

- def __init__(self, configs, options, progress, devnull, make_cmd, reference_src_dir): + def __init__(self, configs, options, progress, devnull, make_cmd, + reference_src_dir, db_queue): """Create a new process slot.

Arguments: @@ -957,6 +989,7 @@ class Slot: make_cmd: command name of GNU Make. reference_src_dir: Determine the true starting config state from this source tree. + db_queue: output queue to write config info for the database """ self.options = options self.progress = progress @@ -964,6 +997,7 @@ class Slot: self.devnull = devnull self.make_cmd = (make_cmd, 'O=' + self.build_dir) self.reference_src_dir = reference_src_dir + self.db_queue = db_queue self.parser = KconfigParser(configs, options, self.build_dir) self.state = STATE_IDLE self.failed_boards = set() @@ -1039,6 +1073,8 @@ class Slot: if self.current_src_dir: self.current_src_dir = None self.do_defconfig() + elif self.options.build_db: + self.do_build_db() else: self.do_savedefconfig() elif self.state == STATE_SAVEDEFCONFIG: @@ -1088,6 +1124,17 @@ class Slot: cwd=self.current_src_dir) self.state = STATE_AUTOCONF

+ def do_build_db(self): + """Add the board to the database""" + configs = {} + with open(os.path.join(self.build_dir, AUTO_CONF_PATH)) as fd: + for line in fd.readlines(): + if line.startswith('CONFIG'): + config, value = line.split('=', 1) + configs[config] = value.rstrip() + self.db_queue.put([self.defconfig, configs]) + self.finish(True) + def do_savedefconfig(self): """Update the .config and run 'make savedefconfig'."""

@@ -1170,7 +1217,7 @@ class Slots:

"""Controller of the array of subprocess slots."""

- def __init__(self, configs, options, progress, reference_src_dir): + def __init__(self, configs, options, progress, reference_src_dir, db_queue): """Create a new slots controller.

Arguments: @@ -1179,6 +1226,7 @@ class Slots: progress: A progress indicator. reference_src_dir: Determine the true starting config state from this source tree. + db_queue: output queue to write config info for the database """ self.options = options self.slots = [] @@ -1186,7 +1234,7 @@ class Slots: make_cmd = get_make_cmd() for i in range(options.jobs): self.slots.append(Slot(configs, options, progress, devnull, - make_cmd, reference_src_dir)) + make_cmd, reference_src_dir, db_queue))

def add(self, defconfig): """Add a new subprocess if a vacant slot is found. @@ -1298,7 +1346,7 @@ class ReferenceSource:

return self.src_dir

-def move_config(configs, options): +def move_config(configs, options, db_queue): """Move config options to defconfig files.

Arguments: @@ -1308,6 +1356,8 @@ def move_config(configs, options): if len(configs) == 0: if options.force_sync: print 'No CONFIG is specified. You are probably syncing defconfigs.', + elif options.build_db: + print 'Building %s database' % CONFIG_DATABASE else: print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.', else: @@ -1326,7 +1376,7 @@ def move_config(configs, options): defconfigs = get_all_defconfigs()

progress = Progress(len(defconfigs)) - slots = Slots(configs, options, progress, reference_src_dir) + slots = Slots(configs, options, progress, reference_src_dir, db_queue)

# Main loop to process defconfig files: # Add a new subprocess into a vacant slot. @@ -1353,6 +1403,8 @@ def main():

parser = optparse.OptionParser() # Add options here + parser.add_option('-b', '--build-db', action='store_true', default=False, + help='build a CONFIG database') parser.add_option('-c', '--color', action='store_true', default=False, help='display the log in color') parser.add_option('-C', '--commit', action='store_true', default=False, @@ -1385,7 +1437,7 @@ def main():

(options, configs) = parser.parse_args()

- if len(configs) == 0 and not options.force_sync: + if len(configs) == 0 and not any((options.force_sync, options.build_db)): parser.print_usage() sys.exit(1)

@@ -1395,10 +1447,17 @@ def main():

check_top_directory()

+ config_db = {} + db_queue = Queue.Queue() + t = DatabaseThread(config_db, db_queue) + t.setDaemon(True) + t.start() + if not options.cleanup_headers_only: check_clean_directory() update_cross_compile(options.color) - move_config(configs, options) + move_config(configs, options, db_queue) + db_queue.join()

if configs: cleanup_headers(configs, options) @@ -1418,5 +1477,13 @@ def main(): msg += '\n\nRsync all defconfig files using moveconfig.py' subprocess.call(['git', 'commit', '-s', '-m', msg])

+ if options.build_db: + with open(CONFIG_DATABASE, 'w') as fd: + for defconfig, configs in config_db.iteritems(): + print >>fd, '%s' % defconfig

Not sure how much we care but the python folks would say

fd.write("%s\n" % defconfig)

is more natural and will work for python 2 and 3.

+ for config in sorted(configs.keys()): + print >>fd, ' %s=%s' % (config, configs[config]) + print >>fd + if __name__ == '__main__': main() -- 2.13.0.rc2.291.g57267f2277-goog