scan_configs.py is tool that allow to check how some options are used for a particular subset of platforms. The purpose is to identify the targets that are actually using one or more options of interest. For example, it can tell what targets are still using CONFIG_DM_I2_COMPAT. This is much slower that greping the configs directory but is actually more accurate as it relies on the information found in u-boot.cfg instead. It can also perform diffs between u-boot.cfg and spl/u-boot.cfg (sometimes platforms change the CONFIG options in the header file for the SPL) and diffs between .config and u-boot.cfg. This last is useful to identify options that could be moved to defconfigs.

usage: scan_configs.py [-h] [--soc SOC] [--vendor VENDOR] [--arch ARCH] [--cpu CPU] [--board BOARD] [--target TARGET] [-j JOBS] [-v] [-p] [--keep] [--nobuild] [--uboot | --spl | --spldiff | --dotconfdiff] OPTION [OPTION ...]

all filtering parameters (OPTION, vendor, arch, ...) accept regexp. ex: scan_configs.py .*DM_I2C.* --soc 'omap[2345]|k3' will match CONFIG_DM_I2C and CONFIG_DM_I2C_COMPAT and look for it only for targets using the omap2, omap3, omap4, omap5 or k3 SOCs.

Signed-off-by: Jean-Jacques Hiblot jjhiblot@ti.com

--- I wrote this tool because I was struggling keeping in mind all the platforms that were impacted by a WIP on DM_I2C_COMPAT for TI platforms. You may find it useful too.

limitations: - must be executed at the root of the source tree - the source tree must be clean (make mrproper) - only supports CSV format. visualization in terminal sould be added. But LibreOffice calc is better suited with all its ordering/ filtering capabilities.

tools/scan_configs.py | 379 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 379 insertions(+) create mode 100755 tools/scan_configs.py

diff --git a/tools/scan_configs.py b/tools/scan_configs.py new file mode 100755 index 0000000..9f9e71e --- /dev/null +++ b/tools/scan_configs.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0+ +# +# Author: JJ Hiblot jjhiblot@ti.com +# + +""" +scan the configuration of specified targets (ie defconfigs) and outputs a summary in a csv file. +Useful tool to check what platform is using a particular set of options. + +examples: + +1) Get an overview of the usage of CONFIG_DM, CONFIG_SPL_DM, and DM/I2C related + options for platforms with omap5 or k3 SOC + +$ scan_configs.py CONFIG_SPL_DM CONFIG_DM CONFIG_DM_I2C.* --vendor ti \ + --soc 'omap5|k3' > dummy.csv + +target soc vendor CONFIG_SPL_DM CONFIG_DM CONFIG_DM_I2C_COMPAT CONFIG_DM_I2C +am65x_evm_a53 k3 ti X X +am57xx_evm omap5 ti X X X X +am57xx_hs_evm omap5 ti X X X X +dra7xx_evm omap5 ti X X X X +dra7xx_hs_evm omap5 ti X X X X +omap5_uevm omap5 ti + +This shows quickly that DM is not supported at all for omap5_uevm, and that +only am65x_evm_a53 in not using DM_I2C. +Also all the other platforms use CONFIG_DM_I2C_COMPAT. + +2) Check differences in config between SPL and u-boot (--spldiff option) + +Some platforms may disable/enable stuff in the configuration header files if +in SPl. This makes it hard to know the usage of a variable by just looking at +the .config. This is specially true for DM stuff. + +$ scan_configs.py CONFIG_SPL_DM CONFIG_DM CONFIG_DM_I2C.* --vendor ti \ + --soc 'omap5|k3' --spldiff > dummy.csv + +target soc vendor CONFIG_DM_I2C_COMPAT CONFIG_DM_I2C +am65x_evm_a53 k3 ti +am57xx_evm omap5 ti U-Boot U-Boot +am57xx_hs_evm omap5 ti U-Boot U-Boot +dra7xx_evm omap5 ti U-Boot U-Boot +dra7xx_hs_evm omap5 ti U-Boot U-Boot +omap5_uevm omap5 ti + +3) Check differences between .config and u-boot.cfg (--dotconfigdiff option) + +This option can be used to help identify CONFIG options that should be moved +out of the header configuration file to the defconfig. + +$ scan_configs.py .*EEPROM.* --vendor ti --soc 'omap5|k3' --dotconfdiff > dummy.csv + +target soc vendor CONFIG_SYS_I2C_EEPROM_ADDR CONFIG_SYS_I2C_EEPROM_ADDR_LEN CONFIG_ENV_EEPROM_IS_ON_I2C +am65x_evm_a53 k3 ti +am57xx_evm omap5 ti U-Boot.cfg U-Boot.cfg U-Boot.cfg +am57xx_hs_evm omap5 ti U-Boot.cfg U-Boot.cfg U-Boot.cfg +dra7xx_evm omap5 ti U-Boot.cfg U-Boot.cfg U-Boot.cfg +dra7xx_hs_evm omap5 ti U-Boot.cfg U-Boot.cfg U-Boot.cfg +omap5_uevm omap5 ti + +""" + +import queue +import signal +import threading +import re +import shutil +import csv +import sys +import argparse +import os +from tempfile import TemporaryDirectory +from collections import OrderedDict +import multiprocessing + +left_columns = ['target','soc','vendor'] + +class cfg_parser: + """ Parser for a cfg file """ + def __init__(self, config_filters): + """ initialize the parser to look for limited set of options + + The config_filters is a list of regexp to match the options we are + interested in. + """ + self.rules = [re.compile("#define {} ".format(cfg_opt)) + for cfg_opt in config_filters] + + def parse(self, filename): + """ Read a cfg file, and for each line, check if match any of our + criteria. If so add the option and its value to a dictionary. + The result is a dictionary that looks like: + { "CONFIG_DM":"1", "CONFIG_SYS_LOAD_ADDR":"0x82000000" } + """ + dic = {} + with open(filename, 'r') as f: + lines = f.readlines() + for r in self.rules: + for l in lines: + m = r.match(l) + if m: + dic[l.split()[1]] = ' '.join(l.split()[2:]) + return dic + +class dotconfig_parser: + def __init__(self, config_filters): + """ initialize the parser to look for limited set of options + + The config_filters is a list of regexp to match the options we are + interested in. + """ + self.rules = [re.compile("{}=".format(cfg_opt)) + for cfg_opt in config_filters] + def parse(self, filename): + """ Read a .config file, and for each line, check if match any of our + criteria. If so add the option and its value to a dictionary. + The result is a dictionary that looks like: + { "CONFIG_DM":"1", "CONFIG_SYS_LOAD_ADDR":"0x82000000" } + """ + dic = {} + with open(filename, 'r') as f: + lines = f.readlines() + for r in self.rules: + for l in lines: + m = r.match(l) + if m: + cfg = l.split('=')[0] + val = l[len(cfg)+1:].strip() + dic[cfg] = val if val != "y" else "1" + return dic + +def cmp_options_dic(dic1,name1,dic2,name2): + """ Compare 2 options dictionaries. + The result is another dictionary that summarises the difference (values differ, only in dic1, only in dic2""" + s1 = set(dic1.keys()) + s2 = set(dic2.keys()) + + common_options = list(s1 & s2) + only_in_dic1 = list(s1 - s2) + only_in_dic2 = list(s2 - s1) + + diff_dic = {} + for opt in common_options: + if dic1[opt] != dic2[opt]: + diff_dic[opt] = "Diff '{}' vs '{}'".format(dic1[opt],dic2[opt]) + + for opt in only_in_dic1: + diff_dic[opt] = "{} only".format(name1) + for opt in only_in_dic2: + diff_dic[opt] = "{} only".format(name2) + return diff_dic + +class board: + def __init__(self, status, arch, cpu, soc, vendor, board, target): + self.status = status + self.arch = arch + self.cpu = cpu + self.soc = soc + self.vendor = vendor + self.board = board + self.target = target + self.result_dic = None + self.temp_dir = "./.scan_config_workdir/{}".format(self.target) + + def __repr__(self): + return "{} {} {}".format(self.target, self.vendor,self.soc) + + def cleanup(self): + """ remove the directory in which the cfg files have been built """ + shutil.rmtree(self.temp_dir) + + def match(self,rules): + """ return True if the board match all the criteria """ + for prop,r in rules: + if not r.match(getattr(self,prop)): + return False + return True + + def make_uboot_cfg(self, u_boot, spl): + """ configure for the board/target and build the required cfg files""" + u_boot_target = "u-boot.cfg" if u_boot else "" + spl_target = "spl/u-boot.cfg" if spl else "" + + r = os.system("make {}_config {} {} KBUILD_OUTPUT={} \ + > /dev/null".format(self.target, u_boot_target, + spl_target,self.temp_dir)) + + if r == signal.SIGINT: + raise KeyboardInterrupt + + if r == 0: + return True + else: + return False + + def path(self, f): + return self.temp_dir + '/' + f + +def get_boards(): + """ extract a list of boards from 'boards.cfg' """ + result = [] + if not os.path.isfile("boards.cfg"): + os.system('tools/genboardscfg.py') + + with open('boards.cfg', 'r') as f: + for l in f.readlines(): + if not l or l[0] == "#": + continue + props = l.split() + if not props: + continue + result.append(board(props[0], props[1], props[2], props[3], + props[4], props[5],props[6])) + return result + +def check_clean_directory(): + """Exit if the source tree is not clean.""" + for f in ('.config', 'include/config'): + if os.path.exists(f): + sys.exit("source tree is not clean, please run 'make mrproper'") + +def main(): + parser = argparse.ArgumentParser(description="Show CONFIG options usage") + parser.add_argument("options", metavar='OPTION', type=str, nargs='+', + help="regexp to filter on options.\ + ex: CONFIG_DM_I2C_COMPAT or '.*DM_MMC.*'") + parser.add_argument("--soc", + help="regexp to filter on SoC.\ + ex: 'omap[45]' to inspect omap5 and omap5 targets") + parser.add_argument("--vendor",help="regexp to filter on Vendor.") + parser.add_argument("--arch", help="regexp to filter on Arch") + parser.add_argument("--cpu", help="regexp to filter on CPU") + parser.add_argument("--board", help="regexp to filter on Board") + parser.add_argument("--target", + help="regexp to filter on Target (config file)") + parser.add_argument("-j","--jobs", + help="number of jobs. default to nb of cpus", + default=multiprocessing.cpu_count()) + parser.add_argument("-v","--values", + help="show the values instead of a 'X' in the csv", + action="store_true") + parser.add_argument("-p","--progress", + help="display progress information", + action="store_true") + parser.add_argument("--keep", help="Don't erase generated configs.", + action="store_true") + parser.add_argument("--nobuild", + help="Don't generate configs, use existing ones. \ + (imply --keep)", action="store_true") + group = parser.add_mutually_exclusive_group() + group.add_argument("--uboot", help="use the u-boot config (default)", + action="store_true", default=True) + group.add_argument("--spl", help="use the SPL config", + action="store_true") + group.add_argument("--spldiff", + help="show only the options that differs between \ + u-boot and SPL", action="store_true") + group.add_argument("--dotconfdiff", + help="show only the options that differs between \ + u-boot.cfg and .config", + action="store_true") + args = parser.parse_args() + + # for some reason making u-boot.cfg doesn't work well if the directory + # is not clean + check_clean_directory() + + #compile a list of regexp used to filter the targets + rules = [] + for f in ["soc","vendor","arch","cpu","board","target"]: + if getattr(args,f): + rules.append((f,re.compile("\b{}\b".format(getattr(args,f))))) + + #display the rules to filter the targets + if rules and args.progress: + print("filtering on:", file=sys.stderr) + for prop,r in rules: + print("- {}: {}".format(prop,r.pattern), file=sys.stderr) + + #get a list of boards matching the rules + boards = [b for b in get_boards() if b.match(rules)] + + #create the parsers + cfg_p = cfg_parser(args.options) + dotconfig_p = dotconfig_parser(args.options) + + def process_one_board(b): + if args.dotconfdiff: + if args.nobuild or b.make_uboot_cfg(u_boot = True, spl = True): + uboot_cfg_dic = cfg_p.parse(b.path("u-boot.cfg")) + uboot_dotconf_dic = dotconfig_p.parse(b.path(".config")) + b.result_dic = cmp_options_dic(uboot_cfg_dic,"u-boot.cfg", + uboot_dotconf_dic,".config") + elif args.spldiff: + if args.nobuild or b.make_uboot_cfg(u_boot = True, spl = True): + uboot_dic = cfg_p.parse(b.path("u-boot.cfg")) + spl_dic = cfg_p.parse(b.path("spl/u-boot.cfg")) + b.result_dic = cmp_options_dic(uboot_dic, "U-Boot", + spl_dic, "SPL") + elif args.spl: + if args.nobuild or b.make_uboot_cfg(u_boot = False, spl = True): + b.result_dic = cfg_p.parse(b.path("spl/u-boot.cfg")) + else: + if args.nobuild or b.make_uboot_cfg(u_boot = True, spl = False): + b.result_dic = cfg_p.parse(b.path("u-boot.cfg")) + def worker(): + while True: + b = q.get() + if b is None: + break + try: + if args.progress: + print("{} / {}. Processing {}".format( + len(boards) - (q.qsize() - args.jobs), + len(boards), b.target), file=sys.stderr) + process_one_board(b) + except KeyboardInterrupt: + break + if not args.keep and not args.nobuild: + b.cleanup() + q.task_done() + + q = queue.Queue() + threads = [] + for b in boards: + q.put(b) + # insert "stop worker" markers + for i in range(args.jobs): + q.put(None) + + for i in range(args.jobs): + t = threading.Thread(target=worker) + t.start() + threads.append(t) + + for t in threads: + t.join() + + # create a list of all the options matching our rules and used by at + # least one target + all_options = [] + for b in boards: + if b.result_dic: + all_options.extend(list(b.result_dic.keys())) + + + #generate the CSV file + if args.progress: + print("Generating the CSV", file=sys.stderr) + + csv_columns = list(left_columns) + csv_columns.extend(list(OrderedDict.fromkeys(all_options))) + + dict_data = [] + for b in boards: + d = dict() + if not b.result_dic: + d.update({"target": "{} build failed".format(b.target)}) + continue + for e in left_columns: + d.update({e:getattr(b,e)}) + if args.values: + d.update({k:v if len(v) else "-defined-" for (k,v) in b.result_dic.items()}) + else: + if args.spldiff or args.dotconfdiff: + d.update({k:v.split()[0] for (k,v) in b.result_dic.items()}) + else: + d.update({k:'X' for (k,v) in b.result_dic.items()}) + dict_data.append(d) + + writer = csv.DictWriter(sys.stdout, fieldnames=csv_columns) + writer.writeheader() + for data in dict_data: + writer.writerow(data) + +if __name__ == '__main__': + main()