At present the only straightforward way to write tests that need a slightly different configuration is to create a new board with its own configuration. This is cumbersome.
It would be useful if buildman could adjust the configuration of a build on the fly. In preparation for this, add a utility library which can modify a .config file according to various parameters passed to it.
Signed-off-by: Simon Glass sjg@chromium.org ---
(no changes since v1)
tools/buildman/cfgutil.py | 235 ++++++++++++++++++++++++++++++++++++ tools/buildman/func_test.py | 4 +- tools/buildman/test.py | 123 +++++++++++++++++++ 3 files changed, 360 insertions(+), 2 deletions(-) create mode 100644 tools/buildman/cfgutil.py
diff --git a/tools/buildman/cfgutil.py b/tools/buildman/cfgutil.py new file mode 100644 index 00000000000..4eba50868f5 --- /dev/null +++ b/tools/buildman/cfgutil.py @@ -0,0 +1,235 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2022 Google LLC +# Written by Simon Glass sjg@chromium.org +# + +"""Utility functions for dealing with Kconfig .confing files""" + +import re + +from patman import tools + +RE_LINE = re.compile(r'(# )?CONFIG_([A-Z0-9_]+)(=(.*)| is not set)') +RE_CFG = re.compile(r'(~?)(CONFIG_)?([A-Z0-9_]+)(=.*)?') + +def make_cfg_line(opt, adj): + """Make a new config line for an option + + Args: + opt (str): Option to process, without CONFIG_ prefix + adj (str): Adjustment to make (C is config option without prefix): + C to enable C + ~C to disable C + C=val to set the value of C (val must have quotes if C is + a string Kconfig) + + Returns: + str: New line to use, one of: + CONFIG_opt=y - option is enabled + # CONFIG_opt is not set - option is disabled + CONFIG_opt=val - option is getting a new value (val is + in quotes if this is a string) + """ + if adj[0] == '~': + return f'# CONFIG_{opt} is not set' + if '=' in adj: + return f'CONFIG_{adj}' + return f'CONFIG_{opt}=y' + +def adjust_cfg_line(line, adjust_cfg, done=None): + """Make an adjustment to a single of line from a .config file + + This processes a .config line, producing a new line if a change for this + CONFIG is requested in adjust_cfg + + Args: + line (str): line to process, e.g. '# CONFIG_FRED is not set' or + 'CONFIG_FRED=y' or 'CONFIG_FRED=0x123' or 'CONFIG_FRED="fred"' + adjust_cfg (dict of str): Changes to make to .config file before + building: + key: str config to change, without the CONFIG_ prefix, e.g. + FRED + value: str change to make (C is config option without prefix): + C to enable C + ~C to disable C + C=val to set the value of C (val must have quotes if C is + a string Kconfig) + done (set of set): Adds the config option to this set if it is changed + in some way. This is used to track which ones have been processed. + None to skip. + + Returns: + tuple: + str: New string for this line (maybe unchanged) + str: Adjustment string that was used + """ + out_line = line + m_line = RE_LINE.match(line) + adj = None + if m_line: + _, opt, _, _ = m_line.groups() + adj = adjust_cfg.get(opt) + if adj: + out_line = make_cfg_line(opt, adj) + if done is not None: + done.add(opt) + + return out_line, adj + +def adjust_cfg_lines(lines, adjust_cfg): + """Make adjustments to a list of lines from a .config file + + Args: + lines (list of str): List of lines to process + adjust_cfg (dict of str): Changes to make to .config file before + building: + key: str config to change, without the CONFIG_ prefix, e.g. + FRED + value: str change to make (C is config option without prefix): + C to enable C + ~C to disable C + C=val to set the value of C (val must have quotes if C is + a string Kconfig) + + Returns: + list of str: New list of lines resulting from the processing + """ + out_lines = [] + done = set() + for line in lines: + out_line, _ = adjust_cfg_line(line, adjust_cfg, done) + out_lines.append(out_line) + + for opt in adjust_cfg: + if opt not in done: + adj = adjust_cfg.get(opt) + out_line = make_cfg_line(opt, adj) + out_lines.append(out_line) + + return out_lines + +def adjust_cfg_file(fname, adjust_cfg): + """Make adjustments to a .config file + + Args: + fname (str): Filename of .config file to change + adjust_cfg (dict of str): Changes to make to .config file before + building: + key: str config to change, without the CONFIG_ prefix, e.g. + FRED + value: str change to make (C is config option without prefix): + C to enable C + ~C to disable C + C=val to set the value of C (val must have quotes if C is + a string Kconfig) + """ + lines = tools.ReadFile(fname, binary=False).splitlines() + out_lines = adjust_cfg_lines(lines, adjust_cfg) + out = '\n'.join(out_lines) + '\n' + tools.WriteFile(fname, out, binary=False) + +def convert_list_to_dict(adjust_cfg_list): + """Convert a list of config changes into the dict used by adjust_cfg_file() + + Args: + adjust_cfg_list (list of str): List of changes to make to .config file + before building. Each is one of (where C is the config option with + or without the CONFIG_ prefix) + + C to enable C + ~C to disable C + C=val to set the value of C (val must have quotes if C is + a string Kconfig + + Returns: + dict of str: Changes to make to .config file before building: + key: str config to change, without the CONFIG_ prefix, e.g. FRED + value: str change to make (C is config option without prefix): + C to enable C + ~C to disable C + C=val to set the value of C (val must have quotes if C is + a string Kconfig) + + Raises: + ValueError: if an item in adjust_cfg_list has invalid syntax + """ + result = {} + for cfg in adjust_cfg_list or []: + m_cfg = RE_CFG.match(cfg) + if not m_cfg: + raise ValueError(f"Invalid CONFIG adjustment '{cfg}'") + negate, _, opt, val = m_cfg.groups() + result[opt] = f'%s{opt}%s' % (negate or '', val or '') + + return result + +def check_cfg_lines(lines, adjust_cfg): + """Check that lines do not conflict with the requested changes + + If a line enables a CONFIG which was requested to be disabled, etc., then + this is an error. This function finds such errors. + + Args: + lines (list of str): List of lines to process + adjust_cfg (dict of str): Changes to make to .config file before + building: + key: str config to change, without the CONFIG_ prefix, e.g. + FRED + value: str change to make (C is config option without prefix): + C to enable C + ~C to disable C + C=val to set the value of C (val must have quotes if C is + a string Kconfig) + + Returns: + list of tuple: list of errors, each a tuple: + str: cfg adjustment requested + str: line of the config that conflicts + """ + bad = [] + done = set() + for line in lines: + out_line, adj = adjust_cfg_line(line, adjust_cfg, done) + if out_line != line: + bad.append([adj, line]) + + for opt in adjust_cfg: + if opt not in done: + adj = adjust_cfg.get(opt) + out_line = make_cfg_line(opt, adj) + bad.append([adj, f'Missing expected line: {out_line}']) + + return bad + +def check_cfg_file(fname, adjust_cfg): + """Check that a config file has been adjusted according to adjust_cfg + + Args: + fname (str): Filename of .config file to change + adjust_cfg (dict of str): Changes to make to .config file before + building: + key: str config to change, without the CONFIG_ prefix, e.g. + FRED + value: str change to make (C is config option without prefix): + C to enable C + ~C to disable C + C=val to set the value of C (val must have quotes if C is + a string Kconfig) + + Returns: + str: None if OK, else an error string listing the problems + """ + lines = tools.ReadFile(fname, binary=False).splitlines() + bad_cfgs = check_cfg_lines(lines, adjust_cfg) + if bad_cfgs: + out = [f'{cfg:20} {line}' for cfg, line in bad_cfgs] + content = '\n'.join(out) + return f''' +Some CONFIG adjustments did not take effect. This may be because +the request CONFIGs do not exist or conflict with others. + +Failed adjustments: + +{content} +''' + return None diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 7edbee0652f..e09ccb742e8 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -182,11 +182,11 @@ class TestFunctional(unittest.TestCase): self._buildman_pathname = sys.argv[0] self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0])) command.test_result = self._HandleCommand + bsettings.Setup(None) + bsettings.AddFile(settings_data) self.setupToolchains() self._toolchains.Add('arm-gcc', test=False) self._toolchains.Add('powerpc-gcc', test=False) - bsettings.Setup(None) - bsettings.AddFile(settings_data) self._boards = board.Boards() for brd in boards: self._boards.AddBoard(board.Board(*brd)) diff --git a/tools/buildman/test.py b/tools/buildman/test.py index b9c65c0d326..2751377e879 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -12,6 +12,7 @@ import unittest from buildman import board from buildman import bsettings from buildman import builder +from buildman import cfgutil from buildman import control from buildman import toolchain from patman import commit @@ -624,5 +625,127 @@ class TestBuild(unittest.TestCase): expected = set([os.path.join(base_dir, f) for f in to_remove]) self.assertEqual(expected, result)
+ def test_adjust_cfg_nop(self): + """check various adjustments of config that are nops""" + # enable an enabled CONFIG + self.assertEqual( + 'CONFIG_FRED=y', + cfgutil.adjust_cfg_line('CONFIG_FRED=y', {'FRED':'FRED'})[0]) + + # disable a disabled CONFIG + self.assertEqual( + '# CONFIG_FRED is not set', + cfgutil.adjust_cfg_line( + '# CONFIG_FRED is not set', {'FRED':'~FRED'})[0]) + + # use the adjust_cfg_lines() function + self.assertEqual( + ['CONFIG_FRED=y'], + cfgutil.adjust_cfg_lines(['CONFIG_FRED=y'], {'FRED':'FRED'})) + self.assertEqual( + ['# CONFIG_FRED is not set'], + cfgutil.adjust_cfg_lines(['CONFIG_FRED=y'], {'FRED':'~FRED'})) + + # handling an empty line + self.assertEqual('#', cfgutil.adjust_cfg_line('#', {'FRED':'~FRED'})[0]) + + def test_adjust_cfg(self): + """check various adjustments of config""" + # disable a CONFIG + self.assertEqual( + '# CONFIG_FRED is not set', + cfgutil.adjust_cfg_line('CONFIG_FRED=1' , {'FRED':'~FRED'})[0]) + + # enable a disabled CONFIG + self.assertEqual( + 'CONFIG_FRED=y', + cfgutil.adjust_cfg_line( + '# CONFIG_FRED is not set', {'FRED':'FRED'})[0]) + + # enable a CONFIG that doesn't exist + self.assertEqual( + ['CONFIG_FRED=y'], + cfgutil.adjust_cfg_lines([], {'FRED':'FRED'})) + + # disable a CONFIG that doesn't exist + self.assertEqual( + ['# CONFIG_FRED is not set'], + cfgutil.adjust_cfg_lines([], {'FRED':'~FRED'})) + + # disable a value CONFIG + self.assertEqual( + '# CONFIG_FRED is not set', + cfgutil.adjust_cfg_line('CONFIG_FRED="fred"' , {'FRED':'~FRED'})[0]) + + # setting a value CONFIG + self.assertEqual( + 'CONFIG_FRED="fred"', + cfgutil.adjust_cfg_line('# CONFIG_FRED is not set' , + {'FRED':'FRED="fred"'})[0]) + + # changing a value CONFIG + self.assertEqual( + 'CONFIG_FRED="fred"', + cfgutil.adjust_cfg_line('CONFIG_FRED="ernie"' , + {'FRED':'FRED="fred"'})[0]) + + # setting a value for a CONFIG that doesn't exist + self.assertEqual( + ['CONFIG_FRED="fred"'], + cfgutil.adjust_cfg_lines([], {'FRED':'FRED="fred"'})) + + def test_convert_adjust_cfg_list(self): + """Check conversion of the list of changes into a dict""" + self.assertEqual({}, cfgutil.convert_list_to_dict(None)) + + expect = { + 'FRED':'FRED', + 'MARY':'~MARY', + 'JOHN':'JOHN=0x123', + 'ALICE':'ALICE="alice"', + 'AMY':'AMY', + 'ABE':'~ABE', + 'MARK':'MARK=0x456', + 'ANNA':'ANNA="anna"', + } + actual = cfgutil.convert_list_to_dict( + ['FRED', '~MARY', 'JOHN=0x123', 'ALICE="alice"', + 'CONFIG_AMY', '~CONFIG_ABE', 'CONFIG_MARK=0x456', + 'CONFIG_ANNA="anna"']) + self.assertEqual(expect, actual) + + def test_check_cfg_file(self): + """Test check_cfg_file detects conflicts as expected""" + # Check failure to disable CONFIG + result = cfgutil.check_cfg_lines(['CONFIG_FRED=1'], {'FRED':'~FRED'}) + self.assertEqual([['~FRED', 'CONFIG_FRED=1']], result) + + result = cfgutil.check_cfg_lines( + ['CONFIG_FRED=1', 'CONFIG_MARY="mary"'], {'FRED':'~FRED'}) + self.assertEqual([['~FRED', 'CONFIG_FRED=1']], result) + + result = cfgutil.check_cfg_lines( + ['CONFIG_FRED=1', 'CONFIG_MARY="mary"'], {'MARY':'~MARY'}) + self.assertEqual([['~MARY', 'CONFIG_MARY="mary"']], result) + + # Check failure to enable CONFIG + result = cfgutil.check_cfg_lines( + ['# CONFIG_FRED is not set'], {'FRED':'FRED'}) + self.assertEqual([['FRED', '# CONFIG_FRED is not set']], result) + + # Check failure to set CONFIG value + result = cfgutil.check_cfg_lines( + ['# CONFIG_FRED is not set', 'CONFIG_MARY="not"'], + {'MARY':'MARY="mary"', 'FRED':'FRED'}) + self.assertEqual([ + ['FRED', '# CONFIG_FRED is not set'], + ['MARY="mary"', 'CONFIG_MARY="not"']], result) + + # Check failure to add CONFIG value + result = cfgutil.check_cfg_lines([], {'MARY':'MARY="mary"'}) + self.assertEqual([ + ['MARY="mary"', 'Missing expected line: CONFIG_MARY="mary"']], result) + + if __name__ == "__main__": unittest.main()