[PATCH 00/14] buildman: Introduce some order to config fragments

From: Simon Glass <sjg@chromium.org> U-Boot supports config fragments, separate files which can be used to amend the config for an existing board. So far, buildman has not supported this feature. It is not been possible to buildman boards with fragments, nor has there been any file which describes which fragments are valid with which boards. This series attempts to improve this situation. It updates buildman in two ways: - Provides an option to pass fragment files to the buildman build - Provides a file which associates boards with supported fragments With this, extended boards are built in CI (Gitlab only). Heinrich Schuchardt (3): buildman: allow specifying configuration fragments buildman: unit test for configuration fragments doc/buildman: describe using fragments Simon Glass (11): buildman: Document the format for extended boards buildman: Add a parser for the extended-board file qconfig: Refactor to allow buildman to read the database qconfig: Rename do_build_db() qconfig: Drop args from KconfigParser qconfig: Document cmdline args used by some functions qconfig: Allow buildman to obtain the qconfig database buildman: Add a option to include extended boards buildman: Support building an extended board Provide a basic .buildman file CI: Checking building of extended boards .gitlab-ci.yml | 16 +-- configs/acpi.config | 1 + configs/extended.buildman | 12 +++ tools/buildman/board.py | 8 +- tools/buildman/boards.py | 181 +++++++++++++++++++++++++++++++- tools/buildman/builder.py | 5 +- tools/buildman/builderthread.py | 26 +++-- tools/buildman/buildman.rst | 80 +++++++++++--- tools/buildman/cmdline.py | 4 + tools/buildman/control.py | 9 +- tools/buildman/func_test.py | 104 ++++++++++++++++++ tools/buildman/test.py | 2 +- tools/qconfig.py | 94 +++++++++++++---- 13 files changed, 488 insertions(+), 54 deletions(-) create mode 100644 configs/acpi.config create mode 100644 configs/extended.buildman -- 2.43.0 base-commit: 05840f9593b476d7500e1641723a899c57b26233 branch: bui

From: Heinrich Schuchardt <heinrich.schuchardt@canonical.com> Currently we are no able to build with configuration fragments in our CI. With this patch buildman gets a new argument --fragments for passing a comma-separated list of configuration fragments to add to the board defconfigs, e.g. tools/buildman/buildman \ -o build \ -k qemu-riscv64_smode \ --fragments acpi.config Signed-off-by: Heinrich Schuchardt <heinrich.schuchardt@canonical.com> --- tools/buildman/builder.py | 5 ++++- tools/buildman/builderthread.py | 18 +++++++++++------- tools/buildman/cmdline.py | 2 ++ tools/buildman/control.py | 3 ++- tools/buildman/test.py | 2 +- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 9516e25e215..3511dba4e71 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -1789,7 +1789,8 @@ class Builder: shutil.rmtree(dirname) terminal.print_clear() - def build_boards(self, commits, board_selected, keep_outputs, verbose): + def build_boards(self, commits, board_selected, keep_outputs, verbose, + fragments): """Build all commits for a list of boards Args: @@ -1798,6 +1799,7 @@ class Builder: value is Board object keep_outputs: True to save build output files verbose: Display build results as they are completed + fragments (str): config fragments added to defconfig Returns: Tuple containing: - number of boards that failed to build @@ -1827,6 +1829,7 @@ class Builder: job.keep_outputs = keep_outputs job.work_in_output = self.work_in_output job.adjust_cfg = self.adjust_cfg + job.fragments = fragments job.step = self._step if self.num_threads: self.queue.put(job) diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index a249174f158..a037c23fb81 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -389,7 +389,7 @@ class BuilderThread(threading.Thread): def _config_and_build(self, commit_upto, brd, work_dir, do_config, mrproper, config_only, adjust_cfg, commit, out_dir, out_rel_dir, - result): + fragments, result): """Do the build, configuring first if necessary Args: @@ -404,6 +404,7 @@ class BuilderThread(threading.Thread): out_dir (str): Output directory for the build, or None to use current out_rel_dir (str): Output directory relatie to the current dir + fragments (str): config fragments added to defconfig result (CommandResult): Previous result Returns: @@ -420,6 +421,8 @@ class BuilderThread(threading.Thread): args, cwd, src_dir = self._build_args(brd, out_dir, out_rel_dir, work_dir, commit_upto) config_args = [f'{brd.target}_defconfig'] + if fragments != None: + config_args.extend(fragments.split(',')) config_out = io.StringIO() _remove_old_outputs(out_dir) @@ -458,7 +461,7 @@ class BuilderThread(threading.Thread): def run_commit(self, commit_upto, brd, work_dir, do_config, mrproper, config_only, force_build, force_build_failures, - work_in_output, adjust_cfg): + work_in_output, adjust_cfg, fragments): """Build a particular commit. If the build is already done, and we are not forcing a build, we skip @@ -483,6 +486,7 @@ class BuilderThread(threading.Thread): ~C to disable C C=val to set the value of C (val must have quotes if C is a string Kconfig + fragments (str): config fragments added to defconfig Returns: tuple containing: @@ -512,7 +516,7 @@ class BuilderThread(threading.Thread): result, do_config = self._config_and_build( commit_upto, brd, work_dir, do_config, mrproper, config_only, adjust_cfg, commit, out_dir, out_rel_dir, - result) + fragments, result) result.already_done = False result.toolchain = self.toolchain @@ -710,7 +714,7 @@ class BuilderThread(threading.Thread): self.builder.config_only, force_build or self.builder.force_build, self.builder.force_build_failures, - job.work_in_output, job.adjust_cfg) + job.work_in_output, job.adjust_cfg, job.fragments) failed = result.return_code or result.stderr did_config = do_config if failed and not do_config and not self.mrproper: @@ -721,7 +725,7 @@ class BuilderThread(threading.Thread): brd, work_dir, True, self.mrproper or self.builder.fallback_mrproper, False, True, False, job.work_in_output, - job.adjust_cfg) + job.adjust_cfg, job.fragments) did_config = True if not self.builder.force_reconfig: do_config = request_config @@ -767,14 +771,14 @@ class BuilderThread(threading.Thread): result, request_config = self.run_commit(None, brd, work_dir, True, self.mrproper, self.builder.config_only, True, self.builder.force_build_failures, job.work_in_output, - job.adjust_cfg) + job.adjust_cfg, job.fragments) failed = result.return_code or result.stderr if failed and not self.mrproper: result, request_config = self.run_commit(None, brd, work_dir, True, self.builder.fallback_mrproper, self.builder.config_only, True, self.builder.force_build_failures, - job.work_in_output, job.adjust_cfg) + job.work_in_output, job.adjust_cfg, job.fragments) result.commit_upto = 0 self._write_result(result, job.keep_outputs, job.work_in_output) diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index 9236d6187cf..e194712c37f 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -63,6 +63,8 @@ def add_upto_m(parser): help="Fetch a toolchain for architecture FETCH_ARCH ('list' to list)." ' You can also fetch several toolchains separate by comma, or' " 'all' to download all") + parser.add_argument('--fragments', type=str, + help="Comma separated list of configuration fragments to be applied") parser.add_argument( '--full-check', action='store_true', help='Check maintainer entries and TARGET configs') diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 4dedd333551..69ee4074273 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -564,7 +564,8 @@ def run_builder(builder, commits, board_selected, args): builder.show_summary(commits, board_selected) else: fail, warned, excs = builder.build_boards( - commits, board_selected, args.keep_outputs, args.verbose) + commits, board_selected, args.keep_outputs, args.verbose, + args.fragments) if excs: return 102 if fail: diff --git a/tools/buildman/test.py b/tools/buildman/test.py index 7ee9496ffb3..735203e934c 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -237,7 +237,7 @@ class TestBuild(unittest.TestCase): # Build the boards for the pre-defined commits and warnings/errors # associated with each. This calls our Make() to inject the fake output. build.build_boards(self.commits, board_selected, keep_outputs=False, - verbose=False) + verbose=False, fragments='') lines = terminal.get_print_test_lines() count = 0 for line in lines: -- 2.43.0

From: Heinrich Schuchardt <heinrich.schuchardt@canonical.com> Check that configuration fragments provided to buildman as comma- separated list are passed on to the make command of the builder. In the %_defconfig: target of scripts/kconfig/Makefile the defconfig file and the fragments are combined via the gcc preprocessor. Modeling this step in the buildman testbench without invoking the actual Makefile would not provide better test coverage. Reviewed-by: Simon Glass <sjg@chromium.org> Signed-off-by: Heinrich Schuchardt <heinrich.schuchardt@canonical.com> --- tools/buildman/func_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 51c6855420e..7e837fe075e 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -798,6 +798,21 @@ Some images are invalid''' lines = self.check_command('-L')[0] self.assertIn(b'NO_LTO=1', lines[0]) + def testFragments(self): + """Test passing of configuration fragments to the make command""" + # Single fragment passed as argument + extra_args = ['board0', '--fragments', 'f1.config'] + lines, cfg_data = self.check_command(*extra_args) + self.assertRegex(lines[0].decode('utf-8'), + r'make O=/.*board0_defconfig\s+f1\.config', + 'Test single fragment') + # Multiple fragments passed as comma-separated list + extra_args = ['board0', '--fragments', 'f1.config,f2.config'] + lines, cfg_data = self.check_command(*extra_args) + self.assertRegex(lines[0].decode('utf-8'), + r'make O=/.*board0_defconfig\s+f1\.config\s+f2\.config', + 'Test multiple fragments') + def testReproducible(self): """Test that the -r flag works""" lines, cfg_data = self.check_command('-r') -- 2.43.0

From: Heinrich Schuchardt <heinrich.schuchardt@canonical.com> Describe the new --fragments parameter Signed-off-by: Heinrich Schuchardt <heinrich.schuchardt@canonical.com> --- tools/buildman/buildman.rst | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst index 8c45a841024..df9e19c8a0f 100644 --- a/tools/buildman/buildman.rst +++ b/tools/buildman/buildman.rst @@ -1126,27 +1126,23 @@ later comparison. defconfig fragments ------------------- -Buildman provides some initial support for configuration fragments. It can scan -these when present in defconfig files and handle the resuiting Kconfig -correctly. Thus it is possible to build a board which has a ``#include`` in the -defconfig file. +A comma-separated list of configuration fragments can be added via the +--fragments parameter to all processed defconfig files, e.g. -For now, Buildman simply includes the files to produce a single output file, -using the C preprocessor. It does not call the ``merge_config.sh`` script. The -redefined/redundant logic in that script could fairly easily be repeated in -Buildman, to detect potential problems. For now it is not clear that this is -useful. +.. code-block:: bash -To specify the C preprocessor to use, set the ``CPP`` environment variable. The -default is ``cpp``. + buildman -k qemu-riscv64_smode --fragments acpi.config -Note that Buildman does not support adding fragments to existing boards, e.g. -like:: +Buildman invokes ``make`` passing the defconfig file and the fragment files as +target arguments. In ``scripts/kconfig/Makefile`` the script ``merge_config.sh`` +is called for each fragment file to add it to the configuration. - make qemu_riscv64_defconfig acpi.config +Buildman also supports configuration fragments that are included in defconfig +files via an ``#include`` statement. It can scan these and handle the resulting +Kconfig correctly. -This is partly because there is no way for Buildman to know which fragments are -valid on which boards. +To specify the C preprocessor to use, set the ``CPP`` environment variable. The +default is ``cpp``. Building with clang ------------------- -- 2.43.0

From: Simon Glass <sjg@chromium.org> It has become more common to use config fragments to extend or adjust the functionality of boards in U-Boot. Add some documentation for how to use this with buildman. Signed-off-by: Simon Glass <sjg@chromium.org> --- tools/buildman/buildman.rst | 56 +++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst index df9e19c8a0f..487e9d67a4b 100644 --- a/tools/buildman/buildman.rst +++ b/tools/buildman/buildman.rst @@ -1144,6 +1144,62 @@ Kconfig correctly. To specify the C preprocessor to use, set the ``CPP`` environment variable. The default is ``cpp``. +Specifying the build matrix with fragments +------------------------------------------ + +In order to build boards which can use fragments, Buildman needs to know which +fragments are valid with which boards. + +In ``defconfig/``, files with a '.buildman' suffix are used to effectively +create new boards for Buildman to build. All such files are processed, but it +might be best to put all the information in a single file for now, e.g. +``extended.buildman``. + +The syntax consists of a number of sections, each introduced by a name. For each +section the fragment file is named. This name cannot include spaces. When +applied to a board, a new 'extended' board is created, its name consisting of +the original name, with one or more of these names prepended, with a comma +between each. For example, if the base board is `am62x_beagleplay_a53`, with +a name of `am62x_dfu` an extended board called `am62x_dfu,am62x_beagleplay_a53` +is created, which can be selected by buildman just like a normal board. + +A `desc` field provides a human-readable description, ignored by buildman. + +The behaviour of the extented config must be specified. First, the fragments +which make it up must be listed, on separate `fragment: <config>' lines, where +<config> specifies the fragment file in the defconfigs directory, with an +implied `.config` suffix. So a fragment of `acpi` indicates that +`configs/acpi.config` should be added to the base defconfig for the board. +Multiple fragments can be specified. When building, fragments are applied in +the order they are specified. + +Following the fragments, the targets which can accept that fragment are +specified, either by their board name, with wildcards, or a set of ``CONFIG`` +options to check. All ``CONFIG`` options must match for a board to be included +in the set. The syntax is `CONFIG_<config>=<value>` where `<config>` is the +Kconfig name and `<value>` is the value. Strings must be quoted. For boolean +options, use a value of `y` or `n`. + +For example:: + + name: acpi_boards + desc: Build RISC-V QEMU builds with ACPI + fragment: acpi + targets: + qemu_riscv* + + name: am62x_dfu + desc: Build Android variant of 'k3' boards, with DFU + fragment: am62x_r5_usbdfu + fragment: am62x_a53_android + targets: + CONFIG_SYS_SOC="k3" + +Buildman normally ignores these files. To request that Buildman process these +extended new 'boards', use the ``-X / --extend`` option. Note that this may +significantly increase the number of boards which Buildman builds, so you may +need to add additional terms to limit this. + Building with clang ------------------- -- 2.43.0

From: Simon Glass <sjg@chromium.org> We wish to support .buildman files in the config/ directory which can associate boards with config fragments. Add a parser for this. Signed-off-by: Simon Glass <sjg@chromium.org> --- tools/buildman/boards.py | 89 ++++++++++++++++++++++++++++++++++++- tools/buildman/func_test.py | 89 +++++++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 1 deletion(-) diff --git a/tools/buildman/boards.py b/tools/buildman/boards.py index 2fe43c3fc89..99d9c5fb9bd 100644 --- a/tools/buildman/boards.py +++ b/tools/buildman/boards.py @@ -5,7 +5,7 @@ """Maintains a list of boards and allows them to be selected""" -from collections import OrderedDict +from collections import OrderedDict, namedtuple import errno import fnmatch import glob @@ -35,6 +35,8 @@ COMMENT_BLOCK = f'''# ''' +Extended = namedtuple('Extended', 'name,desc,fragments,targets') + def try_remove(fname): """Remove a file ignoring 'No such file or directory' error. @@ -903,3 +905,88 @@ class Boards: print(warn, file=sys.stderr) self.format_and_output(params_list, output) return not warnings + + +class ExtendedParser: + """Parser for extended-board (.buildman) files""" + def __init__(self): + self.extended = [] + self.name = None + self.fragments = [] + self.targets = [] + self.in_targets = False + self.desc = None + + def start(self): + """Start a new extended board""" + self.name = None + self.fragments = [] + self.targets = [] + self.in_targets = False + self.desc = None + + def finish(self): + """Finish any pending extended board""" + if self.name: + self.extended.append(Extended(self.name, self.desc, self.fragments, + self.targets)) + self.start() + + @staticmethod + def parse_file(fname): + """Parse a file and return the result""" + return ExtendedParser.parse_data(fname, + tools.read_file(fname, binary=False)) + + @staticmethod + def parse_data(fname, data): + """Parse a file and return the result""" + parser = ExtendedParser() + parser.parse(fname, data) + return parser.extended + + def parse(self, fname, data): + """Parse the file""" + self.start() + for seq, line in enumerate(data.splitlines()): + linenum = seq + 1 + if not line.strip() or line[0] == '#': + continue + if line[0] == ' ': + if not self.in_targets: + raise ValueError(f'{fname}:{linenum}: Unexpected indent') + if '=' in line: + pair = line.split('=') + if len(pair) != 2: + raise ValueError(f'{fname}:{linenum}: Invalid CONFIG syntax') + first, rest = pair + cfg = first.strip() + value = rest.strip() + self.targets.append([cfg, value]) + else: + target = line.strip() + if ' ' in target: + raise ValueError(f'{fname}:{linenum}: Invalid target regex') + self.targets.append(['regex', line.strip()]) + else: + pair = line.split(':') + if len(pair) != 2: + raise ValueError(f'{fname}:{linenum}: Invalid tag') + tag, rest = pair + value = rest.strip() + if tag == 'name': + self.finish() + if ' ' in value: + raise ValueError(f'{fname}:{linenum}: Invalid name') + self.name = value + elif tag == 'desc': + self.desc = value + elif tag == 'fragment': + self.fragments.append(value) + elif tag == 'targets': + self.in_targets = True + else: + raise ValueError(f"{fname}:{linenum}: Unknown tag '{tag}'") + + self.finish() + return self.extended diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 7e837fe075e..968765b713c 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -14,6 +14,7 @@ import unittest from buildman import board from buildman import boards +from buildman.boards import Extended from buildman import bsettings from buildman import cmdline from buildman import control @@ -1177,3 +1178,91 @@ CONFIG_SOC="fred" # It should appear at the end of the build line self.assertEqual(b'u-boot.dtb', lines[1].split()[-1]) + + def test_extended(self): + """Test parsing of extended (.buildman) files""" + data = ''' +name: acpi_boards +desc: Build RISC-V QEMU builds with ACPI +fragment: acpi +targets: + qemu_riscv* + +name: am62x_dfu +desc: Build Android variant of 'k3' boards, with DFU +fragment: am62x_r5_usbdfu +fragment: am62x_a53_android +targets: + CONFIG_SYS_SOC="k3" +''' + fname = os.path.join(self._base_dir, 'try.buildman') + tools.write_file(fname, data.encode('utf-8')) + result = boards.ExtendedParser.parse_file(fname) + self.maxDiff = None + self.assertEqual([ + Extended(name='acpi_boards', + desc='Build RISC-V QEMU builds with ACPI', + fragments=['acpi'], + targets=[ + ['regex', 'qemu_riscv*']]), + Extended(name='am62x_dfu', + desc="Build Android variant of 'k3' boards, with DFU", + fragments=['am62x_r5_usbdfu', 'am62x_a53_android'], + targets=[ + ['CONFIG_SYS_SOC', '"k3"']] + )], result) + + def test_extended_bad_indent(self): + """Test unexpected indentation""" + with self.assertRaises(ValueError) as exc: + boards.ExtendedParser.parse_data('mary', ' name: fred') + self.assertEqual('mary:1: Unexpected indent', + str(exc.exception)) + + def test_extended_invalid_config(self): + """Test a bad CONFIG_xxx= line""" + with self.assertRaises(ValueError) as exc: + boards.ExtendedParser.parse_data('anna', ''' +name: joan +targets: + CONFIG_SOMETHING=val=" +''') + self.assertEqual('anna:4: Invalid CONFIG syntax', + str(exc.exception)) + + def test_extended_invalid_target(self): + """Test a bad target regex""" + with self.assertRaises(ValueError) as exc: + boards.ExtendedParser.parse_data('john', ''' +name: fred +targets: + qemu* *riscv +''') + self.assertEqual('john:4: Invalid target regex', + str(exc.exception)) + + def test_extended_invalid_tag(self): + """Test a bad tag""" + with self.assertRaises(ValueError) as exc: + boards.ExtendedParser.parse_data('hannah', ''' +name: frank :was here +''') + self.assertEqual('hannah:2: Invalid tag', + str(exc.exception)) + + def test_extended_unknown_tag(self): + """Test a unknown tag""" + with self.assertRaises(ValueError) as exc: + boards.ExtendedParser.parse_data('susan', ''' +name: allan +something: me +''') + self.assertEqual("susan:3: Unknown tag 'something'", + str(exc.exception)) + + def test_extended_bad_name(self): + """Test a name with an invalid char""" + with self.assertRaises(ValueError) as exc: + boards.ExtendedParser.parse_data('bert', 'name: katie was here') + self.assertEqual('bert:1: Invalid name', + str(exc.exception)) -- 2.43.0

From: Simon Glass <sjg@chromium.org> The qconfig tool has the ability to search for CONFIGs used by boards. Refactor the code slightly so that buildman obtain the database. Signed-off-by: Simon Glass <sjg@chromium.org> --- tools/qconfig.py | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/tools/qconfig.py b/tools/qconfig.py index d2ba77bbfce..0b489b7e69f 100755 --- a/tools/qconfig.py +++ b/tools/qconfig.py @@ -1118,23 +1118,21 @@ def defconfig_matches(configs, re_match, re_val): return True return False -def do_find_config(config_list, list_format): - """Find boards with a given combination of CONFIGs +def find_config(dbase, config_list): + """Find all defconfigs which match a config list Args: config_list (list of str): List of CONFIG options to check (each a regex consisting of a config option, with or without a CONFIG_ prefix. If an option is preceded by a tilde (~) then it must be false, otherwise it must be true) - list_format (bool): True to write in 'list' format, one board name per - line - Returns: - int: exit code (0 for success) + Return: + set: matching defconfig, without the '_defconfig' suffix """ - _, all_defconfigs, config_db, _ = read_database() - # Start with all defconfigs + _, all_defconfigs, config_db, _ = dbase + out = all_defconfigs # Work through each config in turn @@ -1161,10 +1159,31 @@ def do_find_config(config_list, list_format): has_cfg = defconfig_matches(config_db[defc], re_match, re_val) if has_cfg == want: out.add(defc) + + result = {c.split('_defconfig')[0] for c in out} + + return result + +def do_find_config(config_list, list_format): + """Find boards with a given combination of CONFIGs + + Args: + config_list (list of str): List of CONFIG options to check (each a regex + consisting of a config option, with or without a CONFIG_ prefix. If + an option is preceded by a tilde (~) then it must be false, + otherwise it must be true) + list_format (bool): True to write in 'list' format, one board name per + line + + Returns: + int: exit code (0 for success) + """ + dbase = read_database() + out = find_config(dbase, config_list) if not list_format: print(f'{len(out)} matches') sep = '\n' if list_format else ' ' - print(sep.join(item.split('_defconfig')[0] for item in sorted(list(out)))) + print(sep.join(sorted(list(out)))) return 0 -- 2.43.0

From: Simon Glass <sjg@chromium.org> This function does not actually build the database; it just adds a board to the datbase. Rename it to do_add_to_db() to reflect that. Signed-off-by: Simon Glass <sjg@chromium.org> --- tools/qconfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/qconfig.py b/tools/qconfig.py index 0b489b7e69f..4390f61468c 100755 --- a/tools/qconfig.py +++ b/tools/qconfig.py @@ -471,7 +471,7 @@ class Slot: self.current_src_dir = None self.do_defconfig() elif self.args.build_db: - self.do_build_db() + self.do_add_to_db() else: self.do_savedefconfig() elif self.state == STATE_SAVEDEFCONFIG: @@ -525,7 +525,7 @@ class Slot: cwd=self.current_src_dir) self.state = STATE_AUTOCONF - def do_build_db(self): + def do_add_to_db(self): """Add the board to the database""" configs = {} for line in read_file(os.path.join(self.build_dir, AUTO_CONF_PATH)): -- 2.43.0

From: Simon Glass <sjg@chromium.org> This class does not actually use the program arguments, so don't pass them in. Signed-off-by: Simon Glass <sjg@chromium.org> --- tools/qconfig.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/qconfig.py b/tools/qconfig.py index 4390f61468c..81dc80169ae 100755 --- a/tools/qconfig.py +++ b/tools/qconfig.py @@ -290,14 +290,12 @@ class KconfigParser: re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') - def __init__(self, args, build_dir): + def __init__(self, build_dir): """Create a new parser. Args: - args (Namespace): program arguments build_dir: Build directory. """ - self.args = args self.dotconfig = os.path.join(build_dir, '.config') self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') self.spl_autoconf = os.path.join(build_dir, 'spl', 'include', @@ -392,7 +390,7 @@ class Slot: self.reference_src_dir = reference_src_dir self.db_queue = db_queue self.col = progress.col - self.parser = KconfigParser(args, self.build_dir) + self.parser = KconfigParser(self.build_dir) self.state = STATE_IDLE self.failed_boards = set() self.defconfig = None -- 2.43.0

From: Simon Glass <sjg@chromium.org> It might better to use separate arguments for each item, but for now, document which args are used by Slot, Slots and move_config() Signed-off-by: Simon Glass <sjg@chromium.org> --- tools/qconfig.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/qconfig.py b/tools/qconfig.py index 81dc80169ae..177ffa59111 100755 --- a/tools/qconfig.py +++ b/tools/qconfig.py @@ -372,7 +372,8 @@ class Slot: Args: toolchains: Toolchains object containing toolchains. - args: Program arguments + args: Program arguments; this class uses build_db, verbose, + force_sync, dry_run, exit_on_error progress: A progress indicator. devnull: A file object of '/dev/null'. make_cmd: command name of GNU Make. @@ -615,7 +616,8 @@ class Slots: Args: toolchains (Toolchains): Toolchains object containing toolchains - args (Namespace): Program arguments + args (Namespace): Program arguments; this class uses build_db, + verbose, force_sync, dry_run, exit_on_error, jobs, progress (Progress): A progress indicator. reference_src_dir (str): Determine the true starting config state from this source tree (None for none) @@ -719,7 +721,9 @@ def move_config(args): """Build database or sync config options to defconfig files. Args: - args (Namespace): Program arguments + args (Namespace): Program arguments; this class uses build_db, + verbose, force_sync, dry_run, exit_on_error, jobs, git_ref, + defconfigs, defconfiglist, nocolour Returns: tuple: -- 2.43.0

From: Simon Glass <sjg@chromium.org> Provided a convenient function for buildman to use, which builds the database if needed, then returns it. Signed-off-by: Simon Glass <sjg@chromium.org> --- tools/qconfig.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/tools/qconfig.py b/tools/qconfig.py index 177ffa59111..eca6f5b1e1c 100755 --- a/tools/qconfig.py +++ b/tools/qconfig.py @@ -9,7 +9,7 @@ Author: Masahiro Yamada <yamada.masahiro@socionext.com> Author: Simon Glass <sjg@chromium.org> """ -from argparse import ArgumentParser +from argparse import ArgumentParser, Namespace import collections from contextlib import ExitStack import doctest @@ -1717,6 +1717,41 @@ def do_tests(): return 0 +def ensure_database(threads): + """Return a qconfig database so that Kconfig options can be queried + + If a database exists, it is assumed to be up-to-date. If not, one is built, + which can take a few minutes. + + Args: + threads (int): Number of threads to use when processing + + Returns: + tuple: + set of all config options seen (each a str) + set of all defconfigs seen (each a str) + dict of configs for each defconfig: + key: defconfig name, e.g. "MPC8548CDS_legacy_defconfig" + value: dict: + key: CONFIG option + value: Value of option + dict of defconfigs for each config: + key: CONFIG option + value: set of boards using that option + """ + if not os.path.exists(CONFIG_DATABASE): + print('Building qconfig.db database') + args = Namespace(build_db=True, verbose=False, force_sync=False, + dry_run=False, exit_on_error=False, jobs=threads, + git_ref=None, defconfigs=None, defconfiglist=None, + nocolour=False) + config_db, progress = move_config(args) + + write_db(config_db, progress) + + return read_database() + + def main(): """Main program""" parser, args = parse_args() -- 2.43.0

From: Simon Glass <sjg@chromium.org> Provide a new -X option which uses any available .buildman files to augment the list of boards supported by U-Boot with 'extended boards', which consist of a base board with one or more config fragments. Signed-off-by: Simon Glass <sjg@chromium.org> --- tools/buildman/board.py | 8 +++- tools/buildman/boards.py | 94 ++++++++++++++++++++++++++++++++++++++- tools/buildman/cmdline.py | 2 + tools/buildman/control.py | 6 +++ 4 files changed, 108 insertions(+), 2 deletions(-) diff --git a/tools/buildman/board.py b/tools/buildman/board.py index 248d8bfff18..c061bf56039 100644 --- a/tools/buildman/board.py +++ b/tools/buildman/board.py @@ -6,7 +6,8 @@ class Board: """A particular board that we can build""" - def __init__(self, status, arch, cpu, soc, vendor, board_name, target, cfg_name): + def __init__(self, status, arch, cpu, soc, vendor, board_name, target, + cfg_name, extended=None, orig_target=None): """Create a new board type. Args: @@ -18,8 +19,11 @@ class Board: board_name: Name of board (e.g. integrator) target: Target name (use make <target>_defconfig to configure) cfg_name: Config-file name (in includes/configs/) + extended (boards.Extended): Extended board, if this board is one + orig_target (str): Name of target this extended board is based on """ self.target = target + self.status = status self.arch = arch self.cpu = cpu self.soc = soc @@ -29,3 +33,5 @@ class Board: self.props = [self.target, self.arch, self.cpu, self.board_name, self.vendor, self.soc, self.cfg_name] self.build_it = False + self.extended = extended + self.orig_target = orig_target diff --git a/tools/buildman/boards.py b/tools/buildman/boards.py index 99d9c5fb9bd..ce6cafe975e 100644 --- a/tools/buildman/boards.py +++ b/tools/buildman/boards.py @@ -19,6 +19,7 @@ import time from buildman import board from buildman import kconfiglib +import qconfig from u_boot_pylib import command from u_boot_pylib.terminal import print_clear, tprint from u_boot_pylib import tools @@ -906,6 +907,92 @@ class Boards: self.format_and_output(params_list, output) return not warnings + def parse_all_extended(self, dbase): + """Parse any .buildman files to find boards composed of fragments + + Args: + dbase (tuple): + set of all config options seen (each a str) + set of all defconfigs seen (each a str) + dict of configs for each defconfig: + key: defconfig name, e.g. "MPC8548CDS_legacy_defconfig" + value: dict: + key: CONFIG option + value: Value of option + dict of defconfigs for each config: + key: CONFIG option + value: set of boards using that option + """ + for fname in glob.glob('configs/*.buildman'): + self.parse_extended(dbase, fname) + + def find_by_target(self, target): + """Find a board given its target name + + Args: + target (str): Target string to search for + + Return: + Board: board found + + Raises: + ValueError: Board was not found + """ + for b in self._boards: + if b.target == target: + return b + raise ValueError(f"Board '{target}' not found") + + def parse_extended(self, dbase, fname): + """Parse a single 'extended' file""" + result = ExtendedParser.parse_file(fname) + for ext in result: + ext_boards = self.scan_extended(dbase, ext) + for name in ext_boards: + # Find the base board + brd = self.find_by_target(name) + newb = board.Board(brd.status, brd.arch, brd.cpu, brd.soc, + brd.vendor, brd.board_name, + f'{ext.name},{brd.target}', + brd.cfg_name, ext, brd.target) + + self.add_board(newb) + + def scan_extended(self, dbase, ext): + """Scan for extended boards""" + # First check the fragments + frags = [] + for frag in ext.fragments: + fname = os.path.join(f'configs/{frag}.config') + frags.append(tools.read_file(fname, binary=False)) + + # Now get a list of defconfigs (without the _defconfig suffix) + defconfigs = set() + cfg_list = [] + for first, val in ext.targets: + if first == 'regex': + pattern = f'configs/{val}' + fnames = glob.glob(pattern) + if not fnames: + print(f"'Warning: No configs matching '{pattern}'") + for fname in fnames: + m_cfg = re.match(r'^configs/(.*)_defconfig$', fname) + defconfigs.add(m_cfg.group(1)) + else: + if val == 'n': + cfg_list.append(f'~{first}') + elif val == 'y': + cfg_list.append(f'{first}') + else: + cfg_list.append(f'{first}={val}') + + # Search for boards with the given configs + boards = qconfig.find_config(dbase, cfg_list) + if defconfigs: + boards &= defconfigs + + return boards + class ExtendedParser: """Parser for extended-board (.buildman) files""" @@ -946,7 +1033,12 @@ class ExtendedParser: return parser.extended def parse(self, fname, data): - """Parse the file""" + """Parse the file + + Args: + fname (str): Filename to parse (used for error messages) + data (str): Contents of the file + """ self.start() for seq, line in enumerate(data.splitlines()): linenum = seq + 1 diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index e194712c37f..ad07e6cac39 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -173,6 +173,8 @@ def add_after_m(parser): parser.add_argument('-x', '--exclude', dest='exclude', type=str, action='append', help='Specify a list of boards to exclude, separated by comma') + parser.add_argument('-X', '--extend', action='store_true', + default=False, help='Include boards based on fragments (configs/*.buildman)') parser.add_argument('-y', '--filter-dtb-warnings', action='store_true', default=False, help='Filter out device-tree-compiler warnings from output') diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 69ee4074273..01a618b989a 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -21,6 +21,7 @@ from buildman import cfgutil from buildman import toolchain from buildman.builder import Builder from patman import patchstream +import qconfig from u_boot_pylib import command from u_boot_pylib import gitutil from u_boot_pylib import terminal @@ -766,6 +767,11 @@ def do_buildman(args, toolchains=None, make_func=None, brds=None, if isinstance(brds, int): return brds + if args.extend: + dbase = qconfig.ensure_database( + args.threads or multiprocessing.cpu_count()) + brds.parse_all_extended(dbase) + selected, why_selected, board_warnings = determine_boards( brds, args.terms, col, args.boards, args.exclude) -- 2.43.0

From: Simon Glass <sjg@chromium.org> Adjust the build logic to automatically deal with adding config fragments to an existing board, to fully support the -X option. Signed-off-by: Simon Glass <sjg@chromium.org> --- tools/buildman/builderthread.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index a037c23fb81..e14f11d2a26 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -420,7 +420,13 @@ class BuilderThread(threading.Thread): args, cwd, src_dir = self._build_args(brd, out_dir, out_rel_dir, work_dir, commit_upto) - config_args = [f'{brd.target}_defconfig'] + if brd.extended: + config_args = [f'{brd.orig_target}_defconfig'] + for frag in brd.extended.fragments: + fname = os.path.join(f'{frag}.config') + config_args.append(fname) + else: + config_args = [f'{brd.target}_defconfig'] if fragments != None: config_args.extend(fragments.split(',')) config_out = io.StringIO() -- 2.43.0

From: Simon Glass <sjg@chromium.org> Add a file containing a few extended boards to illustrate this new buildman feature. Signed-off-by: Simon Glass <sjg@chromium.org> --- configs/acpi.config | 1 + configs/extended.buildman | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 configs/acpi.config create mode 100644 configs/extended.buildman diff --git a/configs/acpi.config b/configs/acpi.config new file mode 100644 index 00000000000..dde31cb1985 --- /dev/null +++ b/configs/acpi.config @@ -0,0 +1 @@ +CONFIG_ACPIGEN=y diff --git a/configs/extended.buildman b/configs/extended.buildman new file mode 100644 index 00000000000..6dd12e4492d --- /dev/null +++ b/configs/extended.buildman @@ -0,0 +1,12 @@ +# Build RISC-V QEMU builds with ACPI +name: acpi +fragment: acpi +targets: + qemu-riscv* + +# Build Android variant of 'k3' boards, with DFU +name: am62x_dfu +fragment: am62x_r5_usbdfu +fragment: am62x_a53_android +targets: + CONFIG_SYS_SOC="k3" -- 2.43.0

From: Simon Glass <sjg@chromium.org> Enable building of extended boards in the world build, so we can ensure these don't collect failures over time. Signed-off-by: Simon Glass <sjg@chromium.org> --- .gitlab-ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5c96c7f80cd..de73ab7a917 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -159,9 +159,9 @@ build all 32bit ARM platforms: -r tools/buildman/requirements.txt - ret=0; git config --global --add safe.directory "${CI_PROJECT_DIR}"; - ./tools/buildman/buildman -o /tmp -PEWM arm -x aarch64 || ret=$?; + ./tools/buildman/buildman -o /tmp -PEWM arm -x aarch64 -X || ret=$?; if [[ $ret -ne 0 ]]; then - ./tools/buildman/buildman -o /tmp -seP; + ./tools/buildman/buildman -o /tmp -sePX; exit $ret; fi; @@ -178,9 +178,9 @@ build all 64bit ARM platforms: - ret=0; git config --global --add safe.directory "${CI_PROJECT_DIR}"; pip install -r tools/buildman/requirements.txt; - ./tools/buildman/buildman -o /tmp -PEWM aarch64 || ret=$?; + ./tools/buildman/buildman -o /tmp -PEWM aarch64 -X || ret=$?; if [[ $ret -ne 0 ]]; then - ./tools/buildman/buildman -o /tmp -seP; + ./tools/buildman/buildman -o /tmp -sePX; exit $ret; fi; @@ -196,9 +196,9 @@ build all PowerPC platforms: -r tools/buildman/requirements.txt - ret=0; git config --global --add safe.directory "${CI_PROJECT_DIR}"; - ./tools/buildman/buildman -o /tmp -P -E -W powerpc || ret=$?; + ./tools/buildman/buildman -o /tmp -P -E -W -X powerpc || ret=$?; if [[ $ret -ne 0 ]]; then - ./tools/buildman/buildman -o /tmp -seP; + ./tools/buildman/buildman -o /tmp -sePX; exit $ret; fi; @@ -216,9 +216,9 @@ build all other platforms: -r tools/buildman/requirements.txt - ret=0; git config --global --add safe.directory "${CI_PROJECT_DIR}"; - ./tools/buildman/buildman -o /tmp -PEWM -x arm,powerpc,xtensa || ret=$?; + ./tools/buildman/buildman -o /tmp -PEWM -x arm,powerpc,xtensa -X || ret=$?; if [[ $ret -ne 0 ]]; then - ./tools/buildman/buildman -o /tmp -seP; + ./tools/buildman/buildman -o /tmp -sePX; exit $ret; fi; -- 2.43.0
participants (1)
-
Simon Glass