From: Simon Glass <simon.glass@canonical.com> The _show_not_built() function does not actually show boards that fail to build due to missing toolchains. This is because toolchain errors create a done file with return_code=10, resulting in OUTCOME_ERROR rather than OUTCOME_UNKNOWN. Update the function to detect toolchain errors by checking for "Tool chain error" in the error lines of OUTCOME_ERROR results, in addition to checking for OUTCOME_UNKNOWN. Update the unit tests to use mock outcomes with err_lines, and add tests for the new toolchain error detection. Update the functional test to verify the "Boards not built" message appears in the summary. Co-developed-by: Claude Opus 4.5 <noreply@anthropic.com> Signed-off-by: Simon Glass <simon.glass@canonical.com> --- tools/buildman/builder.py | 23 ++++++--- tools/buildman/func_test.py | 12 ++--- tools/buildman/test_builder.py | 89 +++++++++++++++++++++++++++++++--- 3 files changed, 102 insertions(+), 22 deletions(-) diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 99aac80d95e..307249b5e13 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -1938,19 +1938,28 @@ class Builder: def _show_not_built(board_selected, board_dict): """Show boards that were not built - This reports boards that are in board_selected but have no outcome in - board_dict. In practice this is unlikely to happen since - get_result_summary() creates an outcome for every board, even if just - OUTCOME_UNKNOWN. + This reports boards that couldn't be built due to toolchain issues. + These have OUTCOME_UNKNOWN (no result file) or OUTCOME_ERROR with + "Tool chain error" in the error lines. Args: board_selected (dict): Dict of selected boards, keyed by target board_dict (dict): Dict of boards that were built, keyed by target """ not_built = [] - for brd in board_selected: - if brd not in board_dict: - not_built.append(brd) + for target in board_selected: + if target not in board_dict: + not_built.append(target) + else: + outcome = board_dict[target] + if outcome.rc == OUTCOME_UNKNOWN: + not_built.append(target) + elif outcome.rc == OUTCOME_ERROR: + # Check for toolchain error in the error lines + for line in outcome.err_lines: + if 'Tool chain error' in line: + not_built.append(target) + break if not_built: tprint(f"Boards not built ({len(not_built)}): " f"{', '.join(not_built)}") diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 21c700aa073..fa946c55645 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -593,26 +593,24 @@ Some images are invalid''' def testToolchainErrors(self): """Test that toolchain errors are reported in the summary - When toolchains are missing, boards fail to build. The summary - should report which boards had errors, grouped by architecture. + When toolchains are missing, boards cannot be built. The summary + should report which boards were not built. """ self.setupToolchains() # Build with missing toolchains - only sandbox will succeed self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir) - # Now show summary - should report boards with errors + # Now show summary - should report boards not built terminal.get_print_test_lines() # Clear self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, '-s', clean_dir=False) lines = terminal.get_print_test_lines() text = '\n'.join(line.text for line in lines) - # Check that boards with missing toolchains are shown with errors - # The '+' prefix indicates new errors for these boards - self.assertIn('arm:', text) + # Check that boards with missing toolchains are reported as not built + self.assertIn('Boards not built', text) self.assertIn('board0', text) self.assertIn('board1', text) - self.assertIn('powerpc:', text) self.assertIn('board2', text) def testBranch(self): diff --git a/tools/buildman/test_builder.py b/tools/buildman/test_builder.py index d31c0080863..78c80aa6c43 100644 --- a/tools/buildman/test_builder.py +++ b/tools/buildman/test_builder.py @@ -350,10 +350,20 @@ class TestShowNotBuilt(unittest.TestCase): """Clean up after tests""" terminal.set_print_test_mode(False) + def _make_outcome(self, rc, err_lines=None): + """Create a mock outcome with a given return code""" + outcome = mock.Mock() + outcome.rc = rc + outcome.err_lines = err_lines if err_lines else [] + return outcome + def test_all_boards_built(self): - """Test when all selected boards were built""" + """Test when all selected boards were built successfully""" board_selected = {'board1': None, 'board2': None} - board_dict = {'board1': None, 'board2': None} + board_dict = { + 'board1': self._make_outcome(builder.OUTCOME_OK), + 'board2': self._make_outcome(builder.OUTCOME_OK), + } terminal.get_print_test_lines() # Clear builder.Builder._show_not_built(board_selected, board_dict) @@ -362,10 +372,14 @@ class TestShowNotBuilt(unittest.TestCase): # No output when all boards were built self.assertEqual(len(lines), 0) - def test_some_boards_not_built(self): - """Test when some boards were not built""" + def test_some_boards_unknown(self): + """Test when some boards have OUTCOME_UNKNOWN (e.g. missing toolchain)""" board_selected = {'board1': None, 'board2': None, 'board3': None} - board_dict = {'board1': None} # Only board1 was built + board_dict = { + 'board1': self._make_outcome(builder.OUTCOME_OK), + 'board2': self._make_outcome(builder.OUTCOME_UNKNOWN), + 'board3': self._make_outcome(builder.OUTCOME_UNKNOWN), + } terminal.get_print_test_lines() # Clear builder.Builder._show_not_built(board_selected, board_dict) @@ -377,10 +391,13 @@ class TestShowNotBuilt(unittest.TestCase): self.assertIn('board2', lines[0].text) self.assertIn('board3', lines[0].text) - def test_no_boards_built(self): - """Test when no boards were built""" + def test_all_boards_unknown(self): + """Test when all boards have OUTCOME_UNKNOWN""" board_selected = {'board1': None, 'board2': None} - board_dict = {} # No boards built + board_dict = { + 'board1': self._make_outcome(builder.OUTCOME_UNKNOWN), + 'board2': self._make_outcome(builder.OUTCOME_UNKNOWN), + } terminal.get_print_test_lines() # Clear builder.Builder._show_not_built(board_selected, board_dict) @@ -391,6 +408,62 @@ class TestShowNotBuilt(unittest.TestCase): self.assertIn('board1', lines[0].text) self.assertIn('board2', lines[0].text) + def test_build_error_not_counted(self): + """Test that build errors (not toolchain) are not counted as 'not built'""" + board_selected = {'board1': None, 'board2': None} + board_dict = { + 'board1': self._make_outcome(builder.OUTCOME_OK), + 'board2': self._make_outcome(builder.OUTCOME_ERROR, + ['error: some build error']), + } + + terminal.get_print_test_lines() # Clear + builder.Builder._show_not_built(board_selected, board_dict) + lines = terminal.get_print_test_lines() + + # Build errors are still "built", just with errors + self.assertEqual(len(lines), 0) + + def test_toolchain_error_counted(self): + """Test that toolchain errors are counted as 'not built'""" + board_selected = {'board1': None, 'board2': None, 'board3': None} + board_dict = { + 'board1': self._make_outcome(builder.OUTCOME_OK), + 'board2': self._make_outcome(builder.OUTCOME_ERROR, + ['Tool chain error for arm: not found']), + 'board3': self._make_outcome(builder.OUTCOME_ERROR, + ['error: some build error']), + } + + terminal.get_print_test_lines() # Clear + builder.Builder._show_not_built(board_selected, board_dict) + lines = terminal.get_print_test_lines() + + # Only toolchain errors count as "not built" + self.assertEqual(len(lines), 1) + self.assertIn('Boards not built', lines[0].text) + self.assertIn('1', lines[0].text) + self.assertIn('board2', lines[0].text) + self.assertNotIn('board3', lines[0].text) + + def test_board_not_in_dict(self): + """Test that boards missing from board_dict are counted as 'not built'""" + board_selected = {'board1': None, 'board2': None, 'board3': None} + board_dict = { + 'board1': self._make_outcome(builder.OUTCOME_OK), + # board2 and board3 are not in board_dict + } + + terminal.get_print_test_lines() # Clear + builder.Builder._show_not_built(board_selected, board_dict) + lines = terminal.get_print_test_lines() + + self.assertEqual(len(lines), 1) + self.assertIn('Boards not built', lines[0].text) + self.assertIn('2', lines[0].text) + self.assertIn('board2', lines[0].text) + self.assertIn('board3', lines[0].text) + class TestPrepareOutputSpace(unittest.TestCase): """Tests for Builder._prepare_output_space() and _get_output_space_removals()""" -- 2.43.0