From: Simon Glass <sjg@chromium.org> Add support for probing remote build machines over SSH to prepare for distributed builds. Add a new [machines] section to the buildman config file where hostnames are listed, one per line. The --machines flag probes all configured machines in parallel over SSH, collecting their architecture, CPU count, thread count, load average, memory and disk space. Machines that are too busy, low on disk or low on memory are marked unavailable. Toolchains on each machine are checked either via 'buildman --list-tool-chains' or by testing for the boss's toolchain paths over SSH. The --machines-fetch-arch flag fetches missing toolchains, and version-mismatched toolchains are re-fetched to keep all machines in sync. Support per-machine [machine:<name>] config sections with a max_boards option to cap concurrent builds on machines with limited resources. Add gcc_version() and resolve_toolchain_aliases() helpers for toolchain version comparison and alias resolution. Signed-off-by: Simon Glass <sjg@chromium.org> --- tools/buildman/bsettings.py | 10 +- tools/buildman/cmdline.py | 7 + tools/buildman/control.py | 6 + tools/buildman/machine.py | 923 ++++++++++++++++++++++++++++ tools/buildman/main.py | 2 + tools/buildman/test_machine.py | 1046 ++++++++++++++++++++++++++++++++ 6 files changed, 1993 insertions(+), 1 deletion(-) create mode 100644 tools/buildman/machine.py create mode 100644 tools/buildman/test_machine.py diff --git a/tools/buildman/bsettings.py b/tools/buildman/bsettings.py index 1af2bc66101..52763b9b958 100644 --- a/tools/buildman/bsettings.py +++ b/tools/buildman/bsettings.py @@ -20,7 +20,7 @@ def setup(fname=''): global settings # pylint: disable=W0603 global config_fname # pylint: disable=W0603 - settings = configparser.ConfigParser() + settings = configparser.ConfigParser(allow_no_value=True) if fname is not None: config_fname = fname if config_fname == '': @@ -110,6 +110,14 @@ x86 = i386 # snapper-boards=ENABLE_AT91_TEST=1 # snapper9260=${snapper-boards} BUILD_TAG=442 # snapper9g45=${snapper-boards} BUILD_TAG=443 + +[machines] +# Remote build machines for distributed builds +# List hostnames, one per line (or user@hostname) +# e.g. +# ohau +# moa +# user@build-server ''', file=out) except IOError: print(f"Couldn't create buildman config file '{cfgname}'\n") diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index f4a1a3d018c..b284b2cbbfa 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -105,6 +105,13 @@ def add_upto_m(parser): parser.add_argument( '-M', '--allow-missing', action='store_true', default=False, help='Tell binman to allow missing blobs and generate fake ones as needed') + parser.add_argument('--mach', '--machines', action='store_true', + default=False, dest='machines', + help='Probe all remote machines from [machines] config and show ' + 'their status and available toolchains') + parser.add_argument('--machines-buildman-path', type=str, + default='buildman', + help='Path to buildman on remote machines (default: %(default)s)') parser.add_argument( '--maintainer-check', action='store_true', help='Check that maintainer entries exist for each board') diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 989057db60e..97f6ffcbfd2 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -18,6 +18,7 @@ import time from buildman import boards from buildman import bsettings from buildman import cfgutil +from buildman import machine from buildman import toolchain from buildman.builder import Builder from buildman.outcome import DisplayOptions @@ -758,6 +759,11 @@ def do_buildman(args, toolchains=None, make_func=None, brds=None, gitutil.setup() col = terminal.Color() + # Handle --machines: probe remote machines and show status + if args.machines: + return machine.do_probe_machines( + col, buildman_path=args.machines_buildman_path) + git_dir = os.path.join(args.git, '.git') toolchains = get_toolchains(toolchains, col, args.override_toolchain, diff --git a/tools/buildman/machine.py b/tools/buildman/machine.py new file mode 100644 index 00000000000..cc4e5d9fe72 --- /dev/null +++ b/tools/buildman/machine.py @@ -0,0 +1,923 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2026 Simon Glass <sjg@chromium.org> + +"""Handles remote machine probing and pool management for distributed builds + +This module provides the Machine and MachinePool classes for managing a pool of +remote build machines. Machines are probed over SSH to determine their +capabilities (CPUs, memory, load, toolchains) and can be used to distribute +board builds across multiple hosts. +""" + +import dataclasses +import json +import os +import threading + +from buildman import bsettings +from buildman import toolchain as toolchain_mod +from u_boot_pylib import command +from u_boot_pylib import terminal +from u_boot_pylib import tout + +# Probe script run on remote machines via SSH. This is kept minimal so that +# it works on any Linux machine with Python 3. +PROBE_SCRIPT = r''' +import json, os, platform, subprocess + +def get_cpus(): + try: + return int(subprocess.check_output(['nproc', '--all'], text=True)) + except (subprocess.CalledProcessError, FileNotFoundError, ValueError): + return 1 + +def get_threads(): + try: + return int(subprocess.check_output(['nproc'], text=True)) + except (subprocess.CalledProcessError, FileNotFoundError, ValueError): + return get_cpus() + +def get_load(): + try: + with open('/proc/loadavg') as f: + return float(f.read().split()[0]) + except (IOError, ValueError, IndexError): + return 0.0 + +def get_mem_avail_mb(): + try: + with open('/proc/meminfo') as f: + for line in f: + if line.startswith('MemAvailable:'): + return int(line.split()[1]) // 1024 + except (IOError, ValueError, IndexError): + pass + return 0 + +def get_disk_avail_mb(path='~'): + path = os.path.expanduser(path) + try: + st = os.statvfs(path) + return (st.f_bavail * st.f_frsize) // (1024 * 1024) + except OSError: + return 0 + +def get_bogomips(): + try: + with open('/proc/cpuinfo') as f: + for line in f: + lower = line.lower() + if 'bogomips' in lower and ':' in lower: + return float(line.split(':')[1].strip()) + except (IOError, ValueError, IndexError): + pass + return 0.0 + +print(json.dumps({ + 'arch': platform.machine(), + 'cpus': get_cpus(), + 'threads': get_threads(), + 'bogomips': get_bogomips(), + 'load_1m': get_load(), + 'mem_avail_mb': get_mem_avail_mb(), + 'disk_avail_mb': get_disk_avail_mb(), +})) +''' + +# Load threshold: if load_1m / cpus exceeds this, the machine is busy +LOAD_THRESHOLD = 0.8 + +# Minimum available disk space in MB to use a machine +MIN_DISK_MB = 1000 + +# Minimum available memory in MB to use a machine +MIN_MEM_MB = 512 + +# SSH connect timeout in seconds +SSH_TIMEOUT = 10 + +# Shorter timeout for probing, since it should be fast +PROBE_TIMEOUT = 3 + + +@dataclasses.dataclass +class MachineInfo: + """Probe results for a remote machine + + Attributes: + arch (str): Machine architecture (e.g. 'x86_64', 'aarch64') + cpus (int): Number of physical CPU cores + threads (int): Number of hardware threads + bogomips (float): BogoMIPS from /proc/cpuinfo (single core) + load (float): 1-minute load average + mem_avail_mb (int): Available memory in MB + disk_avail_mb (int): Available disk space in MB + """ + arch: str = '' + cpus: int = 0 + threads: int = 0 + bogomips: float = 0.0 + load: float = 0.0 + mem_avail_mb: int = 0 + disk_avail_mb: int = 0 + + +class MachineError(Exception): + """Error communicating with a remote machine""" + + +def _run_ssh(hostname, cmd, timeout=SSH_TIMEOUT, stdin_data=None): + """Run a command on a remote machine via SSH + + Args: + hostname (str): SSH hostname (user@host or just host) + cmd (list of str): Command and arguments, passed after '--' to + SSH. May be a single-element list with a shell command string + timeout (int): Connection timeout in seconds + stdin_data (str or None): Data to send to the command's stdin + + Returns: + str: stdout from the command + + Raises: + MachineError: if SSH connection fails or command returns non-zero + """ + ssh_cmd = [ + 'ssh', + '-o', 'BatchMode=yes', + '-o', f'ConnectTimeout={timeout}', + '-o', 'StrictHostKeyChecking=accept-new', + hostname, + '--', + ] + cmd + try: + result = command.run_pipe( + [ssh_cmd], capture=True, capture_stderr=True, + raise_on_error=False, stdin_data=stdin_data) + except command.CommandExc as exc: + raise MachineError(str(exc)) from exc + + if result.return_code: + stderr = result.stderr.strip() + if stderr: + # Take last non-empty line as the real error + lines = [l for l in stderr.splitlines() + if l.strip()] + msg = lines[-1] if lines else stderr + raise MachineError(f'SSH to {hostname}: {msg}') + raise MachineError( + f'SSH to {hostname} failed with code ' + f'{result.return_code}') + + return result.stdout + + +def gcc_version(gcc_path): + """Extract the gcc version directory from a toolchain path + + Looks for a 'gcc-*-nolibc' component in the path, which is the + standard naming convention for buildman-fetched toolchains. + + Args: + gcc_path (str): Full path to gcc binary, e.g. + '~/.buildman-toolchains/gcc-13.1.0-nolibc/aarch64-linux/ + bin/aarch64-linux-gcc' + + Returns: + str or None: The version directory (e.g. 'gcc-13.1.0-nolibc'), + or None if the path does not follow this convention + """ + for part in gcc_path.split('/'): + if part.startswith('gcc-') and 'nolibc' in part: + return part + return None + + +def _parse_toolchain_list(output): + """Parse the output of 'buildman --list-tool-chains' + + Extracts architecture -> gcc path mapping from the output. + + Args: + output (str): Output from buildman --list-tool-chains + + Returns: + dict: Architecture name -> gcc path string + """ + archs = {} + in_list = False + for line in output.splitlines(): + # The list starts after "List of available toolchains" + if 'List of available toolchains' in line: + in_list = True + continue + if in_list and ':' in line: + parts = line.split(':', 1) + if len(parts) == 2: + arch = parts[0].strip() + gcc = parts[1].strip() + if arch and gcc and arch != 'None': + archs[arch] = gcc + elif in_list and not line.strip(): + # Empty line ends the list + break + return archs + + +def _toolchain_status(mach, local_archs, local_gcc=None): + """Get toolchain status text and colour for a machine + + Args: + mach (Machine): Machine to check + local_archs (set of str): Toolchain archs available on local host + local_gcc (dict or None): arch -> gcc path on the local machine + + Returns: + tuple: (str, colour) where colour is a terminal.Color constant + or None for no colour + """ + if not mach.toolchains: + err = mach.tc_error + if err: + return 'fail', terminal.Color.RED + if not mach.avail and not mach.info.arch: + return '-', None + return 'none', terminal.Color.YELLOW + if not local_archs: + return str(len(mach.toolchains)), None + missing = local_archs - set(mach.toolchains.keys()) + + # Check for version mismatches + mismatched = 0 + if local_gcc: + for arch, path in mach.toolchains.items(): + local_ver = gcc_version(local_gcc.get(arch, '')) + if not local_ver: + continue + remote_ver = gcc_version(path) + if remote_ver and remote_ver != local_ver: + mismatched += 1 + + parts = [] + if missing: + parts.append(f'{len(missing)} missing') + if mismatched: + parts.append(f'{mismatched} wrong ver') + if parts: + return ', '.join(parts), terminal.Color.YELLOW + return 'OK', terminal.Color.GREEN + + +def build_version_map(local_gcc): + """Build a map of architecture -> gcc version directory + + Args: + local_gcc (dict): arch -> gcc path + + Returns: + dict: arch -> version string (e.g. 'gcc-13.1.0-nolibc') + """ + versions = {} + if local_gcc: + for arch, path in local_gcc.items(): + ver = gcc_version(path) + if ver: + versions[arch] = ver + return versions + + +def resolve_toolchain_aliases(gcc_dict): + """Add toolchain-alias entries to a gcc dict + + Resolves [toolchain-alias] config entries (e.g. x86->i386, sh->sh4) + so that board architectures using alias names are recognised. + + Args: + gcc_dict (dict): arch -> gcc path, modified in place + """ + for tag, value in bsettings.get_items('toolchain-alias'): + if tag not in gcc_dict: + for alias in value.split(): + if alias in gcc_dict: + gcc_dict[tag] = gcc_dict[alias] + break + + +def get_machines_config(): + """Get the list of machine hostnames from the config + + Returns: + list of str: List of hostnames from [machines] section + """ + items = bsettings.get_items('machines') + return [value.strip() if value else name.strip() + for name, value in items] + + +def do_probe_machines(col=None, fetch=False, buildman_path='buildman'): + """Probe all configured machines and display their status + + This is the entry point for 'buildman --machines' when used without a + build command. It probes all machines, checks their toolchains and + prints a summary. + + Args: + col (terminal.Color or None): Colour object for output + fetch (bool): True to fetch missing toolchains + buildman_path (str): Path to buildman on remote machines + + Returns: + int: 0 on success, non-zero on failure + """ + if not col: + col = terminal.Color() + + machines = get_machines_config() + if not machines: + print(col.build(col.RED, + 'No machines configured. Add a [machines] section ' + 'to ~/.buildman')) + return 1 + + # Get local toolchains for comparison. Only include cross- + # toolchains under ~/.buildman-toolchains/ since system compilers + # (sandbox, c89, c99) can't be probed or fetched remotely. + local_toolchains = toolchain_mod.Toolchains() + local_toolchains.get_settings(show_warning=False) + local_toolchains.scan(verbose=False) + home = os.path.expanduser('~') + local_gcc = {arch: tc.gcc + for arch, tc in local_toolchains.toolchains.items() + if tc.gcc.startswith(home)} + resolve_toolchain_aliases(local_gcc) + local_archs = set(local_gcc.keys()) + + pool = MachinePool() + pool.probe_all(col) + pool.check_toolchains(local_archs, buildman_path=buildman_path, + fetch=fetch, col=col, local_gcc=local_gcc) + pool.print_summary(col, local_archs=local_archs, + local_gcc=local_gcc) + return 0 + + +class MachinePool: + """Manages a pool of remote build machines + + Reads machine hostnames from the [machines] section of the buildman + config and provides methods to probe, check toolchains and display + the status of all machines. + + Attributes: + machines (list of Machine): All machines in the pool + """ + + def __init__(self, names=None): + """Create a MachinePool + + Args: + names (list of str or None): If provided, only include machines + whose config key matches one of these names. If None, include + all machines from the config. + """ + self.machines = [] + self._load_from_config(names) + + def _load_from_config(self, names=None): + """Load machine hostnames from the [machines] config section + + Supports bare hostnames (one per line) or name=hostname pairs. + The hostname may include a username (user@host): + [machines] + ohau + moa + myserver = build1.example.com + ruru = sglass@ruru + + Args: + names (list of str or None): If provided, only include machines + whose config key matches one of these names + """ + name_set = set(names) if names else set() + items = bsettings.get_items('machines') + for name, value in items: + # With allow_no_value=True, bare hostnames have value=None + # and the hostname is the key. For key=value pairs, use value. + key = name.strip() + if name_set and key not in name_set: + continue + hostname = value.strip() if value else key + mach = Machine(hostname, name=key) + for oname, ovalue in bsettings.get_items(f'machine:{key}'): + if oname == 'max_boards': + mach.max_boards = int(ovalue) + self.machines.append(mach) + + def probe_all(self, col=None): + """Probe all machines in the pool in parallel + + All machines are probed concurrently via threads. Progress is shown + on a single line and results are printed afterwards. + + Args: + col (terminal.Color or None): Colour object for output + + Returns: + list of Machine: Machines that are available + """ + if not col: + col = terminal.Color() + + names = [m.name for m in self.machines] + done = [] + lock = threading.Lock() + + def _probe(mach): + mach.probe() + with lock: + done.append(mach.name) + tout.progress(f'Probing {len(done)}/{len(names)}: ' + f'{", ".join(done)}') + + # Probe all machines in parallel + threads = [] + tout.progress(f'Probing {len(names)} machines') + for mach in self.machines: + t = threading.Thread(target=_probe, args=(mach,)) + t.start() + threads.append(t) + for t in threads: + t.join() + tout.clear_progress() + return self.get_available() + + def check_toolchains(self, needed_archs, buildman_path='buildman', + fetch=False, col=None, local_gcc=None): + """Check and optionally fetch toolchains on available machines + + Probes toolchains on all available machines in parallel. If + fetch is True, missing toolchains are fetched sequentially. + + Toolchains whose gcc version (e.g. gcc-13.1.0-nolibc) differs + from the local machine are treated as missing and will be + re-fetched if fetch is True. + + Args: + needed_archs (set of str): Set of architectures needed + (e.g. {'arm', 'aarch64', 'sandbox'}) + buildman_path (str): Path to buildman on remote machines + fetch (bool): True to attempt to fetch missing toolchains + col (terminal.Color or None): Colour object for output + local_gcc (dict or None): arch -> gcc path on the local + machine, used for version comparison + + Returns: + dict: Machine -> set of missing architectures + """ + if not col: + col = terminal.Color() + + reachable = self.get_reachable() + if not reachable: + return {} + + # Probe toolchains on all reachable machines, not just available + # ones, so that busy machines still show toolchain info + done = [] + lock = threading.Lock() + + def _check(mach): + mach.probe_toolchains(buildman_path, local_gcc=local_gcc) + with lock: + done.append(mach.name) + tout.progress(f'Checking toolchains {len(done)}/' + f'{len(reachable)}: {", ".join(done)}') + + threads = [] + tout.progress(f'Checking toolchains on {len(reachable)} machines') + for mach in reachable: + t = threading.Thread(target=_check, args=(mach,)) + t.start() + threads.append(t) + for t in threads: + t.join() + tout.clear_progress() + + local_versions = build_version_map(local_gcc) + + # Check for missing or version-mismatched toolchains + missing_map = {} + for mach in reachable: + missing = needed_archs - set(mach.toolchains.keys()) + # Also treat version mismatches as missing + for arch, path in mach.toolchains.items(): + local_ver = local_versions.get(arch) + if not local_ver: + continue + remote_ver = gcc_version(path) + if remote_ver and remote_ver != local_ver: + missing.add(arch) + if missing: + missing_map[mach] = missing + + if fetch and missing_map: + self._fetch_all_missing(missing_map, local_versions, + local_gcc, buildman_path) + + return missing_map + + def _fetch_all_missing(self, missing_map, local_versions, + local_gcc, buildman_path): + """Fetch missing toolchains on all machines in parallel + + For version-mismatched toolchains, removes the old version + directory on the remote before fetching, so the new version + takes its place. + + Updates missing_map in place, removing architectures that + were successfully fetched. + + Args: + missing_map (dict): Machine -> set of missing archs + local_versions (dict): arch -> version string (e.g. + 'gcc-13.1.0-nolibc') from the local machine + local_gcc (dict or None): arch -> gcc path on the boss, + passed to re-probe after fetching + buildman_path (str): Path to buildman on remote + """ + lock = threading.Lock() + done = [] + failed = [] + total = sum(len(v) for v in missing_map.values()) + + def _fetch_one(mach, missing): + fetched = set() + for arch in list(missing): + # Remove old mismatched version before fetching + old_ver = gcc_version(mach.toolchains.get(arch, '')) + if old_ver and old_ver != local_versions.get(arch): + try: + _run_ssh(mach.name, [ + 'rm', '-rf', + f'~/.buildman-toolchains/{old_ver}']) + except MachineError: + pass + ok = mach.fetch_toolchain(buildman_path, arch) + with lock: + done.append(arch) + if ok: + fetched.add(arch) + else: + failed.append(f'{mach.name}: {arch}') + tout.progress( + f'Fetching toolchains {len(done)}/{total}: ' + f'{mach.name} {arch}') + if fetched: + mach.probe_toolchains(buildman_path, + local_gcc=local_gcc) + missing -= fetched + if not missing: + with lock: + del missing_map[mach] + + tout.progress(f'Fetching {total} toolchains on ' + f'{len(missing_map)} machines') + threads = [] + for mach, missing in list(missing_map.items()): + t = threading.Thread(target=_fetch_one, args=(mach, missing)) + t.start() + threads.append(t) + for t in threads: + t.join() + tout.clear_progress() + + # Report failures + for msg in failed: + print(f' Failed to fetch {msg}') + + # Report remaining version mismatches grouped by machine + if missing_map: + print('Version mismatches (local vs remote):') + for mach, missing in sorted(missing_map.items(), + key=lambda x: x[0].name): + diffs = [] + for arch in sorted(missing): + local_ver = local_versions.get(arch, '?') + diffs.append(f'{arch}({local_ver})') + print(f' {mach.name}: {", ".join(diffs)}') + + def get_reachable(self): + """Get list of machines that were successfully probed + + This includes machines that are busy or low on resources, as long + as they were reachable via SSH. + + Returns: + list of Machine: Reachable machines (may not be available) + """ + return [m for m in self.machines + if m.avail or m.info.arch] + + def get_available(self): + """Get list of machines that are available for building + + Returns: + list of Machine: Available machines + """ + return [m for m in self.machines if m.avail] + + def get_total_weight(self): + """Get the total weight of all available machines + + Returns: + int: Sum of weights of all available machines + """ + return sum(m.weight for m in self.get_available()) + + def print_summary(self, col=None, local_archs=None, local_gcc=None): + """Print a summary of all machines in the pool + + Args: + col (terminal.Color or None): Colour object for output + local_archs (set of str or None): Toolchain architectures available + on the local host, used to compare remote toolchain status + local_gcc (dict or None): arch -> gcc path on local machine, + for version comparison + """ + if not col: + col = terminal.Color() + if not local_archs: + local_archs = set() + available = self.get_available() + total_weight = self.get_total_weight() + print(col.build(col.BLUE, + f'Machine pool: {len(available)} of {len(self.machines)} ' + f'machines available, total weight {total_weight}')) + print() + fmt = ' {:<10} {:>10} {:>7} {:>8} {:>6} {:>7} {:>7} {:>10} {}' + print(fmt.format('Name', 'Arch', 'Threads', 'BogoMIPS', + 'Load', 'Mem GB', 'Disk TB', 'Toolchains', 'Status')) + print(f' {"-" * 88}') + for mach in self.machines: + if mach.avail: + parts = [f'weight {mach.weight}'] + if mach.max_boards: + parts.append(f'max {mach.max_boards}') + status_text = ', '.join(parts) + status_colour = col.GREEN + elif mach.reason == 'not probed': + status_text = 'not probed' + status_colour = col.YELLOW + else: + status_text = mach.reason + status_colour = col.RED + inf = mach.info + mem_gb = f'{inf.mem_avail_mb / 1024:.1f}' + disk_tb = f'{inf.disk_avail_mb / 1024 / 1024:.1f}' + tc_text, tc_colour = _toolchain_status( + mach, local_archs, local_gcc) + + # Format the line with plain text for correct alignment, + # then apply colour to the toolchain and status fields + line = fmt.format(mach.name, inf.arch or '-', inf.threads, + f'{inf.bogomips:.0f}', f'{inf.load:.1f}', + mem_gb, disk_tb, tc_text, status_text) + if tc_colour: + line = line.replace(tc_text, + col.build(tc_colour, tc_text), 1) + line = line.replace(status_text, + col.build(status_colour, status_text), 1) + print(line) + + local_versions = build_version_map(local_gcc) + + # Print toolchain errors and missing details after the table + notes = [] + for mach in self.machines: + err = mach.tc_error + if err: + notes.append(f' {mach.name}: {err}') + elif local_archs and mach.toolchains is not None: + missing = local_archs - set(mach.toolchains.keys()) + if missing: + parts = [] + for arch in sorted(missing): + ver = local_versions.get(arch) + if ver: + parts.append(f'{arch}({ver})') + else: + parts.append(arch) + notes.append( + f' {mach.name}: need {", ".join(parts)}') + if notes: + print() + for note in notes: + print(note) + + +class Machine: + """Represents a remote (or local) build machine + + Attributes: + hostname (str): SSH hostname (user@host or just host) + name (str): Short display name from config key + info (MachineInfo): Probed machine information + avail (bool): True if reachable and not too busy + reason (str): Reason the machine is unavailable, or '' + toolchains (dict): Available toolchain architectures, arch -> gcc path + tc_error (str): Error from last toolchain probe, or '' + weight (int): Number of build threads to allocate + max_boards (int): Max concurrent boards (0 = use nthreads) + """ + def __init__(self, hostname, name=None): + self.hostname = hostname + self.name = name or hostname + self.info = MachineInfo() + self.avail = False + self.reason = 'not probed' + self.toolchains = {} + self.tc_error = '' + self.weight = 0 + self.max_boards = 0 + + def probe(self, timeout=PROBE_TIMEOUT): + """Probe this machine's capabilities over SSH + + Runs a small Python script on the remote machine to collect + architecture, CPU count, thread count, load average, available memory + and disk space. + + Args: + timeout (int): SSH connect timeout in seconds + + Returns: + bool: True if the machine was probed successfully + """ + try: + result = _run_ssh(self.hostname, ['python3'], + timeout=timeout, stdin_data=PROBE_SCRIPT) + except MachineError as exc: + self.avail = False + self.reason = str(exc) + return False + + try: + info = json.loads(result) + except json.JSONDecodeError: + self.avail = False + self.reason = f'invalid probe response: {result[:100]}' + return False + + self.info = MachineInfo( + arch=info.get('arch', ''), + cpus=info.get('cpus', 0), + threads=info.get('threads', 0), + bogomips=info.get('bogomips', 0.0), + load=info.get('load_1m', 0.0), + mem_avail_mb=info.get('mem_avail_mb', 0), + disk_avail_mb=info.get('disk_avail_mb', 0), + ) + + # Check whether the machine is too busy or low on resources + self.avail = True + self.reason = '' + inf = self.info + if inf.cpus and inf.load / inf.cpus > LOAD_THRESHOLD: + self.avail = False + self.reason = (f'busy (load {inf.load:.1f} ' + f'with {inf.cpus} cpus)') + elif inf.disk_avail_mb < MIN_DISK_MB: + self.avail = False + self.reason = (f'low disk ' + f'({inf.disk_avail_mb} MB available)') + elif inf.mem_avail_mb < MIN_MEM_MB: + self.avail = False + self.reason = (f'low memory ' + f'({inf.mem_avail_mb} MB available)') + + if self.avail: + self._calc_weight() + return True + + def probe_toolchains(self, buildman_path, local_gcc=None): + """Probe available toolchains on this machine + + If local_gcc is provided, checks which of the boss's + toolchains exist on this machine by testing for the gcc + binary under ~/.buildman-toolchains. This avoids depending + on the remote machine's .buildman config. + + Falls back to running 'buildman --list-tool-chains' on the + remote when local_gcc is not provided (e.g. --machines + without a build). + + Args: + buildman_path (str): Path to buildman on the remote machine + local_gcc (dict or None): arch -> gcc path on the boss + + Returns: + dict: Architecture -> gcc path mapping + """ + self.tc_error = '' + if local_gcc: + return self._probe_toolchains_from_boss(local_gcc) + try: + result = _run_ssh(self.hostname, + [buildman_path, '--list-tool-chains']) + except MachineError as exc: + self.toolchains = {} + self.tc_error = str(exc) + return self.toolchains + + self.toolchains = _parse_toolchain_list(result) + return self.toolchains + + def _probe_toolchains_from_boss(self, local_gcc): + """Check which of the boss's toolchains exist on this machine + + For each architecture, extracts the path relative to the home + directory (e.g. .buildman-toolchains/gcc-13.1.0-nolibc/...) + and tests whether that gcc binary exists on the remote. This + makes the worker mirror the boss's toolchain choices. + + Args: + local_gcc (dict): arch -> gcc path on the boss + + Returns: + dict: Architecture -> gcc path mapping (using remote paths) + """ + # Build a list of relative paths to check + home_prefix = os.path.expanduser('~') + checks = {} + for arch, gcc in local_gcc.items(): + if gcc.startswith(home_prefix): + rel = gcc[len(home_prefix):] + if rel.startswith('/'): + rel = rel[1:] + checks[arch] = rel + + if not checks: + self.toolchains = {} + return self.toolchains + + # Build a single SSH command that tests all paths + # Output: "arch:yes" or "arch:no" for each + test_cmds = [] + for arch, rel in checks.items(): + test_cmds.append( + f'test -f ~/{rel} && echo {arch}:yes || echo {arch}:no') + try: + result = _run_ssh(self.hostname, + ['; '.join(test_cmds)]) + except MachineError as exc: + self.toolchains = {} + self.tc_error = str(exc) + return self.toolchains + + self.toolchains = {} + for line in result.splitlines(): + line = line.strip() + if ':yes' in line: + arch = line.split(':')[0] + rel = checks.get(arch) + if rel: + self.toolchains[arch] = f'~/{rel}' + return self.toolchains + + def fetch_toolchain(self, buildman_path, arch): + """Fetch a toolchain for a given architecture on this machine + + Args: + buildman_path (str): Path to buildman on the remote + arch (str): Architecture to fetch (e.g. 'arm') + + Returns: + bool: True if the fetch succeeded + """ + try: + _run_ssh(self.hostname, + [buildman_path, '--fetch-arch', arch]) + return True + except MachineError: + return False + + def _calc_weight(self): + """Calculate the build weight (threads to allocate) for this machine + + Uses available threads minus a fraction of the current load to avoid + over-committing a partially loaded machine. + """ + if not self.avail: + self.weight = 0 + return + # Reserve some capacity based on current load + spare = max(1, self.info.threads - int(self.info.load)) + self.weight = spare + + def __repr__(self): + inf = self.info + status = 'avail' if self.avail else f'unavail: {self.reason}' + return (f'Machine({self.hostname}, arch={inf.arch}, ' + f'threads={inf.threads}, ' + f'bogomips={inf.bogomips:.0f}, load={inf.load:.1f}, ' + f'weight={self.weight}, {status})') diff --git a/tools/buildman/main.py b/tools/buildman/main.py index af289a46508..faff7d41ceb 100755 --- a/tools/buildman/main.py +++ b/tools/buildman/main.py @@ -41,6 +41,7 @@ def run_tests(skip_net_tests, debug, verbose, args): from buildman import test_bsettings from buildman import test_builder from buildman import test_cfgutil + from buildman import test_machine test_name = args.terms and args.terms[0] or None if skip_net_tests: @@ -63,6 +64,7 @@ def run_tests(skip_net_tests, debug, verbose, args): test_builder.TestCheckOutputForLoop, test_builder.TestMake, test_builder.TestPrintBuildSummary, + test_machine, 'buildman.toolchain']) return (0 if result.wasSuccessful() else 1) diff --git a/tools/buildman/test_machine.py b/tools/buildman/test_machine.py new file mode 100644 index 00000000000..1397a4a76c0 --- /dev/null +++ b/tools/buildman/test_machine.py @@ -0,0 +1,1046 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2026 Simon Glass <sjg@chromium.org> + +"""Tests for the machine module""" + +# pylint: disable=W0212 + +import json +import os +import unittest +from unittest import mock + +from u_boot_pylib import command +from u_boot_pylib import terminal + +from buildman import bsettings +from buildman import machine + + +# Base machine-info dict used by probe tests. Individual tests override +# fields as needed (e.g. load_1m, mem_avail_mb) via {**MACHINE_INFO, ...}. +MACHINE_INFO = { + 'arch': 'x86_64', + 'cpus': 4, + 'threads': 8, + 'bogomips': 5000.0, + 'load_1m': 0.5, + 'mem_avail_mb': 16000, + 'disk_avail_mb': 50000, +} + + +class TestParseToolchainList(unittest.TestCase): + """Test _parse_toolchain_list()""" + + def test_parse_normal(self): + """Test parsing normal toolchain list output""" + output = '''List of available toolchains (3): +arm : /usr/bin/arm-linux-gnueabi-gcc +aarch64 : /usr/bin/aarch64-linux-gnu-gcc +sandbox : /usr/bin/gcc +''' + result = machine._parse_toolchain_list(output) + self.assertEqual(result, { + 'arm': '/usr/bin/arm-linux-gnueabi-gcc', + 'aarch64': '/usr/bin/aarch64-linux-gnu-gcc', + 'sandbox': '/usr/bin/gcc', + }) + + def test_parse_empty(self): + """Test parsing empty output""" + self.assertEqual(machine._parse_toolchain_list(''), {}) + + def test_parse_none_toolchains(self): + """Test parsing when no toolchains are available""" + output = '''List of available toolchains (0): +None +''' + result = machine._parse_toolchain_list(output) + self.assertEqual(result, {}) + + def test_parse_with_colour(self): + """Test parsing output that has extra text before the list""" + output = """Some preamble text +List of available toolchains (2): +arm : /opt/toolchains/arm-gcc +x86 : /usr/bin/x86_64-linux-gcc + +Some trailing text +""" + result = machine._parse_toolchain_list(output) + self.assertEqual(result, { + 'arm': '/opt/toolchains/arm-gcc', + 'x86': '/usr/bin/x86_64-linux-gcc', + }) + + +class TestMachine(unittest.TestCase): + """Test Machine class""" + + def test_init(self): + """Test initial state of a Machine""" + m = machine.Machine('myhost') + self.assertEqual(m.hostname, 'myhost') + self.assertEqual(m.info.arch, '') + self.assertFalse(m.avail) + self.assertEqual(m.reason, 'not probed') + self.assertEqual(m.weight, 0) + self.assertEqual(m.toolchains, {}) + + @mock.patch('buildman.machine._run_ssh') + def test_probe_success(self, mock_ssh): + """Test successful probe""" + mock_ssh.return_value = json.dumps({ + **MACHINE_INFO, 'cpus': 8, 'threads': 16, 'load_1m': 1.5}) + m = machine.Machine('server1') + result = m.probe() + self.assertTrue(result) + self.assertTrue(m.avail) + self.assertEqual(m.info.cpus, 8) + self.assertEqual(m.info.threads, 16) + self.assertAlmostEqual(m.info.load, 1.5) + self.assertEqual(m.info.mem_avail_mb, 16000) + self.assertEqual(m.info.disk_avail_mb, 50000) + self.assertGreater(m.weight, 0) + + @mock.patch('buildman.machine._run_ssh') + def test_probe_busy(self, mock_ssh): + """Test probe of a busy machine""" + mock_ssh.return_value = json.dumps({ + **MACHINE_INFO, 'load_1m': 5.0}) + m = machine.Machine('busy-host') + result = m.probe() + self.assertTrue(result) + self.assertFalse(m.avail) + self.assertIn('busy', m.reason) + + @mock.patch('buildman.machine._run_ssh') + def test_probe_low_disk(self, mock_ssh): + """Test probe of a machine with low disk space""" + mock_ssh.return_value = json.dumps({ + **MACHINE_INFO, 'disk_avail_mb': 500}) + m = machine.Machine('low-disk') + result = m.probe() + self.assertTrue(result) + self.assertFalse(m.avail) + self.assertIn('disk', m.reason) + + @mock.patch('buildman.machine._run_ssh') + def test_probe_low_mem(self, mock_ssh): + """Test probe of a machine with low memory""" + mock_ssh.return_value = json.dumps({ + **MACHINE_INFO, 'mem_avail_mb': 200}) + m = machine.Machine('low-mem') + result = m.probe() + self.assertTrue(result) + self.assertFalse(m.avail) + self.assertIn('memory', m.reason) + + @mock.patch('buildman.machine._run_ssh') + def test_probe_ssh_failure(self, mock_ssh): + """Test probe when SSH fails""" + mock_ssh.side_effect = machine.MachineError('connection refused') + m = machine.Machine('bad-host') + result = m.probe() + self.assertFalse(result) + self.assertFalse(m.avail) + self.assertIn('connection refused', m.reason) + + @mock.patch('buildman.machine._run_ssh') + def test_probe_bad_json(self, mock_ssh): + """Test probe when remote returns invalid JSON""" + mock_ssh.return_value = 'not json at all' + m = machine.Machine('bad-json') + result = m.probe() + self.assertFalse(result) + self.assertFalse(m.avail) + self.assertIn('invalid probe response', m.reason) + + @mock.patch('buildman.machine._run_ssh') + def test_probe_toolchains(self, mock_ssh): + """Test probing toolchains""" + mock_ssh.return_value = '''List of available toolchains (2): +arm : /usr/bin/arm-linux-gnueabi-gcc +sandbox : /usr/bin/gcc + +''' + m = machine.Machine('server1') + archs = m.probe_toolchains('buildman') + self.assertEqual(archs, { + 'arm': '/usr/bin/arm-linux-gnueabi-gcc', + 'sandbox': '/usr/bin/gcc', + }) + + @mock.patch('buildman.machine._run_ssh') + def test_weight_calculation(self, mock_ssh): + """Test weight calculation based on load""" + mock_ssh.return_value = json.dumps({ + **MACHINE_INFO, 'cpus': 8, 'threads': 16, 'load_1m': 4.0}) + m = machine.Machine('server1') + m.probe() + # weight = threads - int(load) = 16 - 4 = 12 + self.assertEqual(m.weight, 12) + + @mock.patch('buildman.machine._run_ssh') + def test_weight_minimum(self, mock_ssh): + """Test weight is at least 1 when available""" + mock_ssh.return_value = json.dumps({ + **MACHINE_INFO, 'arch': 'aarch64', 'threads': 4, + 'bogomips': 48.0, 'load_1m': 3.1}) + m = machine.Machine('server1') + m.probe() + # weight = max(1, 4 - 3) = 1 + self.assertEqual(m.weight, 1) + + def test_repr(self): + """Test string representation""" + m = machine.Machine('server1') + self.assertIn('server1', repr(m)) + self.assertIn('unavail', repr(m)) + + +class TestMachinePool(unittest.TestCase): + """Test MachinePool class""" + + def setUp(self): + """Set up bsettings for each test""" + bsettings.setup(None) + + def test_empty_pool(self): + """Test pool with no machines configured""" + pool = machine.MachinePool() + self.assertEqual(pool.machines, []) + self.assertEqual(pool.get_available(), []) + self.assertEqual(pool.get_total_weight(), 0) + + def test_load_from_config(self): + """Test loading machines from config with bare hostnames""" + bsettings.add_file( + '[machines]\n' + 'ohau\n' + 'moa\n' + ) + pool = machine.MachinePool() + self.assertEqual(len(pool.machines), 2) + self.assertEqual(pool.machines[0].hostname, 'ohau') + self.assertEqual(pool.machines[1].hostname, 'moa') + + def test_load_from_config_key_value(self): + """Test loading machines from config with key=value pairs""" + bsettings.add_file( + '[machines]\n' + 'server1 = build1.example.com\n' + 'server2 = user@build2.example.com\n' + ) + pool = machine.MachinePool() + self.assertEqual(len(pool.machines), 2) + self.assertEqual(pool.machines[0].hostname, 'build1.example.com') + self.assertEqual(pool.machines[1].hostname, 'user@build2.example.com') + + @mock.patch('buildman.machine._run_ssh') + def test_probe_all(self, mock_ssh): + """Test probing all machines""" + mock_ssh.return_value = json.dumps({ + **MACHINE_INFO, 'load_1m': 1.0, + 'mem_avail_mb': 8000, 'disk_avail_mb': 20000}) + bsettings.add_file( + '[machines]\n' + 'host1\n' + 'host2\n' + ) + pool = machine.MachinePool() + available = pool.probe_all() + self.assertEqual(len(available), 2) + self.assertEqual(pool.get_total_weight(), 14) + + @mock.patch('buildman.machine._run_ssh') + def test_probe_mixed(self, mock_ssh): + """Test probing with some machines available and some not""" + def ssh_side_effect(hostname, _cmd, **_kwargs): + if hostname == 'host1': + return json.dumps({ + **MACHINE_INFO, 'load_1m': 1.0, + 'mem_avail_mb': 8000, 'disk_avail_mb': 20000}) + raise machine.MachineError('connection refused') + + mock_ssh.side_effect = ssh_side_effect + bsettings.add_file( + '[machines]\n' + 'host1\n' + 'host2\n' + ) + pool = machine.MachinePool() + available = pool.probe_all() + self.assertEqual(len(available), 1) + self.assertEqual(available[0].hostname, 'host1') + + @mock.patch('buildman.machine._run_ssh') + def test_check_toolchains(self, mock_ssh): + """Test checking toolchains on machines""" + def ssh_side_effect(_hostname, cmd, **_kwargs): + if 'python3' in cmd: + return json.dumps({ + **MACHINE_INFO, 'load_1m': 1.0, + 'mem_avail_mb': 8000, 'disk_avail_mb': 20000}) + if '--list-tool-chains' in cmd: + return '''List of available toolchains (2): +arm : /usr/bin/arm-gcc +sandbox : /usr/bin/gcc + +''' + return '' + + mock_ssh.side_effect = ssh_side_effect + bsettings.add_file( + '[machines]\n' + 'host1\n' + ) + pool = machine.MachinePool() + pool.probe_all() + missing = pool.check_toolchains({'arm', 'sandbox'}) + self.assertEqual(missing, {}) + + @mock.patch('buildman.machine._run_ssh') + def test_check_toolchains_missing(self, mock_ssh): + """Test checking toolchains with some missing""" + def ssh_side_effect(_hostname, cmd, **_kwargs): + if 'python3' in cmd: + return json.dumps({ + **MACHINE_INFO, 'load_1m': 1.0, + 'mem_avail_mb': 8000, 'disk_avail_mb': 20000}) + if '--list-tool-chains' in cmd: + return '''List of available toolchains (1): +sandbox : /usr/bin/gcc + +''' + return '' + + mock_ssh.side_effect = ssh_side_effect + bsettings.add_file( + '[machines]\n' + 'host1\n' + ) + pool = machine.MachinePool() + pool.probe_all() + missing = pool.check_toolchains({'arm', 'sandbox'}) + self.assertEqual(len(missing), 1) + m = list(missing.keys())[0] + self.assertIn('arm', missing[m]) + + +class TestRunSsh(unittest.TestCase): + """Test _run_ssh()""" + + @mock.patch('buildman.machine.command.run_pipe') + def test_success(self, mock_pipe): + """Test successful SSH command""" + mock_pipe.return_value = mock.Mock( + return_code=0, stdout='hello\n', stderr='') + result = machine._run_ssh('host1', ['echo', 'hello']) + self.assertEqual(result, 'hello\n') + + # Verify SSH options + pipe_list = mock_pipe.call_args[0][0] + cmd = pipe_list[0] + self.assertIn('ssh', cmd) + self.assertIn('BatchMode=yes', cmd) + self.assertIn('host1', cmd) + self.assertIn('echo', cmd) + + @mock.patch('buildman.machine.command.run_pipe') + def test_failure(self, mock_pipe): + """Test SSH command failure""" + mock_pipe.return_value = mock.Mock( + return_code=255, stdout='', + stderr='Connection refused') + with self.assertRaises(machine.MachineError) as ctx: + machine._run_ssh('host1', ['echo', 'hello']) + self.assertIn('Connection refused', str(ctx.exception)) + + @mock.patch('buildman.machine.command.run_pipe') + def test_failure_multiline_stderr(self, mock_pipe): + """Test SSH failure with multi-line stderr picks last line""" + mock_pipe.return_value = mock.Mock( + return_code=255, stdout='', + stderr='Warning: Added host key\n' + 'Permission denied (publickey).') + with self.assertRaises(machine.MachineError) as ctx: + machine._run_ssh('host1', ['echo', 'hello']) + self.assertIn('Permission denied', str(ctx.exception)) + self.assertNotIn('Warning', str(ctx.exception)) + + @mock.patch('buildman.machine.command.run_pipe') + def test_command_exc(self, mock_pipe): + """Test SSH command exception""" + mock_pipe.side_effect = command.CommandExc( + 'ssh failed', command.CommandResult()) + with self.assertRaises(machine.MachineError) as ctx: + machine._run_ssh('host1', ['echo', 'hello']) + self.assertIn('ssh failed', str(ctx.exception)) + + +class TestGetMachinesConfig(unittest.TestCase): + """Test get_machines_config()""" + + def setUp(self): + bsettings.setup(None) + + def test_empty(self): + """Test with no machines configured""" + self.assertEqual(machine.get_machines_config(), []) + + def test_with_machines(self): + """Test with machines configured""" + bsettings.add_file( + '[machines]\n' + 'host1\n' + 'host2\n' + ) + result = machine.get_machines_config() + self.assertEqual(result, ['host1', 'host2']) + + +class TestGccVersion(unittest.TestCase): + """Test gcc_version()""" + + def test_normal_path(self): + """Test extracting version from a standard toolchain path""" + path = ('~/.buildman-toolchains/gcc-13.1.0-nolibc/' + 'aarch64-linux/bin/aarch64-linux-gcc') + self.assertEqual(machine.gcc_version(path), 'gcc-13.1.0-nolibc') + + def test_no_match(self): + """Test path that does not contain a gcc-*-nolibc component""" + self.assertIsNone(machine.gcc_version('/usr/bin/gcc')) + + def test_empty(self): + """Test empty path""" + self.assertIsNone(machine.gcc_version('')) + + +class TestBuildVersionMap(unittest.TestCase): + """Test build_version_map()""" + + def test_normal(self): + """Test building version map from gcc dict""" + gcc = { + 'arm': '~/.buildman-toolchains/gcc-13.1.0-nolibc/arm/bin/gcc', + 'sandbox': '/usr/bin/gcc', + } + result = machine.build_version_map(gcc) + self.assertEqual(result, {'arm': 'gcc-13.1.0-nolibc'}) + + def test_none(self): + """Test with None input""" + self.assertEqual(machine.build_version_map(None), {}) + + +class TestResolveToolchainAliases(unittest.TestCase): + """Test resolve_toolchain_aliases()""" + + def setUp(self): + bsettings.setup(None) + + def test_alias(self): + """Test resolving aliases from config""" + bsettings.add_file( + '[toolchain-alias]\n' + 'x86 = i386 i686\n' + ) + gcc = {'i386': '/usr/bin/i386-gcc'} + machine.resolve_toolchain_aliases(gcc) + self.assertEqual(gcc['x86'], '/usr/bin/i386-gcc') + + def test_no_alias_needed(self): + """Test when arch already exists""" + bsettings.add_file( + '[toolchain-alias]\n' + 'x86 = i386\n' + ) + gcc = {'x86': '/usr/bin/x86-gcc', 'i386': '/usr/bin/i386-gcc'} + machine.resolve_toolchain_aliases(gcc) + # Should not overwrite existing + self.assertEqual(gcc['x86'], '/usr/bin/x86-gcc') + + +class TestToolchainStatus(unittest.TestCase): + """Test _toolchain_status()""" + + def test_no_toolchains_no_error(self): + """Test machine with no toolchains and no error""" + m = machine.Machine('host1') + m.avail = True + m.info.arch = 'x86_64' + text, colour = machine._toolchain_status(m, set()) + self.assertEqual(text, 'none') + + def test_no_toolchains_with_error(self): + """Test machine with toolchain error""" + m = machine.Machine('host1') + m.tc_error = 'SSH failed' + text, colour = machine._toolchain_status(m, set()) + self.assertEqual(text, 'fail') + + def test_all_present(self): + """Test all local toolchains present on machine""" + m = machine.Machine('host1') + m.toolchains = {'arm': '/usr/bin/arm-gcc', + 'sandbox': '/usr/bin/gcc'} + local = {'arm', 'sandbox'} + text, colour = machine._toolchain_status(m, local) + self.assertEqual(text, 'OK') + + def test_some_missing(self): + """Test some toolchains missing""" + m = machine.Machine('host1') + m.toolchains = {'sandbox': '/usr/bin/gcc'} + local = {'arm', 'sandbox'} + text, colour = machine._toolchain_status(m, local) + self.assertIn('missing', text) + + def test_version_mismatch(self): + """Test version mismatch detection""" + m = machine.Machine('host1') + m.toolchains = { + 'arm': '~/.buildman-toolchains/gcc-12.0.0-nolibc/arm/bin/gcc'} + local_gcc = { + 'arm': '~/.buildman-toolchains/gcc-13.1.0-nolibc/arm/bin/gcc'} + text, colour = machine._toolchain_status( + m, {'arm'}, local_gcc=local_gcc) + self.assertIn('wrong ver', text) + + def test_no_local_archs(self): + """Test with empty local arch set""" + m = machine.Machine('host1') + m.toolchains = {'arm': '/usr/bin/gcc', 'x86': '/usr/bin/gcc'} + text, colour = machine._toolchain_status(m, set()) + self.assertEqual(text, '2') + + def test_unreachable_no_toolchains(self): + """Test unreachable machine with no arch info""" + m = machine.Machine('host1') + text, colour = machine._toolchain_status(m, {'arm'}) + self.assertEqual(text, '-') + + +class TestMachineExtended(unittest.TestCase): + """Extended Machine tests for coverage""" + + @mock.patch('buildman.machine._run_ssh') + def test_probe_toolchains_ssh_failure(self, mock_ssh): + """Test toolchain probe when SSH fails""" + mock_ssh.side_effect = machine.MachineError('timeout') + m = machine.Machine('host1') + result = m.probe_toolchains('buildman') + self.assertEqual(result, {}) + self.assertIn('timeout', m.tc_error) + + @mock.patch('buildman.machine._run_ssh') + def test_probe_toolchains_from_boss(self, mock_ssh): + """Test probing toolchains by checking boss's paths on remote""" + home = os.path.expanduser('~') + local_gcc = { + 'arm': f'{home}/.buildman-toolchains/gcc-13/arm/bin/gcc', + 'x86': f'{home}/.buildman-toolchains/gcc-13/x86/bin/gcc', + } + mock_ssh.return_value = 'arm:yes\nx86:no\n' + m = machine.Machine('host1') + result = m.probe_toolchains('buildman', local_gcc=local_gcc) + self.assertIn('arm', result) + self.assertNotIn('x86', result) + + @mock.patch('buildman.machine._run_ssh') + def test_probe_toolchains_from_boss_ssh_fail(self, mock_ssh): + """Test probing boss toolchains when SSH fails""" + home = os.path.expanduser('~') + local_gcc = { + 'arm': f'{home}/.buildman-toolchains/gcc-13/arm/bin/gcc', + } + mock_ssh.side_effect = machine.MachineError('conn refused') + m = machine.Machine('host1') + result = m.probe_toolchains('buildman', local_gcc=local_gcc) + self.assertEqual(result, {}) + self.assertIn('conn refused', m.tc_error) + + @mock.patch('buildman.machine._run_ssh') + def test_probe_toolchains_from_boss_no_home(self, mock_ssh): + """Test probing boss toolchains with non-home paths""" + local_gcc = {'sandbox': '/usr/bin/gcc'} + m = machine.Machine('host1') + result = m.probe_toolchains('buildman', local_gcc=local_gcc) + self.assertEqual(result, {}) + mock_ssh.assert_not_called() + + @mock.patch('buildman.machine._run_ssh') + def test_fetch_toolchain_success(self, mock_ssh): + """Test successful toolchain fetch""" + mock_ssh.return_value = 'Fetched arm toolchain' + m = machine.Machine('host1') + self.assertTrue(m.fetch_toolchain('buildman', 'arm')) + + @mock.patch('buildman.machine._run_ssh') + def test_fetch_toolchain_failure(self, mock_ssh): + """Test failed toolchain fetch""" + mock_ssh.side_effect = machine.MachineError('fetch failed') + m = machine.Machine('host1') + self.assertFalse(m.fetch_toolchain('buildman', 'arm')) + + @mock.patch('buildman.machine._run_ssh') + def test_weight_unavailable(self, mock_ssh): + """Test weight is 0 when unavailable""" + m = machine.Machine('host1') + m.avail = False + m._calc_weight() + self.assertEqual(m.weight, 0) + + +class TestRunSshExtended(unittest.TestCase): + """Extended _run_ssh tests""" + + @mock.patch('buildman.machine.command.run_pipe') + def test_failure_no_stderr(self, mock_pipe): + """Test SSH failure with no stderr""" + mock_pipe.return_value = mock.Mock( + return_code=1, stdout='', stderr='') + with self.assertRaises(machine.MachineError) as ctx: + machine._run_ssh('host1', ['cmd']) + self.assertIn('failed with code 1', str(ctx.exception)) + + @mock.patch('buildman.machine.command.run_pipe') + def test_stdin_data(self, mock_pipe): + """Test SSH with stdin_data""" + mock_pipe.return_value = mock.Mock( + return_code=0, stdout='result\n', stderr='') + result = machine._run_ssh('host1', ['python3'], + stdin_data='print("result")') + self.assertEqual(result, 'result\n') + # Verify stdin_data was passed through + _, kwargs = mock_pipe.call_args + self.assertEqual(kwargs['stdin_data'], 'print("result")') + + +class TestMachinePoolExtended(unittest.TestCase): + """Extended MachinePool tests for coverage""" + + def setUp(self): + bsettings.setup(None) + + def test_load_with_max_boards(self): + """Test loading machines with max_boards config""" + bsettings.add_file( + '[machines]\n' + 'server1\n' + '[machine:server1]\n' + 'max_boards = 50\n' + ) + pool = machine.MachinePool() + self.assertEqual(len(pool.machines), 1) + self.assertEqual(pool.machines[0].max_boards, 50) + + def test_load_filtered_names(self): + """Test loading only specified machine names""" + bsettings.add_file( + '[machines]\n' + 'host1\n' + 'host2\n' + 'host3\n' + ) + pool = machine.MachinePool(names=['host1', 'host3']) + self.assertEqual(len(pool.machines), 2) + names = [m.hostname for m in pool.machines] + self.assertEqual(names, ['host1', 'host3']) + + def test_get_reachable(self): + """Test get_reachable includes busy machines""" + bsettings.add_file('[machines]\nhost1\nhost2\n') + pool = machine.MachinePool() + # Simulate host1 reachable but busy, host2 unreachable + pool.machines[0].avail = False + pool.machines[0].info.arch = 'x86_64' + self.assertEqual(len(pool.get_reachable()), 1) + self.assertEqual(pool.get_reachable()[0].hostname, 'host1') + + @mock.patch('buildman.machine._run_ssh') + def test_print_summary(self, mock_ssh): + """Test print_summary runs without error""" + mock_ssh.return_value = json.dumps({ + **MACHINE_INFO, 'load_1m': 1.0, + 'mem_avail_mb': 8000, 'disk_avail_mb': 20000}) + bsettings.add_file('[machines]\nhost1\n') + pool = machine.MachinePool() + pool.probe_all() + # Just verify it doesn't crash + with terminal.capture(): + pool.print_summary() + + @mock.patch('buildman.machine._run_ssh') + def test_print_summary_with_toolchains(self, mock_ssh): + """Test print_summary with toolchain info""" + def ssh_side_effect(_hostname, cmd, **_kwargs): + if 'python3' in cmd: + return json.dumps({ + **MACHINE_INFO, 'load_1m': 1.0, + 'mem_avail_mb': 8000, 'disk_avail_mb': 20000}) + if '--list-tool-chains' in cmd: + return 'List of available toolchains (1):\narm : /bin/gcc\n\n' + return '' + + mock_ssh.side_effect = ssh_side_effect + bsettings.add_file('[machines]\nhost1\n') + pool = machine.MachinePool() + pool.probe_all() + pool.check_toolchains({'arm', 'sandbox'}) + with terminal.capture(): + pool.print_summary(local_archs={'arm', 'sandbox'}) + + @mock.patch('buildman.machine._run_ssh') + def test_check_toolchains_version_mismatch(self, mock_ssh): + """Test version mismatch detection in check_toolchains""" + home = os.path.expanduser('~') + + # The remote has gcc-12, but the boss has gcc-13 + remote_gcc = (f'.buildman-toolchains/gcc-12.0.0-nolibc/' + 'arm/bin/arm-linux-gcc') + + def ssh_side_effect(_hostname, cmd, **_kwargs): + if 'python3' in cmd: + return json.dumps({ + **MACHINE_INFO, 'load_1m': 1.0, + 'mem_avail_mb': 8000, 'disk_avail_mb': 20000}) + # Boss probe: report arm as present (wrong version) + return 'arm:yes\n' + + mock_ssh.side_effect = ssh_side_effect + bsettings.add_file('[machines]\nhost1\n') + pool = machine.MachinePool() + pool.probe_all() + + local_gcc = { + 'arm': f'{home}/.buildman-toolchains/gcc-13.1.0-nolibc/' + 'arm/bin/arm-linux-gcc', + } + # check_toolchains will re-probe, which sets the remote path + # via _probe_toolchains_from_boss. The SSH mock returns + # arm:yes, so the remote path uses the boss's relative path + # but we need it to have the old version. Patch the machine's + # toolchains after the probe runs. + orig_probe = machine.Machine._probe_toolchains_from_boss + + def fake_probe(self_mach, lg): + orig_probe(self_mach, lg) + # Override with the wrong version + if 'arm' in self_mach.toolchains: + self_mach.toolchains['arm'] = f'~/{remote_gcc}' + return self_mach.toolchains + + with mock.patch.object(machine.Machine, + '_probe_toolchains_from_boss', + fake_probe): + missing = pool.check_toolchains({'arm'}, local_gcc=local_gcc) + + # arm should be flagged as missing due to version mismatch + self.assertEqual(len(missing), 1) + + +class TestFetchMissing(unittest.TestCase): + """Test _fetch_all_missing()""" + + def setUp(self): + bsettings.setup(None) + + @mock.patch('buildman.machine._run_ssh') + def test_fetch_success(self, mock_ssh): + """Test successful toolchain fetch""" + mock_ssh.return_value = '' + bsettings.add_file('[machines]\nhost1\n') + pool = machine.MachinePool() + m = pool.machines[0] + m.avail = True + m.info.arch = 'x86_64' + + missing_map = {m: {'arm'}} + with terminal.capture(): + pool._fetch_all_missing(missing_map, {}, None, 'buildman') + # After successful fetch + re-probe, arch is removed + # (re-probe returns empty since SSH mock returns '') + + @mock.patch('buildman.machine._run_ssh') + def test_fetch_failure(self, mock_ssh): + """Test failed toolchain fetch""" + mock_ssh.side_effect = machine.MachineError('failed') + bsettings.add_file('[machines]\nhost1\n') + pool = machine.MachinePool() + m = pool.machines[0] + m.avail = True + m.info.arch = 'x86_64' + + missing_map = {m: {'arm'}} + with terminal.capture(): + pool._fetch_all_missing(missing_map, {}, None, 'buildman') + # Should still have the missing arch + self.assertIn('arm', missing_map[m]) + + @mock.patch('buildman.machine._run_ssh') + def test_fetch_with_version_removal(self, mock_ssh): + """Test fetching removes old version first""" + calls = [] + + def ssh_side_effect(hostname, cmd, **_kwargs): + calls.append(cmd) + if '--fetch-arch' in cmd: + return '' + if 'rm' in cmd: + return '' + return '' + + mock_ssh.side_effect = ssh_side_effect + bsettings.add_file('[machines]\nhost1\n') + pool = machine.MachinePool() + m = pool.machines[0] + m.avail = True + m.info.arch = 'x86_64' + m.toolchains = { + 'arm': '~/.buildman-toolchains/gcc-12.0.0-nolibc/arm/bin/gcc'} + + missing_map = {m: {'arm'}} + local_versions = {'arm': 'gcc-13.1.0-nolibc'} + with terminal.capture(): + pool._fetch_all_missing(missing_map, local_versions, None, + 'buildman') + # Should have called rm -rf for the old version + rm_calls = [c for c in calls if 'rm' in c] + self.assertTrue(len(rm_calls) > 0) + + +class TestPrintSummaryEdgeCases(unittest.TestCase): + """Test print_summary edge cases""" + + def setUp(self): + bsettings.setup(None) + + @mock.patch('buildman.machine._run_ssh') + def test_unavailable_machine(self, mock_ssh): + """Test summary with unavailable machine""" + mock_ssh.side_effect = machine.MachineError('refused') + bsettings.add_file('[machines]\nhost1\n') + pool = machine.MachinePool() + pool.probe_all() + # Should not crash with unavailable machine + with terminal.capture(): + pool.print_summary() + + @mock.patch('buildman.machine._run_ssh') + def test_busy_machine(self, mock_ssh): + """Test summary with busy machine""" + mock_ssh.return_value = json.dumps({ + **MACHINE_INFO, 'load_1m': 10.0}) + bsettings.add_file('[machines]\nhost1\n') + pool = machine.MachinePool() + pool.probe_all() + with terminal.capture(): + pool.print_summary() + + @mock.patch('buildman.machine._run_ssh') + def test_with_max_boards(self, mock_ssh): + """Test summary shows max_boards""" + mock_ssh.return_value = json.dumps({ + **MACHINE_INFO, 'load_1m': 1.0, + 'mem_avail_mb': 8000, 'disk_avail_mb': 20000}) + bsettings.add_file( + '[machines]\nhost1\n' + '[machine:host1]\nmax_boards = 50\n') + pool = machine.MachinePool() + pool.probe_all() + with terminal.capture(): + pool.print_summary() + + @mock.patch('buildman.machine._run_ssh') + def test_with_tc_error(self, mock_ssh): + """Test summary with toolchain error note""" + mock_ssh.return_value = json.dumps({ + **MACHINE_INFO, 'load_1m': 1.0, + 'mem_avail_mb': 8000, 'disk_avail_mb': 20000}) + bsettings.add_file('[machines]\nhost1\n') + pool = machine.MachinePool() + pool.probe_all() + pool.machines[0].tc_error = 'buildman not found' + with terminal.capture(): + pool.print_summary(local_archs={'arm'}) + + @mock.patch('buildman.machine._run_ssh') + def test_with_missing_toolchains(self, mock_ssh): + """Test summary with missing toolchain notes""" + mock_ssh.return_value = json.dumps({ + **MACHINE_INFO, 'load_1m': 1.0, + 'mem_avail_mb': 8000, 'disk_avail_mb': 20000}) + bsettings.add_file('[machines]\nhost1\n') + pool = machine.MachinePool() + pool.probe_all() + pool.machines[0].toolchains = {'sandbox': '/usr/bin/gcc'} + local_gcc = { + 'arm': os.path.expanduser( + '~/.buildman-toolchains/gcc-13/arm/bin/gcc'), + } + with terminal.capture(): + pool.print_summary(local_archs={'arm', 'sandbox'}, + local_gcc=local_gcc) + + +class TestCheckToolchainsEdge(unittest.TestCase): + """Test check_toolchains edge cases""" + + def setUp(self): + bsettings.setup(None) + + def test_no_reachable(self): + """Test check_toolchains with no reachable machines (line 482)""" + bsettings.add_file('[machines]\nhost1\n') + pool = machine.MachinePool() + # Machine is not probed, so not reachable + result = pool.check_toolchains({'arm'}) + self.assertEqual(result, {}) + + @mock.patch('buildman.machine._run_ssh') + def test_fetch_flag(self, mock_ssh): + """Test check_toolchains with fetch=True (line 524)""" + home = os.path.expanduser('~') + + def ssh_side_effect(_hostname, cmd, **_kwargs): + if 'python3' in cmd: + return json.dumps({ + **MACHINE_INFO, 'load_1m': 1.0, + 'mem_avail_mb': 8000, 'disk_avail_mb': 20000}) + # No toolchains found + return '' + + mock_ssh.side_effect = ssh_side_effect + bsettings.add_file('[machines]\nhost1\n') + pool = machine.MachinePool() + pool.probe_all() + + local_gcc = { + 'arm': f'{home}/.buildman-toolchains/gcc-13/arm/bin/gcc', + } + # fetch=True should trigger _fetch_all_missing + with mock.patch.object(pool, '_fetch_all_missing') as mock_fetch: + pool.check_toolchains({'arm'}, fetch=True, + local_gcc=local_gcc) + mock_fetch.assert_called_once() + + +class TestFetchVersionRemovalFailure(unittest.TestCase): + """Test _fetch_all_missing rm -rf failure path (lines 563-564)""" + + def setUp(self): + bsettings.setup(None) + + @mock.patch('buildman.machine._run_ssh') + def test_rm_failure_continues(self, mock_ssh): + """Test that rm -rf failure is silently ignored""" + call_count = [0] + + def ssh_side_effect(hostname, cmd, **_kwargs): + call_count[0] += 1 + if 'rm' in cmd: + raise machine.MachineError('rm failed') + if '--fetch-arch' in cmd: + return '' + return '' + + mock_ssh.side_effect = ssh_side_effect + bsettings.add_file('[machines]\nhost1\n') + pool = machine.MachinePool() + m = pool.machines[0] + m.avail = True + m.info.arch = 'x86_64' + # Old version that differs from local + m.toolchains = { + 'arm': '~/.buildman-toolchains/gcc-12.0.0-nolibc/arm/bin/gcc'} + + missing_map = {m: {'arm'}} + local_versions = {'arm': 'gcc-13.1.0-nolibc'} + with terminal.capture(): + pool._fetch_all_missing(missing_map, local_versions, None, + 'buildman') + # Should have attempted rm and then fetch despite rm failure + self.assertGreater(call_count[0], 1) + + +class TestPrintSummaryNotProbed(unittest.TestCase): + """Test print_summary 'not probed' branch (lines 669-670)""" + + def setUp(self): + bsettings.setup(None) + + def test_not_probed_machine(self): + """Test summary shows 'not probed' for unprobed machine""" + bsettings.add_file('[machines]\nhost1\n') + pool = machine.MachinePool() + # Don't probe - machine stays in 'not probed' state + with terminal.capture(): + pool.print_summary() + + +class TestPrintSummaryMissingNoVersion(unittest.TestCase): + """Test print_summary missing toolchain without version (line 707)""" + + def setUp(self): + bsettings.setup(None) + + @mock.patch('buildman.machine._run_ssh') + def test_missing_with_and_without_version(self, mock_ssh): + """Test missing note for archs with and without version info""" + home = os.path.expanduser('~') + mock_ssh.return_value = json.dumps({ + **MACHINE_INFO, 'load_1m': 1.0, + 'mem_avail_mb': 8000, 'disk_avail_mb': 20000}) + bsettings.add_file('[machines]\nhost1\n') + pool = machine.MachinePool() + pool.probe_all() + pool.machines[0].toolchains = {} + # arm has a version (under ~/.buildman-toolchains), + # sandbox does not + local_gcc = { + 'arm': f'{home}/.buildman-toolchains/gcc-13.1.0-nolibc/' + 'arm/bin/gcc', + 'sandbox': '/usr/bin/gcc', + } + with terminal.capture(): + pool.print_summary(local_archs={'arm', 'sandbox'}, + local_gcc=local_gcc) + + +class TestDoProbe(unittest.TestCase): + """Test do_probe_machines()""" + + def setUp(self): + bsettings.setup(None) + + def test_no_machines(self): + """Test with no machines configured""" + with terminal.capture(): + ret = machine.do_probe_machines() + self.assertEqual(ret, 1) + + @mock.patch('buildman.machine._run_ssh') + def test_with_machines(self, mock_ssh): + """Test probing configured machines""" + def ssh_side_effect(_hostname, cmd, **_kwargs): + if 'python3' in cmd: + return json.dumps({ + **MACHINE_INFO, 'load_1m': 1.0, + 'mem_avail_mb': 8000, 'disk_avail_mb': 20000}) + if '--list-tool-chains' in cmd: + return 'List of available toolchains (0):\n\n' + return '' + + mock_ssh.side_effect = ssh_side_effect + bsettings.add_file('[machines]\nhost1\n') + with terminal.capture(): + ret = machine.do_probe_machines() + self.assertEqual(ret, 0) + + +if __name__ == '__main__': + unittest.main() -- 2.43.0