[PATCH 0/3] codman: Add category system and CSV output
From: Simon Glass <simon.glass@canonical.com> This series adds a category system to codman for classifying source files by functional area (boot, drivers, networking, etc.). Categories and features are defined in a TOML configuration file (category.cfg). The category system is used by a new CSV output option, which generates machine-readable reports suitable for spreadsheet analysis. Other output formats (terminal, HTML) do not yet use categories. Simon Glass (3): codman: Add category module for file classification codman: Set up categories for the U-Boot codebase codman: Add CSV output with category support tools/codman/category.cfg | 854 ++++++++++++++++++++++++++++++++++ tools/codman/category.py | 147 ++++++ tools/codman/codman.py | 19 +- tools/codman/codman.rst | 89 ++++ tools/codman/output.py | 209 ++++++++- tools/codman/test_category.py | 279 +++++++++++ tools/codman/test_output.py | 212 +++++++++ 7 files changed, 1807 insertions(+), 2 deletions(-) create mode 100644 tools/codman/category.cfg create mode 100644 tools/codman/category.py create mode 100644 tools/codman/test_category.py create mode 100644 tools/codman/test_output.py -- 2.43.0 base-commit: c6b889d3fa570424ac3db8f555d1ca0373ec2145 branch: codmana
From: Simon Glass <simon.glass@canonical.com> When analysing large codebases, it is useful to group source files by functional area such as boot, drivers, or networking. Add category.py module to match source files to categories and features defined in a TOML configuration file. The module supports three pattern types: exact paths, glob patterns, and directory prefixes. An ignore list allows excluding external code from reports. Add unit tests for the category-matching functions and document the category system in codman.rst Co-developed-by: Claude Opus 4.5 <noreply@anthropic.com> Signed-off-by: Simon Glass <simon.glass@canonical.com> --- tools/codman/category.py | 147 +++++++++++++++++++++ tools/codman/codman.rst | 41 ++++++ tools/codman/test_category.py | 234 ++++++++++++++++++++++++++++++++++ 3 files changed, 422 insertions(+) create mode 100644 tools/codman/category.py create mode 100644 tools/codman/test_category.py diff --git a/tools/codman/category.py b/tools/codman/category.py new file mode 100644 index 00000000000..b7b2ecb93ac --- /dev/null +++ b/tools/codman/category.py @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright 2025 Canonical Ltd +# +"""Category and feature management for codman. + +This module provides functions for loading category configuration and +matching source files to features/categories. +""" + +from collections import namedtuple +import fnmatch +import os + +try: + import tomllib +except ImportError: + import tomli as tomllib + +from u_boot_pylib import tools + +# Return type for load_category_config functions +CategoryConfig = namedtuple('CategoryConfig', + ['categories', 'features', 'ignore']) + + +def load_category_config(srcdir): + """Load category configuration from category.cfg. + + Args: + srcdir (str): Root directory of the source tree + + Returns: + CategoryConfig: namedtuple with (categories, features, ignore) or + None if not found + """ + cfg_path = os.path.join(srcdir, 'tools', 'codman', 'category.cfg') + if not os.path.exists(cfg_path): + return None + + try: + data = tools.read_file(cfg_path, binary=False) + config = tomllib.loads(data) + ignore = config.get('ignore', {}).get('files', []) + return CategoryConfig(config.get('categories', {}), + config.get('features', {}), ignore) + except (IOError, tomllib.TOMLDecodeError): + return None + + +def load_config_file(cfg_path): + """Load category configuration from a specific file path. + + Args: + cfg_path (str): Path to the category configuration file + + Returns: + CategoryConfig: namedtuple with (categories, features, ignore) or + None if not found + """ + if not os.path.exists(cfg_path): + return None + + try: + data = tools.read_file(cfg_path, binary=False) + config = tomllib.loads(data) + ignore = config.get('ignore', {}).get('files', []) + return CategoryConfig(config.get('categories', {}), + config.get('features', {}), ignore) + except (IOError, tomllib.TOMLDecodeError): + return None + + +def should_ignore_file(filepath, ignore_patterns): + """Check if a file should be ignored based on ignore patterns. + + Args: + filepath (str): Relative file path to check + ignore_patterns (list): List of patterns to ignore + + Returns: + bool: True if file should be ignored + """ + if not ignore_patterns: + return False + + for pattern in ignore_patterns: + # Directory prefix: pattern ending with '/' matches all files under it + if pattern.endswith('/'): + if filepath.startswith(pattern): + return True + # Exact match or glob pattern + elif fnmatch.fnmatch(filepath, pattern) or filepath == pattern: + return True + return False + + +def get_file_feature(filepath, features): + """Match a file path to a feature based on the feature's file patterns. + + Args: + filepath (str): Relative file path to match + features (dict): Features dict from category config + + Returns: + tuple: (feature_id, category_id) or (None, None) if no match + """ + for feat_id, feat_data in features.items(): + for pattern in feat_data.get('files', []): + # Directory prefix: pattern ending with '/' matches all under it + if pattern.endswith('/'): + if filepath.startswith(pattern): + return feat_id, feat_data.get('category') + # Exact match or glob pattern + elif fnmatch.fnmatch(filepath, pattern) or filepath == pattern: + return feat_id, feat_data.get('category') + return None, None + + +def get_category_desc(categories, category_id): + """Get the description for a category. + + Args: + categories (dict): Categories dict from category config + category_id (str): Category identifier + + Returns: + str: Category description or None if not found + """ + if categories and category_id in categories: + return categories[category_id].get('description') + return None + + +def get_feature_desc(features, feature_id): + """Get the description for a feature. + + Args: + features (dict): Features dict from category config + feature_id (str): Feature identifier + + Returns: + str: Feature description or None if not found + """ + if features and feature_id in features: + return features[feature_id].get('description') + return None diff --git a/tools/codman/codman.rst b/tools/codman/codman.rst index 7f5df3d1c6f..c651fd6514e 100644 --- a/tools/codman/codman.rst +++ b/tools/codman/codman.rst @@ -312,6 +312,47 @@ The HTML report includes: This is useful for sharing reports or exploring large codebases interactively in a web browser. +Categories and Features +----------------------- + +Codman can categorise source files into functional areas using the +``tools/codman/category.cfg`` configuration file. This TOML file defines: + +**Categories**: High-level groupings like "load-boot", "storage", "drivers" + +**Features**: Specific functional areas within categories, with file patterns +that define which source files belong to each feature. + +The configuration uses three types of file patterns: + +* Exact paths: ``"boot/bootm.c"`` +* Glob patterns: ``"drivers/video/*.c"`` +* Directory prefixes: ``"lib/acpi/"`` (matches all files under the directory) + +Example category.cfg structure:: + + [categories.load-boot] + description = "Loading & Boot" + + [features.boot-linux-direct] + category = "load-boot" + description = "Direct Linux boot" + files = [ + "boot/bootm.c", + "boot/bootm_os.c", + "boot/image-board.c", + ] + +**Ignoring External Code** + +The ``[ignore]`` section in category.cfg can exclude external/vendored code +from reports:: + + [ignore] + files = [ + "lib/lwip/lwip/", # External lwIP library + ] + Unused Files (``unused``) ------------------------- diff --git a/tools/codman/test_category.py b/tools/codman/test_category.py new file mode 100644 index 00000000000..3ce89d70b18 --- /dev/null +++ b/tools/codman/test_category.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0+ +# +# Copyright 2025 Canonical Ltd +# +"""Unit tests for category.py module""" + +import os +import shutil +import sys +import tempfile +import unittest + +# Test configuration +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +# Import the module to test +sys.path.insert(0, SCRIPT_DIR) +sys.path.insert(0, os.path.join(SCRIPT_DIR, '..')) +import category # pylint: disable=wrong-import-position +from u_boot_pylib import tools # pylint: disable=wrong-import-position + + +class TestMatchFileToFeature(unittest.TestCase): + """Test cases for get_file_feature function""" + + def test_exact_match(self): + """Test exact file path matching""" + features = { + 'test-feature': { + 'category': 'test-cat', + 'files': ['path/to/file.c'], + } + } + feat_id, cat_id = category.get_file_feature( + 'path/to/file.c', features) + self.assertEqual(feat_id, 'test-feature') + self.assertEqual(cat_id, 'test-cat') + + def test_glob_pattern(self): + """Test glob pattern matching""" + features = { + 'test-feature': { + 'category': 'test-cat', + 'files': ['drivers/video/*.c'], + } + } + feat_id, cat_id = category.get_file_feature( + 'drivers/video/console.c', features) + self.assertEqual(feat_id, 'test-feature') + self.assertEqual(cat_id, 'test-cat') + + def test_directory_prefix(self): + """Test directory prefix matching (pattern ending with /)""" + features = { + 'efi-loader': { + 'category': 'efi', + 'files': ['lib/efi_loader/'], + } + } + # Should match files directly in directory + feat_id, cat_id = category.get_file_feature( + 'lib/efi_loader/efi_acpi.c', features) + self.assertEqual(feat_id, 'efi-loader') + self.assertEqual(cat_id, 'efi') + + # Should match files in subdirectories + feat_id, cat_id = category.get_file_feature( + 'lib/efi_loader/subdir/file.c', features) + self.assertEqual(feat_id, 'efi-loader') + self.assertEqual(cat_id, 'efi') + + def test_no_match(self): + """Test when no feature matches""" + features = { + 'test-feature': { + 'category': 'test-cat', + 'files': ['other/path/*.c'], + } + } + feat_id, cat_id = category.get_file_feature( + 'different/path/file.c', features) + self.assertIsNone(feat_id) + self.assertIsNone(cat_id) + + def test_empty_features(self): + """Test with empty features dict""" + feat_id, cat_id = category.get_file_feature('any/file.c', {}) + self.assertIsNone(feat_id) + self.assertIsNone(cat_id) + + def test_feature_without_files(self): + """Test feature with empty files list""" + features = { + 'test-feature': { + 'category': 'test-cat', + 'files': [], + } + } + feat_id, cat_id = category.get_file_feature( + 'any/file.c', features) + self.assertIsNone(feat_id) + self.assertIsNone(cat_id) + + def test_first_match_wins(self): + """Test that first matching feature is returned""" + features = { + 'feature-a': { + 'category': 'cat-a', + 'files': ['lib/'], + }, + 'feature-b': { + 'category': 'cat-b', + 'files': ['lib/specific.c'], + } + } + # Order depends on dict iteration, but one should match + feat_id, _ = category.get_file_feature( + 'lib/specific.c', features) + self.assertIsNotNone(feat_id) + self.assertIn(feat_id, ['feature-a', 'feature-b']) + + +class TestLoadCategoryConfig(unittest.TestCase): + """Test cases for load_category_config functions""" + + def setUp(self): + """Create temporary directory for test files""" + self.test_dir = tempfile.mkdtemp(prefix='test_category_') + + def tearDown(self): + """Clean up temporary directory""" + if os.path.exists(self.test_dir): + shutil.rmtree(self.test_dir) + + def test_load_valid_config(self): + """Test loading a valid TOML config file""" + cfg_content = ''' +[categories.load-boot] +description = "Loading & Boot" + +[features.boot-direct] +category = "load-boot" +description = "Direct boot" +files = ["boot/bootm.c"] +''' + cfg_path = os.path.join(self.test_dir, 'category.cfg') + tools.write_file(cfg_path, cfg_content, binary=False) + + result = category.load_config_file(cfg_path) + + self.assertIsNotNone(result) + self.assertIn('load-boot', result.categories) + self.assertEqual(result.categories['load-boot']['description'], + 'Loading & Boot') + self.assertIn('boot-direct', result.features) + self.assertEqual(result.features['boot-direct']['category'], + 'load-boot') + + def test_load_missing_file(self): + """Test loading from non-existent file""" + cfg_path = os.path.join(self.test_dir, 'nonexistent.cfg') + result = category.load_config_file(cfg_path) + self.assertIsNone(result) + + def test_load_invalid_toml(self): + """Test loading invalid TOML file""" + cfg_path = os.path.join(self.test_dir, 'invalid.cfg') + tools.write_file(cfg_path, 'this is not valid TOML [[[', binary=False) + result = category.load_config_file(cfg_path) + self.assertIsNone(result) + + def test_load_from_srcdir(self): + """Test load_category_config with srcdir parameter""" + # Create tools/codman directory structure + codman_dir = os.path.join(self.test_dir, 'tools', 'codman') + os.makedirs(codman_dir) + + cfg_content = ''' +[categories.test] +description = "Test category" + +[features.test-feat] +category = "test" +description = "Test feature" +files = [] +''' + cfg_path = os.path.join(codman_dir, 'category.cfg') + tools.write_file(cfg_path, cfg_content, binary=False) + + result = category.load_category_config(self.test_dir) + + self.assertIsNotNone(result) + self.assertIn('test', result.categories) + + +class TestHelperFunctions(unittest.TestCase): + """Test cases for helper functions""" + + def test_get_category_desc(self): + """Test get_category_desc function""" + categories = { + 'load-boot': {'description': 'Loading & Boot'}, + 'storage': {'description': 'Storage'}, + } + desc = category.get_category_desc(categories, 'load-boot') + self.assertEqual(desc, 'Loading & Boot') + + desc = category.get_category_desc(categories, 'nonexistent') + self.assertIsNone(desc) + + desc = category.get_category_desc(None, 'load-boot') + self.assertIsNone(desc) + + def test_get_feature_desc(self): + """Test get_feature_desc function""" + features = { + 'boot-direct': { + 'description': 'Direct boot', + 'category': 'load-boot', + }, + } + desc = category.get_feature_desc(features, 'boot-direct') + self.assertEqual(desc, 'Direct boot') + + desc = category.get_feature_desc(features, 'nonexistent') + self.assertIsNone(desc) + + desc = category.get_feature_desc(None, 'boot-direct') + self.assertIsNone(desc) + + +if __name__ == '__main__': + unittest.main() -- 2.43.0
From: Simon Glass <simon.glass@canonical.com> Provide a basic category.cfg file for U-Boot, focussing solely on the qemu-x86 build. Signed-off-by: Simon Glass <simon.glass@canonical.com> --- tools/codman/category.cfg | 854 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 854 insertions(+) create mode 100644 tools/codman/category.cfg diff --git a/tools/codman/category.cfg b/tools/codman/category.cfg new file mode 100644 index 00000000000..478459d1bb0 --- /dev/null +++ b/tools/codman/category.cfg @@ -0,0 +1,854 @@ +# Codman category and feature configuration +# +# This file defines categories and features for code analysis. +# +# Ignore section lists external code to exclude from reports: +# [ignore] +# files = ["path/to/external/"] +# +# Categories group related features together: +# [categories.<label>] +# description = "Human-readable name" +# +# Features define functional areas with associated source files: +# [features.<label>] +# category = "<category-label>" # must match a category above +# description = "Human-readable name" +# files = [ # list of file patterns to match +# "path/to/file.c", # exact file path +# "path/to/files*", # glob pattern +# "path/to/dir/", # directory prefix (matches all files under it) +# ] +# +# Features should be kept in alphabetical order by label. + +[ignore] +#files = [ +# "lib/lwip/lwip/", +#] + +[categories.efi-loader] +description = "EFI loader" + +[categories.load-boot] +description = "Loading & Boot" + +[categories.storage] +description = "Storage" + +[categories.tables] +description = "Tables & Data Structures" + +[categories.drivers] +description = "Drivers" + +[categories.network] +description = "Network" + +[categories.user-interface] +description = "User Interface" + +[categories.verification] +description = "Verification" + +[categories.other] +description = "Other" + +[features.board-init] +category = "load-boot" +description = "Board initialisation" +files = [ + "common/board_f.c", + "common/board_r.c", + "common/board_info.c", + "common/init/", +] + +[features.block-devices] +category = "storage" +description = "Block devices" +files = [ + "cmd/blkmap.c", + "drivers/block/blk-uclass.c", + "drivers/block/blkcache.c", + "drivers/block/blkmap.c", + "drivers/block/blkmap_helper.c", + "drivers/block/ide.c", +] + +[features.bochs] +category = "user-interface" +description = "Bochs display" +files = [] + +[features.boot-arm-bare] +category = "load-boot" +description = "Boot on bare-metal ARM targets" +files = [] + +[features.boot-arm-qemu] +category = "load-boot" +description = "Boot on ARM QEMU" +files = [] + +[features.boot-efi-x86-arm] +category = "load-boot" +description = "Boot on EFI on x86 & ARM" +files = [] + +[features.boot-linux-direct] +category = "load-boot" +description = "Boot Linux directly" +files = [ + "boot/bootm.c", + "boot/bootm_final.c", + "boot/bootm_os.c", + "boot/bootmeth-uclass.c", + "boot/bootmeth_cros.c", + "boot/bootmeth_efi.c", + "boot/bootmeth_efi_mgr.c", + "boot/bootmeth_extlinux.c", + "boot/bootmeth_pxe.c", + "boot/bootmeth_qfw.c", + "boot/bootmeth_script.c", + "common/bootstage.c", +] + +[features.boot-linux-efi] +category = "load-boot" +description = "Boot Linux via EFI" +files = [ + "cmd/efi_common.c", + "cmd/eficonfig.c", + "cmd/efidebug.c", + "lib/efi/basename.c", + "lib/efi/device_path.c", + "lib/efi/helper.c", + "lib/efi/input.c", + "lib/efi/load_options.c", + "lib/efi/memory.c", + "lib/efi/run.c", + "lib/efi/string.c", +] + +[features.efi-loader] +category = "efi-loader" +description = "Boot Linux via EFI" +files = [ + "lib/efi_loader/", +] + +[features.boot-timing] +category = "load-boot" +description = "Boot timing & pass to userspace" +files = [] + +[features.boot-x86-bare] +category = "load-boot" +description = "Boot on x86 bare metal" +files = [ + "arch/x86/cpu/", + "arch/x86/cpu/cpu.c", + "arch/x86/cpu/cpu_x86.c", + "arch/x86/cpu/i386/call64.S", + "arch/x86/cpu/i386/cpu.c", + "arch/x86/cpu/i386/interrupt.c", + "arch/x86/cpu/i386/setjmp.S", + "arch/x86/cpu/intel_common/cpu.c", + "arch/x86/cpu/intel_common/cpu_from_spl.c", + "arch/x86/cpu/intel_common/fast_spi.c", + "arch/x86/cpu/intel_common/lpc.c", + "arch/x86/cpu/intel_common/lpss.c", + "arch/x86/cpu/intel_common/microcode.c", + "arch/x86/cpu/intel_common/pch.c", + "arch/x86/cpu/ioapic.c", + "arch/x86/cpu/irq.c", + "arch/x86/cpu/lapic.c", + "arch/x86/cpu/mtrr.c", + "arch/x86/cpu/pci.c", + "arch/x86/cpu/qemu/car.S", + "arch/x86/cpu/qemu/cpu.c", + "arch/x86/cpu/qemu/dram.c", + "arch/x86/cpu/qemu/e820.c", + "arch/x86/cpu/qemu/qemu.c", + "arch/x86/cpu/qfw_cpu.c", + "arch/x86/cpu/resetvec.S", + "arch/x86/cpu/sipi_vector.S", + "arch/x86/cpu/start.S", + "arch/x86/cpu/start16.S", + "arch/x86/cpu/start64.S", + "arch/x86/cpu/turbo.c", + "arch/x86/cpu/x86_64/cpu.c", + "arch/x86/cpu/x86_64/interrupts.c", + "arch/x86/cpu/x86_64/misc.c", + "arch/x86/cpu/x86_64/setjmp.S", + "arch/x86/lib/acpi.c", + "arch/x86/lib/bdinfo.c", + "arch/x86/lib/bios.c", + "arch/x86/lib/bios_asm.S", + "arch/x86/lib/bios_interrupts.c", + "arch/x86/lib/bootm.c", + "arch/x86/lib/cmd_boot.c", + "arch/x86/lib/crt0_ia32_efi.S", + "arch/x86/lib/crt0_x86_64_efi.S", + "arch/x86/lib/div64.c", + "arch/x86/lib/e820.c", + "arch/x86/lib/early_cmos.c", + "arch/x86/lib/i8254.c", + "arch/x86/lib/i8259.c", + "arch/x86/lib/init_helpers.c", + "arch/x86/lib/interrupts.c", + "arch/x86/lib/lpc-uclass.c", + "arch/x86/lib/mpspec.c", + "arch/x86/lib/northbridge-uclass.c", + "arch/x86/lib/physmem.c", + "arch/x86/lib/pirq_routing.c", + "arch/x86/lib/reloc_ia32_efi.c", + "arch/x86/lib/reloc_x86_64_efi.c", + "arch/x86/lib/relocate.c", + "arch/x86/lib/sections.c", + "arch/x86/lib/sfi.c", + "arch/x86/lib/spl.c", + "arch/x86/lib/string.c", + "arch/x86/lib/zimage.c", +] + +[features.cli] +category = "other" +description = "CLI for command entry" +files = [ + "cmd/acpi.c", + "cmd/bdinfo.c", + "cmd/blk_common.c", + "cmd/blkcache.c", + "cmd/bloblist.c", + "cmd/boot.c", + "cmd/bootdev.c", + "cmd/bootefi.c", + "cmd/bootflow.c", + "cmd/bootm.c", + "cmd/bootmeth.c", + "cmd/bootstage.c", + "cmd/bootstd.c", + "cmd/cat.c", + "cmd/cls.c", + "cmd/console.c", + "cmd/cpu.c", + "cmd/cyclic.c", + "cmd/date.c", + "cmd/disk.c", + "cmd/dm.c", + "cmd/echo.c", + "cmd/elf.c", + "cmd/exit.c", + "cmd/ext2.c", + "cmd/ext4.c", + "cmd/fat.c", + "cmd/fdt.c", + "cmd/font.c", + "cmd/fs.c", + "cmd/gettime.c", + "cmd/help.c", + "cmd/ide.c", + "cmd/io.c", + "cmd/irq.c", + "cmd/itest.c", + "cmd/legacy-mtd-utils.c", + "cmd/load.c", + "cmd/malloc.c", + "cmd/mem.c", + "cmd/meminfo.c", + "cmd/mii.c", + "cmd/mouse.c", + "cmd/net-common.c", + "cmd/net.c", + "cmd/nvedit.c", + "cmd/nvedit_efi.c", + "cmd/nvme.c", + "cmd/panic.c", + "cmd/part.c", + "cmd/pci.c", + "cmd/qfw.c", + "cmd/rtc.c", + "cmd/scsi.c", + "cmd/sf.c", + "cmd/sleep.c", + "cmd/smbios.c", + "cmd/source.c", + "cmd/spi.c", + "cmd/test.c", + "cmd/time.c", + "cmd/usb.c", + "cmd/vbe.c", + "cmd/version.c", + "cmd/video.c", + "cmd/virtio.c", + "cmd/x86/cpuid.c", + "cmd/x86/msr.c", + "cmd/x86/mtrr.c", + "cmd/x86/qfw_x86.c", + "cmd/x86/zboot.c", + "cmd/ximg.c", + "common/cli.c", + "common/cli_getch.c", + "common/cli_hush.c", + "common/cli_readline.c", + "common/cli_simple.c", + "common/command.c", +] + +[features.text-console] +category = "user-interface" +description = "Text console" +files = [ + "common/console.c", + "common/stdio.c", + "common/iomux.c", +] + +[features.compression] +category = "other" +description = "Compression (lzma, gzip, lzo, zstd)" +files = [ + "lib/gunzip.c", + "lib/lz4.c", + "lib/lz4_wrapper.c", + "lib/lzo/", + "lib/xxhash.c", + "lib/zlib/", + "lib/zstd/", +] + +[features.crc-hash] +category = "verification" +description = "CRC and hash" +files = [ + "common/hash.c", + "drivers/rng/rng-uclass.c", + "lib/crc16-ccitt.c", + "lib/crc16.c", + "lib/crc32.c", + "lib/crc8.c", + "lib/hash-checksum.c", + "lib/hashtable.c", + "lib/md5.c", + "lib/sha1.c", + "lib/sha256.c", + "lib/sha256_common.c", + "lib/sha512.c", +] + +[features.disk-handling] +category = "storage" +description = "Disk handling" +files = [ + "disk/disk-uclass.c", + "disk/part.c", + "disk/part_dos.c", + "disk/part_efi.c", + "disk/part_iso.c", + "disk/part_mac.c", +] + +[features.dos-partition] +category = "storage" +description = "DOS partition read" +files = [] + +[features.driver-model] +category = "drivers" +description = "Driver abstraction/model" +files = [ + "drivers/ata/ahci-pci.c", + "drivers/ata/ahci-uclass.c", + "drivers/ata/ahci.c", + "drivers/ata/libata.c", + "drivers/core/device-remove.c", + "drivers/core/device.c", + "drivers/core/dump.c", + "drivers/core/fdtaddr.c", + "drivers/core/lists.c", + "drivers/core/of_extra.c", + "drivers/core/ofnode.c", + "drivers/core/ofnode_graph.c", + "drivers/core/read_extra.c", + "drivers/core/regmap.c", + "drivers/core/root.c", + "drivers/core/simple-bus.c", + "drivers/core/syscon-uclass.c", + "drivers/core/tag.c", + "drivers/core/uclass.c", + "drivers/core/util.c", + "drivers/cpu/cpu-uclass.c", + "drivers/crypto/fsl/sec.c", + "drivers/gpio/gpio-uclass.c", + "drivers/misc/irq-uclass.c", + "drivers/mtd/mtd_uboot.c", + "drivers/mtd/mtdcore.c", + "drivers/mtd/spi/sf-uclass.c", + "drivers/mtd/spi/sf_probe.c", + "drivers/mtd/spi/spi-nor-core.c", + "drivers/mtd/spi/spi-nor-ids.c", + "drivers/mtd/spi/spi-nor-tiny.c", + "drivers/pch/pch-uclass.c", + "drivers/pch/pch7.c", + "drivers/pch/pch9.c", + "drivers/pinctrl/pinctrl-generic.c", + "drivers/pinctrl/pinctrl-uclass.c", + "drivers/scsi/scsi-uclass.c", + "drivers/scsi/scsi.c", + "drivers/scsi/scsi_bootdev.c", + "drivers/spi/spi-mem.c", + "drivers/spi/spi-uclass.c", + "drivers/sysinfo/sysinfo-uclass.c", + "drivers/sysreset/sysreset-uclass.c", + "drivers/sysreset/sysreset_x86.c", + "drivers/usb/common/common.c", + "drivers/usb/eth/asix.c", + "drivers/usb/eth/smsc95xx.c", + "drivers/usb/eth/usb_ether.c", + "common/event.c", +] + +[features.efi-device-paths] +category = "tables" +description = "EFI device paths" +files = [] + +[features.efi-gop] +category = "user-interface" +description = "EFI GOP display" +files = [] + +[features.efi-ulib-lace] +category = "drivers" +description = "EFI / ulib / Lace driver model" +files = [ + "lib/efi_driver/", + "lib/ulib/", + "lib/uuid.c", +] + +[features.net-lwip] +category = "network" +description = "lwIP network stack" +files = [ + "cmd/net-lwip.c", + "lib/lwip/", + "net/lwip/", +] + +[features.ethernet] +category = "network" +description = "Ethernet" +files = [ + "common/miiphyutil.c", + "drivers/net/e1000.c", + "lib/net_utils.c", + "net/arp.c", + "net/bootp.c", + "net/eth-uclass.c", + "net/eth_bootdev.c", + "net/eth_common.c", + "net/net-common.c", + "net/net.c", + "net/ping.c", + "net/tftp.c", +] + +[features.ext4-readonly] +category = "storage" +description = "EXT4 read-only support" +files = [ + "fs/ext4/dev.c", + "fs/ext4/ext4_common.c", + "fs/ext4/ext4_journal.c", + "fs/ext4/ext4_write.c", + "fs/ext4/ext4fs.c", +] + +[features.ext4-readwrite] +category = "storage" +description = "EXT4 read-write support" +files = [ + "fs/ext4l/", +] + +[features.extlinux-parse] +category = "load-boot" +description = "Extlinux configuration parsing" +files = [ + "boot/pxe_parse.c", + "boot/pxe_utils.c", + "cmd/pxe.c", +] + +[features.fat-read] +category = "storage" +description = "FAT read" +files = [ + "fs/fat/fat.c", + "fs/fat/fat_write.c", +] + +[features.fdt-read] +category = "tables" +description = "FDT reading" +files = [ + "lib/fdtdec.c", + "lib/fdtdec_common.c", +] + +[features.fdt-update] +category = "tables" +description = "FDT updating" +files = [ + "boot/fdt_support.c", +] + +[features.firmware-handoff] +category = "tables" +description = "Firmware handoff" +files = [ + "common/bloblist.c", +] + +[features.fit-load] +category = "load-boot" +description = "FIT image loading" +files = [ + "boot/bootdev-uclass.c", + "boot/bootflow.c", + "boot/bootflow_menu.c", + "boot/bootstd-uclass.c", + "boot/common_fit.c", + "boot/ext_pxe_common.c", + "boot/fdt_region.c", + "boot/fit_print.c", + "boot/image-board.c", + "boot/image-cipher.c", + "boot/image-fdt.c", + "boot/image-fit-sig.c", + "boot/image-fit.c", + "boot/image-host.c", + "boot/image-pre-load.c", + "boot/image.c", + "boot/vbe.c", + "boot/vbe_common.c", + "boot/vbe_request.c", + "boot/vbe_simple.c", + "boot/vbe_simple_os.c", + "lib/fdt-libcrypto.c", + "lib/fdt_print.c", +] + +[features.fit-verify-internal] +category = "verification" +description = "FIT verification (internal)" +files = [ + "lib/aes/", + "lib/ecdsa/ecdsa-libcrypto.c", + "lib/rc4.c", + "lib/rsa/rsa-mod-exp.c", + "lib/rsa/rsa-sign.c", + "lib/rsa/rsa-verify.c", +] + +[features.fit-verify-shim] +category = "verification" +description = "FIT verification via Shim" +files = [] + +[features.fonts] +category = "user-interface" +description = "Font support" +files = [ + "drivers/video/console_truetype.c", +] + +[features.fs-layer] +category = "storage" +description = "Filesystem layer" +files = [ + "fs/dir-uclass.c", + "fs/file-uclass.c", + "fs/fs-uclass.c", + "fs/fs_internal.c", + "fs/fs_legacy.c", +] + +[features.graphical-display] +category = "user-interface" +description = "Graphical display support" +files = [ + "boot/bootflow_menu.c", + "boot/expo.c", + "common/splash.c", + "common/splash_source.c", + "boot/expo_build.c", + "boot/scene.c", + "boot/scene_menu.c", + "boot/scene_textedit.c", + "boot/scene_textline.c", + "drivers/video/backlight-uclass.c", + "drivers/video/bochs.c", + "drivers/video/console_core.c", + "drivers/video/console_normal.c", + "drivers/video/panel-uclass.c", + "drivers/video/simple_panel.c", + "drivers/video/vesa_helper.c", + "drivers/video/vidconsole-uclass.c", + "drivers/video/video-uclass.c", + "drivers/video/video_bmp.c", +] + +[features.https-images] +category = "storage" +description = "HTTPS to read images" +files = [] + +[features.keyboard-decode] +category = "user-interface" +description = "Keyboard decode" +files = [] + +[features.keyboard-input] +category = "user-interface" +description = "Keyboard input support" +files = [ + "common/usb_kbd.c", + "drivers/input/i8042.c", + "drivers/input/input.c", + "drivers/input/key_matrix.c", + "drivers/input/keyboard-uclass.c", + "drivers/input/mouse-uclass.c", + "drivers/input/usb_mouse.c", +] + +[features.logging] +category = "other" +description = "Logging" +files = [ + "common/log.c", + "common/log_console.c", +] + +[features.alloc] +category = "other" +description = "Malloc & Memory" +files = [ + "common/dlmalloc.c", + "common/malloc_simple.c", + "common/memsize.c", + "common/memtop.c", + "lib/lmb.c", + "lib/physmem.c", +] + +[features.memory-images] +category = "tables" +description = "Memory for loaded images" +files = [] + +[features.misc] +category = "other" +description = "Miscellaneous" +files = [ + "common/autoboot.c", + "common/cyclic.c", + "common/exports.c", + "common/main.c", + "common/s_record.c", + "common/version.c", + "common/xyzModem.c", + "lib/binman.c", + "lib/dhry/", + "lib/elf.c", +] + +[features.mouse-input] +category = "user-interface" +description = "Mouse input support" +files = [ + "drivers/input/mouse-uclass*", + "efi_mouse*", + "usb_mouse*", +] + +[features.nvme] +category = "storage" +description = "NVMe support" +files = [ + "drivers/nvme/nvme-uclass.c", + "drivers/nvme/nvme.c", + "drivers/nvme/nvme_pci.c", + "drivers/nvme/nvme_show.c", +] + +[features.pci] +category = "drivers" +description = "PCI support" +files = [ + "drivers/pci/", +] + +[features.qemu-qfw] +category = "drivers" +description = "QEMU / Qfw features" +files = [ + "drivers/qfw/qfw-uclass.c", + "drivers/qfw/qfw.c", + "drivers/qfw/qfw_acpi.c", + "drivers/qfw/qfw_pio.c", + "drivers/qfw/qfw_smbios.c", +] + +[features.schema-access] +category = "tables" +description = "Schema read & access" +files = [ + "boot/bootctl/", +] + +[features.serial-driver] +category = "drivers" +description = "Serial driver" +files = [ + "drivers/serial/ns16550.c", + "drivers/serial/serial-uclass.c", +] + +[features.spl] +category = "load-boot" +description = "SPL (Secondary Program Loader)" +files = [ + "common/spl/", +] + +[features.tests] +category = "other" +description = "Tests" +files = [ + "lib/efi_selftest/", + "test/", +] + +[features.text-console-graphical] +category = "user-interface" +description = "Text console on graphical display" +files = [] + +[features.text-editor] +category = "user-interface" +description = "Text editor" +files = [] + +[features.text-menu] +category = "user-interface" +description = "Text-based menu" +files = [ + "common/menu.c", +] + +[features.timer-rtc] +category = "drivers" +description = "Timer and RTC drivers" +files = [ + "drivers/rtc/mc146818.c", + "drivers/rtc/rtc-uclass.c", + "drivers/timer/timer-uclass.c", + "drivers/timer/tsc_timer.c", + "lib/date.c", + "lib/rtc-lib.c", + "lib/time.c", +] + +[features.truetype-fonts] +category = "user-interface" +description = "Antialiased truetype fonts" +files = [] + +[features.environment] +category = "other" +description = "Environment" +files = [ + "env/", +] + +[features.unmatched] +category = "other" +description = "Unmatched files" +files = [ +] + +[features.usb-core] +category = "drivers" +description = "USB core" +files = [ + "common/usb.c", + "common/usb_hub.c", +] + +[features.usb-storage] +category = "storage" +description = "USB devices (host) storage" +files = [ + "common/usb_storage.c", +] + +[features.utility-lib] +category = "other" +description = "Utility library functions" +files = [ + "lib/abuf.c", + "lib/alist.c", + "lib/charset.c", + "lib/ctype.c", + "lib/display_options.c", + "lib/div64.c", + "lib/errno.c", + "lib/hang.c", + "lib/hexdump.c", + "lib/ldiv.c", + "lib/linux_compat.c", + "lib/linux_string.c", + "lib/list_sort.c", + "lib/membuf.c", + "lib/panic.c", + "lib/qsort.c", + "lib/rand.c", + "lib/slre.c", + "lib/string.c", + "lib/strto.c", + "lib/vsprintf.c", +] + +[features.vesa] +category = "user-interface" +description = "VESA display" +files = [] + +[features.virtio] +category = "drivers" +description = "Virtio drivers" +files = [ + "board/emulation/common/bootcmd.c", + "drivers/virtio/", +] + +[features.x86-tables] +category = "tables" +description = "x86 ACPI and SMBIOS tables" +files = [ + "arch/x86/lib/tables.c", + "lib/acpi/", + "lib/smbios-parser.c", + "lib/tables_csum.c", +] + +[features.xhci-usb-host] +category = "drivers" +description = "XHCI / USB host" +files = [ + "drivers/usb/host/", +] -- 2.43.0
From: Simon Glass <simon.glass@canonical.com> When analysing code by functional area, a machine-readable format is needed for spreadsheet analysis and further processing. Add a --csv option to generate CSV reports with category and feature columns. The category system matches source files to functional areas defined in category.cfg using exact paths, glob patterns, and directory prefixes. Add a -F/--files-only flag for simplified output with just file rows, -u/--show-unmatched to list uncategorised files, and -E/--show-empty-features to list placeholder features. An [ignore] section in category.cfg allows excluding external code from reports. Co-developed-by: Claude Opus 4.5 <noreply@anthropic.com> Signed-off-by: Simon Glass <simon.glass@canonical.com> --- tools/codman/codman.py | 19 ++- tools/codman/codman.rst | 48 ++++++++ tools/codman/output.py | 209 ++++++++++++++++++++++++++++++++- tools/codman/test_category.py | 45 ++++++++ tools/codman/test_output.py | 212 ++++++++++++++++++++++++++++++++++ 5 files changed, 531 insertions(+), 2 deletions(-) create mode 100644 tools/codman/test_output.py diff --git a/tools/codman/codman.py b/tools/codman/codman.py index 893512456b2..7c6da823611 100755 --- a/tools/codman/codman.py +++ b/tools/codman/codman.py @@ -488,6 +488,14 @@ def parse_args(argv=None): help='Show line counts in kilolines (kLOC) instead of lines') dirs.add_argument('--html', type=str, metavar='FILE', help='Output results as HTML to the specified file') + dirs.add_argument('--csv', type=str, metavar='FILE', + help='Output results as CSV to the specified file') + dirs.add_argument('-u', '--show-unmatched', action='store_true', + help='List all files without a category match') + dirs.add_argument('-F', '--files-only', action='store_true', + help='Only output file rows in CSV (exclude directories)') + dirs.add_argument('-E', '--show-empty-features', action='store_true', + help='List features with no files defined') # detail command detail = subparsers.add_parser('detail', @@ -611,8 +619,9 @@ def do_output(args, all_srcs, used, skipped, results, srcdir, analysis_method): elif args.cmd == 'copy-used': ok = output.copy_used_files(used, srcdir, args.copy_used) elif args.cmd == 'dirs': - # Check if HTML output is requested + # Check if HTML or CSV output is requested html_file = getattr(args, 'html', None) + csv_file = getattr(args, 'csv', None) if html_file: ok = output.generate_html_breakdown(all_srcs, used, results, srcdir, args.subdirs, args.show_files, @@ -620,6 +629,14 @@ def do_output(args, all_srcs, used, skipped, results, srcdir, analysis_method): getattr(args, 'kloc', False), html_file, args.board, analysis_method) + elif csv_file: + ok = output.generate_csv( + all_srcs, used, results, srcdir, args.subdirs, + args.show_files, args.show_empty, + getattr(args, 'kloc', False), csv_file, + getattr(args, 'show_unmatched', False), + getattr(args, 'files_only', False), + getattr(args, 'show_empty_features', False)) else: ok = output.show_dir_breakdown(all_srcs, used, results, srcdir, args.subdirs, args.show_files, diff --git a/tools/codman/codman.rst b/tools/codman/codman.rst index c651fd6514e..a9f361c7c70 100644 --- a/tools/codman/codman.rst +++ b/tools/codman/codman.rst @@ -138,6 +138,10 @@ The ``dirs command`` has a few extra options: * ``-e, --show-empty`` - Show directories/files with 0 lines used * ``-k, --kloc`` - Show line counts in kilolines (kLOC) instead of raw lines * ``--html <file>`` - Generate an HTML report with collapsible drill-down +* ``--csv <file>`` - Generate a CSV report for spreadsheet analysis +* ``-F, --files-only`` - Only output file rows in CSV (exclude directories) +* ``-u, --show-unmatched`` - List files without a category match +* ``-E, --show-empty-features`` - List features with no files defined Other: @@ -312,6 +316,39 @@ The HTML report includes: This is useful for sharing reports or exploring large codebases interactively in a web browser. +CSV Reports (``dirs --csv``) +---------------------------- + +Generate a CSV report for spreadsheet analysis or further processing:: + + codman -b qemu-x86 dirs -sf --csv report.csv + +The CSV includes columns for Type, Path, Category, Feature, file counts, and +line statistics:: + + Type,Path,Category,Feature,Files,Used,%Used,%Code,Lines,Used + dir,arch/x86/cpu,,,20,15,75,85,3816,3227 + file,arch/x86/cpu/call32.S,load-boot,boot-x86-bare,,,,100,61,61 + file,arch/x86/cpu/cpu.c,load-boot,boot-x86-bare,,,,88,399,353 + ... + +Use ``-F`` (``--files-only``) for a simplified output with just file rows +(no directory summaries):: + + codman -b qemu-x86 dirs -sf --csv report.csv -F + +This produces cleaner output with columns: Path, Category, Feature, %Code, +Lines, Used:: + + Path,Category,Feature,%Code,Lines,Used + arch/x86/cpu/call32.S,load-boot,boot-x86-bare,100,61,61 + arch/x86/cpu/cpu.c,load-boot,boot-x86-bare,88,399,353 + arch/x86/cpu/cpu_x86.c,load-boot,boot-x86-bare,100,99,99 + ... + +CSV reports include category information from ``category.cfg``. Other output +formats (terminal, HTML) do not yet use categories. + Categories and Features ----------------------- @@ -343,6 +380,17 @@ Example category.cfg structure:: "boot/image-board.c", ] +When generating HTML reports, codman matches each source file to its feature +and category, making it easy to analyse code by functional area. + +Use ``-u`` (``--show-unmatched``) to list files that don't match any feature:: + + codman -b qemu-x86 dirs -sf -u + +Use ``-E`` (``--show-empty-features``) to list features with no files defined:: + + codman -b qemu-x86 dirs -sf -E + **Ignoring External Code** The ``[ignore]`` section in category.cfg can exclude external/vendored code diff --git a/tools/codman/output.py b/tools/codman/output.py index 67d8f98a649..9c855207435 100644 --- a/tools/codman/output.py +++ b/tools/codman/output.py @@ -14,6 +14,7 @@ formats: - File copying operations """ +import csv import os import shutil import sys @@ -23,6 +24,8 @@ from collections import defaultdict sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) from u_boot_pylib import terminal, tout # pylint: disable=wrong-import-position +import category + class DirStats: # pylint: disable=too-few-public-methods """Statistics for a directory. @@ -507,7 +510,7 @@ def generate_html_breakdown(all_sources, used_sources, file_results, srcdir, use_kloc (bool): If True, show line counts in kLOC html_file (str): Path to output HTML file board (str): Board name (optional) - analysis_method (str): Analysis method used ('unifdef', 'lsp', or 'dwarf') + analysis_method (str): Analysis method ('unifdef'/'lsp'/'dwarf') Returns: bool: True on success @@ -907,6 +910,210 @@ def generate_html_breakdown(all_sources, used_sources, file_results, srcdir, return False +def _write_file_row(writer, info, features, ignore_patterns, file_results, + use_kloc, files_only): + """Write a single file row to CSV. + + Args: + writer: CSV writer object + info (dict): File info with 'path', 'total', 'active' keys + features (dict): Features dict from category config + ignore_patterns (list): List of patterns to ignore + file_results (dict): File analysis results (or None) + use_kloc (bool): If True, show line counts in kLOC + files_only (bool): If True, use simplified row format + + Returns: + tuple: (wrote_row, is_matched) - whether row was written, whether file + matched a category + """ + # Skip ignored files (external code) + if category.should_ignore_file(info['path'], ignore_patterns): + return False, True # Not written, but considered matched + + # Match file to feature/category + feat_id, cat_id = None, None + if features: + feat_id, cat_id = category.get_file_feature(info['path'], features) + + is_matched = feat_id is not None + + if file_results: + pct_active = percent(info['active'], info['total']) + + if use_kloc: + total_str = klocs(info['total']) + active_str = klocs(info['active']) + else: + total_str = info['total'] + active_str = info['active'] + + if files_only: + writer.writerow([info['path'], cat_id or '', feat_id or '', + f'{pct_active:.0f}', total_str, active_str]) + else: + writer.writerow(['file', info['path'], cat_id or '', feat_id or '', + '', '', '', f'{pct_active:.0f}', + total_str, active_str]) + + return True, is_matched + + +def _report_matching_stats(features, total_files, unmatched_files, + show_unmatched, show_empty_features): + """Report category matching statistics. + + Args: + features (dict): Features dict from category config + total_files (int): Total number of files processed + unmatched_files (list): List of file paths without category match + show_unmatched (bool): If True, list all unmatched files + show_empty_features (bool): If True, list features with no files + """ + if features and total_files > 0: + matched = total_files - len(unmatched_files) + print(f'Category matching: {matched}/{total_files} files matched, ' + f'{len(unmatched_files)} unmatched') + if show_unmatched and unmatched_files: + print('Unmatched files:') + for filepath in sorted(unmatched_files): + print(f' {filepath}') + + if features and show_empty_features: + empty_features = [ + feat_id for feat_id, feat_data in features.items() + if not feat_data.get('files', []) + ] + if empty_features: + print(f'Features with no files ({len(empty_features)}):') + for feat_id in sorted(empty_features): + print(f' {feat_id}') + + +def generate_csv(all_sources, used_sources, file_results, srcdir, + by_subdirs, show_files, show_empty, use_kloc, csv_file, + show_unmatched=False, files_only=False, + show_empty_features=False): + """Generate CSV output with directory breakdown. + + Args: + all_sources (set): Set of all source file paths + used_sources (set): Set of used source file paths + file_results (dict): Optional dict mapping file paths to line analysis + results (or None) + srcdir (str): Root directory of the source tree + by_subdirs (bool): If True, show full subdirectory breakdown + show_files (bool): If True, show individual files within directories + show_empty (bool): If True, show directories with 0 lines used + use_kloc (bool): If True, show line counts in kLOC + csv_file (str): Path to output CSV file + show_unmatched (bool): If True, list all unmatched files to stdout + files_only (bool): If True, only output file rows (exclude directories) + show_empty_features (bool): If True, list features with no files defined + + Returns: + bool: True on success + """ + + # Load category configuration for file-to-feature matching + cfg = category.load_category_config(srcdir) + features = cfg.features if cfg else None + ignore_patterns = cfg.ignore if cfg else None + + # Collect directory statistics + dir_stats = collect_dir_stats(all_sources, used_sources, file_results, + srcdir, by_subdirs, show_files) + + # Calculate totals + total_lines_all = sum(count_lines(f) for f in all_sources) + if file_results: + total_lines_used = sum(r.active_lines for r in file_results.values()) + else: + total_lines_used = sum(count_lines(f) for f in used_sources) + + # Track unmatched files + unmatched_files = [] + total_files = 0 + + try: + with open(csv_file, 'w', newline='', encoding='utf-8') as f: + writer = csv.writer(f) + + # Write header + lines_header = 'kLOC' if use_kloc else 'Lines' + if files_only: + writer.writerow(['Path', 'Category', 'Feature', '%Code', + lines_header, 'Used']) + else: + writer.writerow(['Type', 'Path', 'Category', 'Feature', 'Files', + 'Used', '%Used', '%Code', lines_header, 'Used']) + + # Sort and output directories + for dir_path in sorted(dir_stats.keys()): + stats = dir_stats[dir_path] + + # Skip directories with 0 lines used unless show_empty is set + if not show_empty and stats.lines_used == 0: + continue + + pct_used = percent(stats.used, stats.total) + pct_code = percent(stats.lines_used, stats.lines_total) + + if use_kloc: + lines_total_str = klocs(stats.lines_total) + lines_used_str = klocs(stats.lines_used) + else: + lines_total_str = stats.lines_total + lines_used_str = stats.lines_used + + if not files_only: + writer.writerow([ + 'dir', dir_path, '', '', stats.total, stats.used, + f'{pct_used:.0f}', f'{pct_code:.0f}', + lines_total_str, lines_used_str]) + + # Output files if requested + if show_files and stats.files: + sorted_files = sorted( + stats.files, key=lambda x: os.path.basename(x['path'])) + + for info in sorted_files: + if not show_empty and info['active'] == 0: + continue + + wrote, matched = _write_file_row( + writer, info, features, ignore_patterns, + file_results, use_kloc, files_only) + if wrote: + total_files += 1 + if not matched: + unmatched_files.append(info['path']) + + # Write totals row + pct_files = percent(len(used_sources), len(all_sources)) + pct_code = percent(total_lines_used, total_lines_all) + + if use_kloc: + total_str = klocs(total_lines_all) + used_str = klocs(total_lines_used) + else: + total_str = total_lines_all + used_str = total_lines_used + + if not files_only: + writer.writerow(['total', 'TOTAL', '', '', len(all_sources), + len(used_sources), f'{pct_files:.0f}', + f'{pct_code:.0f}', total_str, used_str]) + + tout.info(f'CSV report written to: {csv_file}') + _report_matching_stats(features, total_files, unmatched_files, + show_unmatched, show_empty_features) + return True + except IOError as e: + tout.error(f'Failed to write CSV file: {e}') + return False + + def show_statistics(all_sources, used_sources, skipped_sources, file_results, srcdir, top_n): """Show overall statistics about source file usage. diff --git a/tools/codman/test_category.py b/tools/codman/test_category.py index 3ce89d70b18..475c69df75a 100644 --- a/tools/codman/test_category.py +++ b/tools/codman/test_category.py @@ -194,6 +194,51 @@ files = [] self.assertIn('test', result.categories) +class TestShouldIgnoreFile(unittest.TestCase): + """Test cases for should_ignore_file function""" + + def test_ignore_directory_prefix(self): + """Test ignoring files by directory prefix""" + ignore = ['lib/external/'] + self.assertTrue(category.should_ignore_file( + 'lib/external/foo.c', ignore)) + self.assertTrue(category.should_ignore_file( + 'lib/external/sub/bar.c', ignore)) + self.assertFalse(category.should_ignore_file( + 'lib/internal/foo.c', ignore)) + + def test_ignore_exact_path(self): + """Test ignoring files by exact path""" + ignore = ['lib/external/specific.c'] + self.assertTrue(category.should_ignore_file( + 'lib/external/specific.c', ignore)) + self.assertFalse(category.should_ignore_file( + 'lib/external/other.c', ignore)) + + def test_ignore_glob_pattern(self): + """Test ignoring files by glob pattern""" + ignore = ['lib/external/*.c'] + self.assertTrue(category.should_ignore_file( + 'lib/external/foo.c', ignore)) + self.assertFalse(category.should_ignore_file( + 'lib/external/foo.h', ignore)) + + def test_empty_ignore_list(self): + """Test with empty ignore list""" + self.assertFalse(category.should_ignore_file('any/file.c', [])) + self.assertFalse(category.should_ignore_file('any/file.c', None)) + + def test_multiple_ignore_patterns(self): + """Test with multiple ignore patterns""" + ignore = ['lib/external/', 'vendor/*.c'] + self.assertTrue(category.should_ignore_file( + 'lib/external/foo.c', ignore)) + self.assertTrue(category.should_ignore_file( + 'vendor/bar.c', ignore)) + self.assertFalse(category.should_ignore_file( + 'src/main.c', ignore)) + + class TestHelperFunctions(unittest.TestCase): """Test cases for helper functions""" diff --git a/tools/codman/test_output.py b/tools/codman/test_output.py new file mode 100644 index 00000000000..126ea95af57 --- /dev/null +++ b/tools/codman/test_output.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0+ +# +# Copyright 2025 Canonical Ltd +# +"""Unit tests for output.py CSV generation""" + +import csv +import os +import shutil +import sys +import tempfile +import unittest +from collections import namedtuple + +# Test configuration +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +# Import the module to test +sys.path.insert(0, SCRIPT_DIR) +sys.path.insert(0, os.path.join(SCRIPT_DIR, '..')) +import output # pylint: disable=wrong-import-position +from u_boot_pylib import tools # pylint: disable=wrong-import-position + + +# Mock FileResult for testing +FileResult = namedtuple('FileResult', + ['total_lines', 'active_lines', 'inactive_lines']) + + +class TestGenerateCsv(unittest.TestCase): + """Test cases for generate_csv function""" + + def setUp(self): + """Create temporary directory with test files""" + self.test_dir = tempfile.mkdtemp(prefix='test_output_') + + # Create source files + self.src_dir = os.path.join(self.test_dir, 'src') + os.makedirs(os.path.join(self.src_dir, 'boot')) + os.makedirs(os.path.join(self.src_dir, 'drivers', 'net')) + os.makedirs(os.path.join(self.src_dir, 'tools', 'codman')) + + # Create test source files with known content + self.files = { + 'boot/bootm.c': '// boot\n' * 100, + 'boot/image.c': '// image\n' * 50, + 'drivers/net/eth.c': '// eth\n' * 200, + } + for path, content in self.files.items(): + full_path = os.path.join(self.src_dir, path) + tools.write_file(full_path, content, binary=False) + + # Create category.cfg + cfg_content = ''' +[categories.load-boot] +description = "Loading & Boot" + +[categories.drivers] +description = "Drivers" + +[features.boot-core] +category = "load-boot" +description = "Core boot" +files = ["boot/"] + +[features.ethernet] +category = "drivers" +description = "Ethernet" +files = ["drivers/net/"] +''' + cfg_path = os.path.join(self.src_dir, 'tools', 'codman', 'category.cfg') + tools.write_file(cfg_path, cfg_content, binary=False) + + self.csv_file = os.path.join(self.test_dir, 'report.csv') + + def tearDown(self): + """Clean up temporary directory""" + if os.path.exists(self.test_dir): + shutil.rmtree(self.test_dir) + + def test_csv_basic(self): + """Test basic CSV generation""" + all_sources = { + os.path.join(self.src_dir, p) for p in self.files + } + used_sources = all_sources.copy() + + result = output.generate_csv( + all_sources, used_sources, None, self.src_dir, + by_subdirs=True, show_files=True, show_empty=False, + use_kloc=False, csv_file=self.csv_file) + + self.assertTrue(result) + self.assertTrue(os.path.exists(self.csv_file)) + + # Read and verify CSV content + data = tools.read_file(self.csv_file, binary=False) + rows = list(csv.reader(data.splitlines())) + + # Check header + self.assertEqual(rows[0][0], 'Type') + self.assertEqual(rows[0][1], 'Path') + self.assertEqual(rows[0][2], 'Category') + self.assertEqual(rows[0][3], 'Feature') + + def test_csv_files_only(self): + """Test CSV generation with files_only option""" + all_sources = { + os.path.join(self.src_dir, p) for p in self.files + } + used_sources = all_sources.copy() + + result = output.generate_csv( + all_sources, used_sources, None, self.src_dir, + by_subdirs=True, show_files=True, show_empty=False, + use_kloc=False, csv_file=self.csv_file, files_only=True) + + self.assertTrue(result) + + data = tools.read_file(self.csv_file, binary=False) + rows = list(csv.reader(data.splitlines())) + + # Check simplified header for files_only + self.assertEqual(rows[0][0], 'Path') + self.assertEqual(rows[0][1], 'Category') + self.assertEqual(rows[0][2], 'Feature') + self.assertEqual(rows[0][3], '%Code') + + # No 'dir' or 'total' rows + for row in rows[1:]: + self.assertNotIn(row[0], ['dir', 'total']) + + def test_csv_category_matching(self): + """Test that files are matched to correct categories""" + all_sources = { + os.path.join(self.src_dir, p) for p in self.files + } + used_sources = all_sources.copy() + + # Create mock file results + file_results = {} + for path, content in self.files.items(): + full_path = os.path.join(self.src_dir, path) + lines = len(content.split('\n')) + file_results[full_path] = FileResult(lines, lines, 0) + + result = output.generate_csv( + all_sources, used_sources, file_results, self.src_dir, + by_subdirs=True, show_files=True, show_empty=False, + use_kloc=False, csv_file=self.csv_file, files_only=True) + + self.assertTrue(result) + + data = tools.read_file(self.csv_file, binary=False) + rows = list(csv.reader(data.splitlines())) + + # Find boot files and verify category + boot_rows = [r for r in rows[1:] if 'boot/' in r[0]] + self.assertEqual(len(boot_rows), 2) # bootm.c and image.c + for row in boot_rows: + self.assertEqual(row[1], 'load-boot') + self.assertEqual(row[2], 'boot-core') + + # Find driver files and verify category + driver_rows = [r for r in rows[1:] if 'drivers/' in r[0]] + self.assertEqual(len(driver_rows), 1) # eth.c + for row in driver_rows: + self.assertEqual(row[1], 'drivers') + self.assertEqual(row[2], 'ethernet') + + def test_csv_with_ignore(self): + """Test CSV generation with ignored files""" + # Add ignore section to config + cfg_path = os.path.join(self.src_dir, 'tools', 'codman', 'category.cfg') + existing = tools.read_file(cfg_path, binary=False) + tools.write_file(cfg_path, + existing + '\n[ignore]\nfiles = ["drivers/net/"]\n', + binary=False) + + all_sources = { + os.path.join(self.src_dir, p) for p in self.files + } + used_sources = all_sources.copy() + + # Create mock file results + file_results = {} + for path, content in self.files.items(): + full_path = os.path.join(self.src_dir, path) + lines = len(content.split('\n')) + file_results[full_path] = FileResult(lines, lines, 0) + + result = output.generate_csv( + all_sources, used_sources, file_results, self.src_dir, + by_subdirs=True, show_files=True, show_empty=False, + use_kloc=False, csv_file=self.csv_file, files_only=True) + + self.assertTrue(result) + + data = tools.read_file(self.csv_file, binary=False) + rows = list(csv.reader(data.splitlines())) + + # Verify ignored files are not in output + paths = [r[0] for r in rows[1:]] + self.assertFalse(any('drivers/net/' in p for p in paths)) + + # Boot files should still be there + self.assertTrue(any('boot/' in p for p in paths)) + + +if __name__ == '__main__': + unittest.main() -- 2.43.0
participants (1)
-
Simon Glass