From: Simon Glass <sjg@chromium.org> Some pytest tests restart the U-Boot process during execution via ubman.restart_uboot() or ubman.restart_uboot_with_flags() This can be undesirable when debugging the pytests, since you have to reconnect gdb many times. Add a 'restart' pytest marker to all such tests, allowing them to be excluded with some like -k 'not restart' The marker is applied to individual test functions or to test classes where all methods perform a restart. Signed-off-by: Simon Glass <sjg@chromium.org> --- doc/develop/pytest/usage.rst | 20 ++++++++++++++++--- test/py/pytest.ini | 1 + test/py/tests/test_distro.py | 4 ++++ .../test_capsule_firmware_fit.py | 1 + .../test_capsule_firmware_raw.py | 1 + .../test_capsule_firmware_signed_fit.py | 1 + .../test_capsule_firmware_signed_raw.py | 1 + test/py/tests/test_efi_fit.py | 1 + .../py/tests/test_efi_secboot/test_authvar.py | 1 + test/py/tests/test_efi_secboot/test_signed.py | 1 + .../test_efi_secboot/test_signed_intca.py | 1 + .../tests/test_efi_secboot/test_unsigned.py | 1 + test/py/tests/test_efi_selftest.py | 6 ++++++ test/py/tests/test_eficonfig.py | 1 + test/py/tests/test_fpga.py | 1 + test/py/tests/test_fs/test_erofs.py | 2 +- .../test_fs/test_squashfs/test_sqfs_ls.py | 1 + test/py/tests/test_help.py | 2 ++ test/py/tests/test_log.py | 1 + test/py/tests/test_reset.py | 2 ++ test/py/tests/test_sandbox_exit.py | 4 ++++ test/py/tests/test_saveenv.py | 1 + test/py/tests/test_spl.py | 1 + test/py/tests/test_stackprotector.py | 1 + test/py/tests/test_upl.py | 1 + test/py/tests/test_vbe_vpl.py | 1 + test/py/tests/test_vboot.py | 1 + test/py/tests/test_vpl.py | 1 + 28 files changed, 57 insertions(+), 4 deletions(-) diff --git a/doc/develop/pytest/usage.rst b/doc/develop/pytest/usage.rst index 9e9cc38c9b2..087acded0d8 100644 --- a/doc/develop/pytest/usage.rst +++ b/doc/develop/pytest/usage.rst @@ -236,9 +236,10 @@ protocol, or any graphical wrapper around gdb. Some tests deliberately cause the sandbox process to exit, e.g. to test the reset command, or sandbox's CTRL-C handling. When this happens, you will need -to attach the debugger to the new sandbox instance. If these tests are not -relevant to your debugging session, you can skip them using pytest's ``-k`` -command-line option; see the next section. +to attach the debugger to the new sandbox instance. These tests are marked +with the ``restart`` pytest marker, so you can skip them all with:: + + test/py/test.py -B sandbox --gdbserver localhost:1234 -m "not restart" Command-line options -------------------- @@ -606,3 +607,16 @@ set), since the locally launched QEMU is not available in that environment: .. code-block:: python @pytest.mark.localqemu + +The restart marker indicates that a test restarts the U-Boot process +during execution, e.g. via ``ubman.restart_uboot()`` or +``ubman.restart_uboot_with_flags()``. This can be used to skip these tests +when a restart is not desirable: + +.. code-block:: python + + @pytest.mark.restart + +To exclude these tests from a test run:: + + test/py/test.py -B sandbox -m "not restart" diff --git a/test/py/pytest.ini b/test/py/pytest.ini index bebb22cd3d6..a3a8aaf74b5 100644 --- a/test/py/pytest.ini +++ b/test/py/pytest.ini @@ -15,3 +15,4 @@ markers = singlethread: Cannot run in parallel role: U-Boot: Indicates the lab 'role' which can execute this test localqemu: U-Boot: Test launches its own QEMU, skip in lab mode + restart: U-Boot: Test restarts U-Boot during execution diff --git a/test/py/tests/test_distro.py b/test/py/tests/test_distro.py index bc3b511af63..0a0ab8269cc 100644 --- a/test/py/tests/test_distro.py +++ b/test/py/tests/test_distro.py @@ -10,6 +10,7 @@ CONSOLE = 'earlycon=uart8250,io,0x3f8 console=uart8250,io,0x3f8' @pytest.mark.boardspec('qemu-x86_64') @pytest.mark.role('qemu-x86_64') +@pytest.mark.restart def test_distro(ubman): """Test booting into Ubuntu 24.04""" with ubman.log.section('boot'): @@ -64,6 +65,7 @@ def test_distro(ubman): @pytest.mark.boardspec('colibri-imx8x') @pytest.mark.role('colibrimx8') +@pytest.mark.restart def test_distro_script(ubman): """Test that a selected board can boot into Llinux using a script""" with ubman.log.section('boot'): @@ -79,6 +81,7 @@ def test_distro_script(ubman): @pytest.mark.boardspec('efi-arm_app64') @pytest.mark.role('efi-aarch64') +@pytest.mark.restart def test_distro_arm_app_extlinux(ubman): """Test that the ARM EFI app can boot into Ubuntu 25.04 via extlinux""" with ubman.log.section('boot'): @@ -98,6 +101,7 @@ def test_distro_arm_app_extlinux(ubman): @pytest.mark.boardspec('efi-arm_app64') @pytest.mark.role('efi-aarch64') +@pytest.mark.restart def test_distro_arm_app_efi(ubman): """Test that the ARM EFI app can boot into Ubuntu 25.04 via EFI""" with ubman.log.section('boot'): diff --git a/test/py/tests/test_efi_capsule/test_capsule_firmware_fit.py b/test/py/tests/test_efi_capsule/test_capsule_firmware_fit.py index 016274533cd..338554291c1 100644 --- a/test/py/tests/test_efi_capsule/test_capsule_firmware_fit.py +++ b/test/py/tests/test_efi_capsule/test_capsule_firmware_fit.py @@ -28,6 +28,7 @@ from capsule_common import ( @pytest.mark.buildconfigspec('cmd_nvedit_efi') @pytest.mark.buildconfigspec('cmd_sf') @pytest.mark.slow +@pytest.mark.restart class TestEfiCapsuleFirmwareFit(): """Test capsule-on-disk firmware update for FIT images """ diff --git a/test/py/tests/test_efi_capsule/test_capsule_firmware_raw.py b/test/py/tests/test_efi_capsule/test_capsule_firmware_raw.py index b8cb483b380..42fb1ae9954 100644 --- a/test/py/tests/test_efi_capsule/test_capsule_firmware_raw.py +++ b/test/py/tests/test_efi_capsule/test_capsule_firmware_raw.py @@ -29,6 +29,7 @@ from capsule_common import ( @pytest.mark.buildconfigspec('cmd_nvedit_efi') @pytest.mark.buildconfigspec('cmd_sf') @pytest.mark.slow +@pytest.mark.restart class TestEfiCapsuleFirmwareRaw: """ Tests verifying capsule-on-disk firmware update for raw images """ diff --git a/test/py/tests/test_efi_capsule/test_capsule_firmware_signed_fit.py b/test/py/tests/test_efi_capsule/test_capsule_firmware_signed_fit.py index 29545c5080a..f0ad67e2468 100644 --- a/test/py/tests/test_efi_capsule/test_capsule_firmware_signed_fit.py +++ b/test/py/tests/test_efi_capsule/test_capsule_firmware_signed_fit.py @@ -31,6 +31,7 @@ from capsule_common import ( @pytest.mark.buildconfigspec('cmd_nvedit_efi') @pytest.mark.buildconfigspec('cmd_sf') @pytest.mark.slow +@pytest.mark.restart class TestEfiCapsuleFirmwareSignedFit(): """Capsule-on-disk firmware update test """ diff --git a/test/py/tests/test_efi_capsule/test_capsule_firmware_signed_raw.py b/test/py/tests/test_efi_capsule/test_capsule_firmware_signed_raw.py index a500c499bb9..8456f39454f 100644 --- a/test/py/tests/test_efi_capsule/test_capsule_firmware_signed_raw.py +++ b/test/py/tests/test_efi_capsule/test_capsule_firmware_signed_raw.py @@ -29,6 +29,7 @@ from capsule_common import ( @pytest.mark.buildconfigspec('cmd_nvedit_efi') @pytest.mark.buildconfigspec('cmd_sf') @pytest.mark.slow +@pytest.mark.restart class TestEfiCapsuleFirmwareSignedRaw(): """Firmware Update (Signed capsule with raw images) Test """ diff --git a/test/py/tests/test_efi_fit.py b/test/py/tests/test_efi_fit.py index 63ee8e6cef2..efa9ebf5adf 100644 --- a/test/py/tests/test_efi_fit.py +++ b/test/py/tests/test_efi_fit.py @@ -159,6 +159,7 @@ FDT_DATA = ''' @pytest.mark.buildconfigspec('fit') @pytest.mark.notbuildconfigspec('generate_acpi_table') @pytest.mark.requiredtool('dtc') +@pytest.mark.restart def test_efi_fit_launch(ubman): """Test handling of UEFI binaries inside FIT images. diff --git a/test/py/tests/test_efi_secboot/test_authvar.py b/test/py/tests/test_efi_secboot/test_authvar.py index 3750f302dba..03caec65fec 100644 --- a/test/py/tests/test_efi_secboot/test_authvar.py +++ b/test/py/tests/test_efi_secboot/test_authvar.py @@ -16,6 +16,7 @@ import pytest @pytest.mark.buildconfigspec('cmd_fat') @pytest.mark.buildconfigspec('cmd_nvedit_efi') @pytest.mark.slow +@pytest.mark.restart class TestEfiAuthVar(object): def test_efi_var_auth1(self, ubman, efi_boot_env): """ diff --git a/test/py/tests/test_efi_secboot/test_signed.py b/test/py/tests/test_efi_secboot/test_signed.py index e8aaef7090c..78ed2a08fe2 100644 --- a/test/py/tests/test_efi_secboot/test_signed.py +++ b/test/py/tests/test_efi_secboot/test_signed.py @@ -17,6 +17,7 @@ import pytest @pytest.mark.buildconfigspec('cmd_fat') @pytest.mark.buildconfigspec('cmd_nvedit_efi') @pytest.mark.slow +@pytest.mark.restart class TestEfiSignedImage(object): def test_efi_signed_image_auth1(self, ubman, efi_boot_env): """ diff --git a/test/py/tests/test_efi_secboot/test_signed_intca.py b/test/py/tests/test_efi_secboot/test_signed_intca.py index 58f7be03b8b..6e0d7800305 100644 --- a/test/py/tests/test_efi_secboot/test_signed_intca.py +++ b/test/py/tests/test_efi_secboot/test_signed_intca.py @@ -19,6 +19,7 @@ import pytest @pytest.mark.buildconfigspec('cmd_fat') @pytest.mark.buildconfigspec('cmd_nvedit_efi') @pytest.mark.slow +@pytest.mark.restart class TestEfiSignedImageIntca(object): def test_efi_signed_image_intca1(self, ubman, efi_boot_env_intca): """ diff --git a/test/py/tests/test_efi_secboot/test_unsigned.py b/test/py/tests/test_efi_secboot/test_unsigned.py index bd6e1b2dadd..f5624806d88 100644 --- a/test/py/tests/test_efi_secboot/test_unsigned.py +++ b/test/py/tests/test_efi_secboot/test_unsigned.py @@ -17,6 +17,7 @@ import pytest @pytest.mark.buildconfigspec('cmd_fat') @pytest.mark.buildconfigspec('cmd_nvedit_efi') @pytest.mark.slow +@pytest.mark.restart class TestEfiUnsignedImage(object): def test_efi_unsigned_image_auth1(self, ubman, efi_boot_env): """ diff --git a/test/py/tests/test_efi_selftest.py b/test/py/tests/test_efi_selftest.py index 5f29157ae15..2a959ae3406 100644 --- a/test/py/tests/test_efi_selftest.py +++ b/test/py/tests/test_efi_selftest.py @@ -7,6 +7,7 @@ import pytest @pytest.mark.buildconfigspec('cmd_bootefi_selftest') +@pytest.mark.restart def test_efi_selftest_base(ubman): """Run UEFI unit tests @@ -24,6 +25,7 @@ def test_efi_selftest_base(ubman): @pytest.mark.buildconfigspec('hush_parser') @pytest.mark.buildconfigspec('of_control') @pytest.mark.notbuildconfigspec('generate_acpi_table') +@pytest.mark.restart def test_efi_selftest_device_tree(ubman): """Test the device tree support in the UEFI sub-system @@ -44,6 +46,7 @@ def test_efi_selftest_device_tree(ubman): ubman.restart_uboot() @pytest.mark.buildconfigspec('cmd_bootefi_selftest') +@pytest.mark.restart def test_efi_selftest_watchdog_reboot(ubman): """Test the watchdog timer @@ -61,6 +64,7 @@ def test_efi_selftest_watchdog_reboot(ubman): ubman.run_command(cmd='', send_nl=False, wait_for_reboot=True) @pytest.mark.buildconfigspec('cmd_bootefi_selftest') +@pytest.mark.restart def test_efi_selftest_text_input(ubman): """Test the EFI_SIMPLE_TEXT_INPUT_PROTOCOL @@ -116,6 +120,7 @@ def test_efi_selftest_text_input(ubman): ubman.restart_uboot() @pytest.mark.buildconfigspec('cmd_bootefi_selftest') +@pytest.mark.restart def test_efi_selftest_text_input_ex(ubman): """Test the EFI_SIMPLE_TEXT_INPUT_EX_PROTOCOL @@ -180,6 +185,7 @@ def test_efi_selftest_text_input_ex(ubman): @pytest.mark.buildconfigspec('cmd_bootefi_selftest') @pytest.mark.buildconfigspec('efi_tcg2_protocol') @pytest.mark.notbuildconfigspec('sandbox') +@pytest.mark.restart def test_efi_selftest_tcg2(ubman): """Test the EFI_TCG2 PROTOCOL diff --git a/test/py/tests/test_eficonfig.py b/test/py/tests/test_eficonfig.py index 20ea8d9f28f..171c8ec6cc8 100644 --- a/test/py/tests/test_eficonfig.py +++ b/test/py/tests/test_eficonfig.py @@ -12,6 +12,7 @@ from tests import fs_helper @pytest.mark.boardspec('sandbox') @pytest.mark.buildconfigspec('cmd_eficonfig') @pytest.mark.buildconfigspec('cmd_bootefi_bootmgr') +@pytest.mark.restart def test_efi_eficonfig(ubman): def prepare_image(u_boot_config): diff --git a/test/py/tests/test_fpga.py b/test/py/tests/test_fpga.py index 299a8653f74..84998dc2de2 100644 --- a/test/py/tests/test_fpga.py +++ b/test/py/tests/test_fpga.py @@ -97,6 +97,7 @@ expected_usage = 'fpga - loadable FPGA image support' @pytest.mark.xfail @pytest.mark.buildconfigspec('cmd_fpga') +@pytest.mark.restart def test_fpga_fail(ubman): # Test non valid fpga subcommand expected = 'fpga: non existing command' diff --git a/test/py/tests/test_fs/test_erofs.py b/test/py/tests/test_fs/test_erofs.py index a2bb6b505f2..5150cb5cb1f 100644 --- a/test/py/tests/test_fs/test_erofs.py +++ b/test/py/tests/test_fs/test_erofs.py @@ -189,7 +189,7 @@ def erofs_run_all_tests(ubman): @pytest.mark.buildconfigspec('fs_erofs') @pytest.mark.requiredtool('mkfs.erofs') @pytest.mark.requiredtool('md5sum') - +@pytest.mark.restart def test_erofs(ubman): """ Executes the erofs test suite. diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py b/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py index adda3b98cda..50522c31d02 100644 --- a/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py +++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py @@ -106,6 +106,7 @@ def sqfs_run_all_ls_tests(ubman): @pytest.mark.buildconfigspec('fs_squashfs') @pytest.mark.requiredtool('mksquashfs') @pytest.mark.singlethread +@pytest.mark.restart def test_sqfs_ls(ubman): """ Executes the sqfsls test suite. diff --git a/test/py/tests/test_help.py b/test/py/tests/test_help.py index afb57201ba3..ff8de8e8124 100644 --- a/test/py/tests/test_help.py +++ b/test/py/tests/test_help.py @@ -15,6 +15,7 @@ def test_help(ubman): assert lines.splitlines()[0] == "? - alias for 'help'" @pytest.mark.boardspec('sandbox') +@pytest.mark.restart def test_help_no_devicetree(ubman): try: ubman.restart_uboot_with_flags([], use_dtb=False) @@ -26,6 +27,7 @@ def test_help_no_devicetree(ubman): ubman.restart_uboot() @pytest.mark.boardspec('sandbox_vpl') +@pytest.mark.restart def test_vpl_help(ubman): try: ubman.restart_uboot() diff --git a/test/py/tests/test_log.py b/test/py/tests/test_log.py index 4558b037e2a..3fc61e258a0 100644 --- a/test/py/tests/test_log.py +++ b/test/py/tests/test_log.py @@ -41,6 +41,7 @@ def test_log_format(ubman): @pytest.mark.buildconfigspec('debug_uart') @pytest.mark.boardspec('sandbox') +@pytest.mark.restart def test_log_dropped(ubman): """Test dropped 'log' message when debug_uart is activated""" diff --git a/test/py/tests/test_reset.py b/test/py/tests/test_reset.py index af079a70664..4423baee9c3 100644 --- a/test/py/tests/test_reset.py +++ b/test/py/tests/test_reset.py @@ -41,6 +41,7 @@ def setup_reset_env(ubman): pytest.skip('skipping reset test due to jtag bootmode') @pytest.mark.buildconfigspec('hush_parser') +@pytest.mark.restart def test_reset(ubman): """Test the reset command in non-JTAG bootmode. It does COLD reset, which resets CPU, DDR and peripherals @@ -52,6 +53,7 @@ def test_reset(ubman): test_000_version.test_version(ubman) @pytest.mark.buildconfigspec('hush_parser') +@pytest.mark.restart def test_reset_w(ubman): """Test the reset -w command in non-JTAG bootmode. It does WARM reset, which resets CPU but keep DDR/peripherals active. diff --git a/test/py/tests/test_sandbox_exit.py b/test/py/tests/test_sandbox_exit.py index 849fd477941..eb95558f09e 100644 --- a/test/py/tests/test_sandbox_exit.py +++ b/test/py/tests/test_sandbox_exit.py @@ -7,6 +7,7 @@ import signal @pytest.mark.boardspec('sandbox') @pytest.mark.buildconfigspec('sysreset_cmd_poweroff') +@pytest.mark.restart def test_poweroff(ubman): """Test that the "poweroff" command exits sandbox process.""" @@ -14,6 +15,7 @@ def test_poweroff(ubman): assert(ubman.validate_exited()) @pytest.mark.boardspec('sandbox') +@pytest.mark.restart def test_ctrl_c(ubman): """Test that sending SIGINT to sandbox causes it to exit.""" @@ -23,6 +25,7 @@ def test_ctrl_c(ubman): @pytest.mark.boardspec('sandbox') @pytest.mark.buildconfigspec('cmd_exception') @pytest.mark.buildconfigspec('sandbox_crash_reset') +@pytest.mark.restart def test_exception_reset(ubman): """Test that SIGILL causes a reset.""" @@ -38,6 +41,7 @@ def test_exception_reset(ubman): @pytest.mark.boardspec('sandbox') @pytest.mark.buildconfigspec('cmd_exception') @pytest.mark.notbuildconfigspec('sandbox_crash_reset') +@pytest.mark.restart def test_exception_exit(ubman): """Test that SIGILL causes a reset.""" diff --git a/test/py/tests/test_saveenv.py b/test/py/tests/test_saveenv.py index 555284906da..0fb8dcaf1d8 100644 --- a/test/py/tests/test_saveenv.py +++ b/test/py/tests/test_saveenv.py @@ -73,6 +73,7 @@ def set_env(ubman, var_name, var_value): @pytest.mark.buildconfigspec('cmd_saveenv') @pytest.mark.buildconfigspec('hush_parser') +@pytest.mark.restart def test_saveenv(ubman): """Test the saveenv command in non-JTAG bootmode. It saves the U-Boot environment in persistent storage. diff --git a/test/py/tests/test_spl.py b/test/py/tests/test_spl.py index 48407399039..b3a2de9801c 100644 --- a/test/py/tests/test_spl.py +++ b/test/py/tests/test_spl.py @@ -15,6 +15,7 @@ def test_ut_spl_init(ubman): with open(fn, 'wb') as fh: fh.write(data) +@pytest.mark.restart def test_spl(ubman, ut_spl_subtest): """Execute a "ut" subtest. diff --git a/test/py/tests/test_stackprotector.py b/test/py/tests/test_stackprotector.py index a7e20d6307c..2a99ba7b9c9 100644 --- a/test/py/tests/test_stackprotector.py +++ b/test/py/tests/test_stackprotector.py @@ -6,6 +6,7 @@ import signal @pytest.mark.buildconfigspec('cmd_stackprotector_test') @pytest.mark.notbuildconfigspec('asan') +@pytest.mark.restart def test_stackprotector(ubman): """Test that the stackprotector function works.""" diff --git a/test/py/tests/test_upl.py b/test/py/tests/test_upl.py index f2b69078cf1..ea921e0c005 100644 --- a/test/py/tests/test_upl.py +++ b/test/py/tests/test_upl.py @@ -9,6 +9,7 @@ import pytest import utils @pytest.mark.boardspec('sandbox_vpl') +@pytest.mark.restart def test_upl_handoff(ubman): """Test of UPL handoff diff --git a/test/py/tests/test_vbe_vpl.py b/test/py/tests/test_vbe_vpl.py index f011b034f63..6ecd03086cc 100644 --- a/test/py/tests/test_vbe_vpl.py +++ b/test/py/tests/test_vbe_vpl.py @@ -10,6 +10,7 @@ import utils @pytest.mark.boardspec('sandbox_vpl') @pytest.mark.requiredtool('dtc') +@pytest.mark.restart def test_vbe_vpl(ubman): #cmd = [ubman.config.build_dir + fname, '-v'] ram = os.path.join(ubman.config.build_dir, 'ram.bin') diff --git a/test/py/tests/test_vboot.py b/test/py/tests/test_vboot.py index a2545ab3b60..820c8059be3 100644 --- a/test/py/tests/test_vboot.py +++ b/test/py/tests/test_vboot.py @@ -141,6 +141,7 @@ TESTDATA += [pytest.param(*v, marks=pytest.mark.slow) for v in TESTDATA_IN[1:]] @pytest.mark.requiredtool('fdtget') @pytest.mark.requiredtool('fdtput') @pytest.mark.requiredtool('openssl') +@pytest.mark.restart @pytest.mark.parametrize("name,sha_algo,padding,sign_options,required,full_test,algo_arg,global_sign", TESTDATA) def test_vboot_base(ubman, name, sha_algo, padding, sign_options, required, diff --git a/test/py/tests/test_vpl.py b/test/py/tests/test_vpl.py index a269c7c262e..2ea1f0c399d 100644 --- a/test/py/tests/test_vpl.py +++ b/test/py/tests/test_vpl.py @@ -5,6 +5,7 @@ import os.path import pytest +@pytest.mark.restart def test_vpl(ubman, ut_vpl_subtest): """Execute a "ut" subtest. -- 2.43.0