1#!/usr/bin/env python3 2# vim: set syntax=python ts=4 : 3# 4# Copyright (c) 2018-2025 Intel Corporation 5# Copyright 2022 NXP 6# Copyright (c) 2024 Arm Limited (or its affiliates). All rights reserved. 7# 8# SPDX-License-Identifier: Apache-2.0 9 10import argparse 11import json 12import logging 13import os 14import re 15import shutil 16import subprocess 17import sys 18from collections.abc import Generator 19from datetime import datetime, timezone 20from importlib import metadata 21from pathlib import Path 22 23import zephyr_module 24from twisterlib.constants import SUPPORTED_SIMS 25from twisterlib.coverage import supported_coverage_formats 26from twisterlib.error import TwisterRuntimeError 27from twisterlib.log_helper import log_command 28 29logger = logging.getLogger('twister') 30logger.setLevel(logging.DEBUG) 31 32ZEPHYR_BASE = os.getenv("ZEPHYR_BASE") 33if not ZEPHYR_BASE: 34 sys.exit("$ZEPHYR_BASE environment variable undefined") 35 36# Use this for internal comparisons; that's what canonicalization is 37# for. Don't use it when invoking other components of the build system 38# to avoid confusing and hard to trace inconsistencies in error messages 39# and logs, generated Makefiles, etc. compared to when users invoke these 40# components directly. 41# Note "normalization" is different from canonicalization, see os.path. 42canonical_zephyr_base = os.path.realpath(ZEPHYR_BASE) 43 44 45def _get_installed_packages() -> Generator[str, None, None]: 46 """Return list of installed python packages.""" 47 for dist in metadata.distributions(): 48 yield dist.metadata['Name'] 49 50 51def python_version_guard(): 52 min_ver = (3, 10) 53 if sys.version_info < min_ver: 54 min_ver_str = '.'.join([str(v) for v in min_ver]) 55 cur_ver_line = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" 56 print(f"Unsupported Python version {cur_ver_line}.") 57 print(f"Currently, Twister requires at least Python {min_ver_str}.") 58 print("Install a newer Python version and retry.") 59 sys.exit(1) 60 61 62installed_packages: list[str] = list(_get_installed_packages()) 63PYTEST_PLUGIN_INSTALLED = 'pytest-twister-harness' in installed_packages 64 65 66def norm_path(astring): 67 newstring = os.path.normpath(astring).replace(os.sep, '/') 68 return newstring 69 70 71def add_parse_arguments(parser = None) -> argparse.ArgumentParser: 72 if parser is None: 73 parser = argparse.ArgumentParser( 74 description=__doc__, 75 formatter_class=argparse.RawDescriptionHelpFormatter, 76 allow_abbrev=False) 77 parser.fromfile_prefix_chars = "+" 78 79 case_select = parser.add_argument_group("Test case selection", 80 """ 81Artificially long but functional example: 82 $ ./scripts/twister -v \\ 83 --testsuite-root tests/ztest/base \\ 84 --testsuite-root tests/kernel \\ 85 --test tests/ztest/base/testing.ztest.verbose_0 \\ 86 --test tests/kernel/fifo/fifo_api/kernel.fifo 87 88 "kernel.fifo.poll" is one of the test section names in 89 __/fifo_api/testcase.yaml 90 """) 91 92 test_plan_report = parser.add_argument_group( 93 title="Test plan reporting", 94 description="Report the composed test plan details and exit (dry-run)." 95 ) 96 97 test_plan_report_xor = test_plan_report.add_mutually_exclusive_group() 98 99 platform_group_option = parser.add_mutually_exclusive_group() 100 101 run_group_option = parser.add_mutually_exclusive_group() 102 103 device = parser.add_mutually_exclusive_group() 104 105 test_or_build = parser.add_mutually_exclusive_group() 106 107 test_xor_subtest = case_select.add_mutually_exclusive_group() 108 109 test_xor_generator = case_select.add_mutually_exclusive_group() 110 111 valgrind_asan_group = parser.add_mutually_exclusive_group() 112 113 footprint_group = parser.add_argument_group( 114 title="Memory footprint", 115 description="Collect and report ROM/RAM size footprint for the test instance images built.") 116 117 coverage_group = parser.add_argument_group( 118 title="Code coverage", 119 description="Build with code coverage support, collect code coverage statistics " 120 "executing tests, compose code coverage report at the end.\n" 121 "Effective for devices with 'HAS_COVERAGE_SUPPORT'.") 122 123 test_plan_report_xor.add_argument( 124 "-E", 125 "--save-tests", 126 metavar="FILENAME", 127 action="store", 128 help="Write a list of tests and platforms to be run to %(metavar)s file and stop execution." 129 " The resulting file will have the same content as 'testplan.json'." 130 ) 131 132 case_select.add_argument( 133 "-F", 134 "--load-tests", 135 metavar="FILENAME", 136 action="store", 137 help="Load a list of tests and platforms to be run" 138 "from a JSON file ('testplan.json' schema)." 139 ) 140 141 case_select.add_argument( 142 "-T", "--testsuite-root", action="append", default=[], type = norm_path, 143 help="Base directory to recursively search for test cases. All " 144 "testcase.yaml files under here will be processed. May be " 145 "called multiple times. Defaults to the 'samples/' and " 146 "'tests/' directories at the base of the Zephyr tree.") 147 148 case_select.add_argument( 149 "-f", 150 "--only-failed", 151 action="store_true", 152 help="Run only those tests that failed the previous twister run " 153 "invocation.") 154 155 test_plan_report_xor.add_argument("--list-tests", action="store_true", 156 help="""List of all sub-test functions recursively found in 157 all --testsuite-root arguments. The output is flattened and reports detailed 158 sub-test names without their directories. 159 Note: sub-test names can share the same test scenario identifier prefix 160 (section.subsection) even if they are from different test projects. 161 """) 162 163 test_plan_report_xor.add_argument("--test-tree", action="store_true", 164 help="""Output the test plan in a tree form.""") 165 166 platform_group_option.add_argument( 167 "-G", 168 "--integration", 169 action="store_true", 170 help="Run integration tests") 171 172 platform_group_option.add_argument( 173 "--emulation-only", action="store_true", 174 help="Only build and run emulation platforms") 175 176 run_group_option.add_argument( 177 "--device-testing", action="store_true", 178 help="Test on device directly. Specify the serial device to " 179 "use with the --device-serial option.") 180 181 run_group_option.add_argument("--generate-hardware-map", 182 help="""Probe serial devices connected to this platform 183 and create a hardware map file to be used with 184 --device-testing 185 """) 186 187 run_group_option.add_argument( 188 "--simulation", dest="sim_name", choices=SUPPORTED_SIMS, 189 help="Selects which simulation to use. Must match one of the names defined in the board's " 190 "manifest. If multiple simulator are specified in the selected board and this " 191 "argument is not passed, then the first simulator is selected.") 192 193 194 device.add_argument("--device-serial", 195 help="""Serial device for accessing the board 196 (e.g., /dev/ttyACM0) 197 """) 198 199 device.add_argument("--device-serial-pty", 200 help="""Script for controlling pseudoterminal. 201 Twister believes that it interacts with a terminal 202 when it actually interacts with the script. 203 204 E.g "twister --device-testing 205 --device-serial-pty <script> 206 """) 207 208 device.add_argument("--hardware-map", 209 help="""Load hardware map from a file. This will be used 210 for testing on hardware that is listed in the file. 211 """) 212 213 parser.add_argument("--device-flash-timeout", type=int, default=60, 214 help="""Set timeout for the device flash operation in seconds. 215 """) 216 217 parser.add_argument("--device-flash-with-test", action="store_true", 218 help="""Add a test case timeout to the flash operation timeout 219 when flash operation also executes test case on the platform. 220 """) 221 222 parser.add_argument("--flash-before", action="store_true", default=False, 223 help="""Flash device before attaching to serial port. 224 This is useful for devices that share the same port for programming 225 and serial console, or use soft-USB, where flash must come first. 226 """) 227 228 test_or_build.add_argument( 229 "-b", 230 "--build-only", 231 action="store_true", 232 default="--prep-artifacts-for-testing" in sys.argv, 233 help="Only build the code, do not attempt to run the code on targets." 234 ) 235 236 test_or_build.add_argument( 237 "--prep-artifacts-for-testing", action="store_true", 238 help="Generate artifacts for testing, do not attempt to run the" 239 "code on targets.") 240 241 parser.add_argument( 242 "--package-artifacts", 243 help="Package artifacts needed for flashing in a file to be used with --test-only" 244 ) 245 246 test_or_build.add_argument( 247 "--test-only", action="store_true", 248 help="""Only run device tests with current artifacts, do not build 249 the code""") 250 251 parser.add_argument("--timeout-multiplier", type=float, default=1, 252 help="""Globally adjust tests timeouts by specified multiplier. The resulting test 253 timeout would be multiplication of test timeout value, board-level timeout multiplier 254 and global timeout multiplier (this parameter)""") 255 256 test_xor_subtest.add_argument( 257 "-s", "--test", "--scenario", action="append", type = norm_path, 258 help="""Run only the specified test suite scenario. These are named by 259 'path/relative/to/Zephyr/base/section.subsection_in_testcase_yaml', 260 or just 'section.subsection' identifier. With '--testsuite-root' option 261 the scenario will be found faster. 262 """) 263 264 test_xor_subtest.add_argument( 265 "--sub-test", action="append", 266 help="""Recursively find sub-test functions (test cases) and run the entire 267 test scenario (section.subsection) where they were found, including all sibling test 268 functions. Sub-tests are named by: 269 'section.subsection_in_testcase_yaml.ztest_suite.ztest_without_test_prefix'. 270 Example_1: 'kernel.fifo.fifo_api_1cpu.fifo_loop' where 'kernel.fifo' is a test scenario 271 name (section.subsection) and 'fifo_api_1cpu.fifo_loop' is a Ztest 'suite_name.test_name'. 272 Example_2: 'debug.coredump.logging_backend' is a standalone test scenario name. 273 Note: This selection mechanism works only for Ztest suite and test function names in 274 the source files which are not generated by macro-substitutions. 275 Note: With --no-detailed-test-id use only Ztest names without scenario name. 276 """) 277 278 parser.add_argument( 279 "--pytest-args", action="append", 280 help="""Pass additional arguments to the pytest subprocess. This parameter 281 will extend the pytest_args from the harness_config in YAML file. 282 """) 283 284 parser.add_argument( 285 "--ctest-args", action="append", 286 help="""Pass additional arguments to the ctest subprocess. This parameter 287 will extend the ctest_args from the harness_config in YAML file. 288 """) 289 290 valgrind_asan_group.add_argument( 291 "--enable-valgrind", action="store_true", 292 help="""Run binary through valgrind and check for several memory access 293 errors. Valgrind needs to be installed on the host. This option only 294 works with host binaries such as those generated for the native_sim 295 configuration and is mutual exclusive with --enable-asan. 296 """) 297 298 valgrind_asan_group.add_argument( 299 "--enable-asan", action="store_true", 300 help="""Enable address sanitizer to check for several memory access 301 errors. Libasan needs to be installed on the host. This option only 302 works with host binaries such as those generated for the native_sim 303 configuration and is mutual exclusive with --enable-valgrind. 304 """) 305 306 # Start of individual args place them in alpha-beta order 307 308 board_root_list = [f"{ZEPHYR_BASE}/boards", f"{ZEPHYR_BASE}/subsys/testsuite/boards"] 309 310 modules = zephyr_module.parse_modules(ZEPHYR_BASE) 311 for module in modules: 312 board_root = module.meta.get("build", {}).get("settings", {}).get("board_root") 313 if board_root: 314 board_root_list.append(os.path.join(module.project, board_root, "boards")) 315 316 parser.add_argument( 317 "-A", "--board-root", action="append", default=board_root_list, 318 help="""Directory to search for board configuration files. All .yaml 319files in the directory will be processed. The directory should have the same 320structure in the main Zephyr tree: boards/<vendor>/<board_name>/""") 321 322 parser.add_argument( 323 "--allow-installed-plugin", action="store_true", default=None, 324 help="Allow to use pytest plugin installed by pip for pytest tests." 325 ) 326 327 parser.add_argument( 328 "-a", "--arch", action="append", 329 help="Arch filter for testing. Takes precedence over --platform. " 330 "If unspecified, test all arches. Multiple invocations " 331 "are treated as a logical 'or' relationship") 332 333 parser.add_argument( 334 "-B", "--subset", 335 help="Only run a subset of the tests, 1/4 for running the first 25%%, " 336 "3/5 means run the 3rd fifth of the total. " 337 "This option is useful when running a large number of tests on " 338 "different hosts to speed up execution time.") 339 340 parser.add_argument( 341 "--shuffle-tests", action="store_true", default=None, 342 help="""Shuffle test execution order to get randomly distributed tests across subsets. 343 Used only when --subset is provided.""") 344 345 parser.add_argument( 346 "--shuffle-tests-seed", action="store", default=None, 347 help="""Seed value for random generator used to shuffle tests. 348 If not provided, seed in generated by system. 349 Used only when --shuffle-tests is provided.""") 350 351 parser.add_argument( 352 "-c", "--clobber-output", action="store_true", 353 help="Cleaning the output directory will simply delete it instead " 354 "of the default policy of renaming.") 355 356 parser.add_argument( 357 "--cmake-only", action="store_true", 358 help="Only run cmake, do not build or run.") 359 360 coverage_group.add_argument("--enable-coverage", action="store_true", 361 help="Enable code coverage collection using gcov.") 362 363 coverage_group.add_argument("-C", "--coverage", action="store_true", 364 help="Generate coverage reports. Implies " 365 "--enable-coverage to collect the coverage data first.") 366 367 coverage_group.add_argument("--gcov-tool", type=Path, default=None, 368 help="Path to the 'gcov' tool to use for code coverage reports. " 369 "By default it will be chosen in the following order:" 370 " using ZEPHYR_TOOLCHAIN_VARIANT ('llvm': from LLVM_TOOLCHAIN_PATH)," 371 " gcov installed on the host - for 'native' or 'unit' platform types," 372 " using ZEPHYR_SDK_INSTALL_DIR.") 373 374 coverage_group.add_argument("--coverage-basedir", default=ZEPHYR_BASE, 375 help="Base source directory for coverage report.") 376 377 coverage_group.add_argument("--coverage-platform", action="append", default=[], 378 help="Platforms to run coverage reports on. " 379 "This option may be used multiple times. " 380 "Default to what was selected with --platform.") 381 382 coverage_group.add_argument("--coverage-tool", choices=['lcov', 'gcovr'], default='gcovr', 383 help="Tool to use to generate coverage reports (%(default)s - default).") 384 385 coverage_group.add_argument("--coverage-formats", action="store", default=None, 386 help="Output formats to use for generated coverage reports " + 387 "as a comma-separated list without spaces. " + 388 "Valid options for 'gcovr' tool are: " + 389 ','.join(supported_coverage_formats['gcovr']) + " (html - default)." + 390 " Valid options for 'lcov' tool are: " + 391 ','.join(supported_coverage_formats['lcov']) + " (html,lcov - default).") 392 393 coverage_group.add_argument("--coverage-per-instance", action="store_true", default=False, 394 help="""Compose individual coverage reports for each test suite 395 when coverage reporting is enabled; it might run in addition to 396 the default aggregation mode which produces one coverage report for 397 all executed test suites. Default: %(default)s""") 398 399 coverage_group.add_argument("--disable-coverage-aggregation", 400 action="store_true", default=False, 401 help="""Don't aggregate coverage report statistics for all the test suites 402 selected to run with enabled coverage. Requires another reporting mode to be 403 active (`--coverage-per-instance`) to have at least one of these reporting 404 modes. Default: %(default)s""") 405 406 parser.add_argument( 407 "--test-config", 408 action="store", 409 default=os.path.join(ZEPHYR_BASE, "tests", "test_config.yaml"), 410 help="Path to file with plans and test configurations." 411 ) 412 413 parser.add_argument("--level", action="store", 414 help="Test level to be used. By default, no levels are used for filtering" 415 "and do the selection based on existing filters.") 416 417 parser.add_argument( 418 "--device-serial-baud", action="store", default=None, 419 help="Serial device baud rate (default 115200)") 420 421 parser.add_argument( 422 "--disable-suite-name-check", action="store_true", default=False, 423 help="Disable extended test suite name verification at the beginning " 424 "of Ztest test. This option could be useful for tests or " 425 "platforms, which from some reasons cannot print early logs.") 426 427 parser.add_argument("-e", "--exclude-tag", action="append", 428 help="Specify tags of tests that should not run. " 429 "Default is to run all tests with all tags.") 430 431 parser.add_argument( 432 "--enable-lsan", action="store_true", 433 help="""Enable leak sanitizer to check for heap memory leaks. 434 Libasan needs to be installed on the host. This option only 435 works with host binaries such as those generated for the native_sim 436 configuration and when --enable-asan is given. 437 """) 438 439 parser.add_argument( 440 "--enable-ubsan", action="store_true", 441 help="""Enable undefined behavior sanitizer to check for undefined 442 behaviour during program execution. It uses an optional runtime library 443 to provide better error diagnostics. This option only works with host 444 binaries such as those generated for the native_sim configuration. 445 """) 446 447 parser.add_argument( 448 "--filter", choices=['buildable', 'runnable'], 449 default='runnable' if "--device-testing" in sys.argv else 'buildable', 450 help="""Filter tests to be built and executed. By default everything is 451 built and if a test is runnable (emulation or a connected device), it 452 is run. This option allows for example to only build tests that can 453 actually be run. Runnable is a subset of buildable.""") 454 455 parser.add_argument("--force-color", action="store_true", 456 help="Always output ANSI color escape sequences " 457 "even when the output is redirected (not a tty)") 458 459 parser.add_argument("--force-toolchain", action="store_true", 460 help="Do not filter based on toolchain, use the set " 461 " toolchain unconditionally") 462 463 footprint_group.add_argument( 464 "--create-rom-ram-report", 465 action="store_true", 466 help="Generate detailed json reports with ROM/RAM symbol sizes for each test image built " 467 "using additional build option `--target footprint`.") 468 469 footprint_group.add_argument( 470 "--footprint-report", 471 nargs="?", 472 default=None, 473 choices=['all', 'ROM', 'RAM'], 474 const="all", 475 help="Select which memory area symbols' data to collect as 'footprint' property " 476 "of each test suite built, and report in 'twister_footprint.json' together " 477 "with the relevant execution metadata the same way as in `twister.json`. " 478 "Implies '--create-rom-ram-report' to generate the footprint data files. " 479 "No value means '%(const)s'. Default: %(default)s""") 480 481 footprint_group.add_argument( 482 "--enable-size-report", 483 action="store_true", 484 help="Collect and report ROM/RAM section sizes for each test image built.") 485 486 parser.add_argument( 487 "--disable-unrecognized-section-test", 488 action="store_true", 489 default=False, 490 help="Don't error on unrecognized sections in the binary images.") 491 492 footprint_group.add_argument( 493 "--footprint-from-buildlog", 494 action = "store_true", 495 help="Take ROM/RAM sections footprint summary values from the 'build.log' " 496 "instead of 'objdump' results used otherwise." 497 "Requires --enable-size-report or one of the baseline comparison modes." 498 "Warning: the feature will not work correctly with sysbuild.") 499 500 compare_group_option = footprint_group.add_mutually_exclusive_group() 501 502 compare_group_option.add_argument( 503 "-m", "--last-metrics", 504 action="store_true", 505 help="Compare footprints to the previous twister invocation as a baseline " 506 "running in the same output directory. " 507 "Implies --enable-size-report option.") 508 509 compare_group_option.add_argument( 510 "--compare-report", 511 help="Use this report file as a baseline for footprint comparison. " 512 "The file should be of 'twister.json' schema. " 513 "Implies --enable-size-report option.") 514 515 footprint_group.add_argument( 516 "--show-footprint", 517 action="store_true", 518 help="With footprint comparison to a baseline, log ROM/RAM section deltas. ") 519 520 footprint_group.add_argument( 521 "-H", "--footprint-threshold", 522 type=float, 523 default=5.0, 524 help="With footprint comparison to a baseline, " 525 "warn the user for any of the footprint metric change which is greater or equal " 526 "to the specified percentage value. " 527 "Default is %(default)s for %(default)s%% delta from the new footprint value. " 528 "Use zero to warn on any footprint metric increase.") 529 530 footprint_group.add_argument( 531 "-D", "--all-deltas", 532 action="store_true", 533 help="With footprint comparison to a baseline, " 534 "warn on any footprint change, increase or decrease. " 535 "Implies --footprint-threshold=0") 536 537 footprint_group.add_argument( 538 "-z", "--size", 539 action="append", 540 metavar='FILENAME', 541 help="Ignore all other command line options and just produce a report to " 542 "stdout with ROM/RAM section sizes on the specified binary images.") 543 544 parser.add_argument( 545 "-i", "--inline-logs", action="store_true", 546 help="Upon test failure, print relevant log data to stdout " 547 "instead of just a path to it.") 548 549 parser.add_argument("--ignore-platform-key", action="store_true", 550 help="Do not filter based on platform key") 551 552 parser.add_argument( 553 "-j", "--jobs", type=int, 554 help="Number of jobs for building, defaults to number of CPU threads, " 555 "overcommitted by factor 2 when --build-only.") 556 557 parser.add_argument( 558 "-K", "--force-platform", action="store_true", 559 help="""Force testing on selected platforms, 560 even if they are excluded in the test configuration (testcase.yaml).""" 561 ) 562 563 parser.add_argument( 564 "-l", "--all", action="store_true", 565 help="Build/test on all platforms. Any --platform arguments " 566 "ignored.") 567 568 test_plan_report_xor.add_argument("--list-tags", action="store_true", 569 help="List all tags occurring in selected tests.") 570 571 parser.add_argument("--log-file", metavar="FILENAME", action="store", 572 help="Specify a file where to save logs.") 573 574 parser.add_argument( 575 "-M", "--runtime-artifact-cleanup", choices=['pass', 'all'], 576 default=None, const='pass', nargs='?', 577 help="""Cleanup test artifacts. The default behavior is 'pass' 578 which only removes artifacts of passing tests. If you wish to 579 remove all artificats including those of failed tests, use 'all'.""") 580 581 parser.add_argument( 582 "--keep-artifacts", action="append", default=[], 583 help="""Keep specified artifacts when cleaning up at runtime. Multiple invocations 584 are possible.""" 585 ) 586 test_xor_generator.add_argument( 587 "-N", "--ninja", action="store_true", 588 default=not any(a in sys.argv for a in ("-k", "--make")), 589 help="Use the Ninja generator with CMake. (This is the default)") 590 591 test_xor_generator.add_argument( 592 "-k", "--make", action="store_true", 593 help="Use the unix Makefile generator with CMake.") 594 595 parser.add_argument( 596 "-n", "--no-clean", action="store_true", 597 help="Re-use the outdir before building. Will result in " 598 "faster compilation since builds will be incremental.") 599 600 parser.add_argument( 601 "--aggressive-no-clean", action="store_true", 602 help="Re-use the outdir before building and do not re-run cmake. Will result in " 603 "much faster compilation since builds will be incremental. This option might " 604 " result in build failures and inconsistencies if dependencies change or when " 605 " applied on a significantly changed code base. Use on your own " 606 " risk. It is recommended to only use this option for local " 607 " development and when testing localized change in a subsystem.") 608 609 parser.add_argument( 610 '--detailed-test-id', action='store_true', 611 help="Compose each test Suite name from its configuration path (relative to root) and " 612 "the appropriate Scenario name using PATH_TO_TEST_CONFIG/SCENARIO_NAME schema. " 613 "Also (for Ztest only), prefix each test Case name with its Scenario name. " 614 "For example: 'kernel.common.timing' Scenario with test Suite name " 615 "'tests/kernel/sleep/kernel.common.timing' and 'kernel.common.timing.sleep.usleep' " 616 "test Case (where 'sleep' is its Ztest suite name and 'usleep' is Ztest test name.") 617 618 parser.add_argument( 619 "--no-detailed-test-id", dest='detailed_test_id', action="store_false", 620 help="Don't prefix each test Suite name with its configuration path, " 621 "so it is the same as the appropriate Scenario name. " 622 "Also (for Ztest only), don't prefix each Ztest Case name with its Scenario name. " 623 "For example: 'kernel.common.timing' Scenario name, the same Suite name, " 624 "and 'sleep.usleep' test Case (where 'sleep' is its Ztest suite name " 625 "and 'usleep' is Ztest test name.") 626 627 # Include paths in names by default. 628 parser.set_defaults(detailed_test_id=True) 629 630 parser.add_argument( 631 "--detailed-skipped-report", action="store_true", 632 help="Generate a detailed report with all skipped test cases" 633 "including those that are filtered based on testsuite definition." 634 ) 635 636 parser.add_argument( 637 "-O", "--outdir", 638 default=os.path.join(os.getcwd(), "twister-out"), 639 help="Output directory for logs and binaries. " 640 "Default is 'twister-out' in the current directory. " 641 "This directory will be cleaned unless '--no-clean' is set. " 642 "The '--clobber-output' option controls what cleaning does.") 643 644 parser.add_argument( 645 "-o", "--report-dir", 646 help="""Output reports containing results of the test run into the 647 specified directory. 648 The output will be both in JSON and JUNIT format 649 (twister.json and twister.xml). 650 """) 651 652 parser.add_argument("--overflow-as-errors", action="store_true", 653 help="Treat RAM/SRAM overflows as errors.") 654 655 parser.add_argument("--report-filtered", action="store_true", 656 help="Include filtered tests in the reports.") 657 658 parser.add_argument("-P", "--exclude-platform", action="append", default=[], 659 help="""Exclude platforms and do not build or run any tests 660 on those platforms. This option can be called multiple times. 661 """ 662 ) 663 664 parser.add_argument("--persistent-hardware-map", action='store_true', 665 help="""With --generate-hardware-map, tries to use 666 persistent names for serial devices on platforms 667 that support this feature (currently only Linux). 668 """) 669 670 parser.add_argument( 671 "--vendor", action="append", default=[], 672 help="Vendor filter for testing") 673 674 parser.add_argument( 675 "-p", "--platform", action="append", default=[], 676 help="Platform filter for testing. This option may be used multiple " 677 "times. Test suites will only be built/run on the platforms " 678 "specified. If this option is not used, then platforms marked " 679 "as default in the platform metadata file will be chosen " 680 "to build and test. ") 681 682 parser.add_argument( 683 "--platform-reports", action="store_true", 684 help="""Create individual reports for each platform. 685 """) 686 687 parser.add_argument("--pre-script", 688 help="""specify a pre script. This will be executed 689 before device handler open serial port and invoke runner. 690 """) 691 692 parser.add_argument( 693 "--quarantine-list", 694 action="append", 695 metavar="FILENAME", 696 help="Load list of test scenarios under quarantine. The entries in " 697 "the file need to correspond to the test scenarios names as in " 698 "corresponding tests .yaml files. These scenarios " 699 "will be skipped with quarantine as the reason.") 700 701 parser.add_argument( 702 "--quarantine-verify", 703 action="store_true", 704 help="Use the list of test scenarios under quarantine and run them" 705 "to verify their current status.") 706 707 parser.add_argument( 708 "--quit-on-failure", 709 action="store_true", 710 help="""quit twister once there is build / run failure 711 """) 712 713 parser.add_argument( 714 "--report-name", 715 help="""Create a report with a custom name. 716 """) 717 718 parser.add_argument( 719 "--report-summary", action="store", nargs='?', type=int, const=0, 720 help="Show failed/error report from latest run. Default shows all items found. " 721 "However, you can specify the number of items (e.g. --report-summary 15). " 722 "It also works well with the --outdir switch.") 723 724 parser.add_argument( 725 "--report-suffix", 726 help="""Add a suffix to all generated file names, for example to add a 727 version or a commit ID. 728 """) 729 730 parser.add_argument( 731 "--report-all-options", action="store_true", 732 help="""Show all command line options applied, including defaults, as 733 environment.options object in twister.json. Default: show only non-default settings. 734 """) 735 736 parser.add_argument( 737 "--retry-failed", type=int, default=0, 738 help="Retry failing tests again, up to the number of times specified.") 739 740 parser.add_argument( 741 "--retry-interval", type=int, default=60, 742 help="Retry failing tests after specified period of time.") 743 744 parser.add_argument( 745 "--retry-build-errors", action="store_true", 746 help="Retry build errors as well.") 747 748 parser.add_argument( 749 "-S", "--enable-slow", action="store_true", 750 default="--enable-slow-only" in sys.argv, 751 help="Execute time-consuming test cases that have been marked " 752 "as 'slow' in testcase.yaml. Normally these are only built.") 753 754 parser.add_argument( 755 "--enable-slow-only", action="store_true", 756 help="Execute time-consuming test cases that have been marked " 757 "as 'slow' in testcase.yaml only. This also set the option --enable-slow") 758 759 parser.add_argument( 760 "--seed", type=int, 761 help="Seed for native_sim pseudo-random number generator") 762 763 parser.add_argument( 764 "--short-build-path", 765 action="store_true", 766 help="Create shorter build directory paths based on symbolic links. " 767 "The shortened build path will be used by CMake for generating " 768 "the build system and executing the build. Use this option if " 769 "you experience build failures related to path length, for " 770 "example on Windows OS. This option can be used only with " 771 "'--ninja' argument (to use Ninja build generator).") 772 773 parser.add_argument( 774 "-t", "--tag", action="append", 775 help="Specify tags to restrict which tests to run by tag value. " 776 "Default is to not do any tag filtering. Multiple invocations " 777 "are treated as a logical 'or' relationship.") 778 779 parser.add_argument("--timestamps", 780 action="store_true", 781 help="Print all messages with time stamps.") 782 783 parser.add_argument( 784 "-u", 785 "--no-update", 786 action="store_true", 787 help="Do not update the results of the last run. This option " 788 "is only useful when reusing the same output directory of " 789 "twister, for example when re-running failed tests with --only-failed " 790 "or --no-clean. This option is for debugging purposes only.") 791 792 parser.add_argument( 793 "-v", 794 "--verbose", 795 action="count", 796 default=0, 797 help="Call multiple times to increase verbosity.") 798 799 parser.add_argument( 800 "-ll", 801 "--log-level", 802 type=str.upper, 803 default='INFO', 804 choices=['NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], 805 help="Select the threshold event severity for which you'd like to receive logs in console." 806 " Default is INFO.") 807 808 parser.add_argument("-W", "--disable-warnings-as-errors", action="store_true", 809 help="Do not treat warning conditions as errors.") 810 811 parser.add_argument( 812 "--west-flash", nargs='?', const=[], 813 help="""Uses west instead of ninja or make to flash when running with 814 --device-testing. Supports comma-separated argument list. 815 816 E.g "twister --device-testing --device-serial /dev/ttyACM0 817 --west-flash="--board-id=foobar,--erase" 818 will translate to "west flash -- --board-id=foobar --erase" 819 820 NOTE: device-testing must be enabled to use this option. 821 """ 822 ) 823 parser.add_argument( 824 "--west-runner", 825 help="""Uses the specified west runner instead of default when running 826 with --west-flash. 827 828 E.g "twister --device-testing --device-serial /dev/ttyACM0 829 --west-flash --west-runner=pyocd" 830 will translate to "west flash --runner pyocd" 831 832 NOTE: west-flash must be enabled to use this option. 833 """ 834 ) 835 836 parser.add_argument( 837 "-X", "--fixture", action="append", default=[], 838 help="Specify a fixture that a board might support.") 839 840 parser.add_argument( 841 "-x", "--extra-args", action="append", default=[], 842 help="""Extra CMake cache entries to define when building test cases. 843 May be called multiple times. The key-value entries will be 844 prefixed with -D before being passed to CMake. 845 E.g 846 "twister -x=USE_CCACHE=0" 847 will translate to 848 "cmake -DUSE_CCACHE=0" 849 which will ultimately disable ccache. 850 """ 851 ) 852 853 parser.add_argument( 854 "-y", "--dry-run", action="store_true", 855 help="""Create the filtered list of test cases, but don't actually 856 run them. Useful if you're just interested in the test plan 857 generated for every run and saved in the specified output 858 directory (testplan.json). 859 """) 860 861 parser.add_argument("extra_test_args", nargs=argparse.REMAINDER, 862 help="Additional args following a '--' are passed to the test binary") 863 864 parser.add_argument("--alt-config-root", action="append", default=[], 865 help="Alternative test configuration root/s. When a test is found, " 866 "Twister will check if a test configuration file exist in any of " 867 "the alternative test configuration root folders. For example, " 868 "given $test_root/tests/foo/testcase.yaml, Twister will use " 869 "$alt_config_root/tests/foo/testcase.yaml if it exists") 870 871 return parser 872 873 874def parse_arguments( 875 parser: argparse.ArgumentParser, 876 args, 877 options = None, 878 on_init=True 879) -> argparse.Namespace: 880 if options is None: 881 options = parser.parse_args(args) 882 883 # Very early error handling 884 if options.short_build_path and not options.ninja: 885 logger.error("--short-build-path requires Ninja to be enabled") 886 sys.exit(1) 887 888 if options.device_serial_pty and os.name == "nt": # OS is Windows 889 logger.error("--device-serial-pty is not supported on Windows OS") 890 sys.exit(1) 891 892 if options.west_runner and options.west_flash is None: 893 logger.error("west-runner requires west-flash to be enabled") 894 sys.exit(1) 895 896 if options.west_flash and not options.device_testing: 897 logger.error("west-flash requires device-testing to be enabled") 898 sys.exit(1) 899 900 if not options.testsuite_root: 901 # if we specify a test scenario which is part of a suite directly, do 902 # not set testsuite root to default, just point to the test directory 903 # directly. 904 if options.test: 905 for scenario in options.test: 906 if dirname := os.path.dirname(scenario): 907 options.testsuite_root.append(dirname) 908 909 # check again and make sure we have something set 910 if not options.testsuite_root: 911 options.testsuite_root = [os.path.join(ZEPHYR_BASE, "tests"), 912 os.path.join(ZEPHYR_BASE, "samples")] 913 914 if options.last_metrics or options.compare_report: 915 options.enable_size_report = True 916 917 if options.footprint_report: 918 options.create_rom_ram_report = True 919 920 if options.aggressive_no_clean: 921 options.no_clean = True 922 923 if options.coverage: 924 options.enable_coverage = True 925 926 if options.enable_coverage and not options.coverage_platform: 927 options.coverage_platform = options.platform 928 929 if ( 930 (not options.coverage) 931 and (options.disable_coverage_aggregation or options.coverage_per_instance) 932 ): 933 logger.error("Enable coverage reporting to set its aggregation mode.") 934 sys.exit(1) 935 936 if ( 937 options.coverage 938 and options.disable_coverage_aggregation and (not options.coverage_per_instance) 939 ): 940 logger.error("At least one coverage reporting mode should be enabled: " 941 "either aggregation, or per-instance, or both.") 942 sys.exit(1) 943 944 if options.coverage_formats: 945 for coverage_format in options.coverage_formats.split(','): 946 if coverage_format not in supported_coverage_formats[options.coverage_tool]: 947 logger.error(f"Unsupported coverage report formats:'{options.coverage_formats}' " 948 f"for {options.coverage_tool}") 949 sys.exit(1) 950 951 if options.enable_valgrind and not shutil.which("valgrind"): 952 logger.error("valgrind enabled but valgrind executable not found") 953 sys.exit(1) 954 955 if ( 956 (not options.device_testing) 957 and (options.device_serial or options.device_serial_pty or options.hardware_map) 958 ): 959 logger.error( 960 "Use --device-testing with --device-serial, or --device-serial-pty, or --hardware-map." 961 ) 962 sys.exit(1) 963 964 if ( 965 options.device_testing 966 and (options.device_serial or options.device_serial_pty) and len(options.platform) != 1 967 ): 968 logger.error("When --device-testing is used with --device-serial " 969 "or --device-serial-pty, exactly one platform must " 970 "be specified") 971 sys.exit(1) 972 973 if options.device_flash_with_test and not options.device_testing: 974 logger.error("--device-flash-with-test requires --device_testing") 975 sys.exit(1) 976 977 if options.flash_before and options.device_flash_with_test: 978 logger.error("--device-flash-with-test does not apply when --flash-before is used") 979 sys.exit(1) 980 981 if options.flash_before and options.device_serial_pty: 982 logger.error("--device-serial-pty cannot be used when --flash-before is set (for now)") 983 sys.exit(1) 984 985 if options.shuffle_tests and options.subset is None: 986 logger.error("--shuffle-tests requires --subset") 987 sys.exit(1) 988 989 if options.shuffle_tests_seed and options.shuffle_tests is None: 990 logger.error("--shuffle-tests-seed requires --shuffle-tests") 991 sys.exit(1) 992 993 if options.size: 994 from twisterlib.size_calc import SizeCalculator 995 for fn in options.size: 996 sc = SizeCalculator(fn, []) 997 sc.size_report() 998 sys.exit(0) 999 1000 if options.footprint_from_buildlog: 1001 logger.warning("WARNING: Using --footprint-from-buildlog will give inconsistent results " 1002 "for configurations using sysbuild. It is recommended to not use this flag " 1003 "when building configurations using sysbuild.") 1004 if not options.enable_size_report: 1005 logger.error("--footprint-from-buildlog requires --enable-size-report") 1006 sys.exit(1) 1007 1008 if len(options.extra_test_args) > 0: 1009 # extra_test_args is a list of CLI args that Twister did not recognize 1010 # and are intended to be passed through to the ztest executable. This 1011 # list should begin with a "--". If not, there is some extra 1012 # unrecognized arg(s) that shouldn't be there. Tell the user there is a 1013 # syntax error. 1014 if options.extra_test_args[0] != "--": 1015 try: 1016 double_dash = options.extra_test_args.index("--") 1017 except ValueError: 1018 double_dash = len(options.extra_test_args) 1019 unrecognized = " ".join(options.extra_test_args[0:double_dash]) 1020 1021 logger.error( 1022 f"Unrecognized arguments found: '{unrecognized}'." 1023 " Use -- to delineate extra arguments for test binary or pass -h for help." 1024 ) 1025 1026 sys.exit(1) 1027 1028 # Strip off the initial "--" following validation. 1029 options.extra_test_args = options.extra_test_args[1:] 1030 1031 if on_init and not options.allow_installed_plugin and PYTEST_PLUGIN_INSTALLED: 1032 logger.error("By default Twister should work without pytest-twister-harness " 1033 "plugin being installed, so please, uninstall it by " 1034 "`pip uninstall pytest-twister-harness` and `git clean " 1035 "-dxf scripts/pylib/pytest-twister-harness`.") 1036 sys.exit(1) 1037 elif on_init and options.allow_installed_plugin and PYTEST_PLUGIN_INSTALLED: 1038 logger.warning("You work with installed version of " 1039 "pytest-twister-harness plugin.") 1040 1041 return options 1042 1043def strip_ansi_sequences(s: str) -> str: 1044 """Remove ANSI escape sequences from a string.""" 1045 return re.sub(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])', "", s) 1046 1047class TwisterEnv: 1048 1049 def __init__(self, options : argparse.Namespace, default_options=None) -> None: 1050 self.version = "Unknown" 1051 self.toolchain = None 1052 self.commit_date = "Unknown" 1053 self.run_date = None 1054 self.options = options 1055 self.default_options = default_options 1056 1057 if options.ninja: 1058 self.generator_cmd = "ninja" 1059 self.generator = "Ninja" 1060 else: 1061 self.generator_cmd = "make" 1062 self.generator = "Unix Makefiles" 1063 logger.info(f"Using {self.generator}..") 1064 1065 self.test_roots = options.testsuite_root 1066 1067 if not isinstance(options.board_root, list): 1068 self.board_roots = [options.board_root] 1069 else: 1070 self.board_roots = options.board_root 1071 self.outdir = os.path.abspath(options.outdir) 1072 1073 self.snippet_roots = [Path(ZEPHYR_BASE)] 1074 modules = zephyr_module.parse_modules(ZEPHYR_BASE) 1075 for module in modules: 1076 snippet_root = module.meta.get("build", {}).get("settings", {}).get("snippet_root") 1077 if snippet_root: 1078 self.snippet_roots.append(Path(module.project) / snippet_root) 1079 1080 1081 self.soc_roots = [Path(ZEPHYR_BASE), Path(ZEPHYR_BASE) / 'subsys' / 'testsuite'] 1082 self.dts_roots = [Path(ZEPHYR_BASE)] 1083 self.arch_roots = [Path(ZEPHYR_BASE)] 1084 1085 for module in modules: 1086 soc_root = module.meta.get("build", {}).get("settings", {}).get("soc_root") 1087 if soc_root: 1088 self.soc_roots.append(Path(module.project) / Path(soc_root)) 1089 dts_root = module.meta.get("build", {}).get("settings", {}).get("dts_root") 1090 if dts_root: 1091 self.dts_roots.append(Path(module.project) / Path(dts_root)) 1092 arch_root = module.meta.get("build", {}).get("settings", {}).get("arch_root") 1093 if arch_root: 1094 self.arch_roots.append(Path(module.project) / Path(arch_root)) 1095 1096 self.hwm = None 1097 1098 self.test_config = options.test_config 1099 1100 self.alt_config_root = options.alt_config_root 1101 1102 def non_default_options(self) -> dict: 1103 """Returns current command line options which are set to non-default values.""" 1104 diff = {} 1105 if not self.default_options: 1106 return diff 1107 dict_options = vars(self.options) 1108 dict_default = vars(self.default_options) 1109 for k in dict_options: 1110 if k not in dict_default or dict_options[k] != dict_default[k]: 1111 diff[k] = dict_options[k] 1112 return diff 1113 1114 def discover(self): 1115 self.check_zephyr_version() 1116 self.get_toolchain() 1117 self.run_date = datetime.now(timezone.utc).isoformat(timespec='seconds') 1118 1119 def check_zephyr_version(self): 1120 try: 1121 subproc = subprocess.run(["git", "describe", "--abbrev=12", "--always"], 1122 stdout=subprocess.PIPE, 1123 text=True, 1124 cwd=ZEPHYR_BASE) 1125 if subproc.returncode == 0: 1126 _version = subproc.stdout.strip() 1127 if _version: 1128 self.version = _version 1129 logger.info(f"Zephyr version: {self.version}") 1130 except OSError: 1131 logger.exception("Failure while reading Zephyr version.") 1132 1133 if self.version == "Unknown": 1134 logger.warning("Could not determine version") 1135 1136 try: 1137 subproc = subprocess.run(["git", "show", "-s", "--format=%cI", "HEAD"], 1138 stdout=subprocess.PIPE, 1139 text=True, 1140 cwd=ZEPHYR_BASE) 1141 if subproc.returncode == 0: 1142 self.commit_date = subproc.stdout.strip() 1143 except OSError: 1144 logger.exception("Failure while reading head commit date.") 1145 1146 @staticmethod 1147 def run_cmake_script(args=None): 1148 if args is None: 1149 args = [] 1150 script = os.fspath(args[0]) 1151 1152 logger.debug(f"Running cmake script {script}") 1153 1154 cmake_args = ["-D{}".format(a.replace('"', '')) for a in args[1:]] 1155 cmake_args.extend(['-P', script]) 1156 1157 cmake = shutil.which('cmake') 1158 if not cmake: 1159 msg = "Unable to find `cmake` in path" 1160 logger.error(msg) 1161 raise Exception(msg) 1162 cmd = [cmake] + cmake_args 1163 log_command(logger, "Calling cmake", cmd) 1164 1165 kwargs = dict() 1166 kwargs['stdout'] = subprocess.PIPE 1167 # CMake sends the output of message() to stderr unless it's STATUS 1168 kwargs['stderr'] = subprocess.STDOUT 1169 1170 p = subprocess.Popen(cmd, **kwargs) 1171 out, _ = p.communicate() 1172 1173 # It might happen that the environment adds ANSI escape codes like \x1b[0m, 1174 # for instance if twister is executed from inside a makefile. In such a 1175 # scenario it is then necessary to remove them, as otherwise the JSON decoding 1176 # will fail. 1177 out = strip_ansi_sequences(out.decode()) 1178 1179 if p.returncode == 0: 1180 msg = f"Finished running {args[0]}" 1181 logger.debug(msg) 1182 results = {"returncode": p.returncode, "msg": msg, "stdout": out} 1183 1184 else: 1185 logger.error(f"CMake script failure: {args[0]}") 1186 results = {"returncode": p.returncode, "returnmsg": out} 1187 1188 return results 1189 1190 def get_toolchain(self): 1191 toolchain_script = Path(ZEPHYR_BASE) / Path('cmake/verify-toolchain.cmake') 1192 result = self.run_cmake_script([toolchain_script, "FORMAT=json"]) 1193 1194 try: 1195 if result['returncode']: 1196 raise TwisterRuntimeError(f"E: {result['returnmsg']}") 1197 except Exception as e: 1198 print(str(e)) 1199 sys.exit(2) 1200 self.toolchain = json.loads(result['stdout'])['ZEPHYR_TOOLCHAIN_VARIANT'] 1201 logger.info(f"Using '{self.toolchain}' toolchain.") 1202
