1#!/usr/bin/env python3 2# vim: set syntax=python ts=4 : 3# 4# Copyright (c) 2018-2024 Intel Corporation 5# Copyright 2022 NXP 6# Copyright (c) 2024 Arm Limited (or its affiliates). All rights reserved. 7# 8# SPDX-License-Identifier: Apache-2.0 9 10import argparse 11import json 12import logging 13import os 14import re 15import shutil 16import subprocess 17import sys 18from collections.abc import Generator 19from datetime import datetime, timezone 20from importlib import metadata 21from pathlib import Path 22 23import zephyr_module 24from twisterlib.constants import SUPPORTED_SIMS 25from twisterlib.coverage import supported_coverage_formats 26from twisterlib.error import TwisterRuntimeError 27from twisterlib.log_helper import log_command 28 29logger = logging.getLogger('twister') 30logger.setLevel(logging.DEBUG) 31 32ZEPHYR_BASE = os.getenv("ZEPHYR_BASE") 33if not ZEPHYR_BASE: 34 sys.exit("$ZEPHYR_BASE environment variable undefined") 35 36# Use this for internal comparisons; that's what canonicalization is 37# for. Don't use it when invoking other components of the build system 38# to avoid confusing and hard to trace inconsistencies in error messages 39# and logs, generated Makefiles, etc. compared to when users invoke these 40# components directly. 41# Note "normalization" is different from canonicalization, see os.path. 42canonical_zephyr_base = os.path.realpath(ZEPHYR_BASE) 43 44 45def _get_installed_packages() -> Generator[str, None, None]: 46 """Return list of installed python packages.""" 47 for dist in metadata.distributions(): 48 yield dist.metadata['Name'] 49 50 51def python_version_guard(): 52 min_ver = (3, 10) 53 if sys.version_info < min_ver: 54 min_ver_str = '.'.join([str(v) for v in min_ver]) 55 cur_ver_line = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" 56 print(f"Unsupported Python version {cur_ver_line}.") 57 print(f"Currently, Twister requires at least Python {min_ver_str}.") 58 print("Install a newer Python version and retry.") 59 sys.exit(1) 60 61 62installed_packages: list[str] = list(_get_installed_packages()) 63PYTEST_PLUGIN_INSTALLED = 'pytest-twister-harness' in installed_packages 64 65 66def norm_path(astring): 67 newstring = os.path.normpath(astring).replace(os.sep, '/') 68 return newstring 69 70 71def add_parse_arguments(parser = None) -> argparse.ArgumentParser: 72 if parser is None: 73 parser = argparse.ArgumentParser( 74 description=__doc__, 75 formatter_class=argparse.RawDescriptionHelpFormatter, 76 allow_abbrev=False) 77 parser.fromfile_prefix_chars = "+" 78 79 case_select = parser.add_argument_group("Test case selection", 80 """ 81Artificially long but functional example: 82 $ ./scripts/twister -v \\ 83 --testsuite-root tests/ztest/base \\ 84 --testsuite-root tests/kernel \\ 85 --test tests/ztest/base/testing.ztest.verbose_0 \\ 86 --test tests/kernel/fifo/fifo_api/kernel.fifo 87 88 "kernel.fifo.poll" is one of the test section names in 89 __/fifo_api/testcase.yaml 90 """) 91 92 test_plan_report = parser.add_argument_group( 93 title="Test plan reporting", 94 description="Report the composed test plan details and exit (dry-run)." 95 ) 96 97 test_plan_report_xor = test_plan_report.add_mutually_exclusive_group() 98 99 platform_group_option = parser.add_mutually_exclusive_group() 100 101 run_group_option = parser.add_mutually_exclusive_group() 102 103 device = parser.add_mutually_exclusive_group() 104 105 test_or_build = parser.add_mutually_exclusive_group() 106 107 test_xor_subtest = case_select.add_mutually_exclusive_group() 108 109 test_xor_generator = case_select.add_mutually_exclusive_group() 110 111 valgrind_asan_group = parser.add_mutually_exclusive_group() 112 113 footprint_group = parser.add_argument_group( 114 title="Memory footprint", 115 description="Collect and report ROM/RAM size footprint for the test instance images built.") 116 117 test_plan_report_xor.add_argument( 118 "-E", 119 "--save-tests", 120 metavar="FILENAME", 121 action="store", 122 help="Write a list of tests and platforms to be run to %(metavar)s file and stop execution." 123 " The resulting file will have the same content as 'testplan.json'." 124 ) 125 126 case_select.add_argument( 127 "-F", 128 "--load-tests", 129 metavar="FILENAME", 130 action="store", 131 help="Load a list of tests and platforms to be run" 132 "from a JSON file ('testplan.json' schema)." 133 ) 134 135 case_select.add_argument( 136 "-T", "--testsuite-root", action="append", default=[], type = norm_path, 137 help="Base directory to recursively search for test cases. All " 138 "testcase.yaml files under here will be processed. May be " 139 "called multiple times. Defaults to the 'samples/' and " 140 "'tests/' directories at the base of the Zephyr tree.") 141 142 case_select.add_argument( 143 "-f", 144 "--only-failed", 145 action="store_true", 146 help="Run only those tests that failed the previous twister run " 147 "invocation.") 148 149 test_plan_report_xor.add_argument("--list-tests", action="store_true", 150 help="""List of all sub-test functions recursively found in 151 all --testsuite-root arguments. The output is flattened and reports detailed 152 sub-test names without their directories. 153 Note: sub-test names can share the same test scenario identifier prefix 154 (section.subsection) even if they are from different test projects. 155 """) 156 157 test_plan_report_xor.add_argument("--test-tree", action="store_true", 158 help="""Output the test plan in a tree form.""") 159 160 platform_group_option.add_argument( 161 "-G", 162 "--integration", 163 action="store_true", 164 help="Run integration tests") 165 166 platform_group_option.add_argument( 167 "--emulation-only", action="store_true", 168 help="Only build and run emulation platforms") 169 170 run_group_option.add_argument( 171 "--device-testing", action="store_true", 172 help="Test on device directly. Specify the serial device to " 173 "use with the --device-serial option.") 174 175 run_group_option.add_argument("--generate-hardware-map", 176 help="""Probe serial devices connected to this platform 177 and create a hardware map file to be used with 178 --device-testing 179 """) 180 181 run_group_option.add_argument( 182 "--simulation", dest="sim_name", choices=SUPPORTED_SIMS, 183 help="Selects which simulation to use. Must match one of the names defined in the board's " 184 "manifest. If multiple simulator are specified in the selected board and this " 185 "argument is not passed, then the first simulator is selected.") 186 187 188 device.add_argument("--device-serial", 189 help="""Serial device for accessing the board 190 (e.g., /dev/ttyACM0) 191 """) 192 193 device.add_argument("--device-serial-pty", 194 help="""Script for controlling pseudoterminal. 195 Twister believes that it interacts with a terminal 196 when it actually interacts with the script. 197 198 E.g "twister --device-testing 199 --device-serial-pty <script> 200 """) 201 202 device.add_argument("--hardware-map", 203 help="""Load hardware map from a file. This will be used 204 for testing on hardware that is listed in the file. 205 """) 206 207 parser.add_argument("--device-flash-timeout", type=int, default=60, 208 help="""Set timeout for the device flash operation in seconds. 209 """) 210 211 parser.add_argument("--device-flash-with-test", action="store_true", 212 help="""Add a test case timeout to the flash operation timeout 213 when flash operation also executes test case on the platform. 214 """) 215 216 parser.add_argument("--flash-before", action="store_true", default=False, 217 help="""Flash device before attaching to serial port. 218 This is useful for devices that share the same port for programming 219 and serial console, or use soft-USB, where flash must come first. 220 """) 221 222 test_or_build.add_argument( 223 "-b", 224 "--build-only", 225 action="store_true", 226 default="--prep-artifacts-for-testing" in sys.argv, 227 help="Only build the code, do not attempt to run the code on targets." 228 ) 229 230 test_or_build.add_argument( 231 "--prep-artifacts-for-testing", action="store_true", 232 help="Generate artifacts for testing, do not attempt to run the" 233 "code on targets.") 234 235 parser.add_argument( 236 "--package-artifacts", 237 help="Package artifacts needed for flashing in a file to be used with --test-only" 238 ) 239 240 test_or_build.add_argument( 241 "--test-only", action="store_true", 242 help="""Only run device tests with current artifacts, do not build 243 the code""") 244 245 parser.add_argument("--timeout-multiplier", type=float, default=1, 246 help="""Globally adjust tests timeouts by specified multiplier. The resulting test 247 timeout would be multiplication of test timeout value, board-level timeout multiplier 248 and global timeout multiplier (this parameter)""") 249 250 test_xor_subtest.add_argument( 251 "-s", "--test", "--scenario", action="append", type = norm_path, 252 help="""Run only the specified test suite scenario. These are named by 253 'path/relative/to/Zephyr/base/section.subsection_in_testcase_yaml', 254 or just 'section.subsection' identifier. With '--testsuite-root' option 255 the scenario will be found faster. 256 """) 257 258 test_xor_subtest.add_argument( 259 "--sub-test", action="append", 260 help="""Recursively find sub-test functions (test cases) and run the entire 261 test scenario (section.subsection) where they were found, including all sibling test 262 functions. Sub-tests are named by: 263 'section.subsection_in_testcase_yaml.ztest_suite.ztest_without_test_prefix'. 264 Example_1: 'kernel.fifo.fifo_api_1cpu.fifo_loop' where 'kernel.fifo' is a test scenario 265 name (section.subsection) and 'fifo_api_1cpu.fifo_loop' is a Ztest 'suite_name.test_name'. 266 Example_2: 'debug.coredump.logging_backend' is a standalone test scenario name. 267 Note: This selection mechanism works only for Ztest suite and test function names in 268 the source files which are not generated by macro-substitutions. 269 Note: With --no-detailed-test-id use only Ztest names without scenario name. 270 """) 271 272 parser.add_argument( 273 "--pytest-args", action="append", 274 help="""Pass additional arguments to the pytest subprocess. This parameter 275 will extend the pytest_args from the harness_config in YAML file. 276 """) 277 278 valgrind_asan_group.add_argument( 279 "--enable-valgrind", action="store_true", 280 help="""Run binary through valgrind and check for several memory access 281 errors. Valgrind needs to be installed on the host. This option only 282 works with host binaries such as those generated for the native_sim 283 configuration and is mutual exclusive with --enable-asan. 284 """) 285 286 valgrind_asan_group.add_argument( 287 "--enable-asan", action="store_true", 288 help="""Enable address sanitizer to check for several memory access 289 errors. Libasan needs to be installed on the host. This option only 290 works with host binaries such as those generated for the native_sim 291 configuration and is mutual exclusive with --enable-valgrind. 292 """) 293 294 # Start of individual args place them in alpha-beta order 295 296 board_root_list = [f"{ZEPHYR_BASE}/boards", f"{ZEPHYR_BASE}/subsys/testsuite/boards"] 297 298 modules = zephyr_module.parse_modules(ZEPHYR_BASE) 299 for module in modules: 300 board_root = module.meta.get("build", {}).get("settings", {}).get("board_root") 301 if board_root: 302 board_root_list.append(os.path.join(module.project, board_root, "boards")) 303 304 parser.add_argument( 305 "-A", "--board-root", action="append", default=board_root_list, 306 help="""Directory to search for board configuration files. All .yaml 307files in the directory will be processed. The directory should have the same 308structure in the main Zephyr tree: boards/<vendor>/<board_name>/""") 309 310 parser.add_argument( 311 "--allow-installed-plugin", action="store_true", default=None, 312 help="Allow to use pytest plugin installed by pip for pytest tests." 313 ) 314 315 parser.add_argument( 316 "-a", "--arch", action="append", 317 help="Arch filter for testing. Takes precedence over --platform. " 318 "If unspecified, test all arches. Multiple invocations " 319 "are treated as a logical 'or' relationship") 320 321 parser.add_argument( 322 "-B", "--subset", 323 help="Only run a subset of the tests, 1/4 for running the first 25%%, " 324 "3/5 means run the 3rd fifth of the total. " 325 "This option is useful when running a large number of tests on " 326 "different hosts to speed up execution time.") 327 328 parser.add_argument( 329 "--shuffle-tests", action="store_true", default=None, 330 help="""Shuffle test execution order to get randomly distributed tests across subsets. 331 Used only when --subset is provided.""") 332 333 parser.add_argument( 334 "--shuffle-tests-seed", action="store", default=None, 335 help="""Seed value for random generator used to shuffle tests. 336 If not provided, seed in generated by system. 337 Used only when --shuffle-tests is provided.""") 338 339 parser.add_argument("-C", "--coverage", action="store_true", 340 help="Generate coverage reports. Implies " 341 "--enable-coverage.") 342 343 parser.add_argument( 344 "-c", "--clobber-output", action="store_true", 345 help="Cleaning the output directory will simply delete it instead " 346 "of the default policy of renaming.") 347 348 parser.add_argument( 349 "--cmake-only", action="store_true", 350 help="Only run cmake, do not build or run.") 351 352 parser.add_argument("--coverage-basedir", default=ZEPHYR_BASE, 353 help="Base source directory for coverage report.") 354 355 parser.add_argument("--coverage-platform", action="append", default=[], 356 help="Platforms to run coverage reports on. " 357 "This option may be used multiple times. " 358 "Default to what was selected with --platform.") 359 360 parser.add_argument("--coverage-tool", choices=['lcov', 'gcovr'], default='gcovr', 361 help="Tool to use to generate coverage report.") 362 363 parser.add_argument( 364 "--coverage-formats", 365 action="store", 366 default=None, # default behavior is set in run_coverage 367 help="Output formats to use for generated coverage reports, as a comma-separated list. " + 368 "Valid options for 'gcovr' tool are: " + 369 ','.join(supported_coverage_formats['gcovr']) + " (html - default)." + 370 " Valid options for 'lcov' tool are: " + 371 ','.join(supported_coverage_formats['lcov']) + " (html,lcov - default)." 372 ) 373 374 parser.add_argument( 375 "--test-config", 376 action="store", 377 default=os.path.join(ZEPHYR_BASE, "tests", "test_config.yaml"), 378 help="Path to file with plans and test configurations." 379 ) 380 381 parser.add_argument("--level", action="store", 382 help="Test level to be used. By default, no levels are used for filtering" 383 "and do the selection based on existing filters.") 384 385 parser.add_argument( 386 "--device-serial-baud", action="store", default=None, 387 help="Serial device baud rate (default 115200)") 388 389 parser.add_argument( 390 "--disable-suite-name-check", action="store_true", default=False, 391 help="Disable extended test suite name verification at the beginning " 392 "of Ztest test. This option could be useful for tests or " 393 "platforms, which from some reasons cannot print early logs.") 394 395 parser.add_argument("-e", "--exclude-tag", action="append", 396 help="Specify tags of tests that should not run. " 397 "Default is to run all tests with all tags.") 398 399 parser.add_argument("--enable-coverage", action="store_true", 400 help="Enable code coverage using gcov.") 401 402 parser.add_argument( 403 "--enable-lsan", action="store_true", 404 help="""Enable leak sanitizer to check for heap memory leaks. 405 Libasan needs to be installed on the host. This option only 406 works with host binaries such as those generated for the native_sim 407 configuration and when --enable-asan is given. 408 """) 409 410 parser.add_argument( 411 "--enable-ubsan", action="store_true", 412 help="""Enable undefined behavior sanitizer to check for undefined 413 behaviour during program execution. It uses an optional runtime library 414 to provide better error diagnostics. This option only works with host 415 binaries such as those generated for the native_sim configuration. 416 """) 417 418 parser.add_argument( 419 "--filter", choices=['buildable', 'runnable'], 420 default='runnable' if "--device-testing" in sys.argv else 'buildable', 421 help="""Filter tests to be built and executed. By default everything is 422 built and if a test is runnable (emulation or a connected device), it 423 is run. This option allows for example to only build tests that can 424 actually be run. Runnable is a subset of buildable.""") 425 426 parser.add_argument("--force-color", action="store_true", 427 help="Always output ANSI color escape sequences " 428 "even when the output is redirected (not a tty)") 429 430 parser.add_argument("--force-toolchain", action="store_true", 431 help="Do not filter based on toolchain, use the set " 432 " toolchain unconditionally") 433 434 parser.add_argument("--gcov-tool", type=Path, default=None, 435 help="Path to the gcov tool to use for code coverage " 436 "reports") 437 438 footprint_group.add_argument( 439 "--create-rom-ram-report", 440 action="store_true", 441 help="Generate detailed json reports with ROM/RAM symbol sizes for each test image built " 442 "using additional build option `--target footprint`.") 443 444 footprint_group.add_argument( 445 "--footprint-report", 446 nargs="?", 447 default=None, 448 choices=['all', 'ROM', 'RAM'], 449 const="all", 450 help="Select which memory area symbols' data to collect as 'footprint' property " 451 "of each test suite built, and report in 'twister_footprint.json' together " 452 "with the relevant execution metadata the same way as in `twister.json`. " 453 "Implies '--create-rom-ram-report' to generate the footprint data files. " 454 "No value means '%(const)s'. Default: %(default)s""") 455 456 footprint_group.add_argument( 457 "--enable-size-report", 458 action="store_true", 459 help="Collect and report ROM/RAM section sizes for each test image built.") 460 461 parser.add_argument( 462 "--disable-unrecognized-section-test", 463 action="store_true", 464 default=False, 465 help="Don't error on unrecognized sections in the binary images.") 466 467 footprint_group.add_argument( 468 "--footprint-from-buildlog", 469 action = "store_true", 470 help="Take ROM/RAM sections footprint summary values from the 'build.log' " 471 "instead of 'objdump' results used otherwise." 472 "Requires --enable-size-report or one of the baseline comparison modes." 473 "Warning: the feature will not work correctly with sysbuild.") 474 475 compare_group_option = footprint_group.add_mutually_exclusive_group() 476 477 compare_group_option.add_argument( 478 "-m", "--last-metrics", 479 action="store_true", 480 help="Compare footprints to the previous twister invocation as a baseline " 481 "running in the same output directory. " 482 "Implies --enable-size-report option.") 483 484 compare_group_option.add_argument( 485 "--compare-report", 486 help="Use this report file as a baseline for footprint comparison. " 487 "The file should be of 'twister.json' schema. " 488 "Implies --enable-size-report option.") 489 490 footprint_group.add_argument( 491 "--show-footprint", 492 action="store_true", 493 help="With footprint comparison to a baseline, log ROM/RAM section deltas. ") 494 495 footprint_group.add_argument( 496 "-H", "--footprint-threshold", 497 type=float, 498 default=5.0, 499 help="With footprint comparison to a baseline, " 500 "warn the user for any of the footprint metric change which is greater or equal " 501 "to the specified percentage value. " 502 "Default is %(default)s for %(default)s%% delta from the new footprint value. " 503 "Use zero to warn on any footprint metric increase.") 504 505 footprint_group.add_argument( 506 "-D", "--all-deltas", 507 action="store_true", 508 help="With footprint comparison to a baseline, " 509 "warn on any footprint change, increase or decrease. " 510 "Implies --footprint-threshold=0") 511 512 footprint_group.add_argument( 513 "-z", "--size", 514 action="append", 515 metavar='FILENAME', 516 help="Ignore all other command line options and just produce a report to " 517 "stdout with ROM/RAM section sizes on the specified binary images.") 518 519 parser.add_argument( 520 "-i", "--inline-logs", action="store_true", 521 help="Upon test failure, print relevant log data to stdout " 522 "instead of just a path to it.") 523 524 parser.add_argument("--ignore-platform-key", action="store_true", 525 help="Do not filter based on platform key") 526 527 parser.add_argument( 528 "-j", "--jobs", type=int, 529 help="Number of jobs for building, defaults to number of CPU threads, " 530 "overcommitted by factor 2 when --build-only.") 531 532 parser.add_argument( 533 "-K", "--force-platform", action="store_true", 534 help="""Force testing on selected platforms, 535 even if they are excluded in the test configuration (testcase.yaml).""" 536 ) 537 538 parser.add_argument( 539 "-l", "--all", action="store_true", 540 help="Build/test on all platforms. Any --platform arguments " 541 "ignored.") 542 543 test_plan_report_xor.add_argument("--list-tags", action="store_true", 544 help="List all tags occurring in selected tests.") 545 546 parser.add_argument("--log-file", metavar="FILENAME", action="store", 547 help="Specify a file where to save logs.") 548 549 parser.add_argument( 550 "-M", "--runtime-artifact-cleanup", choices=['pass', 'all'], 551 default=None, const='pass', nargs='?', 552 help="""Cleanup test artifacts. The default behavior is 'pass' 553 which only removes artifacts of passing tests. If you wish to 554 remove all artificats including those of failed tests, use 'all'.""") 555 556 test_xor_generator.add_argument( 557 "-N", "--ninja", action="store_true", 558 default=not any(a in sys.argv for a in ("-k", "--make")), 559 help="Use the Ninja generator with CMake. (This is the default)") 560 561 test_xor_generator.add_argument( 562 "-k", "--make", action="store_true", 563 help="Use the unix Makefile generator with CMake.") 564 565 parser.add_argument( 566 "-n", "--no-clean", action="store_true", 567 help="Re-use the outdir before building. Will result in " 568 "faster compilation since builds will be incremental.") 569 570 parser.add_argument( 571 "--aggressive-no-clean", action="store_true", 572 help="Re-use the outdir before building and do not re-run cmake. Will result in " 573 "much faster compilation since builds will be incremental. This option might " 574 " result in build failures and inconsistencies if dependencies change or when " 575 " applied on a significantly changed code base. Use on your own " 576 " risk. It is recommended to only use this option for local " 577 " development and when testing localized change in a subsystem.") 578 579 parser.add_argument( 580 '--detailed-test-id', action='store_true', 581 help="Compose each test Suite name from its configuration path (relative to root) and " 582 "the appropriate Scenario name using PATH_TO_TEST_CONFIG/SCENARIO_NAME schema. " 583 "Also (for Ztest only), prefix each test Case name with its Scenario name. " 584 "For example: 'kernel.common.timing' Scenario with test Suite name " 585 "'tests/kernel/sleep/kernel.common.timing' and 'kernel.common.timing.sleep.usleep' " 586 "test Case (where 'sleep' is its Ztest suite name and 'usleep' is Ztest test name.") 587 588 parser.add_argument( 589 "--no-detailed-test-id", dest='detailed_test_id', action="store_false", 590 help="Don't prefix each test Suite name with its configuration path, " 591 "so it is the same as the appropriate Scenario name. " 592 "Also (for Ztest only), don't prefix each Ztest Case name with its Scenario name. " 593 "For example: 'kernel.common.timing' Scenario name, the same Suite name, " 594 "and 'sleep.usleep' test Case (where 'sleep' is its Ztest suite name " 595 "and 'usleep' is Ztest test name.") 596 597 # Include paths in names by default. 598 parser.set_defaults(detailed_test_id=True) 599 600 parser.add_argument( 601 "--detailed-skipped-report", action="store_true", 602 help="Generate a detailed report with all skipped test cases" 603 "including those that are filtered based on testsuite definition." 604 ) 605 606 parser.add_argument( 607 "-O", "--outdir", 608 default=os.path.join(os.getcwd(), "twister-out"), 609 help="Output directory for logs and binaries. " 610 "Default is 'twister-out' in the current directory. " 611 "This directory will be cleaned unless '--no-clean' is set. " 612 "The '--clobber-output' option controls what cleaning does.") 613 614 parser.add_argument( 615 "-o", "--report-dir", 616 help="""Output reports containing results of the test run into the 617 specified directory. 618 The output will be both in JSON and JUNIT format 619 (twister.json and twister.xml). 620 """) 621 622 parser.add_argument("--overflow-as-errors", action="store_true", 623 help="Treat RAM/SRAM overflows as errors.") 624 625 parser.add_argument("--report-filtered", action="store_true", 626 help="Include filtered tests in the reports.") 627 628 parser.add_argument("-P", "--exclude-platform", action="append", default=[], 629 help="""Exclude platforms and do not build or run any tests 630 on those platforms. This option can be called multiple times. 631 """ 632 ) 633 634 parser.add_argument("--persistent-hardware-map", action='store_true', 635 help="""With --generate-hardware-map, tries to use 636 persistent names for serial devices on platforms 637 that support this feature (currently only Linux). 638 """) 639 640 parser.add_argument( 641 "--vendor", action="append", default=[], 642 help="Vendor filter for testing") 643 644 parser.add_argument( 645 "-p", "--platform", action="append", default=[], 646 help="Platform filter for testing. This option may be used multiple " 647 "times. Test suites will only be built/run on the platforms " 648 "specified. If this option is not used, then platforms marked " 649 "as default in the platform metadata file will be chosen " 650 "to build and test. ") 651 652 parser.add_argument( 653 "--platform-reports", action="store_true", 654 help="""Create individual reports for each platform. 655 """) 656 657 parser.add_argument("--pre-script", 658 help="""specify a pre script. This will be executed 659 before device handler open serial port and invoke runner. 660 """) 661 662 parser.add_argument( 663 "--quarantine-list", 664 action="append", 665 metavar="FILENAME", 666 help="Load list of test scenarios under quarantine. The entries in " 667 "the file need to correspond to the test scenarios names as in " 668 "corresponding tests .yaml files. These scenarios " 669 "will be skipped with quarantine as the reason.") 670 671 parser.add_argument( 672 "--quarantine-verify", 673 action="store_true", 674 help="Use the list of test scenarios under quarantine and run them" 675 "to verify their current status.") 676 677 parser.add_argument( 678 "--quit-on-failure", 679 action="store_true", 680 help="""quit twister once there is build / run failure 681 """) 682 683 parser.add_argument( 684 "--report-name", 685 help="""Create a report with a custom name. 686 """) 687 688 parser.add_argument( 689 "--report-summary", action="store", nargs='?', type=int, const=0, 690 help="Show failed/error report from latest run. Default shows all items found. " 691 "However, you can specify the number of items (e.g. --report-summary 15). " 692 "It also works well with the --outdir switch.") 693 694 parser.add_argument( 695 "--report-suffix", 696 help="""Add a suffix to all generated file names, for example to add a 697 version or a commit ID. 698 """) 699 700 parser.add_argument( 701 "--report-all-options", action="store_true", 702 help="""Show all command line options applied, including defaults, as 703 environment.options object in twister.json. Default: show only non-default settings. 704 """) 705 706 parser.add_argument( 707 "--retry-failed", type=int, default=0, 708 help="Retry failing tests again, up to the number of times specified.") 709 710 parser.add_argument( 711 "--retry-interval", type=int, default=60, 712 help="Retry failing tests after specified period of time.") 713 714 parser.add_argument( 715 "--retry-build-errors", action="store_true", 716 help="Retry build errors as well.") 717 718 parser.add_argument( 719 "-S", "--enable-slow", action="store_true", 720 default="--enable-slow-only" in sys.argv, 721 help="Execute time-consuming test cases that have been marked " 722 "as 'slow' in testcase.yaml. Normally these are only built.") 723 724 parser.add_argument( 725 "--enable-slow-only", action="store_true", 726 help="Execute time-consuming test cases that have been marked " 727 "as 'slow' in testcase.yaml only. This also set the option --enable-slow") 728 729 parser.add_argument( 730 "--seed", type=int, 731 help="Seed for native_sim pseudo-random number generator") 732 733 parser.add_argument( 734 "--short-build-path", 735 action="store_true", 736 help="Create shorter build directory paths based on symbolic links. " 737 "The shortened build path will be used by CMake for generating " 738 "the build system and executing the build. Use this option if " 739 "you experience build failures related to path length, for " 740 "example on Windows OS. This option can be used only with " 741 "'--ninja' argument (to use Ninja build generator).") 742 743 parser.add_argument( 744 "-t", "--tag", action="append", 745 help="Specify tags to restrict which tests to run by tag value. " 746 "Default is to not do any tag filtering. Multiple invocations " 747 "are treated as a logical 'or' relationship.") 748 749 parser.add_argument("--timestamps", 750 action="store_true", 751 help="Print all messages with time stamps.") 752 753 parser.add_argument( 754 "-u", 755 "--no-update", 756 action="store_true", 757 help="Do not update the results of the last run. This option " 758 "is only useful when reusing the same output directory of " 759 "twister, for example when re-running failed tests with --only-failed " 760 "or --no-clean. This option is for debugging purposes only.") 761 762 parser.add_argument( 763 "-v", 764 "--verbose", 765 action="count", 766 default=0, 767 help="Call multiple times to increase verbosity.") 768 769 parser.add_argument( 770 "-ll", 771 "--log-level", 772 type=str.upper, 773 default='INFO', 774 choices=['NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], 775 help="Select the threshold event severity for which you'd like to receive logs in console." 776 " Default is INFO.") 777 778 parser.add_argument("-W", "--disable-warnings-as-errors", action="store_true", 779 help="Do not treat warning conditions as errors.") 780 781 parser.add_argument( 782 "--west-flash", nargs='?', const=[], 783 help="""Uses west instead of ninja or make to flash when running with 784 --device-testing. Supports comma-separated argument list. 785 786 E.g "twister --device-testing --device-serial /dev/ttyACM0 787 --west-flash="--board-id=foobar,--erase" 788 will translate to "west flash -- --board-id=foobar --erase" 789 790 NOTE: device-testing must be enabled to use this option. 791 """ 792 ) 793 parser.add_argument( 794 "--west-runner", 795 help="""Uses the specified west runner instead of default when running 796 with --west-flash. 797 798 E.g "twister --device-testing --device-serial /dev/ttyACM0 799 --west-flash --west-runner=pyocd" 800 will translate to "west flash --runner pyocd" 801 802 NOTE: west-flash must be enabled to use this option. 803 """ 804 ) 805 806 parser.add_argument( 807 "-X", "--fixture", action="append", default=[], 808 help="Specify a fixture that a board might support.") 809 810 parser.add_argument( 811 "-x", "--extra-args", action="append", default=[], 812 help="""Extra CMake cache entries to define when building test cases. 813 May be called multiple times. The key-value entries will be 814 prefixed with -D before being passed to CMake. 815 E.g 816 "twister -x=USE_CCACHE=0" 817 will translate to 818 "cmake -DUSE_CCACHE=0" 819 which will ultimately disable ccache. 820 """ 821 ) 822 823 parser.add_argument( 824 "-y", "--dry-run", action="store_true", 825 help="""Create the filtered list of test cases, but don't actually 826 run them. Useful if you're just interested in the test plan 827 generated for every run and saved in the specified output 828 directory (testplan.json). 829 """) 830 831 parser.add_argument("extra_test_args", nargs=argparse.REMAINDER, 832 help="Additional args following a '--' are passed to the test binary") 833 834 parser.add_argument("--alt-config-root", action="append", default=[], 835 help="Alternative test configuration root/s. When a test is found, " 836 "Twister will check if a test configuration file exist in any of " 837 "the alternative test configuration root folders. For example, " 838 "given $test_root/tests/foo/testcase.yaml, Twister will use " 839 "$alt_config_root/tests/foo/testcase.yaml if it exists") 840 841 return parser 842 843 844def parse_arguments( 845 parser: argparse.ArgumentParser, 846 args, 847 options = None, 848 on_init=True 849) -> argparse.Namespace: 850 if options is None: 851 options = parser.parse_args(args) 852 853 # Very early error handling 854 if options.short_build_path and not options.ninja: 855 logger.error("--short-build-path requires Ninja to be enabled") 856 sys.exit(1) 857 858 if options.device_serial_pty and os.name == "nt": # OS is Windows 859 logger.error("--device-serial-pty is not supported on Windows OS") 860 sys.exit(1) 861 862 if options.west_runner and options.west_flash is None: 863 logger.error("west-runner requires west-flash to be enabled") 864 sys.exit(1) 865 866 if options.west_flash and not options.device_testing: 867 logger.error("west-flash requires device-testing to be enabled") 868 sys.exit(1) 869 870 if not options.testsuite_root: 871 # if we specify a test scenario which is part of a suite directly, do 872 # not set testsuite root to default, just point to the test directory 873 # directly. 874 if options.test: 875 for scenario in options.test: 876 if dirname := os.path.dirname(scenario): 877 options.testsuite_root.append(dirname) 878 879 # check again and make sure we have something set 880 if not options.testsuite_root: 881 options.testsuite_root = [os.path.join(ZEPHYR_BASE, "tests"), 882 os.path.join(ZEPHYR_BASE, "samples")] 883 884 if options.last_metrics or options.compare_report: 885 options.enable_size_report = True 886 887 if options.footprint_report: 888 options.create_rom_ram_report = True 889 890 if options.aggressive_no_clean: 891 options.no_clean = True 892 893 if options.coverage: 894 options.enable_coverage = True 895 896 if options.enable_coverage and not options.coverage_platform: 897 options.coverage_platform = options.platform 898 899 if options.coverage_formats: 900 for coverage_format in options.coverage_formats.split(','): 901 if coverage_format not in supported_coverage_formats[options.coverage_tool]: 902 logger.error(f"Unsupported coverage report formats:'{options.coverage_formats}' " 903 f"for {options.coverage_tool}") 904 sys.exit(1) 905 906 if options.enable_valgrind and not shutil.which("valgrind"): 907 logger.error("valgrind enabled but valgrind executable not found") 908 sys.exit(1) 909 910 if ( 911 (not options.device_testing) 912 and (options.device_serial or options.device_serial_pty or options.hardware_map) 913 ): 914 logger.error( 915 "Use --device-testing with --device-serial, or --device-serial-pty, or --hardware-map." 916 ) 917 sys.exit(1) 918 919 if ( 920 options.device_testing 921 and (options.device_serial or options.device_serial_pty) and len(options.platform) != 1 922 ): 923 logger.error("When --device-testing is used with --device-serial " 924 "or --device-serial-pty, exactly one platform must " 925 "be specified") 926 sys.exit(1) 927 928 if options.device_flash_with_test and not options.device_testing: 929 logger.error("--device-flash-with-test requires --device_testing") 930 sys.exit(1) 931 932 if options.flash_before and options.device_flash_with_test: 933 logger.error("--device-flash-with-test does not apply when --flash-before is used") 934 sys.exit(1) 935 936 if options.flash_before and options.device_serial_pty: 937 logger.error("--device-serial-pty cannot be used when --flash-before is set (for now)") 938 sys.exit(1) 939 940 if options.shuffle_tests and options.subset is None: 941 logger.error("--shuffle-tests requires --subset") 942 sys.exit(1) 943 944 if options.shuffle_tests_seed and options.shuffle_tests is None: 945 logger.error("--shuffle-tests-seed requires --shuffle-tests") 946 sys.exit(1) 947 948 if options.size: 949 from twisterlib.size_calc import SizeCalculator 950 for fn in options.size: 951 sc = SizeCalculator(fn, []) 952 sc.size_report() 953 sys.exit(0) 954 955 if options.footprint_from_buildlog: 956 logger.warning("WARNING: Using --footprint-from-buildlog will give inconsistent results " 957 "for configurations using sysbuild. It is recommended to not use this flag " 958 "when building configurations using sysbuild.") 959 if not options.enable_size_report: 960 logger.error("--footprint-from-buildlog requires --enable-size-report") 961 sys.exit(1) 962 963 if len(options.extra_test_args) > 0: 964 # extra_test_args is a list of CLI args that Twister did not recognize 965 # and are intended to be passed through to the ztest executable. This 966 # list should begin with a "--". If not, there is some extra 967 # unrecognized arg(s) that shouldn't be there. Tell the user there is a 968 # syntax error. 969 if options.extra_test_args[0] != "--": 970 try: 971 double_dash = options.extra_test_args.index("--") 972 except ValueError: 973 double_dash = len(options.extra_test_args) 974 unrecognized = " ".join(options.extra_test_args[0:double_dash]) 975 976 logger.error( 977 f"Unrecognized arguments found: '{unrecognized}'." 978 " Use -- to delineate extra arguments for test binary or pass -h for help." 979 ) 980 981 sys.exit(1) 982 983 # Strip off the initial "--" following validation. 984 options.extra_test_args = options.extra_test_args[1:] 985 986 if on_init and not options.allow_installed_plugin and PYTEST_PLUGIN_INSTALLED: 987 logger.error("By default Twister should work without pytest-twister-harness " 988 "plugin being installed, so please, uninstall it by " 989 "`pip uninstall pytest-twister-harness` and `git clean " 990 "-dxf scripts/pylib/pytest-twister-harness`.") 991 sys.exit(1) 992 elif on_init and options.allow_installed_plugin and PYTEST_PLUGIN_INSTALLED: 993 logger.warning("You work with installed version of " 994 "pytest-twister-harness plugin.") 995 996 return options 997 998def strip_ansi_sequences(s: str) -> str: 999 """Remove ANSI escape sequences from a string.""" 1000 return re.sub(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])', "", s) 1001 1002class TwisterEnv: 1003 1004 def __init__(self, options : argparse.Namespace, default_options=None) -> None: 1005 self.version = "Unknown" 1006 self.toolchain = None 1007 self.commit_date = "Unknown" 1008 self.run_date = None 1009 self.options = options 1010 self.default_options = default_options 1011 1012 if options.ninja: 1013 self.generator_cmd = "ninja" 1014 self.generator = "Ninja" 1015 else: 1016 self.generator_cmd = "make" 1017 self.generator = "Unix Makefiles" 1018 logger.info(f"Using {self.generator}..") 1019 1020 self.test_roots = options.testsuite_root 1021 1022 if not isinstance(options.board_root, list): 1023 self.board_roots = [options.board_root] 1024 else: 1025 self.board_roots = options.board_root 1026 self.outdir = os.path.abspath(options.outdir) 1027 1028 self.snippet_roots = [Path(ZEPHYR_BASE)] 1029 modules = zephyr_module.parse_modules(ZEPHYR_BASE) 1030 for module in modules: 1031 snippet_root = module.meta.get("build", {}).get("settings", {}).get("snippet_root") 1032 if snippet_root: 1033 self.snippet_roots.append(Path(module.project) / snippet_root) 1034 1035 1036 self.soc_roots = [Path(ZEPHYR_BASE), Path(ZEPHYR_BASE) / 'subsys' / 'testsuite'] 1037 self.dts_roots = [Path(ZEPHYR_BASE)] 1038 self.arch_roots = [Path(ZEPHYR_BASE)] 1039 1040 for module in modules: 1041 soc_root = module.meta.get("build", {}).get("settings", {}).get("soc_root") 1042 if soc_root: 1043 self.soc_roots.append(Path(module.project) / Path(soc_root)) 1044 dts_root = module.meta.get("build", {}).get("settings", {}).get("dts_root") 1045 if dts_root: 1046 self.dts_roots.append(Path(module.project) / Path(dts_root)) 1047 arch_root = module.meta.get("build", {}).get("settings", {}).get("arch_root") 1048 if arch_root: 1049 self.arch_roots.append(Path(module.project) / Path(arch_root)) 1050 1051 self.hwm = None 1052 1053 self.test_config = options.test_config 1054 1055 self.alt_config_root = options.alt_config_root 1056 1057 def non_default_options(self) -> dict: 1058 """Returns current command line options which are set to non-default values.""" 1059 diff = {} 1060 if not self.default_options: 1061 return diff 1062 dict_options = vars(self.options) 1063 dict_default = vars(self.default_options) 1064 for k in dict_options: 1065 if k not in dict_default or dict_options[k] != dict_default[k]: 1066 diff[k] = dict_options[k] 1067 return diff 1068 1069 def discover(self): 1070 self.check_zephyr_version() 1071 self.get_toolchain() 1072 self.run_date = datetime.now(timezone.utc).isoformat(timespec='seconds') 1073 1074 def check_zephyr_version(self): 1075 try: 1076 subproc = subprocess.run(["git", "describe", "--abbrev=12", "--always"], 1077 stdout=subprocess.PIPE, 1078 text=True, 1079 cwd=ZEPHYR_BASE) 1080 if subproc.returncode == 0: 1081 _version = subproc.stdout.strip() 1082 if _version: 1083 self.version = _version 1084 logger.info(f"Zephyr version: {self.version}") 1085 except OSError: 1086 logger.exception("Failure while reading Zephyr version.") 1087 1088 if self.version == "Unknown": 1089 logger.warning("Could not determine version") 1090 1091 try: 1092 subproc = subprocess.run(["git", "show", "-s", "--format=%cI", "HEAD"], 1093 stdout=subprocess.PIPE, 1094 text=True, 1095 cwd=ZEPHYR_BASE) 1096 if subproc.returncode == 0: 1097 self.commit_date = subproc.stdout.strip() 1098 except OSError: 1099 logger.exception("Failure while reading head commit date.") 1100 1101 @staticmethod 1102 def run_cmake_script(args=None): 1103 if args is None: 1104 args = [] 1105 script = os.fspath(args[0]) 1106 1107 logger.debug(f"Running cmake script {script}") 1108 1109 cmake_args = ["-D{}".format(a.replace('"', '')) for a in args[1:]] 1110 cmake_args.extend(['-P', script]) 1111 1112 cmake = shutil.which('cmake') 1113 if not cmake: 1114 msg = "Unable to find `cmake` in path" 1115 logger.error(msg) 1116 raise Exception(msg) 1117 cmd = [cmake] + cmake_args 1118 log_command(logger, "Calling cmake", cmd) 1119 1120 kwargs = dict() 1121 kwargs['stdout'] = subprocess.PIPE 1122 # CMake sends the output of message() to stderr unless it's STATUS 1123 kwargs['stderr'] = subprocess.STDOUT 1124 1125 p = subprocess.Popen(cmd, **kwargs) 1126 out, _ = p.communicate() 1127 1128 # It might happen that the environment adds ANSI escape codes like \x1b[0m, 1129 # for instance if twister is executed from inside a makefile. In such a 1130 # scenario it is then necessary to remove them, as otherwise the JSON decoding 1131 # will fail. 1132 out = strip_ansi_sequences(out.decode()) 1133 1134 if p.returncode == 0: 1135 msg = f"Finished running {args[0]}" 1136 logger.debug(msg) 1137 results = {"returncode": p.returncode, "msg": msg, "stdout": out} 1138 1139 else: 1140 logger.error(f"CMake script failure: {args[0]}") 1141 results = {"returncode": p.returncode, "returnmsg": out} 1142 1143 return results 1144 1145 def get_toolchain(self): 1146 toolchain_script = Path(ZEPHYR_BASE) / Path('cmake/verify-toolchain.cmake') 1147 result = self.run_cmake_script([toolchain_script, "FORMAT=json"]) 1148 1149 try: 1150 if result['returncode']: 1151 raise TwisterRuntimeError(f"E: {result['returnmsg']}") 1152 except Exception as e: 1153 print(str(e)) 1154 sys.exit(2) 1155 self.toolchain = json.loads(result['stdout'])['ZEPHYR_TOOLCHAIN_VARIANT'] 1156 logger.info(f"Using '{self.toolchain}' toolchain.") 1157
