1#!/usr/bin/env python3 2# vim: set syntax=python ts=4 : 3# 4# Copyright (c) 2018 Intel Corporation 5# Copyright 2022 NXP 6# SPDX-License-Identifier: Apache-2.0 7 8import os 9import pkg_resources 10import sys 11from pathlib import Path 12import json 13import logging 14import subprocess 15import shutil 16import re 17import argparse 18from datetime import datetime, timezone 19from twisterlib.coverage import supported_coverage_formats 20 21logger = logging.getLogger('twister') 22logger.setLevel(logging.DEBUG) 23 24from twisterlib.error import TwisterRuntimeError 25from twisterlib.log_helper import log_command 26 27ZEPHYR_BASE = os.getenv("ZEPHYR_BASE") 28if not ZEPHYR_BASE: 29 sys.exit("$ZEPHYR_BASE environment variable undefined") 30 31sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/")) 32 33import zephyr_module 34 35# Use this for internal comparisons; that's what canonicalization is 36# for. Don't use it when invoking other components of the build system 37# to avoid confusing and hard to trace inconsistencies in error messages 38# and logs, generated Makefiles, etc. compared to when users invoke these 39# components directly. 40# Note "normalization" is different from canonicalization, see os.path. 41canonical_zephyr_base = os.path.realpath(ZEPHYR_BASE) 42 43installed_packages = [pkg.project_name for pkg in pkg_resources.working_set] # pylint: disable=not-an-iterable 44PYTEST_PLUGIN_INSTALLED = 'pytest-twister-harness' in installed_packages 45 46 47def add_parse_arguments(parser = None): 48 if parser is None: 49 parser = argparse.ArgumentParser( 50 description=__doc__, 51 formatter_class=argparse.RawDescriptionHelpFormatter, 52 allow_abbrev=False) 53 parser.fromfile_prefix_chars = "+" 54 55 case_select = parser.add_argument_group("Test case selection", 56 """ 57Artificially long but functional example: 58 $ ./scripts/twister -v \\ 59 --testsuite-root tests/ztest/base \\ 60 --testsuite-root tests/kernel \\ 61 --test tests/ztest/base/testing.ztest.verbose_0 \\ 62 --test tests/kernel/fifo/fifo_api/kernel.fifo 63 64 "kernel.fifo.poll" is one of the test section names in 65 __/fifo_api/testcase.yaml 66 """) 67 68 compare_group_option = parser.add_mutually_exclusive_group() 69 70 platform_group_option = parser.add_mutually_exclusive_group() 71 72 run_group_option = parser.add_mutually_exclusive_group() 73 74 device = parser.add_mutually_exclusive_group(required="--device-testing" in sys.argv) 75 76 test_or_build = parser.add_mutually_exclusive_group() 77 78 test_xor_subtest = case_select.add_mutually_exclusive_group() 79 80 test_xor_generator = case_select.add_mutually_exclusive_group() 81 82 valgrind_asan_group = parser.add_mutually_exclusive_group() 83 84 case_select.add_argument( 85 "-E", 86 "--save-tests", 87 metavar="FILENAME", 88 action="store", 89 help="Write a list of tests and platforms to be run to file.") 90 91 case_select.add_argument( 92 "-F", 93 "--load-tests", 94 metavar="FILENAME", 95 action="store", 96 help="Load a list of tests and platforms to be run from file.") 97 98 case_select.add_argument( 99 "-T", "--testsuite-root", action="append", default=[], 100 help="Base directory to recursively search for test cases. All " 101 "testcase.yaml files under here will be processed. May be " 102 "called multiple times. Defaults to the 'samples/' and " 103 "'tests/' directories at the base of the Zephyr tree.") 104 105 case_select.add_argument( 106 "-f", 107 "--only-failed", 108 action="store_true", 109 help="Run only those tests that failed the previous twister run " 110 "invocation.") 111 112 case_select.add_argument("--list-tests", action="store_true", 113 help="""List of all sub-test functions recursively found in 114 all --testsuite-root arguments. Note different sub-tests can share 115 the same section name and come from different directories. 116 The output is flattened and reports --sub-test names only, 117 not their directories. For instance net.socket.getaddrinfo_ok 118 and net.socket.fd_set belong to different directories. 119 """) 120 121 case_select.add_argument("--test-tree", action="store_true", 122 help="""Output the test plan in a tree form""") 123 124 compare_group_option.add_argument("--compare-report", 125 help="Use this report file for size comparison") 126 127 compare_group_option.add_argument( 128 "-m", "--last-metrics", action="store_true", 129 help="Compare with the results of the previous twister " 130 "invocation") 131 132 platform_group_option.add_argument( 133 "-G", 134 "--integration", 135 action="store_true", 136 help="Run integration tests") 137 138 platform_group_option.add_argument( 139 "--emulation-only", action="store_true", 140 help="Only build and run emulation platforms") 141 142 run_group_option.add_argument( 143 "--device-testing", action="store_true", 144 help="Test on device directly. Specify the serial device to " 145 "use with the --device-serial option.") 146 147 run_group_option.add_argument("--generate-hardware-map", 148 help="""Probe serial devices connected to this platform 149 and create a hardware map file to be used with 150 --device-testing 151 """) 152 153 device.add_argument("--device-serial", 154 help="""Serial device for accessing the board 155 (e.g., /dev/ttyACM0) 156 """) 157 158 device.add_argument("--device-serial-pty", 159 help="""Script for controlling pseudoterminal. 160 Twister believes that it interacts with a terminal 161 when it actually interacts with the script. 162 163 E.g "twister --device-testing 164 --device-serial-pty <script> 165 """) 166 167 device.add_argument("--hardware-map", 168 help="""Load hardware map from a file. This will be used 169 for testing on hardware that is listed in the file. 170 """) 171 172 parser.add_argument("--device-flash-timeout", type=int, default=60, 173 help="""Set timeout for the device flash operation in seconds. 174 """) 175 176 parser.add_argument("--device-flash-with-test", action="store_true", 177 help="""Add a test case timeout to the flash operation timeout 178 when flash operation also executes test case on the platform. 179 """) 180 181 test_or_build.add_argument( 182 "-b", "--build-only", action="store_true", default="--prep-artifacts-for-testing" in sys.argv, 183 help="Only build the code, do not attempt to run the code on targets.") 184 185 test_or_build.add_argument( 186 "--prep-artifacts-for-testing", action="store_true", 187 help="Generate artifacts for testing, do not attempt to run the" 188 "code on targets.") 189 190 parser.add_argument( 191 "--package-artifacts", 192 help="Package artifacts needed for flashing in a file to be used with --test-only" 193 ) 194 195 test_or_build.add_argument( 196 "--test-only", action="store_true", 197 help="""Only run device tests with current artifacts, do not build 198 the code""") 199 200 parser.add_argument("--timeout-multiplier", type=float, default=1, 201 help="""Globally adjust tests timeouts by specified multiplier. The resulting test 202 timeout would be multiplication of test timeout value, board-level timeout multiplier 203 and global timeout multiplier (this parameter)""") 204 205 test_xor_subtest.add_argument( 206 "-s", "--test", "--scenario", action="append", 207 help="Run only the specified testsuite scenario. These are named by " 208 "<path/relative/to/Zephyr/base/section.name.in.testcase.yaml>") 209 210 test_xor_subtest.add_argument( 211 "--sub-test", action="append", 212 help="""Recursively find sub-test functions and run the entire 213 test section where they were found, including all sibling test 214 functions. Sub-tests are named by: 215 section.name.in.testcase.yaml.function_name_without_test_prefix 216 Example: In kernel.fifo.fifo_loop: 'kernel.fifo' is a section name 217 and 'fifo_loop' is a name of a function found in main.c without test prefix. 218 """) 219 220 parser.add_argument( 221 "--pytest-args", action="append", 222 help="""Pass additional arguments to the pytest subprocess. This parameter 223 will override the pytest_args from the harness_config in YAML file. 224 """) 225 226 valgrind_asan_group.add_argument( 227 "--enable-valgrind", action="store_true", 228 help="""Run binary through valgrind and check for several memory access 229 errors. Valgrind needs to be installed on the host. This option only 230 works with host binaries such as those generated for the native_sim 231 configuration and is mutual exclusive with --enable-asan. 232 """) 233 234 valgrind_asan_group.add_argument( 235 "--enable-asan", action="store_true", 236 help="""Enable address sanitizer to check for several memory access 237 errors. Libasan needs to be installed on the host. This option only 238 works with host binaries such as those generated for the native_sim 239 configuration and is mutual exclusive with --enable-valgrind. 240 """) 241 242 # Start of individual args place them in alpha-beta order 243 244 board_root_list = ["%s/boards" % ZEPHYR_BASE, 245 "%s/scripts/pylib/twister/boards" % ZEPHYR_BASE] 246 247 modules = zephyr_module.parse_modules(ZEPHYR_BASE) 248 for module in modules: 249 board_root = module.meta.get("build", {}).get("settings", {}).get("board_root") 250 if board_root: 251 board_root_list.append(os.path.join(module.project, board_root, "boards")) 252 253 parser.add_argument( 254 "-A", "--board-root", action="append", default=board_root_list, 255 help="""Directory to search for board configuration files. All .yaml 256files in the directory will be processed. The directory should have the same 257structure in the main Zephyr tree: boards/<arch>/<board_name>/""") 258 259 parser.add_argument( 260 "--allow-installed-plugin", action="store_true", default=None, 261 help="Allow to use pytest plugin installed by pip for pytest tests." 262 ) 263 264 parser.add_argument( 265 "-a", "--arch", action="append", 266 help="Arch filter for testing. Takes precedence over --platform. " 267 "If unspecified, test all arches. Multiple invocations " 268 "are treated as a logical 'or' relationship") 269 270 parser.add_argument( 271 "-B", "--subset", 272 help="Only run a subset of the tests, 1/4 for running the first 25%%, " 273 "3/5 means run the 3rd fifth of the total. " 274 "This option is useful when running a large number of tests on " 275 "different hosts to speed up execution time.") 276 277 parser.add_argument( 278 "--shuffle-tests", action="store_true", default=None, 279 help="""Shuffle test execution order to get randomly distributed tests across subsets. 280 Used only when --subset is provided.""") 281 282 parser.add_argument( 283 "--shuffle-tests-seed", action="store", default=None, 284 help="""Seed value for random generator used to shuffle tests. 285 If not provided, seed in generated by system. 286 Used only when --shuffle-tests is provided.""") 287 288 parser.add_argument("-C", "--coverage", action="store_true", 289 help="Generate coverage reports. Implies " 290 "--enable-coverage.") 291 292 parser.add_argument( 293 "-c", "--clobber-output", action="store_true", 294 help="Cleaning the output directory will simply delete it instead " 295 "of the default policy of renaming.") 296 297 parser.add_argument( 298 "--cmake-only", action="store_true", 299 help="Only run cmake, do not build or run.") 300 301 parser.add_argument("--coverage-basedir", default=ZEPHYR_BASE, 302 help="Base source directory for coverage report.") 303 304 parser.add_argument("--coverage-platform", action="append", default=[], 305 help="Platforms to run coverage reports on. " 306 "This option may be used multiple times. " 307 "Default to what was selected with --platform.") 308 309 parser.add_argument("--coverage-tool", choices=['lcov', 'gcovr'], default='gcovr', 310 help="Tool to use to generate coverage report.") 311 312 parser.add_argument("--coverage-formats", action="store", default=None, # default behavior is set in run_coverage 313 help="Output formats to use for generated coverage reports, as a comma-separated list. " + 314 "Valid options for 'gcovr' tool are: " + 315 ','.join(supported_coverage_formats['gcovr']) + " (html - default)." + 316 " Valid options for 'lcov' tool are: " + 317 ','.join(supported_coverage_formats['lcov']) + " (html,lcov - default).") 318 319 parser.add_argument("--test-config", action="store", default=os.path.join(ZEPHYR_BASE, "tests", "test_config.yaml"), 320 help="Path to file with plans and test configurations.") 321 322 parser.add_argument("--level", action="store", 323 help="Test level to be used. By default, no levels are used for filtering" 324 "and do the selection based on existing filters.") 325 326 parser.add_argument( 327 "-D", "--all-deltas", action="store_true", 328 help="Show all footprint deltas, positive or negative. Implies " 329 "--footprint-threshold=0") 330 331 parser.add_argument( 332 "--device-serial-baud", action="store", default=None, 333 help="Serial device baud rate (default 115200)") 334 335 parser.add_argument("--disable-asserts", action="store_false", 336 dest="enable_asserts", 337 help="deprecated, left for compatibility") 338 339 parser.add_argument( 340 "--disable-unrecognized-section-test", action="store_true", 341 default=False, 342 help="Skip the 'unrecognized section' test.") 343 344 parser.add_argument( 345 "--disable-suite-name-check", action="store_true", default=False, 346 help="Disable extended test suite name verification at the beginning " 347 "of Ztest test. This option could be useful for tests or " 348 "platforms, which from some reasons cannot print early logs.") 349 350 parser.add_argument("-e", "--exclude-tag", action="append", 351 help="Specify tags of tests that should not run. " 352 "Default is to run all tests with all tags.") 353 354 parser.add_argument("--enable-coverage", action="store_true", 355 help="Enable code coverage using gcov.") 356 357 parser.add_argument( 358 "--enable-lsan", action="store_true", 359 help="""Enable leak sanitizer to check for heap memory leaks. 360 Libasan needs to be installed on the host. This option only 361 works with host binaries such as those generated for the native_sim 362 configuration and when --enable-asan is given. 363 """) 364 365 parser.add_argument( 366 "--enable-ubsan", action="store_true", 367 help="""Enable undefined behavior sanitizer to check for undefined 368 behaviour during program execution. It uses an optional runtime library 369 to provide better error diagnostics. This option only works with host 370 binaries such as those generated for the native_sim configuration. 371 """) 372 373 parser.add_argument("--enable-size-report", action="store_true", 374 help="Enable expensive computation of RAM/ROM segment sizes.") 375 376 parser.add_argument("--create-rom-ram-report", action="store_true", 377 help="Generate detailed ram/rom json reports for " 378 "each build, via cmake build calls with the " 379 "`--target footprint` argument") 380 381 parser.add_argument( 382 "--filter", choices=['buildable', 'runnable'], 383 default='buildable', 384 help="""Filter tests to be built and executed. By default everything is 385 built and if a test is runnable (emulation or a connected device), it 386 is run. This option allows for example to only build tests that can 387 actually be run. Runnable is a subset of buildable.""") 388 389 parser.add_argument("--force-color", action="store_true", 390 help="Always output ANSI color escape sequences " 391 "even when the output is redirected (not a tty)") 392 393 parser.add_argument("--force-toolchain", action="store_true", 394 help="Do not filter based on toolchain, use the set " 395 " toolchain unconditionally") 396 397 parser.add_argument("--gcov-tool", type=Path, default=None, 398 help="Path to the gcov tool to use for code coverage " 399 "reports") 400 401 parser.add_argument( 402 "-H", "--footprint-threshold", type=float, default=5, 403 help="When checking test case footprint sizes, warn the user if " 404 "the new app size is greater then the specified percentage " 405 "from the last release. Default is 5. 0 to warn on any " 406 "increase on app size.") 407 408 parser.add_argument( 409 "-i", "--inline-logs", action="store_true", 410 help="Upon test failure, print relevant log data to stdout " 411 "instead of just a path to it.") 412 413 parser.add_argument("--ignore-platform-key", action="store_true", 414 help="Do not filter based on platform key") 415 416 parser.add_argument( 417 "-j", "--jobs", type=int, 418 help="Number of jobs for building, defaults to number of CPU threads, " 419 "overcommitted by factor 2 when --build-only.") 420 421 parser.add_argument( 422 "-K", "--force-platform", action="store_true", 423 help="""Force testing on selected platforms, 424 even if they are excluded in the test configuration (testcase.yaml).""" 425 ) 426 427 parser.add_argument( 428 "-l", "--all", action="store_true", 429 help="Build/test on all platforms. Any --platform arguments " 430 "ignored.") 431 432 parser.add_argument("--list-tags", action="store_true", 433 help="List all tags occurring in selected tests.") 434 435 parser.add_argument("--log-file", metavar="FILENAME", action="store", 436 help="Specify a file where to save logs.") 437 438 parser.add_argument( 439 "-M", "--runtime-artifact-cleanup", choices=['pass', 'all'], 440 default=None, const='pass', nargs='?', 441 help="""Cleanup test artifacts. The default behavior is 'pass' 442 which only removes artifacts of passing tests. If you wish to 443 remove all artificats including those of failed tests, use 'all'.""") 444 445 test_xor_generator.add_argument( 446 "-N", "--ninja", action="store_true", 447 default=not any(a in sys.argv for a in ("-k", "--make")), 448 help="Use the Ninja generator with CMake. (This is the default)") 449 450 test_xor_generator.add_argument( 451 "-k", "--make", action="store_true", 452 help="Use the unix Makefile generator with CMake.") 453 454 parser.add_argument( 455 "-n", "--no-clean", action="store_true", 456 help="Re-use the outdir before building. Will result in " 457 "faster compilation since builds will be incremental.") 458 459 parser.add_argument( 460 "--aggressive-no-clean", action="store_true", 461 help="Re-use the outdir before building and do not re-run cmake. Will result in " 462 "much faster compilation since builds will be incremental. This option might " 463 " result in build failures and inconsistencies if dependencies change or when " 464 " applied on a significantly changed code base. Use on your own " 465 " risk. It is recommended to only use this option for local " 466 " development and when testing localized change in a subsystem.") 467 468 parser.add_argument( 469 '--detailed-test-id', action='store_true', 470 help="Include paths to tests' locations in tests' names. Names will follow " 471 "PATH_TO_TEST/SCENARIO_NAME schema " 472 "e.g. samples/hello_world/sample.basic.helloworld") 473 474 parser.add_argument( 475 "--no-detailed-test-id", dest='detailed_test_id', action="store_false", 476 help="Don't put paths into tests' names. " 477 "With this arg a test name will be a scenario name " 478 "e.g. sample.basic.helloworld.") 479 480 # Include paths in names by default. 481 parser.set_defaults(detailed_test_id=True) 482 483 # To be removed in favor of --detailed-skipped-report 484 parser.add_argument( 485 "--no-skipped-report", action="store_true", 486 help="""Do not report skipped test cases in junit output. [Experimental] 487 """) 488 489 parser.add_argument( 490 "--detailed-skipped-report", action="store_true", 491 help="Generate a detailed report with all skipped test cases" 492 "including those that are filtered based on testsuite definition." 493 ) 494 495 parser.add_argument( 496 "-O", "--outdir", 497 default=os.path.join(os.getcwd(), "twister-out"), 498 help="Output directory for logs and binaries. " 499 "Default is 'twister-out' in the current directory. " 500 "This directory will be cleaned unless '--no-clean' is set. " 501 "The '--clobber-output' option controls what cleaning does.") 502 503 parser.add_argument( 504 "-o", "--report-dir", 505 help="""Output reports containing results of the test run into the 506 specified directory. 507 The output will be both in JSON and JUNIT format 508 (twister.json and twister.xml). 509 """) 510 511 parser.add_argument("--overflow-as-errors", action="store_true", 512 help="Treat RAM/SRAM overflows as errors.") 513 514 515 parser.add_argument("-P", "--exclude-platform", action="append", default=[], 516 help="""Exclude platforms and do not build or run any tests 517 on those platforms. This option can be called multiple times. 518 """ 519 ) 520 521 parser.add_argument("--persistent-hardware-map", action='store_true', 522 help="""With --generate-hardware-map, tries to use 523 persistent names for serial devices on platforms 524 that support this feature (currently only Linux). 525 """) 526 527 parser.add_argument( 528 "--vendor", action="append", default=[], 529 help="Vendor filter for testing") 530 531 parser.add_argument( 532 "-p", "--platform", action="append", default=[], 533 help="Platform filter for testing. This option may be used multiple " 534 "times. Test suites will only be built/run on the platforms " 535 "specified. If this option is not used, then platforms marked " 536 "as default in the platform metadata file will be chosen " 537 "to build and test. ") 538 539 parser.add_argument( 540 "--platform-reports", action="store_true", 541 help="""Create individual reports for each platform. 542 """) 543 544 parser.add_argument("--pre-script", 545 help="""specify a pre script. This will be executed 546 before device handler open serial port and invoke runner. 547 """) 548 549 parser.add_argument("-Q", "--error-on-deprecations", action="store_false", 550 help="Error on deprecation warnings.") 551 552 parser.add_argument( 553 "--quarantine-list", 554 action="append", 555 metavar="FILENAME", 556 help="Load list of test scenarios under quarantine. The entries in " 557 "the file need to correspond to the test scenarios names as in " 558 "corresponding tests .yaml files. These scenarios " 559 "will be skipped with quarantine as the reason.") 560 561 parser.add_argument( 562 "--quarantine-verify", 563 action="store_true", 564 help="Use the list of test scenarios under quarantine and run them" 565 "to verify their current status.") 566 567 parser.add_argument("-R", "--enable-asserts", action="store_true", 568 default=True, 569 help="deprecated, left for compatibility") 570 571 parser.add_argument( 572 "--report-name", 573 help="""Create a report with a custom name. 574 """) 575 576 parser.add_argument( 577 "--report-suffix", 578 help="""Add a suffix to all generated file names, for example to add a 579 version or a commit ID. 580 """) 581 582 parser.add_argument( 583 "--retry-failed", type=int, default=0, 584 help="Retry failing tests again, up to the number of times specified.") 585 586 parser.add_argument( 587 "--retry-interval", type=int, default=60, 588 help="Retry failing tests after specified period of time.") 589 590 parser.add_argument( 591 "--retry-build-errors", action="store_true", 592 help="Retry build errors as well.") 593 594 parser.add_argument( 595 "-S", "--enable-slow", action="store_true", 596 default="--enable-slow-only" in sys.argv, 597 help="Execute time-consuming test cases that have been marked " 598 "as 'slow' in testcase.yaml. Normally these are only built.") 599 600 parser.add_argument( 601 "--enable-slow-only", action="store_true", 602 help="Execute time-consuming test cases that have been marked " 603 "as 'slow' in testcase.yaml only. This also set the option --enable-slow") 604 605 parser.add_argument( 606 "--seed", type=int, 607 help="Seed for native_sim pseudo-random number generator") 608 609 parser.add_argument( 610 "--short-build-path", 611 action="store_true", 612 help="Create shorter build directory paths based on symbolic links. " 613 "The shortened build path will be used by CMake for generating " 614 "the build system and executing the build. Use this option if " 615 "you experience build failures related to path length, for " 616 "example on Windows OS. This option can be used only with " 617 "'--ninja' argument (to use Ninja build generator).") 618 619 parser.add_argument( 620 "--show-footprint", 621 action="store_true", 622 required = "--footprint-from-buildlog" in sys.argv, 623 help="Show footprint statistics and deltas since last release." 624 ) 625 626 parser.add_argument( 627 "-t", "--tag", action="append", 628 help="Specify tags to restrict which tests to run by tag value. " 629 "Default is to not do any tag filtering. Multiple invocations " 630 "are treated as a logical 'or' relationship.") 631 632 parser.add_argument("--timestamps", 633 action="store_true", 634 help="Print all messages with time stamps.") 635 636 parser.add_argument( 637 "-u", 638 "--no-update", 639 action="store_true", 640 help="Do not update the results of the last run of twister.") 641 642 parser.add_argument( 643 "-v", 644 "--verbose", 645 action="count", 646 default=0, 647 help="Emit debugging information, call multiple times to increase " 648 "verbosity.") 649 650 parser.add_argument("-W", "--disable-warnings-as-errors", action="store_true", 651 help="Do not treat warning conditions as errors.") 652 653 parser.add_argument( 654 "--west-flash", nargs='?', const=[], 655 help="""Uses west instead of ninja or make to flash when running with 656 --device-testing. Supports comma-separated argument list. 657 658 E.g "twister --device-testing --device-serial /dev/ttyACM0 659 --west-flash="--board-id=foobar,--erase" 660 will translate to "west flash -- --board-id=foobar --erase" 661 662 NOTE: device-testing must be enabled to use this option. 663 """ 664 ) 665 parser.add_argument( 666 "--west-runner", 667 help="""Uses the specified west runner instead of default when running 668 with --west-flash. 669 670 E.g "twister --device-testing --device-serial /dev/ttyACM0 671 --west-flash --west-runner=pyocd" 672 will translate to "west flash --runner pyocd" 673 674 NOTE: west-flash must be enabled to use this option. 675 """ 676 ) 677 678 parser.add_argument( 679 "-X", "--fixture", action="append", default=[], 680 help="Specify a fixture that a board might support.") 681 682 parser.add_argument( 683 "-x", "--extra-args", action="append", default=[], 684 help="""Extra CMake cache entries to define when building test cases. 685 May be called multiple times. The key-value entries will be 686 prefixed with -D before being passed to CMake. 687 E.g 688 "twister -x=USE_CCACHE=0" 689 will translate to 690 "cmake -DUSE_CCACHE=0" 691 which will ultimately disable ccache. 692 """ 693 ) 694 695 parser.add_argument( 696 "-y", "--dry-run", action="store_true", 697 help="""Create the filtered list of test cases, but don't actually 698 run them. Useful if you're just interested in the test plan 699 generated for every run and saved in the specified output 700 directory (testplan.json). 701 """) 702 703 parser.add_argument( 704 "-z", "--size", action="append", 705 help="Don't run twister. Instead, produce a report to " 706 "stdout detailing RAM/ROM sizes on the specified filenames. " 707 "All other command line arguments ignored.") 708 709 parser.add_argument( 710 "--footprint-from-buildlog", 711 action = "store_true", 712 help="Get information about memory footprint from generated build.log. " 713 "Requires using --show-footprint option.") 714 715 parser.add_argument("extra_test_args", nargs=argparse.REMAINDER, 716 help="Additional args following a '--' are passed to the test binary") 717 718 parser.add_argument("--alt-config-root", action="append", default=[], 719 help="Alternative test configuration root/s. When a test is found, " 720 "Twister will check if a test configuration file exist in any of " 721 "the alternative test configuration root folders. For example, " 722 "given $test_root/tests/foo/testcase.yaml, Twister will use " 723 "$alt_config_root/tests/foo/testcase.yaml if it exists") 724 725 return parser 726 727 728def parse_arguments(parser, args, options = None): 729 if options is None: 730 options = parser.parse_args(args) 731 732 # Very early error handling 733 if options.short_build_path and not options.ninja: 734 logger.error("--short-build-path requires Ninja to be enabled") 735 sys.exit(1) 736 737 if options.device_serial_pty and os.name == "nt": # OS is Windows 738 logger.error("--device-serial-pty is not supported on Windows OS") 739 sys.exit(1) 740 741 if options.west_runner and options.west_flash is None: 742 logger.error("west-runner requires west-flash to be enabled") 743 sys.exit(1) 744 745 if options.west_flash and not options.device_testing: 746 logger.error("west-flash requires device-testing to be enabled") 747 sys.exit(1) 748 749 if not options.testsuite_root: 750 # if we specify a test scenario which is part of a suite directly, do 751 # not set testsuite root to default, just point to the test directory 752 # directly. 753 if options.test: 754 for scenario in options.test: 755 if dirname := os.path.dirname(scenario): 756 options.testsuite_root.append(dirname) 757 758 # check again and make sure we have something set 759 if not options.testsuite_root: 760 options.testsuite_root = [os.path.join(ZEPHYR_BASE, "tests"), 761 os.path.join(ZEPHYR_BASE, "samples")] 762 763 if options.show_footprint or options.compare_report: 764 options.enable_size_report = True 765 766 if options.aggressive_no_clean: 767 options.no_clean = True 768 769 if options.coverage: 770 options.enable_coverage = True 771 772 if not options.coverage_platform: 773 options.coverage_platform = options.platform 774 775 if options.coverage_formats: 776 for coverage_format in options.coverage_formats.split(','): 777 if coverage_format not in supported_coverage_formats[options.coverage_tool]: 778 logger.error(f"Unsupported coverage report formats:'{options.coverage_formats}' " 779 f"for {options.coverage_tool}") 780 sys.exit(1) 781 782 if options.enable_valgrind and not shutil.which("valgrind"): 783 logger.error("valgrind enabled but valgrind executable not found") 784 sys.exit(1) 785 786 if options.device_testing and (options.device_serial or options.device_serial_pty) and len(options.platform) > 1: 787 logger.error("""When --device-testing is used with 788 --device-serial or --device-serial-pty, 789 only one platform is allowed""") 790 sys.exit(1) 791 792 if options.device_flash_with_test and not options.device_testing: 793 logger.error("--device-flash-with-test requires --device_testing") 794 sys.exit(1) 795 796 if options.shuffle_tests and options.subset is None: 797 logger.error("--shuffle-tests requires --subset") 798 sys.exit(1) 799 800 if options.shuffle_tests_seed and options.shuffle_tests is None: 801 logger.error("--shuffle-tests-seed requires --shuffle-tests") 802 sys.exit(1) 803 804 if options.size: 805 from twisterlib.size_calc import SizeCalculator 806 for fn in options.size: 807 sc = SizeCalculator(fn, []) 808 sc.size_report() 809 sys.exit(0) 810 811 if len(options.extra_test_args) > 0: 812 # extra_test_args is a list of CLI args that Twister did not recognize 813 # and are intended to be passed through to the ztest executable. This 814 # list should begin with a "--". If not, there is some extra 815 # unrecognized arg(s) that shouldn't be there. Tell the user there is a 816 # syntax error. 817 if options.extra_test_args[0] != "--": 818 try: 819 double_dash = options.extra_test_args.index("--") 820 except ValueError: 821 double_dash = len(options.extra_test_args) 822 unrecognized = " ".join(options.extra_test_args[0:double_dash]) 823 824 logger.error("Unrecognized arguments found: '%s'. Use -- to " 825 "delineate extra arguments for test binary or pass " 826 "-h for help.", 827 unrecognized) 828 829 sys.exit(1) 830 831 # Strip off the initial "--" following validation. 832 options.extra_test_args = options.extra_test_args[1:] 833 834 if not options.allow_installed_plugin and PYTEST_PLUGIN_INSTALLED: 835 logger.error("By default Twister should work without pytest-twister-harness " 836 "plugin being installed, so please, uninstall it by " 837 "`pip uninstall pytest-twister-harness` and `git clean " 838 "-dxf scripts/pylib/pytest-twister-harness`.") 839 sys.exit(1) 840 elif options.allow_installed_plugin and PYTEST_PLUGIN_INSTALLED: 841 logger.warning("You work with installed version of " 842 "pytest-twister-harness plugin.") 843 844 return options 845 846 847class TwisterEnv: 848 849 def __init__(self, options=None) -> None: 850 self.version = "Unknown" 851 self.toolchain = None 852 self.commit_date = "Unknown" 853 self.run_date = None 854 self.options = options 855 856 if options and options.ninja: 857 self.generator_cmd = "ninja" 858 self.generator = "Ninja" 859 else: 860 self.generator_cmd = "make" 861 self.generator = "Unix Makefiles" 862 logger.info(f"Using {self.generator}..") 863 864 self.test_roots = options.testsuite_root if options else None 865 866 if options: 867 if not isinstance(options.board_root, list): 868 self.board_roots = [self.options.board_root] 869 else: 870 self.board_roots = self.options.board_root 871 self.outdir = os.path.abspath(options.outdir) 872 else: 873 self.board_roots = None 874 self.outdir = None 875 876 self.snippet_roots = [Path(ZEPHYR_BASE)] 877 modules = zephyr_module.parse_modules(ZEPHYR_BASE) 878 for module in modules: 879 snippet_root = module.meta.get("build", {}).get("settings", {}).get("snippet_root") 880 if snippet_root: 881 self.snippet_roots.append(Path(module.project) / snippet_root) 882 883 self.hwm = None 884 885 self.test_config = options.test_config if options else None 886 887 self.alt_config_root = options.alt_config_root if options else None 888 889 def discover(self): 890 self.check_zephyr_version() 891 self.get_toolchain() 892 self.run_date = datetime.now(timezone.utc).isoformat(timespec='seconds') 893 894 def check_zephyr_version(self): 895 try: 896 subproc = subprocess.run(["git", "describe", "--abbrev=12", "--always"], 897 stdout=subprocess.PIPE, 898 universal_newlines=True, 899 cwd=ZEPHYR_BASE) 900 if subproc.returncode == 0: 901 _version = subproc.stdout.strip() 902 if _version: 903 self.version = _version 904 logger.info(f"Zephyr version: {self.version}") 905 except OSError: 906 logger.exception("Failure while reading Zephyr version.") 907 908 if self.version == "Unknown": 909 logger.warning("Could not determine version") 910 911 try: 912 subproc = subprocess.run(["git", "show", "-s", "--format=%cI", "HEAD"], 913 stdout=subprocess.PIPE, 914 universal_newlines=True, 915 cwd=ZEPHYR_BASE) 916 if subproc.returncode == 0: 917 self.commit_date = subproc.stdout.strip() 918 except OSError: 919 logger.exception("Failure while reading head commit date.") 920 921 @staticmethod 922 def run_cmake_script(args=[]): 923 script = os.fspath(args[0]) 924 925 logger.debug("Running cmake script %s", script) 926 927 cmake_args = ["-D{}".format(a.replace('"', '')) for a in args[1:]] 928 cmake_args.extend(['-P', script]) 929 930 cmake = shutil.which('cmake') 931 if not cmake: 932 msg = "Unable to find `cmake` in path" 933 logger.error(msg) 934 raise Exception(msg) 935 cmd = [cmake] + cmake_args 936 log_command(logger, "Calling cmake", cmd) 937 938 kwargs = dict() 939 kwargs['stdout'] = subprocess.PIPE 940 # CMake sends the output of message() to stderr unless it's STATUS 941 kwargs['stderr'] = subprocess.STDOUT 942 943 p = subprocess.Popen(cmd, **kwargs) 944 out, _ = p.communicate() 945 946 # It might happen that the environment adds ANSI escape codes like \x1b[0m, 947 # for instance if twister is executed from inside a makefile. In such a 948 # scenario it is then necessary to remove them, as otherwise the JSON decoding 949 # will fail. 950 ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') 951 out = ansi_escape.sub('', out.decode()) 952 953 if p.returncode == 0: 954 msg = "Finished running %s" % (args[0]) 955 logger.debug(msg) 956 results = {"returncode": p.returncode, "msg": msg, "stdout": out} 957 958 else: 959 logger.error("Cmake script failure: %s" % (args[0])) 960 results = {"returncode": p.returncode, "returnmsg": out} 961 962 return results 963 964 def get_toolchain(self): 965 toolchain_script = Path(ZEPHYR_BASE) / Path('cmake/verify-toolchain.cmake') 966 result = self.run_cmake_script([toolchain_script, "FORMAT=json"]) 967 968 try: 969 if result['returncode']: 970 raise TwisterRuntimeError(f"E: {result['returnmsg']}") 971 except Exception as e: 972 print(str(e)) 973 sys.exit(2) 974 self.toolchain = json.loads(result['stdout'])['ZEPHYR_TOOLCHAIN_VARIANT'] 975 logger.info(f"Using '{self.toolchain}' toolchain.") 976
