1#!/usr/bin/env python3
2# vim: set syntax=python ts=4 :
3# Copyright (c) 2020 Intel Corporation
4# SPDX-License-Identifier: Apache-2.0
5"""Zephyr Test Runner (twister)
6
7Also check the "User and Developer Guides" at https://docs.zephyrproject.org/
8
9This script scans for the set of unit test applications in the git
10repository and attempts to execute them. By default, it tries to
11build each test case on one platform per architecture, using a precedence
12list defined in an architecture configuration file, and if possible
13run the tests in any available emulators or simulators on the system.
14
15Test cases are detected by the presence of a 'testcase.yaml' or a sample.yaml
16files in the application's project directory. This file may contain one or more
17blocks, each identifying a test scenario. The title of the block is a name for
18the test case, which only needs to be unique for the test cases specified in
19that testsuite meta-data. The full canonical name for each test case is <path to
20test case>/<block>.
21
22Each test block in the testsuite meta data can define the following key/value
23pairs:
24
25  tags: <list of tags> (required)
26    A set of string tags for the testsuite. Usually pertains to
27    functional domains but can be anything. Command line invocations
28    of this script can filter the set of tests to run based on tag.
29
30  skip: <True|False> (default False)
31    skip testsuite unconditionally. This can be used for broken tests.
32
33  slow: <True|False> (default False)
34    Don't build or run this test case unless --enable-slow was passed
35    in on the command line. Intended for time-consuming test cases
36    that are only run under certain circumstances, like daily
37    builds.
38
39  extra_args: <list of extra arguments>
40    Extra cache entries to pass to CMake when building or running the
41    test case.
42
43  extra_configs: <list of extra configurations>
44    Extra configuration options to be merged with a master prj.conf
45    when building or running the test case.
46
47  required_snippets: <list of snippets>
48    Snippets that must be applied for the test case to run.
49
50  sysbuild: <True|False> (default False)
51    If true, build the sample using the sysbuild infrastructure. Filtering
52    will only be enabled for the main project, and is not supported for
53    other projects included by sysbuild.
54
55  build_only: <True|False> (default False)
56    If true, don't try to run the test even if the selected platform
57    supports it.
58
59  build_on_all: <True|False> (default False)
60    If true, attempt to build test on all available platforms.
61
62  depends_on: <list of features>
63    A board or platform can announce what features it supports, this option
64    will enable the test only those platforms that provide this feature.
65
66  min_ram: <integer>
67    minimum amount of RAM needed for this test to build and run. This is
68    compared with information provided by the board metadata.
69
70  min_flash: <integer>
71    minimum amount of ROM needed for this test to build and run. This is
72    compared with information provided by the board metadata.
73
74  modules: <list of modules>
75    Add list of modules needed for this sample to build and run.
76
77  timeout: <number of seconds>
78    Length of time to run test in emulator before automatically killing it.
79    Default to 60 seconds.
80
81  arch_allow: <list of arches, such as x86, arm, arc>
82    Set of architectures that this test case should only be run for.
83
84  arch_exclude: <list of arches, such as x86, arm, arc>
85    Set of architectures that this test case should not run on.
86
87  platform_allow: <list of platforms>
88    Set of platforms that this test case should only be run for.
89
90  platform_exclude: <list of platforms>
91    Set of platforms that this test case should not run on.
92
93  simulation_exclude: <list of simulators>
94    Set of simulators that this test case should not run on.
95
96  extra_sections: <list of extra binary sections>
97    When computing sizes, twister will report errors if it finds
98    extra, unexpected sections in the Zephyr binary unless they are named
99    here. They will not be included in the size calculation.
100
101  filter: <expression>
102    Filter whether the testsuite should be run by evaluating an expression
103    against an environment containing the following values:
104
105    { ARCH : <architecture>,
106      PLATFORM : <platform>,
107      <all CONFIG_* key/value pairs in the test's generated defconfig>,
108      <all DT_* key/value pairs in the test's generated device tree file>,
109      <all CMake key/value pairs in the test's generated CMakeCache.txt file>,
110      *<env>: any environment variable available
111    }
112
113    The grammar for the expression language is as follows:
114
115    expression ::= expression "and" expression
116                 | expression "or" expression
117                 | "not" expression
118                 | "(" expression ")"
119                 | symbol "==" constant
120                 | symbol "!=" constant
121                 | symbol "<" number
122                 | symbol ">" number
123                 | symbol ">=" number
124                 | symbol "<=" number
125                 | symbol "in" list
126                 | symbol ":" string
127                 | symbol
128
129    list ::= "[" list_contents "]"
130
131    list_contents ::= constant
132                    | list_contents "," constant
133
134    constant ::= number
135               | string
136
137
138    For the case where expression ::= symbol, it evaluates to true
139    if the symbol is defined to a non-empty string.
140
141    Operator precedence, starting from lowest to highest:
142
143        or (left associative)
144        and (left associative)
145        not (right associative)
146        all comparison operators (non-associative)
147
148    The ':' operator compiles the string argument as a regular expression,
149    and then returns a true value only if the symbol's value in the environment
150    matches. For example, if CONFIG_SOC="stm32f107xc" then
151
152        filter = CONFIG_SOC : "stm.*"
153
154    Would match it.
155
156    Note that arch_allow, arch_exclude, platform_allow, platform_exclude
157    are not just syntactic sugar for filter expressions. For instance
158
159        arch_exclude = x86 arc
160
161    Can appear at first glance to have a similar effect to
162
163        filter = not ARCH in ["x86", "arc"]
164
165    but unlike "filter", these cause platforms to be filtered already during the testplan
166    generation. While "filter" does not exclue platforms at the testplan generation, and instead
167    relies on the result of running the build configuration stage. That is, to evaluate the filter
168    expression, cmake is run for that target, and then the filter evaluated as a gate for the
169    build and run steps.
170    Therefore filtering by using {platform|arch}_{exclude|allow} is much faster.
171
172The set of test cases that actually run depends on directives in the testsuite
173files and options passed in on the command line. If there is any confusion,
174running with -v or examining the test plan report (testplan.json)
175can help show why particular test cases were skipped.
176
177To load arguments from a file, write '+' before the file name, e.g.,
178+file_name. File content must be one or more valid arguments separated by
179line break instead of white spaces.
180
181Most everyday users will run with no arguments.
182
183"""
184
185import os
186import sys
187from pathlib import Path
188
189
190ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
191if not ZEPHYR_BASE:
192    # This file has been zephyr/scripts/twister for years,
193    # and that is not going to change anytime soon. Let the user
194    # run this script as ./scripts/twister without making them
195    # set ZEPHYR_BASE.
196    ZEPHYR_BASE = str(Path(__file__).resolve().parents[1])
197
198    # Propagate this decision to child processes.
199    os.environ['ZEPHYR_BASE'] = ZEPHYR_BASE
200
201    print(f'ZEPHYR_BASE unset, using "{ZEPHYR_BASE}"')
202
203sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister/"))
204sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/build_helpers"))
205
206from twisterlib.environment import add_parse_arguments, parse_arguments
207from twisterlib.twister_main import main
208
209if __name__ == "__main__":
210    ret = 0
211    try:
212        parser = add_parse_arguments()
213        options = parse_arguments(parser, sys.argv[1:])
214        ret = main(options)
215    finally:
216        if (os.name != "nt") and os.isatty(1):
217            # (OS is not Windows) and (stdout is interactive)
218            os.system("stty sane <&1")
219
220    sys.exit(ret)
221