1# vim: set syntax=python ts=4 :
2#
3# Copyright (c) 2018-2022 Intel Corporation
4# SPDX-License-Identifier: Apache-2.0
5
6import copy
7import warnings
8
9import scl
10from twisterlib.error import ConfigurationError
11
12
13def extract_fields_from_arg_list(target_fields: set, arg_list: str | list):
14    """
15    Given a list of "FIELD=VALUE" args, extract values of args with a
16    given field name and return the remaining args separately.
17    """
18    extracted_fields = {f : list() for f in target_fields}
19    other_fields = []
20
21    if isinstance(arg_list, str):
22        args = arg_list.strip().split()
23    else:
24        args = arg_list
25
26    for field in args:
27        try:
28            name, val = field.split("=", 1)
29        except ValueError:
30            # Can't parse this. Just pass it through
31            other_fields.append(field)
32            continue
33
34        if name in target_fields:
35            extracted_fields[name].append(val.strip('\'"'))
36        else:
37            # Move to other_fields
38            other_fields.append(field)
39
40    return extracted_fields, other_fields
41
42class TwisterConfigParser:
43    """Class to read testsuite yaml files with semantic checking
44    """
45
46    testsuite_valid_keys = {"tags": {"type": "set", "required": False},
47                       "type": {"type": "str", "default": "integration"},
48                       "extra_args": {"type": "list"},
49                       "extra_configs": {"type": "list"},
50                       "extra_conf_files": {"type": "list", "default": []},
51                       "extra_overlay_confs" : {"type": "list", "default": []},
52                       "extra_dtc_overlay_files": {"type": "list", "default": []},
53                       "required_snippets": {"type": "list"},
54                       "build_only": {"type": "bool", "default": False},
55                       "build_on_all": {"type": "bool", "default": False},
56                       "skip": {"type": "bool", "default": False},
57                       "slow": {"type": "bool", "default": False},
58                       "timeout": {"type": "int", "default": 60},
59                       "min_ram": {"type": "int", "default": 16},
60                       "modules": {"type": "list", "default": []},
61                       "depends_on": {"type": "set"},
62                       "min_flash": {"type": "int", "default": 32},
63                       "arch_allow": {"type": "set"},
64                       "arch_exclude": {"type": "set"},
65                       "vendor_allow": {"type": "set"},
66                       "vendor_exclude": {"type": "set"},
67                       "extra_sections": {"type": "list", "default": []},
68                       "integration_platforms": {"type": "list", "default": []},
69                       "ignore_faults": {"type": "bool", "default": False },
70                       "ignore_qemu_crash": {"type": "bool", "default": False },
71                       "testcases": {"type": "list", "default": []},
72                       "platform_type": {"type": "list", "default": []},
73                       "platform_exclude": {"type": "set"},
74                       "platform_allow": {"type": "set"},
75                       "platform_key": {"type": "list", "default": []},
76                       "simulation_exclude": {"type": "list", "default": []},
77                       "toolchain_exclude": {"type": "set"},
78                       "toolchain_allow": {"type": "set"},
79                       "filter": {"type": "str"},
80                       "levels": {"type": "list", "default": []},
81                       "harness": {"type": "str", "default": "test"},
82                       "harness_config": {"type": "map", "default": {}},
83                       "seed": {"type": "int", "default": 0},
84                       "sysbuild": {"type": "bool", "default": False}
85                       }
86
87    def __init__(self, filename, schema):
88        """Instantiate a new TwisterConfigParser object
89
90        @param filename Source .yaml file to read
91        """
92        self.data = {}
93        self.schema = schema
94        self.filename = filename
95        self.scenarios = {}
96        self.common = {}
97
98    def load(self):
99        data = scl.yaml_load_verify(self.filename, self.schema)
100        self.data = data
101
102        if 'tests' in self.data:
103            self.scenarios = self.data['tests']
104        if 'common' in self.data:
105            self.common = self.data['common']
106        return data
107
108    def _cast_value(self, value, typestr):
109        if typestr == "str":
110            return value.strip()
111
112        elif typestr == "float":
113            return float(value)
114
115        elif typestr == "int":
116            return int(value)
117
118        elif typestr == "bool":
119            return value
120
121        elif typestr.startswith("list"):
122            if isinstance(value, list):
123                return value
124            elif isinstance(value, str):
125                value = value.strip()
126                return [value] if value else list()
127            else:
128                raise ValueError
129
130        elif typestr.startswith("set"):
131            if isinstance(value, list):
132                return set(value)
133            elif isinstance(value, str):
134                value = value.strip()
135                return {value} if value else set()
136            else:
137                raise ValueError
138
139        elif typestr.startswith("map"):
140            return value
141        else:
142            raise ConfigurationError(self.filename, f"unknown type '{value}'")
143
144    def get_scenario(self, name):
145        """Get a dictionary representing the keys/values within a scenario
146
147        @param name The scenario in the .yaml file to retrieve data from
148        @return A dictionary containing the scenario key-value pairs with
149            type conversion and default values filled in per valid_keys
150        """
151
152        # "CONF_FILE", "OVERLAY_CONFIG", and "DTC_OVERLAY_FILE" fields from each
153        # of the extra_args lines
154        extracted_common = {}
155        extracted_testsuite = {}
156
157        d = {}
158        for k, v in self.common.items():
159            if k == "extra_args":
160                # Pull out these fields and leave the rest
161                extracted_common, d[k] = extract_fields_from_arg_list(
162                    {"CONF_FILE", "OVERLAY_CONFIG", "DTC_OVERLAY_FILE"}, v
163                )
164            else:
165                # Copy common value to avoid mutating it with test specific values below
166                d[k] = copy.copy(v)
167
168        for k, v in self.scenarios[name].items():
169            if k == "extra_args":
170                # Pull out these fields and leave the rest
171                extracted_testsuite, v = extract_fields_from_arg_list(
172                    {"CONF_FILE", "OVERLAY_CONFIG", "DTC_OVERLAY_FILE"}, v
173                )
174            if k in d:
175                if k == "filter":
176                    d[k] = f"({d[k]}) and ({v})"
177                elif k not in ("extra_conf_files", "extra_overlay_confs",
178                               "extra_dtc_overlay_files"):
179                    if isinstance(d[k], str) and isinstance(v, list):
180                        d[k] = [d[k]] + v
181                    elif isinstance(d[k], list) and isinstance(v, str):
182                        d[k] += [v]
183                    elif isinstance(d[k], list) and isinstance(v, list):
184                        d[k] += v
185                    elif isinstance(d[k], str) and isinstance(v, str):
186                        # overwrite if type is string, otherwise merge into a list
187                        type = self.testsuite_valid_keys[k]["type"]
188                        if type == "str":
189                            d[k] = v
190                        elif type in ("list", "set"):
191                            d[k] = [d[k], v]
192                        else:
193                            raise ValueError
194                    else:
195                        # replace value if not str/list (e.g. integer)
196                        d[k] = v
197            else:
198                d[k] = v
199
200        # Compile conf files in to a single list. The order to apply them is:
201        #  (1) CONF_FILEs extracted from common['extra_args']
202        #  (2) common['extra_conf_files']
203        #  (3) CONF_FILES extracted from scenarios[name]['extra_args']
204        #  (4) scenarios[name]['extra_conf_files']
205        d["extra_conf_files"] = \
206            extracted_common.get("CONF_FILE", []) + \
207            self.common.get("extra_conf_files", []) + \
208            extracted_testsuite.get("CONF_FILE", []) + \
209            self.scenarios[name].get("extra_conf_files", [])
210
211        # Repeat the above for overlay confs and DTC overlay files
212        d["extra_overlay_confs"] = \
213            extracted_common.get("OVERLAY_CONFIG", []) + \
214            self.common.get("extra_overlay_confs", []) + \
215            extracted_testsuite.get("OVERLAY_CONFIG", []) + \
216            self.scenarios[name].get("extra_overlay_confs", [])
217
218        d["extra_dtc_overlay_files"] = \
219            extracted_common.get("DTC_OVERLAY_FILE", []) + \
220            self.common.get("extra_dtc_overlay_files", []) + \
221            extracted_testsuite.get("DTC_OVERLAY_FILE", []) + \
222            self.scenarios[name].get("extra_dtc_overlay_files", [])
223
224        if any({len(x) > 0 for x in extracted_common.values()}) or \
225           any({len(x) > 0 for x in extracted_testsuite.values()}):
226            warnings.warn(
227                "Do not specify CONF_FILE, OVERLAY_CONFIG, or DTC_OVERLAY_FILE "
228                "in extra_args. This feature is deprecated and will soon "
229                "result in an error. Use extra_conf_files, extra_overlay_confs "
230                "or extra_dtc_overlay_files YAML fields instead",
231                DeprecationWarning,
232                stacklevel=2
233            )
234
235        for k, kinfo in self.testsuite_valid_keys.items():
236            if k not in d:
237                required = kinfo.get("required", False)
238
239                if required:
240                    raise ConfigurationError(
241                        self.filename,
242                        f"missing required value for '{k}' in test '{name}'"
243                    )
244                else:
245                    if "default" in kinfo:
246                        default = kinfo["default"]
247                    else:
248                        default = self._cast_value("", kinfo["type"])
249                    d[k] = default
250            else:
251                try:
252                    d[k] = self._cast_value(d[k], kinfo["type"])
253                except ValueError:
254                    raise ConfigurationError(
255                        self.filename,
256                        f"bad {kinfo['type']} value '{d[k]}' for key '{k}' in name '{name}'"
257                    ) from None
258
259        return d
260