1# vim: set syntax=python ts=4 :
2#
3# Copyright (c) 2018-2022 Intel Corporation
4# SPDX-License-Identifier: Apache-2.0
5
6import copy
7import scl
8import warnings
9from typing import Union
10from twisterlib.error import ConfigurationError
11
12def extract_fields_from_arg_list(target_fields: set, arg_list: Union[str, list]):
13    """
14    Given a list of "FIELD=VALUE" args, extract values of args with a
15    given field name and return the remaining args separately.
16    """
17    extracted_fields = {f : list() for f in target_fields}
18    other_fields = []
19
20    if isinstance(arg_list, str):
21        args = arg_list.strip().split()
22    else:
23        args = arg_list
24
25    for field in args:
26        try:
27            name, val = field.split("=", 1)
28        except ValueError:
29            # Can't parse this. Just pass it through
30            other_fields.append(field)
31            continue
32
33        if name in target_fields:
34            extracted_fields[name].append(val.strip('\'"'))
35        else:
36            # Move to other_fields
37            other_fields.append(field)
38
39    return extracted_fields, " ".join(other_fields)
40
41class TwisterConfigParser:
42    """Class to read testsuite yaml files with semantic checking
43    """
44
45    testsuite_valid_keys = {"tags": {"type": "set", "required": False},
46                       "type": {"type": "str", "default": "integration"},
47                       "extra_args": {"type": "list"},
48                       "extra_configs": {"type": "list"},
49                       "extra_conf_files": {"type": "list", "default": []},
50                       "extra_overlay_confs" : {"type": "list", "default": []},
51                       "extra_dtc_overlay_files": {"type": "list", "default": []},
52                       "required_snippets": {"type": "list"},
53                       "build_only": {"type": "bool", "default": False},
54                       "build_on_all": {"type": "bool", "default": False},
55                       "skip": {"type": "bool", "default": False},
56                       "slow": {"type": "bool", "default": False},
57                       "timeout": {"type": "int", "default": 60},
58                       "min_ram": {"type": "int", "default": 8},
59                       "modules": {"type": "list", "default": []},
60                       "depends_on": {"type": "set"},
61                       "min_flash": {"type": "int", "default": 32},
62                       "arch_allow": {"type": "set"},
63                       "arch_exclude": {"type": "set"},
64                       "extra_sections": {"type": "list", "default": []},
65                       "integration_platforms": {"type": "list", "default": []},
66                       "ignore_faults": {"type": "bool", "default": False },
67                       "ignore_qemu_crash": {"type": "bool", "default": False },
68                       "testcases": {"type": "list", "default": []},
69                       "platform_type": {"type": "list", "default": []},
70                       "platform_exclude": {"type": "set"},
71                       "platform_allow": {"type": "set"},
72                       "platform_key": {"type": "list", "default": []},
73                       "simulation_exclude": {"type": "list", "default": []},
74                       "toolchain_exclude": {"type": "set"},
75                       "toolchain_allow": {"type": "set"},
76                       "filter": {"type": "str"},
77                       "levels": {"type": "list", "default": []},
78                       "harness": {"type": "str", "default": "test"},
79                       "harness_config": {"type": "map", "default": {}},
80                       "seed": {"type": "int", "default": 0},
81                       "sysbuild": {"type": "bool", "default": False}
82                       }
83
84    def __init__(self, filename, schema):
85        """Instantiate a new TwisterConfigParser object
86
87        @param filename Source .yaml file to read
88        """
89        self.data = {}
90        self.schema = schema
91        self.filename = filename
92        self.scenarios = {}
93        self.common = {}
94
95    def load(self):
96        self.data = scl.yaml_load_verify(self.filename, self.schema)
97
98        if 'tests' in self.data:
99            self.scenarios = self.data['tests']
100        if 'common' in self.data:
101            self.common = self.data['common']
102
103    def _cast_value(self, value, typestr):
104        if isinstance(value, str):
105            v = value.strip()
106        if typestr == "str":
107            return v
108
109        elif typestr == "float":
110            return float(value)
111
112        elif typestr == "int":
113            return int(value)
114
115        elif typestr == "bool":
116            return value
117
118        elif typestr.startswith("list"):
119            if isinstance(value, list):
120                return value
121            elif isinstance(value, str):
122                vs = v.split()
123
124                if len(vs) > 1:
125                    warnings.warn(
126                        "Space-separated lists are deprecated, use YAML lists instead",
127                        DeprecationWarning)
128
129                if len(typestr) > 4 and typestr[4] == ":":
130                    return [self._cast_value(vsi, typestr[5:]) for vsi in vs]
131                else:
132                    return vs
133            else:
134                raise ValueError
135
136        elif typestr.startswith("set"):
137            if isinstance(value, list):
138                return set(value)
139            elif isinstance(value, str):
140                vs = v.split()
141
142                if len(vs) > 1:
143                    warnings.warn(
144                        "Space-separated lists are deprecated, use YAML lists instead",
145                        DeprecationWarning)
146
147                if len(typestr) > 3 and typestr[3] == ":":
148                    return {self._cast_value(vsi, typestr[4:]) for vsi in vs}
149                else:
150                    return set(vs)
151            else:
152                raise ValueError
153
154        elif typestr.startswith("map"):
155            return value
156        else:
157            raise ConfigurationError(
158                self.filename, "unknown type '%s'" % value)
159
160    def get_scenario(self, name):
161        """Get a dictionary representing the keys/values within a scenario
162
163        @param name The scenario in the .yaml file to retrieve data from
164        @return A dictionary containing the scenario key-value pairs with
165            type conversion and default values filled in per valid_keys
166        """
167
168        # "CONF_FILE", "OVERLAY_CONFIG", and "DTC_OVERLAY_FILE" fields from each
169        # of the extra_args lines
170        extracted_common = {}
171        extracted_testsuite = {}
172
173        d = {}
174        for k, v in self.common.items():
175            if k == "extra_args":
176                # Pull out these fields and leave the rest
177                extracted_common, d[k] = extract_fields_from_arg_list(
178                    {"CONF_FILE", "OVERLAY_CONFIG", "DTC_OVERLAY_FILE"}, v
179                )
180            else:
181                # Copy common value to avoid mutating it with test specific values below
182                d[k] = copy.copy(v)
183
184        for k, v in self.scenarios[name].items():
185            if k == "extra_args":
186                # Pull out these fields and leave the rest
187                extracted_testsuite, v = extract_fields_from_arg_list(
188                    {"CONF_FILE", "OVERLAY_CONFIG", "DTC_OVERLAY_FILE"}, v
189                )
190            if k in d:
191                if k == "filter":
192                    d[k] = "(%s) and (%s)" % (d[k], v)
193                elif k not in ("extra_conf_files", "extra_overlay_confs",
194                               "extra_dtc_overlay_files"):
195                    if isinstance(d[k], str) and isinstance(v, list):
196                        d[k] = d[k].split() + v
197                    elif isinstance(d[k], list) and isinstance(v, str):
198                        d[k] += v.split()
199                    elif isinstance(d[k], list) and isinstance(v, list):
200                        d[k] += v
201                    elif isinstance(d[k], str) and isinstance(v, str):
202                        d[k] += " " + v
203                    else:
204                        # replace value if not str/list (e.g. integer)
205                        d[k] = v
206            else:
207                d[k] = v
208
209        # Compile conf files in to a single list. The order to apply them is:
210        #  (1) CONF_FILEs extracted from common['extra_args']
211        #  (2) common['extra_conf_files']
212        #  (3) CONF_FILES extracted from scenarios[name]['extra_args']
213        #  (4) scenarios[name]['extra_conf_files']
214        d["extra_conf_files"] = \
215            extracted_common.get("CONF_FILE", []) + \
216            self.common.get("extra_conf_files", []) + \
217            extracted_testsuite.get("CONF_FILE", []) + \
218            self.scenarios[name].get("extra_conf_files", [])
219
220        # Repeat the above for overlay confs and DTC overlay files
221        d["extra_overlay_confs"] = \
222            extracted_common.get("OVERLAY_CONFIG", []) + \
223            self.common.get("extra_overlay_confs", []) + \
224            extracted_testsuite.get("OVERLAY_CONFIG", []) + \
225            self.scenarios[name].get("extra_overlay_confs", [])
226
227        d["extra_dtc_overlay_files"] = \
228            extracted_common.get("DTC_OVERLAY_FILE", []) + \
229            self.common.get("extra_dtc_overlay_files", []) + \
230            extracted_testsuite.get("DTC_OVERLAY_FILE", []) + \
231            self.scenarios[name].get("extra_dtc_overlay_files", [])
232
233        if any({len(x) > 0 for x in extracted_common.values()}) or \
234           any({len(x) > 0 for x in extracted_testsuite.values()}):
235            warnings.warn(
236                "Do not specify CONF_FILE, OVERLAY_CONFIG, or DTC_OVERLAY_FILE "
237                "in extra_args. This feature is deprecated and will soon "
238                "result in an error. Use extra_conf_files, extra_overlay_confs "
239                "or extra_dtc_overlay_files YAML fields instead",
240                DeprecationWarning
241            )
242
243        for k, kinfo in self.testsuite_valid_keys.items():
244            if k not in d:
245                if "required" in kinfo:
246                    required = kinfo["required"]
247                else:
248                    required = False
249
250                if required:
251                    raise ConfigurationError(
252                        self.filename,
253                        "missing required value for '%s' in test '%s'" %
254                        (k, name))
255                else:
256                    if "default" in kinfo:
257                        default = kinfo["default"]
258                    else:
259                        default = self._cast_value("", kinfo["type"])
260                    d[k] = default
261            else:
262                try:
263                    d[k] = self._cast_value(d[k], kinfo["type"])
264                except ValueError:
265                    raise ConfigurationError(
266                        self.filename, "bad %s value '%s' for key '%s' in name '%s'" %
267                                       (kinfo["type"], d[k], k, name))
268
269        return d
270