1#!/usr/bin/env python3
2# SPDX-License-Identifier: Apache-2.0
3# Copyright (c) 2021 Intel Corporation
4
5# A script to generate twister options based on modified files.
6
7import re, os
8import argparse
9import yaml
10import fnmatch
11import subprocess
12import json
13import logging
14import sys
15import glob
16from pathlib import Path
17from git import Repo
18from west.manifest import Manifest
19
20try:
21    from yaml import CSafeLoader as SafeLoader
22except ImportError:
23    from yaml import SafeLoader
24
25if "ZEPHYR_BASE" not in os.environ:
26    exit("$ZEPHYR_BASE environment variable undefined.")
27
28# These are globaly used variables. They are assigned in __main__ and are visible in further methods
29# however, pylint complains that it doesn't recognized them when used (used-before-assignment).
30zephyr_base = Path(os.environ['ZEPHYR_BASE'])
31repository_path = zephyr_base
32repo_to_scan = Repo(zephyr_base)
33args = None
34logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
35logging.getLogger("pykwalify.core").setLevel(50)
36
37sys.path.append(os.path.join(zephyr_base, 'scripts'))
38import list_boards
39
40from pylib.twister.twisterlib.statuses import TwisterStatus
41
42
43def _get_match_fn(globs, regexes):
44    # Constructs a single regex that tests for matches against the globs in
45    # 'globs' and the regexes in 'regexes'. Parts are joined with '|' (OR).
46    # Returns the search() method of the compiled regex.
47    #
48    # Returns None if there are neither globs nor regexes, which should be
49    # interpreted as no match.
50
51    if not (globs or regexes):
52        return None
53
54    regex = ""
55
56    if globs:
57        glob_regexes = []
58        for glob in globs:
59            # Construct a regex equivalent to the glob
60            glob_regex = glob.replace(".", "\\.").replace("*", "[^/]*") \
61                             .replace("?", "[^/]")
62
63            if not glob.endswith("/"):
64                # Require a full match for globs that don't end in /
65                glob_regex += "$"
66
67            glob_regexes.append(glob_regex)
68
69        # The glob regexes must anchor to the beginning of the path, since we
70        # return search(). (?:) is a non-capturing group.
71        regex += "^(?:{})".format("|".join(glob_regexes))
72
73    if regexes:
74        if regex:
75            regex += "|"
76        regex += "|".join(regexes)
77
78    return re.compile(regex).search
79
80class Tag:
81    """
82    Represents an entry for a tag in tags.yaml.
83
84    These attributes are available:
85
86    name:
87        List of GitHub labels for the area. Empty if the area has no 'labels'
88        key.
89
90    description:
91        Text from 'description' key, or None if the area has no 'description'
92        key
93    """
94    def _contains(self, path):
95        # Returns True if the area contains 'path', and False otherwise
96
97        return self._match_fn and self._match_fn(path) and not \
98            (self._exclude_match_fn and self._exclude_match_fn(path))
99
100    def __repr__(self):
101        return "<Tag {}>".format(self.name)
102
103class Filters:
104    def __init__(self, modified_files, ignore_path, alt_tags, testsuite_root,
105                 pull_request=False, platforms=[], detailed_test_id=True, quarantine_list=None, tc_roots_th=20):
106        self.modified_files = modified_files
107        self.testsuite_root = testsuite_root
108        self.resolved_files = []
109        self.twister_options = []
110        self.full_twister = False
111        self.all_tests = []
112        self.tag_options = []
113        self.pull_request = pull_request
114        self.platforms = platforms
115        self.detailed_test_id = detailed_test_id
116        self.ignore_path = ignore_path
117        self.tag_cfg_file = alt_tags
118        self.quarantine_list = quarantine_list
119        self.tc_roots_th = tc_roots_th
120
121    def process(self):
122        self.find_modules()
123        self.find_tags()
124        self.find_tests()
125        if not self.platforms:
126            # disable for now, this is generating lots of churn when changing
127            # architectures that is otherwise covered elsewhere.
128            #self.find_archs()
129            self.find_boards()
130        else:
131            for file in self.modified_files:
132                if file.startswith(("boards/", "dts/")):
133                    self.resolved_files.append(file)
134
135        self.find_excludes()
136
137    def get_plan(self, options, integration=False, use_testsuite_root=True):
138        fname = "_test_plan_partial.json"
139        cmd = [f"{zephyr_base}/scripts/twister", "-c"] + options + ["--save-tests", fname ]
140        if not self.detailed_test_id:
141            cmd += ["--no-detailed-test-id"]
142        if self.testsuite_root and use_testsuite_root:
143            for root in self.testsuite_root:
144                cmd+=["-T", root]
145        if integration:
146            cmd.append("--integration")
147        if self.quarantine_list:
148            for q in self.quarantine_list:
149                cmd += ["--quarantine-list", q]
150
151        logging.info(" ".join(cmd))
152        _ = subprocess.call(cmd)
153        with open(fname, newline='') as jsonfile:
154            json_data = json.load(jsonfile)
155            suites = json_data.get("testsuites", [])
156            self.all_tests.extend(suites)
157        if os.path.exists(fname):
158            os.remove(fname)
159
160    def find_modules(self):
161        if 'west.yml' in self.modified_files and args.commits is not None:
162            print(f"Manifest file 'west.yml' changed")
163            print("=========")
164            old_manifest_content = repo_to_scan.git.show(f"{args.commits[:-2]}:west.yml")
165            with open("west_old.yml", "w") as manifest:
166                manifest.write(old_manifest_content)
167            old_manifest = Manifest.from_file("west_old.yml")
168            new_manifest = Manifest.from_file("west.yml")
169            old_projs = set((p.name, p.revision) for p in old_manifest.projects)
170            new_projs = set((p.name, p.revision) for p in new_manifest.projects)
171            logging.debug(f'old_projs: {old_projs}')
172            logging.debug(f'new_projs: {new_projs}')
173            # Removed projects
174            rprojs = set(filter(lambda p: p[0] not in list(p[0] for p in new_projs),
175                old_projs - new_projs))
176            # Updated projects
177            uprojs = set(filter(lambda p: p[0] in list(p[0] for p in old_projs),
178                new_projs - old_projs))
179            # Added projects
180            aprojs = new_projs - old_projs - uprojs
181
182            # All projs
183            projs = rprojs | uprojs | aprojs
184            projs_names = [name for name, rev in projs]
185
186            logging.info(f'rprojs: {rprojs}')
187            logging.info(f'uprojs: {uprojs}')
188            logging.info(f'aprojs: {aprojs}')
189            logging.info(f'project: {projs_names}')
190
191            if not projs_names:
192                return
193            _options = []
194            for p in projs_names:
195                _options.extend(["-t", p ])
196
197            if self.platforms:
198                for platform in self.platforms:
199                    _options.extend(["-p", platform])
200
201            self.get_plan(_options, True)
202
203
204    def find_archs(self):
205        # we match both arch/<arch>/* and include/zephyr/arch/<arch> and skip common.
206        archs = set()
207
208        for f in self.modified_files:
209            p = re.match(r"^arch\/([^/]+)\/", f)
210            if not p:
211                p = re.match(r"^include\/zephyr\/arch\/([^/]+)\/", f)
212            if p:
213                if p.group(1) != 'common':
214                    archs.add(p.group(1))
215                    # Modified file is treated as resolved, since a matching scope was found
216                    self.resolved_files.append(f)
217
218        _options = []
219        for arch in archs:
220            _options.extend(["-a", arch ])
221
222        if _options:
223            logging.info(f'Potential architecture filters...')
224            if self.platforms:
225                for platform in self.platforms:
226                    _options.extend(["-p", platform])
227
228                self.get_plan(_options, True)
229            else:
230                self.get_plan(_options, True)
231
232    def find_boards(self):
233        changed_boards = set()
234        matched_boards = {}
235        resolved_files = []
236
237        for file in self.modified_files:
238            if file.endswith(".rst") or file.endswith(".png") or file.endswith(".jpg"):
239                continue
240            if file.startswith("boards/"):
241                changed_boards.add(file)
242                resolved_files.append(file)
243
244        roots = [zephyr_base]
245        if repository_path != zephyr_base:
246            roots.append(repository_path)
247
248        # Look for boards in monitored repositories
249        lb_args = argparse.Namespace(**{'arch_roots': roots, 'board_roots': roots, 'board': None, 'soc_roots':roots,
250                                        'board_dir': None})
251        known_boards = list_boards.find_v2_boards(lb_args).values()
252
253        for changed in changed_boards:
254            for board in known_boards:
255                c = (zephyr_base / changed).resolve()
256                if c.is_relative_to(board.dir.resolve()):
257                    for file in glob.glob(os.path.join(board.dir, f"{board.name}*.yaml")):
258                        with open(file, 'r', encoding='utf-8') as f:
259                            b = yaml.load(f.read(), Loader=SafeLoader)
260                            matched_boards[b['identifier']] = board
261
262
263        logging.info(f"found boards: {','.join(matched_boards.keys())}")
264        # If modified file is caught by "find_boards" workflow (change in "boards" dir AND board recognized)
265        # it means a proper testing scope for this file was found and this file can be removed
266        # from further consideration
267        for _, board in matched_boards.items():
268            self.resolved_files.extend(list(filter(lambda f: str(board.dir.relative_to(zephyr_base)) in f, resolved_files)))
269
270        _options = []
271        if len(matched_boards) > 20:
272            logging.warning(f"{len(matched_boards)} boards changed, this looks like a global change, skipping test handling, revert to default.")
273            self.full_twister = True
274            return
275
276        for board in matched_boards:
277            _options.extend(["-p", board ])
278
279        if _options:
280            logging.info(f'Potential board filters...')
281            self.get_plan(_options)
282
283    def find_tests(self):
284        tests = set()
285        for f in self.modified_files:
286            if f.endswith(".rst"):
287                continue
288            d = os.path.dirname(f)
289            scope_found = False
290            while not scope_found and d:
291                head, tail = os.path.split(d)
292                if os.path.exists(os.path.join(d, "testcase.yaml")) or \
293                    os.path.exists(os.path.join(d, "sample.yaml")):
294                    tests.add(d)
295                    # Modified file is treated as resolved, since a matching scope was found
296                    self.resolved_files.append(f)
297                    scope_found = True
298                elif tail == "common":
299                    # Look for yamls in directories collocated with common
300
301                    yamls_found = [yaml for yaml in glob.iglob(head + '/**/testcase.yaml', recursive=True)]
302                    yamls_found.extend([yaml for yaml in glob.iglob(head + '/**/sample.yaml', recursive=True)])
303                    if yamls_found:
304                        for yaml in yamls_found:
305                            tests.add(os.path.dirname(yaml))
306                        self.resolved_files.append(f)
307                        scope_found = True
308                    else:
309                        d = os.path.dirname(d)
310                else:
311                    d = os.path.dirname(d)
312
313        _options = []
314        for t in tests:
315            _options.extend(["-T", t ])
316
317        if len(tests) > self.tc_roots_th:
318            logging.warning(f"{len(tests)} tests changed, this looks like a global change, skipping test handling, revert to default")
319            self.full_twister = True
320            return
321
322        if _options:
323            logging.info(f'Potential test filters...({len(tests)} changed...)')
324            if self.platforms:
325                for platform in self.platforms:
326                    _options.extend(["-p", platform])
327            self.get_plan(_options, use_testsuite_root=False)
328
329    def find_tags(self):
330
331        with open(self.tag_cfg_file, 'r') as ymlfile:
332            tags_config = yaml.safe_load(ymlfile)
333
334        tags = {}
335        for t,x in tags_config.items():
336            tag = Tag()
337            tag.exclude = True
338            tag.name = t
339
340            # tag._match_fn(path) tests if the path matches files and/or
341            # files-regex
342            tag._match_fn = _get_match_fn(x.get("files"), x.get("files-regex"))
343
344            # Like tag._match_fn(path), but for files-exclude and
345            # files-regex-exclude
346            tag._exclude_match_fn = \
347                _get_match_fn(x.get("files-exclude"), x.get("files-regex-exclude"))
348
349            tags[tag.name] = tag
350
351        for f in self.modified_files:
352            for t in tags.values():
353                if t._contains(f):
354                    t.exclude = False
355
356        exclude_tags = set()
357        for t in tags.values():
358            if t.exclude:
359                exclude_tags.add(t.name)
360
361        for tag in exclude_tags:
362            self.tag_options.extend(["-e", tag ])
363
364        if exclude_tags:
365            logging.info(f'Potential tag based filters: {exclude_tags}')
366
367    def find_excludes(self, skip=[]):
368        with open(self.ignore_path, "r") as twister_ignore:
369            ignores = twister_ignore.read().splitlines()
370            ignores = filter(lambda x: not x.startswith("#"), ignores)
371
372        found = set()
373        files_not_resolved = list(filter(lambda x: x not in self.resolved_files, self.modified_files))
374
375        for pattern in ignores:
376            if pattern:
377                found.update(fnmatch.filter(files_not_resolved, pattern))
378
379        logging.debug(found)
380        logging.debug(files_not_resolved)
381
382        # Full twister run can be ordered by detecting great number of tests/boards changed
383        # or if not all modified files were resolved (corresponding scope found)
384        self.full_twister = self.full_twister or sorted(files_not_resolved) != sorted(found)
385
386        if self.full_twister:
387            _options = []
388            logging.info(f'Need to run full or partial twister...')
389            if self.platforms:
390                for platform in self.platforms:
391                    _options.extend(["-p", platform])
392
393                _options.extend(self.tag_options)
394                self.get_plan(_options)
395            else:
396                _options.extend(self.tag_options)
397                self.get_plan(_options, True)
398        else:
399            logging.info(f'No twister needed or partial twister run only...')
400
401def parse_args():
402    parser = argparse.ArgumentParser(
403                description="Generate twister argument files based on modified file",
404                allow_abbrev=False)
405    parser.add_argument('-c', '--commits', default=None,
406            help="Commit range in the form: a..b")
407    parser.add_argument('-m', '--modified-files', default=None,
408            help="File with information about changed/deleted/added files.")
409    parser.add_argument('-o', '--output-file', default="testplan.json",
410            help="JSON file with the test plan to be passed to twister")
411    parser.add_argument('-P', '--pull-request', action="store_true",
412            help="This is a pull request")
413    parser.add_argument('-p', '--platform', action="append",
414            help="Limit this for a platform or a list of platforms.")
415    parser.add_argument('-t', '--tests_per_builder', default=700, type=int,
416            help="Number of tests per builder")
417    parser.add_argument('-n', '--default-matrix', default=10, type=int,
418            help="Number of tests per builder")
419    parser.add_argument('--testcase-roots-threshold', default=20, type=int,
420            help="Threshold value for number of modified testcase roots, up to which an optimized scope is still applied."
421                 "When exceeded, full scope will be triggered")
422    parser.add_argument('--detailed-test-id', action='store_true',
423            help="Include paths to tests' locations in tests' names.")
424    parser.add_argument("--no-detailed-test-id", dest='detailed_test_id', action="store_false",
425            help="Don't put paths into tests' names.")
426    parser.add_argument('-r', '--repo-to-scan', default=None,
427            help="Repo to scan")
428    parser.add_argument('--ignore-path',
429            default=os.path.join(zephyr_base, 'scripts', 'ci', 'twister_ignore.txt'),
430            help="Path to a text file with patterns of files to be matched against changed files")
431    parser.add_argument('--alt-tags',
432            default=os.path.join(zephyr_base, 'scripts', 'ci', 'tags.yaml'),
433            help="Path to a file describing relations between directories and tags")
434    parser.add_argument(
435            "-T", "--testsuite-root", action="append", default=[],
436            help="Base directory to recursively search for test cases. All "
437                "testcase.yaml files under here will be processed. May be "
438                "called multiple times. Defaults to the 'samples/' and "
439                "'tests/' directories at the base of the Zephyr tree.")
440    parser.add_argument(
441            "--quarantine-list", action="append", metavar="FILENAME",
442            help="Load list of test scenarios under quarantine. The entries in "
443                "the file need to correspond to the test scenarios names as in "
444                "corresponding tests .yaml files. These scenarios "
445                "will be skipped with quarantine as the reason.")
446
447    # Include paths in names by default.
448    parser.set_defaults(detailed_test_id=True)
449
450    return parser.parse_args()
451
452
453if __name__ == "__main__":
454
455    args = parse_args()
456    files = []
457    errors = 0
458    if args.repo_to_scan:
459        repository_path = Path(args.repo_to_scan)
460        repo_to_scan = Repo(repository_path)
461    if args.commits:
462        commit = repo_to_scan.git.diff("--name-only", args.commits)
463        files = commit.split("\n")
464    elif args.modified_files:
465        with open(args.modified_files, "r") as fp:
466            files = json.load(fp)
467
468    if files:
469        print("Changed files:\n=========")
470        print("\n".join(files))
471        print("=========")
472
473    f = Filters(files, args.ignore_path, args.alt_tags, args.testsuite_root,
474                args.pull_request, args.platform, args.detailed_test_id, args.quarantine_list,
475                args.testcase_roots_threshold)
476    f.process()
477
478    # remove dupes and filtered cases
479    dup_free = []
480    dup_free_set = set()
481    logging.info(f'Total tests gathered: {len(f.all_tests)}')
482    for ts in f.all_tests:
483        if TwisterStatus(ts.get('status')) == TwisterStatus.FILTER:
484            continue
485        n = ts.get("name")
486        a = ts.get("arch")
487        p = ts.get("platform")
488        if TwisterStatus(ts.get('status')) == TwisterStatus.ERROR:
489            logging.info(f"Error found: {n} on {p} ({ts.get('reason')})")
490            errors += 1
491        if (n, a, p,) not in dup_free_set:
492            dup_free.append(ts)
493            dup_free_set.add((n, a, p,))
494
495    logging.info(f'Total tests to be run: {len(dup_free)}')
496    with open(".testplan", "w") as tp:
497        total_tests = len(dup_free)
498        if total_tests and total_tests < args.tests_per_builder:
499            nodes = 1
500        else:
501            nodes = round(total_tests / args.tests_per_builder)
502
503        tp.write(f"TWISTER_TESTS={total_tests}\n")
504        tp.write(f"TWISTER_NODES={nodes}\n")
505        tp.write(f"TWISTER_FULL={f.full_twister}\n")
506        logging.info(f'Total nodes to launch: {nodes}')
507
508    header = ['test', 'arch', 'platform', 'status', 'extra_args', 'handler',
509            'handler_time', 'used_ram', 'used_rom']
510
511    # write plan
512    if dup_free:
513        data = {}
514        data['testsuites'] = dup_free
515        with open(args.output_file, 'w', newline='') as json_file:
516            json.dump(data, json_file, indent=4, separators=(',',':'))
517
518    sys.exit(errors)
519