1#!/usr/bin/env python3
2# SPDX-License-Identifier: Apache-2.0
3# Copyright (c) 2021 Intel Corporation
4
5# A script to generate twister options based on modified files.
6
7import argparse
8import fnmatch
9import glob
10import json
11import logging
12import os
13import re
14import subprocess
15import sys
16from pathlib import Path
17
18import yaml
19from git import Repo
20from west.manifest import Manifest
21
22try:
23    from yaml import CSafeLoader as SafeLoader
24except ImportError:
25    from yaml import SafeLoader
26
27if "ZEPHYR_BASE" not in os.environ:
28    exit("$ZEPHYR_BASE environment variable undefined.")
29
30# These are globaly used variables. They are assigned in __main__ and are visible in further methods
31# however, pylint complains that it doesn't recognized them when used (used-before-assignment).
32zephyr_base = Path(os.environ['ZEPHYR_BASE'])
33repository_path = zephyr_base
34repo_to_scan = Repo(zephyr_base)
35args = None
36logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
37logging.getLogger("pykwalify.core").setLevel(50)
38
39sys.path.append(os.path.join(zephyr_base, 'scripts'))
40import list_boards  # noqa: E402
41from pylib.twister.twisterlib.statuses import TwisterStatus  # noqa: E402
42
43
44def _get_match_fn(globs, regexes):
45    # Constructs a single regex that tests for matches against the globs in
46    # 'globs' and the regexes in 'regexes'. Parts are joined with '|' (OR).
47    # Returns the search() method of the compiled regex.
48    #
49    # Returns None if there are neither globs nor regexes, which should be
50    # interpreted as no match.
51
52    if not (globs or regexes):
53        return None
54
55    regex = ""
56
57    if globs:
58        glob_regexes = []
59        for glob in globs:
60            # Construct a regex equivalent to the glob
61            glob_regex = glob.replace(".", "\\.").replace("*", "[^/]*").replace("?", "[^/]")
62
63            if not glob.endswith("/"):
64                # Require a full match for globs that don't end in /
65                glob_regex += "$"
66
67            glob_regexes.append(glob_regex)
68
69        # The glob regexes must anchor to the beginning of the path, since we
70        # return search(). (?:) is a non-capturing group.
71        regex += "^(?:{})".format("|".join(glob_regexes))
72
73    if regexes:
74        if regex:
75            regex += "|"
76        regex += "|".join(regexes)
77
78    return re.compile(regex).search
79
80
81class Tag:
82    """
83    Represents an entry for a tag in tags.yaml.
84
85    These attributes are available:
86
87    name:
88        List of GitHub labels for the area. Empty if the area has no 'labels'
89        key.
90
91    description:
92        Text from 'description' key, or None if the area has no 'description'
93        key
94    """
95
96    def _contains(self, path):
97        # Returns True if the area contains 'path', and False otherwise
98
99        return (
100            self._match_fn
101            and self._match_fn(path)
102            and not (self._exclude_match_fn and self._exclude_match_fn(path))
103        )
104
105    def __repr__(self):
106        return f"<Tag {self.name}>"
107
108
109class Filters:
110    def __init__(
111        self,
112        modified_files,
113        ignore_path,
114        alt_tags,
115        testsuite_root,
116        pull_request=False,
117        platforms=None,
118        detailed_test_id=False,
119        quarantine_list=None,
120        tc_roots_th=20,
121    ):
122        self.modified_files = modified_files
123        self.testsuite_root = testsuite_root
124        self.resolved_files = []
125        self.twister_options = []
126        self.full_twister = False
127        self.all_tests = []
128        self.tag_options = []
129        self.pull_request = pull_request
130        self.platforms = platforms if platforms else []
131        self.detailed_test_id = detailed_test_id
132        self.ignore_path = ignore_path
133        self.tag_cfg_file = alt_tags
134        self.quarantine_list = quarantine_list
135        self.tc_roots_th = tc_roots_th
136
137    def process(self):
138        self.find_modules()
139        self.find_tags()
140        self.find_tests()
141        if not self.platforms:
142            # disable for now, this is generating lots of churn when changing
143            # architectures that is otherwise covered elsewhere.
144            # self.find_archs()
145            self.find_boards()
146        else:
147            for file in self.modified_files:
148                if file.startswith(("boards/", "dts/")):
149                    self.resolved_files.append(file)
150
151        self.find_excludes()
152
153    def get_plan(self, options, integration=False, use_testsuite_root=True):
154        fname = "_test_plan_partial.json"
155        cmd = [f"{zephyr_base}/scripts/twister", "-c"] + options + ["--save-tests", fname]
156        if self.detailed_test_id:
157            cmd += ["--detailed-test-id"]
158        if self.testsuite_root and use_testsuite_root:
159            for root in self.testsuite_root:
160                cmd += ["-T", root]
161        if integration:
162            cmd.append("--integration")
163        if self.quarantine_list:
164            for q in self.quarantine_list:
165                cmd += ["--quarantine-list", q]
166
167        logging.info(" ".join(cmd))
168        _ = subprocess.call(cmd)
169        with open(fname, newline='') as jsonfile:
170            json_data = json.load(jsonfile)
171            suites = json_data.get("testsuites", [])
172            self.all_tests.extend(suites)
173        if os.path.exists(fname):
174            os.remove(fname)
175
176    def find_modules(self):
177        if 'west.yml' in self.modified_files and args.commits is not None:
178            print("Manifest file 'west.yml' changed")
179            print("=========")
180            old_manifest_content = repo_to_scan.git.show(f"{args.commits[:-2]}:west.yml")
181            with open("west_old.yml", "w") as manifest:
182                manifest.write(old_manifest_content)
183            old_manifest = Manifest.from_file("west_old.yml")
184            new_manifest = Manifest.from_file("west.yml")
185            old_projs = set((p.name, p.revision) for p in old_manifest.projects)
186            new_projs = set((p.name, p.revision) for p in new_manifest.projects)
187            logging.debug(f'old_projs: {old_projs}')
188            logging.debug(f'new_projs: {new_projs}')
189            # Removed projects
190            rprojs = set(
191                filter(lambda p: p[0] not in list(p[0] for p in new_projs), old_projs - new_projs)
192            )
193            # Updated projects
194            uprojs = set(
195                filter(lambda p: p[0] in list(p[0] for p in old_projs), new_projs - old_projs)
196            )
197            # Added projects
198            aprojs = new_projs - old_projs - uprojs
199
200            # All projs
201            projs = rprojs | uprojs | aprojs
202            projs_names = [name for name, rev in projs]
203
204            logging.info(f'rprojs: {rprojs}')
205            logging.info(f'uprojs: {uprojs}')
206            logging.info(f'aprojs: {aprojs}')
207            logging.info(f'project: {projs_names}')
208
209            if not projs_names:
210                return
211            _options = []
212            for p in projs_names:
213                _options.extend(["-t", p])
214
215            if self.platforms:
216                for platform in self.platforms:
217                    _options.extend(["-p", platform])
218
219            self.get_plan(_options, True)
220
221    def find_archs(self):
222        # we match both arch/<arch>/* and include/zephyr/arch/<arch> and skip common.
223        archs = set()
224
225        for f in self.modified_files:
226            p = re.match(r"^arch\/([^/]+)\/", f)
227            if not p:
228                p = re.match(r"^include\/zephyr\/arch\/([^/]+)\/", f)
229            if p and p.group(1) != 'common':
230                archs.add(p.group(1))
231                # Modified file is treated as resolved, since a matching scope was found
232                self.resolved_files.append(f)
233
234        _options = []
235        for arch in archs:
236            _options.extend(["-a", arch])
237
238        if _options:
239            logging.info('Potential architecture filters...')
240            if self.platforms:
241                for platform in self.platforms:
242                    _options.extend(["-p", platform])
243
244                self.get_plan(_options, True)
245            else:
246                self.get_plan(_options, True)
247
248    def find_boards(self):
249        changed_boards = set()
250        matched_boards = {}
251        resolved_files = []
252
253        for file in self.modified_files:
254            if file.endswith(".rst") or file.endswith(".png") or file.endswith(".jpg"):
255                continue
256            if file.startswith("boards/"):
257                changed_boards.add(file)
258                resolved_files.append(file)
259
260        roots = [zephyr_base]
261        if repository_path != zephyr_base:
262            roots.append(repository_path)
263
264        # Look for boards in monitored repositories
265        lb_args = argparse.Namespace(
266            **{
267                'arch_roots': roots,
268                'board_roots': roots,
269                'board': None,
270                'soc_roots': roots,
271                'board_dir': None,
272            }
273        )
274        known_boards = list_boards.find_v2_boards(lb_args).values()
275
276        for changed in changed_boards:
277            for board in known_boards:
278                c = (zephyr_base / changed).resolve()
279                if c.is_relative_to(board.dir.resolve()):
280                    for file in glob.glob(os.path.join(board.dir, f"{board.name}*.yaml")):
281                        with open(file, encoding='utf-8') as f:
282                            b = yaml.load(f.read(), Loader=SafeLoader)
283                            matched_boards[b['identifier']] = board
284
285        logging.info(f"found boards: {','.join(matched_boards.keys())}")
286        # If modified file is caught by "find_boards" workflow (change in "boards" dir AND board
287        # recognized) it means a proper testing scope for this file was found and this file can
288        # be removed from further consideration
289        for _, board in matched_boards.items():
290            relative_board_dir = str(board.dir.relative_to(zephyr_base))
291            rel_resolved_files = [f for f in resolved_files if relative_board_dir in f]
292            self.resolved_files.extend(rel_resolved_files)
293
294        _options = []
295        if len(matched_boards) > 20:
296            msg = f"{len(matched_boards)} boards changed, this looks like a global change, "
297            msg += "skipping test handling, revert to default."
298            logging.warning(msg)
299            self.full_twister = True
300            return
301
302        for board in matched_boards:
303            _options.extend(["-p", board])
304
305        if _options:
306            logging.info('Potential board filters...')
307            self.get_plan(_options)
308
309    def find_tests(self):
310        tests = set()
311        for f in self.modified_files:
312            if f.endswith(".rst"):
313                continue
314            d = os.path.dirname(f)
315            scope_found = False
316            while not scope_found and d:
317                head, tail = os.path.split(d)
318                if os.path.exists(os.path.join(d, "testcase.yaml")) or os.path.exists(
319                    os.path.join(d, "sample.yaml")
320                ):
321                    tests.add(d)
322                    # Modified file is treated as resolved, since a matching scope was found
323                    self.resolved_files.append(f)
324                    scope_found = True
325                elif tail == "common":
326                    # Look for yamls in directories collocated with common
327                    testcase_yamls = glob.iglob(head + '/**/testcase.yaml', recursive=True)
328                    sample_yamls = glob.iglob(head + '/**/sample.yaml', recursive=True)
329
330                    yamls_found = [*testcase_yamls, *sample_yamls]
331                    if yamls_found:
332                        for yaml in yamls_found:
333                            tests.add(os.path.dirname(yaml))
334                        self.resolved_files.append(f)
335                        scope_found = True
336                    else:
337                        d = os.path.dirname(d)
338                else:
339                    d = os.path.dirname(d)
340
341        _options = []
342        for t in tests:
343            _options.extend(["-T", t])
344
345        if len(tests) > self.tc_roots_th:
346            msg = f"{len(tests)} tests changed, this looks like a global change, "
347            msg += "skipping test handling, revert to default"
348            logging.warning(msg)
349            self.full_twister = True
350            return
351
352        if _options:
353            logging.info(f'Potential test filters...({len(tests)} changed...)')
354            if self.platforms:
355                for platform in self.platforms:
356                    _options.extend(["-p", platform])
357            self.get_plan(_options, use_testsuite_root=False)
358
359    def find_tags(self):
360        with open(self.tag_cfg_file) as ymlfile:
361            tags_config = yaml.safe_load(ymlfile)
362
363        tags = {}
364        for t, x in tags_config.items():
365            tag = Tag()
366            tag.exclude = True
367            tag.name = t
368
369            # tag._match_fn(path) tests if the path matches files and/or
370            # files-regex
371            tag._match_fn = _get_match_fn(x.get("files"), x.get("files-regex"))
372
373            # Like tag._match_fn(path), but for files-exclude and
374            # files-regex-exclude
375            tag._exclude_match_fn = _get_match_fn(
376                x.get("files-exclude"), x.get("files-regex-exclude")
377            )
378
379            tags[tag.name] = tag
380
381        for f in self.modified_files:
382            for t in tags.values():
383                if t._contains(f):
384                    t.exclude = False
385
386        exclude_tags = set()
387        for t in tags.values():
388            if t.exclude:
389                exclude_tags.add(t.name)
390
391        for tag in exclude_tags:
392            self.tag_options.extend(["-e", tag])
393
394        if exclude_tags:
395            logging.info(f'Potential tag based filters: {exclude_tags}')
396
397    def find_excludes(self):
398        with open(self.ignore_path) as twister_ignore:
399            ignores = twister_ignore.read().splitlines()
400            ignores = filter(lambda x: not x.startswith("#"), ignores)
401
402        found = set()
403        not_resolved = lambda x: x not in self.resolved_files  # noqa: E731
404        files_not_resolved = list(filter(not_resolved, self.modified_files))
405
406        for pattern in ignores:
407            if pattern:
408                found.update(fnmatch.filter(files_not_resolved, pattern))
409
410        logging.debug(found)
411        logging.debug(files_not_resolved)
412
413        # Full twister run can be ordered by detecting great number of tests/boards changed
414        # or if not all modified files were resolved (corresponding scope found)
415        self.full_twister = self.full_twister or sorted(files_not_resolved) != sorted(found)
416
417        if self.full_twister:
418            _options = []
419            logging.info('Need to run full or partial twister...')
420            if self.platforms:
421                for platform in self.platforms:
422                    _options.extend(["-p", platform])
423
424                _options.extend(self.tag_options)
425                self.get_plan(_options)
426            else:
427                _options.extend(self.tag_options)
428                self.get_plan(_options, True)
429        else:
430            logging.info('No twister needed or partial twister run only...')
431
432
433def parse_args():
434    parser = argparse.ArgumentParser(
435        description="Generate twister argument files based on modified file", allow_abbrev=False
436    )
437    parser.add_argument('-c', '--commits', default=None, help="Commit range in the form: a..b")
438    parser.add_argument(
439        '-m',
440        '--modified-files',
441        default=None,
442        help="File with information about changed/deleted/added files.",
443    )
444    parser.add_argument(
445        '-o',
446        '--output-file',
447        default="testplan.json",
448        help="JSON file with the test plan to be passed to twister",
449    )
450    parser.add_argument('-P', '--pull-request', action="store_true", help="This is a pull request")
451    parser.add_argument(
452        '-p',
453        '--platform',
454        action="append",
455        help="Limit this for a platform or a list of platforms.",
456    )
457    parser.add_argument(
458        '-t', '--tests_per_builder', default=700, type=int, help="Number of tests per builder"
459    )
460    parser.add_argument(
461        '-n', '--default-matrix', default=10, type=int, help="Number of tests per builder"
462    )
463    parser.add_argument(
464        '--testcase-roots-threshold',
465        default=20,
466        type=int,
467        help="Threshold value for number of modified testcase roots, "
468        "up to which an optimized scope is still applied. "
469        "When exceeded, full scope will be triggered",
470    )
471    parser.add_argument(
472        '--detailed-test-id',
473        action='store_true',
474        help="Include paths to tests' locations in tests' names.",
475    )
476    parser.add_argument(
477        "--no-detailed-test-id",
478        dest='detailed_test_id',
479        action="store_false",
480        help="Don't put paths into tests' names.",
481    )
482    parser.add_argument('-r', '--repo-to-scan', default=None, help="Repo to scan")
483    parser.add_argument(
484        '--ignore-path',
485        default=os.path.join(zephyr_base, 'scripts', 'ci', 'twister_ignore.txt'),
486        help="Path to a text file with patterns of files to be matched against changed files",
487    )
488    parser.add_argument(
489        '--alt-tags',
490        default=os.path.join(zephyr_base, 'scripts', 'ci', 'tags.yaml'),
491        help="Path to a file describing relations between directories and tags",
492    )
493    parser.add_argument(
494        "-T",
495        "--testsuite-root",
496        action="append",
497        default=[],
498        help="Base directory to recursively search for test cases. All "
499        "testcase.yaml files under here will be processed. May be "
500        "called multiple times. Defaults to the 'samples/' and "
501        "'tests/' directories at the base of the Zephyr tree.",
502    )
503    parser.add_argument(
504        "--quarantine-list",
505        action="append",
506        metavar="FILENAME",
507        help="Load list of test scenarios under quarantine. The entries in "
508        "the file need to correspond to the test scenarios names as in "
509        "corresponding tests .yaml files. These scenarios "
510        "will be skipped with quarantine as the reason.",
511    )
512
513    # Do not include paths in names by default.
514    parser.set_defaults(detailed_test_id=False)
515
516    return parser.parse_args()
517
518
519if __name__ == "__main__":
520    args = parse_args()
521    files = []
522    errors = 0
523    if args.repo_to_scan:
524        repository_path = Path(args.repo_to_scan)
525        repo_to_scan = Repo(repository_path)
526    if args.commits:
527        commit = repo_to_scan.git.diff("--name-only", args.commits)
528        files = commit.split("\n")
529    elif args.modified_files:
530        with open(args.modified_files) as fp:
531            files = json.load(fp)
532
533    if files:
534        print("Changed files:\n=========")
535        print("\n".join(files))
536        print("=========")
537
538    f = Filters(
539        files,
540        args.ignore_path,
541        args.alt_tags,
542        args.testsuite_root,
543        args.pull_request,
544        args.platform,
545        args.detailed_test_id,
546        args.quarantine_list,
547        args.testcase_roots_threshold,
548    )
549    f.process()
550
551    # remove dupes and filtered cases
552    dup_free = []
553    dup_free_set = set()
554    logging.info(f'Total tests gathered: {len(f.all_tests)}')
555    for ts in f.all_tests:
556        if TwisterStatus(ts.get('status')) == TwisterStatus.FILTER:
557            continue
558        n = ts.get("name")
559        a = ts.get("arch")
560        p = ts.get("platform")
561        t = ts.get("toolchain")
562        if TwisterStatus(ts.get('status')) == TwisterStatus.ERROR:
563            logging.info(f"Error found: {n} on {p} ({ts.get('reason')})")
564            errors += 1
565        if (n, a, p, t) not in dup_free_set:
566            dup_free.append(ts)
567            dup_free_set.add(
568                (
569                    n,
570                    a,
571                    p,
572                    t,
573                )
574            )
575
576    logging.info(f'Total tests to be run: {len(dup_free)}')
577    with open(".testplan", "w") as tp:
578        total_tests = len(dup_free)
579        if total_tests and total_tests < args.tests_per_builder:
580            nodes = 1
581        else:
582            nodes = round(total_tests / args.tests_per_builder)
583
584        tp.write(f"TWISTER_TESTS={total_tests}\n")
585        tp.write(f"TWISTER_NODES={nodes}\n")
586        tp.write(f"TWISTER_FULL={f.full_twister}\n")
587        logging.info(f'Total nodes to launch: {nodes}')
588
589    header = [
590        'test',
591        'arch',
592        'platform',
593        'status',
594        'extra_args',
595        'handler',
596        'handler_time',
597        'used_ram',
598        'used_rom',
599    ]
600
601    # write plan
602    if dup_free:
603        data = {}
604        data['testsuites'] = dup_free
605        with open(args.output_file, 'w', newline='') as json_file:
606            json.dump(data, json_file, indent=4, separators=(',', ':'))
607
608    sys.exit(errors)
609