1#!/usr/bin/env python3
2# Copyright (c) 2024 Intel Corporation
3#
4# SPDX-License-Identifier: Apache-2.0
5"""
6Blackbox tests for twister's command line functions related to the quarantine.
7"""
8
9import importlib
10import mock
11import os
12import pytest
13import re
14import sys
15import json
16
17# pylint: disable=duplicate-code
18from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock
19from twisterlib.testplan import TestPlan
20
21
22class TestQuarantine:
23    @classmethod
24    def setup_class(cls):
25        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
26        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
27        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
28        cls.twister_module = importlib.util.module_from_spec(cls.spec)
29
30    @classmethod
31    def teardown_class(cls):
32        pass
33
34    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
35    def test_quarantine_verify(self, out_path):
36        test_platforms = ['qemu_x86', 'intel_adl_crb']
37        path = os.path.join(TEST_DATA, 'tests', 'dummy')
38        quarantine_path = os.path.join(TEST_DATA, 'twister-quarantine-list.yml')
39        args = ['-i', '--outdir', out_path, '-T', path, '-y'] + \
40               ['--quarantine-verify'] + \
41               ['--quarantine-list', quarantine_path] + \
42               [val for pair in zip(
43                   ['-p'] * len(test_platforms), test_platforms
44               ) for val in pair]
45
46        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
47                pytest.raises(SystemExit) as sys_exit:
48            self.loader.exec_module(self.twister_module)
49
50        with open(os.path.join(out_path, 'testplan.json')) as f:
51            j = json.load(f)
52        filtered_j = [
53           (ts['platform'], ts['name'], tc['identifier']) \
54               for ts in j['testsuites'] \
55               for tc in ts['testcases'] if 'reason' not in tc
56        ]
57
58        assert str(sys_exit.value) == '0'
59
60        assert len(filtered_j) == 2
61
62    @pytest.mark.parametrize(
63        'test_path, test_platforms, quarantine_directory',
64        [
65            (
66                os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'),
67                ['qemu_x86', 'qemu_x86_64', 'intel_adl_crb'],
68                os.path.join(TEST_DATA, 'twister-quarantine-list.yml'),
69            ),
70        ],
71        ids=[
72            'quarantine',
73        ],
74    )
75    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
76    def test_quarantine_list(self, capfd, out_path, test_path, test_platforms, quarantine_directory):
77        args = ['--outdir', out_path, '-T', test_path] +\
78               ['--quarantine-list', quarantine_directory] + \
79               ['-vv', '-ll', 'DEBUG'] + \
80               [val for pair in zip(
81                   ['-p'] * len(test_platforms), test_platforms
82               ) for val in pair]
83
84        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
85            pytest.raises(SystemExit) as sys_exit:
86            self.loader.exec_module(self.twister_module)
87
88        out, err = capfd.readouterr()
89        sys.stdout.write(out)
90        sys.stderr.write(err)
91
92        board1_match1 = re.search('agnostic/group2/dummy.agnostic.group2 FILTERED: Quarantine: test '
93                               'intel_adl_crb', err)
94        board1_match2 = re.search(
95            'agnostic/group1/subgroup2/dummy.agnostic.group1.subgroup2 FILTERED: Quarantine: test '
96            'intel_adl_crb',
97            err)
98        qemu_64_match = re.search(
99            'agnostic/group1/subgroup2/dummy.agnostic.group1.subgroup2 FILTERED: Quarantine: test '
100            'qemu_x86_64',
101            err)
102        all_platforms_match = re.search(
103            'agnostic/group1/subgroup1/dummy.agnostic.group1.subgroup1 FILTERED: Quarantine: test '
104            'all platforms',
105            err)
106        all_platforms_match2 = re.search(
107            'agnostic/group1/subgroup1/dummy.agnostic.group1.subgroup1 FILTERED: Quarantine: test '
108            'all platforms',
109            err)
110        all_platforms_match3 = re.search(
111            'agnostic/group1/subgroup1/dummy.agnostic.group1.subgroup1 FILTERED: Quarantine: test '
112            'all platforms',
113            err)
114
115        assert board1_match1 and board1_match2, 'platform quarantine not working properly'
116        assert qemu_64_match, 'platform quarantine on scenario not working properly'
117        assert all_platforms_match and all_platforms_match2 and all_platforms_match3, 'scenario ' \
118                                                                                      'quarantine' \
119                                                                                      ' not work ' \
120                                                                                      'properly'
121
122        assert str(sys_exit.value) == '0'
123