1#!/usr/bin/env python3
2# Copyright (c) 2024 Intel Corporation
3#
4# SPDX-License-Identifier: Apache-2.0
5"""
6Blackbox tests for twister's command line functions concerning addons to normal functions
7"""
8
9import importlib
10import mock
11import os
12import pkg_resources
13import pytest
14import re
15import shutil
16import subprocess
17import sys
18
19from conftest import ZEPHYR_BASE, TEST_DATA, sample_filename_mock, testsuite_filename_mock
20from twisterlib.testplan import TestPlan
21
22
23class TestAddon:
24    @classmethod
25    def setup_class(cls):
26        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
27        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
28        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
29        cls.twister_module = importlib.util.module_from_spec(cls.spec)
30
31    @classmethod
32    def teardown_class(cls):
33        pass
34
35    @pytest.mark.parametrize(
36        'ubsan_flags, expected_exit_value',
37        [
38            # No sanitiser, no problem
39            ([], '0'),
40            # Sanitiser catches a mistake, error is raised
41            (['--enable-ubsan'], '1')
42        ],
43        ids=['no sanitiser', 'ubsan']
44    )
45    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
46    def test_enable_ubsan(self, out_path, ubsan_flags, expected_exit_value):
47        test_platforms = ['native_sim']
48        test_path = os.path.join(TEST_DATA, 'tests', 'san', 'ubsan')
49        args = ['-i', '--outdir', out_path, '-T', test_path] + \
50               ubsan_flags + \
51               [] + \
52               [val for pair in zip(
53                   ['-p'] * len(test_platforms), test_platforms
54               ) for val in pair]
55
56        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
57                pytest.raises(SystemExit) as sys_exit:
58            self.loader.exec_module(self.twister_module)
59
60        assert str(sys_exit.value) == expected_exit_value
61
62    @pytest.mark.parametrize(
63        'lsan_flags, expected_exit_value',
64        [
65            # No sanitiser, no problem
66            ([], '0'),
67            # Sanitiser catches a mistake, error is raised
68            (['--enable-asan', '--enable-lsan'], '1')
69        ],
70        ids=['no sanitiser', 'lsan']
71    )
72    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
73    def test_enable_lsan(self, out_path, lsan_flags, expected_exit_value):
74        test_platforms = ['native_sim']
75        test_path = os.path.join(TEST_DATA, 'tests', 'san', 'lsan')
76        args = ['-i', '--outdir', out_path, '-T', test_path] + \
77               lsan_flags + \
78               [] + \
79               [val for pair in zip(
80                   ['-p'] * len(test_platforms), test_platforms
81               ) for val in pair]
82
83        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
84                pytest.raises(SystemExit) as sys_exit:
85            self.loader.exec_module(self.twister_module)
86
87        assert str(sys_exit.value) == expected_exit_value
88
89    @pytest.mark.parametrize(
90        'asan_flags, expected_exit_value, expect_asan',
91        [
92            # No sanitiser, no problem
93            # Note that on some runs it may fail,
94            # as the process is killed instead of ending normally.
95            # This is not 100% repeatable, so this test is removed for now.
96            # ([], '0', False),
97            # Sanitiser catches a mistake, error is raised
98            (['--enable-asan'], '1', True)
99        ],
100        ids=[
101            #'no sanitiser',
102            'asan'
103        ]
104    )
105    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
106    def test_enable_asan(self, capfd, out_path, asan_flags, expected_exit_value, expect_asan):
107        test_platforms = ['native_sim']
108        test_path = os.path.join(TEST_DATA, 'tests', 'san', 'asan')
109        args = ['-i', '-W', '--outdir', out_path, '-T', test_path] + \
110               asan_flags + \
111               [] + \
112               [val for pair in zip(
113                   ['-p'] * len(test_platforms), test_platforms
114               ) for val in pair]
115
116        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
117                pytest.raises(SystemExit) as sys_exit:
118            self.loader.exec_module(self.twister_module)
119
120        assert str(sys_exit.value) == expected_exit_value
121
122        out, err = capfd.readouterr()
123        sys.stdout.write(out)
124        sys.stderr.write(err)
125
126        asan_template = r'^==\d+==ERROR:\s+AddressSanitizer:'
127        assert expect_asan == bool(re.search(asan_template, err, re.MULTILINE))
128
129    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
130    def test_extra_test_args(self, capfd, out_path):
131        test_platforms = ['native_sim']
132        test_path = os.path.join(TEST_DATA, 'tests', 'params', 'dummy')
133        args = ['-i', '--outdir', out_path, '-T', test_path] + \
134               [] + \
135               ['-vvv'] + \
136               [val for pair in zip(
137                   ['-p'] * len(test_platforms), test_platforms
138               ) for val in pair] + \
139               ['--', '-list']
140
141        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
142                pytest.raises(SystemExit) as sys_exit:
143            self.loader.exec_module(self.twister_module)
144
145        # Use of -list makes tests not run.
146        # Thus, the tests 'failed'.
147        assert str(sys_exit.value) == '1'
148
149        out, err = capfd.readouterr()
150        sys.stdout.write(out)
151        sys.stderr.write(err)
152
153        expected_test_names = [
154            'param_tests::test_assert1',
155            'param_tests::test_assert2',
156            'param_tests::test_assert3',
157        ]
158        assert all([testname in err for testname in expected_test_names])
159
160    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
161    def test_extra_args(self, caplog, out_path):
162        test_platforms = ['qemu_x86', 'intel_adl_crb']
163        path = os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2')
164        args = ['--outdir', out_path, '-T', path] + \
165               ['--extra-args', 'USE_CCACHE=0', '--extra-args', 'DUMMY=1'] + \
166               [] + \
167               [val for pair in zip(
168                   ['-p'] * len(test_platforms), test_platforms
169               ) for val in pair]
170
171        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
172                pytest.raises(SystemExit) as sys_exit:
173            self.loader.exec_module(self.twister_module)
174
175        assert str(sys_exit.value) == '0'
176
177        with open(os.path.join(out_path, 'twister.log')) as f:
178            twister_log = f.read()
179
180        pattern_cache = r'Calling cmake: [^\n]+ -DUSE_CCACHE=0 [^\n]+\n'
181        pattern_dummy = r'Calling cmake: [^\n]+ -DDUMMY=1 [^\n]+\n'
182
183        assert ' -DUSE_CCACHE=0 ' in twister_log
184        res = re.search(pattern_cache, twister_log)
185        assert res
186
187        assert ' -DDUMMY=1 ' in twister_log
188        res = re.search(pattern_dummy, twister_log)
189        assert res
190
191    # This test is not side-effect free.
192    # It installs and uninstalls pytest-twister-harness using pip
193    # It uses pip to check whether that plugin is previously installed
194    # and reinstalls it if detected at the start of its run.
195    # However, it does NOT restore the original plugin, ONLY reinstalls it.
196    @pytest.mark.parametrize(
197        'allow_flags, do_install, expected_exit_value, expected_logs',
198        [
199            ([], True, '1', ['By default Twister should work without pytest-twister-harness'
200                             ' plugin being installed, so please, uninstall it by'
201                             ' `pip uninstall pytest-twister-harness` and'
202                             ' `git clean -dxf scripts/pylib/pytest-twister-harness`.']),
203            (['--allow-installed-plugin'], True, '0', ['You work with installed version'
204                                                       ' of pytest-twister-harness plugin.']),
205            ([], False, '0', []),
206            (['--allow-installed-plugin'], False, '0', []),
207        ],
208        ids=['installed, but not allowed', 'installed, allowed',
209             'not installed, not allowed', 'not installed, but allowed']
210    )
211    @mock.patch.object(TestPlan, 'SAMPLE_FILENAME', sample_filename_mock)
212    def test_allow_installed_plugin(self, caplog, out_path, allow_flags, do_install,
213                                    expected_exit_value, expected_logs):
214        environment_twister_module = importlib.import_module('twisterlib.environment')
215        harness_twister_module = importlib.import_module('twisterlib.harness')
216        runner_twister_module = importlib.import_module('twisterlib.runner')
217
218        pth_path = os.path.join(ZEPHYR_BASE, 'scripts', 'pylib', 'pytest-twister-harness')
219        check_installed_command = [sys.executable, '-m', 'pip', 'list']
220        install_command = [sys.executable, '-m', 'pip', 'install', '--no-input', pth_path]
221        uninstall_command = [sys.executable, '-m', 'pip', 'uninstall', '--yes',
222                             'pytest-twister-harness']
223
224        def big_uninstall():
225            pth_path = os.path.join(ZEPHYR_BASE, 'scripts', 'pylib', 'pytest-twister-harness')
226
227            subprocess.run(uninstall_command, check=True,)
228
229            # For our registration to work, we have to delete the installation cache
230            additional_cache_paths = [
231                # Plugin cache
232                os.path.join(pth_path, 'src', 'pytest_twister_harness.egg-info'),
233                # Additional caches
234                os.path.join(pth_path, 'src', 'pytest_twister_harness', '__pycache__'),
235                os.path.join(pth_path, 'src', 'pytest_twister_harness', 'device', '__pycache__'),
236                os.path.join(pth_path, 'src', 'pytest_twister_harness', 'helpers', '__pycache__'),
237                os.path.join(pth_path, 'src', 'pytest_twister_harness', 'build'),
238            ]
239
240            for additional_cache_path in additional_cache_paths:
241                if os.path.exists(additional_cache_path):
242                    if os.path.isfile(additional_cache_path):
243                        os.unlink(additional_cache_path)
244                    else:
245                        shutil.rmtree(additional_cache_path)
246
247        # To refresh the PYTEST_PLUGIN_INSTALLED global variable
248        def refresh_plugin_installed_variable():
249            pkg_resources._initialize_master_working_set()
250            importlib.reload(environment_twister_module)
251            importlib.reload(harness_twister_module)
252            importlib.reload(runner_twister_module)
253
254        check_installed_result = subprocess.run(check_installed_command, check=True,
255                                                capture_output=True, text=True)
256        previously_installed = 'pytest-twister-harness' in check_installed_result.stdout
257
258        # To ensure consistent test start
259        big_uninstall()
260
261        if do_install:
262            subprocess.run(install_command, check=True)
263
264        # Refresh before the test, no matter the testcase
265        refresh_plugin_installed_variable()
266
267        test_platforms = ['native_sim']
268        test_path = os.path.join(TEST_DATA, 'samples', 'pytest', 'shell')
269        args = ['-i', '--outdir', out_path, '-T', test_path] + \
270               allow_flags + \
271               [] + \
272               [val for pair in zip(
273                   ['-p'] * len(test_platforms), test_platforms
274               ) for val in pair]
275
276        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
277                pytest.raises(SystemExit) as sys_exit:
278            self.loader.exec_module(self.twister_module)
279
280        # To ensure consistent test exit, prevent dehermetisation
281        if do_install:
282            big_uninstall()
283
284        # To restore previously-installed plugin as well as we can
285        if previously_installed:
286            subprocess.run(install_command, check=True)
287
288        if previously_installed or do_install:
289            refresh_plugin_installed_variable()
290
291        assert str(sys_exit.value) == expected_exit_value
292
293        assert all([log in caplog.text for log in expected_logs])
294
295    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
296    def test_pytest_args(self, out_path):
297        test_platforms = ['native_sim']
298        test_path = os.path.join(TEST_DATA, 'tests', 'pytest')
299        args = ['-i', '--outdir', out_path, '-T', test_path] + \
300               ['--pytest-args=--custom-pytest-arg', '--pytest-args=foo',
301                '--pytest-args=--cmdopt', '--pytest-args=.'] + \
302               [] + \
303               [val for pair in zip(
304                   ['-p'] * len(test_platforms), test_platforms
305               ) for val in pair]
306
307        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
308                pytest.raises(SystemExit) as sys_exit:
309            self.loader.exec_module(self.twister_module)
310
311        # YAML was modified so that the test will fail without command line override.
312        assert str(sys_exit.value) == '0'
313
314    @pytest.mark.parametrize(
315        'valgrind_flags, expected_exit_value',
316        [
317            # No sanitiser, leak is ignored
318            ([], '0'),
319            # Sanitiser catches a mistake, error is raised
320            (['--enable-valgrind'], '1')
321        ],
322        ids=['no valgrind', 'valgrind']
323    )
324    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
325    def test_enable_valgrind(self, capfd, out_path, valgrind_flags, expected_exit_value):
326        test_platforms = ['native_sim']
327        test_path = os.path.join(TEST_DATA, 'tests', 'san', 'val')
328        args = ['-i', '--outdir', out_path, '-T', test_path] + \
329               valgrind_flags + \
330               [] + \
331               [val for pair in zip(
332                   ['-p'] * len(test_platforms), test_platforms
333               ) for val in pair]
334
335        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
336                pytest.raises(SystemExit) as sys_exit:
337            self.loader.exec_module(self.twister_module)
338
339        assert str(sys_exit.value) == expected_exit_value
340