1#!/usr/bin/env python3
2# Copyright (c) 2024 Intel Corporation
3#
4# SPDX-License-Identifier: Apache-2.0
5"""
6Blackbox tests for twister's command line functions changing the output files.
7"""
8
9import importlib
10import re
11import mock
12import os
13import shutil
14import pytest
15import sys
16import tarfile
17
18# pylint: disable=no-name-in-module
19from conftest import ZEPHYR_BASE, TEST_DATA, sample_filename_mock, testsuite_filename_mock
20from twisterlib.testplan import TestPlan
21
22
23@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
24@mock.patch.object(TestPlan, 'SAMPLE_FILENAME', sample_filename_mock)
25class TestOutfile:
26    @classmethod
27    def setup_class(cls):
28        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
29        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
30        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
31        cls.twister_module = importlib.util.module_from_spec(cls.spec)
32
33    @classmethod
34    def teardown_class(cls):
35        pass
36
37    @pytest.mark.parametrize(
38        'flag_section, clobber, expect_straggler',
39        [
40            ([], True, False),
41            (['--clobber-output'], False, False),
42            (['--no-clean'], False, True),
43            (['--clobber-output', '--no-clean'], False, True),
44        ],
45        ids=['clobber', 'do not clobber', 'do not clean', 'do not clobber, do not clean']
46    )
47    def test_clobber_output(self, out_path, flag_section, clobber, expect_straggler):
48        test_platforms = ['qemu_x86', 'intel_adl_crb']
49        path = os.path.join(TEST_DATA, 'tests', 'dummy')
50        args = ['-i', '--outdir', out_path, '-T', path, '-y'] + \
51               flag_section + \
52               [val for pair in zip(
53                   ['-p'] * len(test_platforms), test_platforms
54               ) for val in pair]
55
56        # We create an empty 'blackbox-out' to trigger the clobbering
57        os.mkdir(os.path.join(out_path))
58        # We want to have a single straggler to check for
59        straggler_name = 'atavi.sm'
60        straggler_path = os.path.join(out_path, straggler_name)
61        open(straggler_path, 'a').close()
62
63        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
64                pytest.raises(SystemExit) as sys_exit:
65            self.loader.exec_module(self.twister_module)
66
67        assert str(sys_exit.value) == '0'
68
69        expected_dirs = ['blackbox-out']
70        if clobber:
71            expected_dirs += ['blackbox-out.1']
72        current_dirs = os.listdir(os.path.normpath(os.path.join(out_path, '..')))
73        print(current_dirs)
74        assert sorted(current_dirs) == sorted(expected_dirs)
75
76        out_contents = os.listdir(os.path.join(out_path))
77        print(out_contents)
78        if expect_straggler:
79            assert straggler_name in out_contents
80        else:
81            assert straggler_name not in out_contents
82
83    def test_runtime_artifact_cleanup(self, out_path):
84        test_platforms = ['qemu_x86', 'intel_adl_crb']
85        path = os.path.join(TEST_DATA, 'samples', 'hello_world')
86        args = ['-i', '--outdir', out_path, '-T', path] + \
87               ['--runtime-artifact-cleanup'] + \
88               [] + \
89               [val for pair in zip(
90                   ['-p'] * len(test_platforms), test_platforms
91               ) for val in pair]
92
93        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
94                pytest.raises(SystemExit) as sys_exit:
95            self.loader.exec_module(self.twister_module)
96
97        assert str(sys_exit.value) == '0'
98
99        relpath = os.path.relpath(path, ZEPHYR_BASE)
100        sample_path = os.path.join(out_path, 'qemu_x86_atom', relpath, 'sample.basic.helloworld')
101        listdir = os.listdir(sample_path)
102        zephyr_listdir = os.listdir(os.path.join(sample_path, 'zephyr'))
103
104        expected_contents = ['CMakeFiles', 'handler.log', 'build.ninja', 'CMakeCache.txt',
105                             'zephyr', 'build.log']
106        expected_zephyr_contents = ['.config']
107
108        assert all([content in expected_zephyr_contents for content in zephyr_listdir]), \
109               'Cleaned zephyr directory has unexpected files.'
110        assert all([content in expected_contents for content in listdir]), \
111               'Cleaned directory has unexpected files.'
112
113    def test_short_build_path(self, out_path):
114        test_platforms = ['qemu_x86']
115        path = os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2')
116        # twister_links dir does not exist in a dry run.
117        args = ['-i', '--outdir', out_path, '-T', path] + \
118               ['--short-build-path'] + \
119               ['--ninja'] + \
120               [val for pair in zip(
121                   ['-p'] * len(test_platforms), test_platforms
122               ) for val in pair]
123
124        relative_test_path = os.path.relpath(path, ZEPHYR_BASE)
125        test_result_path = os.path.join(out_path, 'qemu_x86_atom',
126                                        relative_test_path, 'dummy.agnostic.group2')
127
128        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
129                pytest.raises(SystemExit) as sys_exit:
130            self.loader.exec_module(self.twister_module)
131
132        assert str(sys_exit.value) == '0'
133
134        with open(os.path.join(out_path, 'twister.log')) as f:
135            twister_log = f.read()
136
137        pattern_running = r'Running\s+cmake\s+on\s+(?P<full_path>[\\\/].*)\s+for\s+qemu_x86/atom\s*\n'
138        res_running = re.search(pattern_running, twister_log)
139        assert res_running
140
141        # Spaces, forward slashes, etc. in the path as well as CMake peculiarities
142        # require us to forgo simple RegExes.
143        pattern_calling_line = r'Calling cmake: [^\n]+$'
144        res_calling = re.search(pattern_calling_line, twister_log[res_running.end():], re.MULTILINE)
145        calling_line = res_calling.group()
146
147        # HIGHLY DANGEROUS pattern!
148        # If the checked text is not CMake flags only, it is exponential!
149        # Where N is the length of non-flag space-delimited text section.
150        flag_pattern = r'(?:\S+(?: \\)?)+- '
151        cmake_path = shutil.which('cmake')
152        if not cmake_path:
153            assert False, 'CMake not found.'
154
155        cmake_call_section = r'^Calling cmake: ' + re.escape(cmake_path)
156        calling_line = re.sub(cmake_call_section, '', calling_line)
157        calling_line = calling_line[::-1]
158        flag_iterable = re.finditer(flag_pattern, calling_line)
159
160        for match in flag_iterable:
161            reversed_flag = match.group()
162            flag = reversed_flag[::-1]
163
164            # Build flag
165            if flag.startswith(' -B'):
166                flag_value = flag[3:]
167                build_filename = os.path.basename(os.path.normpath(flag_value))
168                unshortened_build_path = os.path.join(test_result_path, build_filename)
169                assert flag_value != unshortened_build_path, 'Build path unchanged.'
170                assert len(flag_value) < len(unshortened_build_path), 'Build path not shortened.'
171
172            # Pipe flag
173            if flag.startswith(' -DQEMU_PIPE='):
174                flag_value = flag[13:]
175                pipe_filename = os.path.basename(os.path.normpath(flag_value))
176                unshortened_pipe_path = os.path.join(test_result_path, pipe_filename)
177                assert flag_value != unshortened_pipe_path, 'Pipe path unchanged.'
178                assert len(flag_value) < len(unshortened_pipe_path), 'Pipe path not shortened.'
179
180    def test_prep_artifacts_for_testing(self, out_path):
181        test_platforms = ['qemu_x86', 'intel_adl_crb']
182        path = os.path.join(TEST_DATA, 'samples', 'hello_world')
183        relative_test_path = os.path.relpath(path, ZEPHYR_BASE)
184        zephyr_out_path = os.path.join(out_path, 'qemu_x86_atom', relative_test_path,
185                                       'sample.basic.helloworld', 'zephyr')
186        args = ['-i', '--outdir', out_path, '-T', path] + \
187               ['--prep-artifacts-for-testing'] + \
188               [val for pair in zip(
189                   ['-p'] * len(test_platforms), test_platforms
190               ) for val in pair]
191
192        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
193                pytest.raises(SystemExit) as sys_exit:
194            self.loader.exec_module(self.twister_module)
195
196        assert str(sys_exit.value) == '0'
197
198        zephyr_artifact_list = os.listdir(zephyr_out_path)
199
200        # --build-only and normal run leave more files than --prep-artifacts-for-testing
201        # However, the cost of testing that this leaves less seems to outweigh the benefits.
202        # So we'll only check for the most important artifact.
203        assert 'zephyr.elf' in zephyr_artifact_list
204
205    def test_package_artifacts(self, out_path):
206        test_platforms = ['qemu_x86']
207        path = os.path.join(TEST_DATA, 'samples', 'hello_world')
208        package_name = 'PACKAGE'
209        package_path = os.path.join(out_path, package_name)
210        args = ['-i', '--outdir', out_path, '-T', path] + \
211               ['--package-artifacts', package_path] + \
212               [val for pair in zip(
213                   ['-p'] * len(test_platforms), test_platforms
214               ) for val in pair]
215
216        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
217                pytest.raises(SystemExit) as sys_exit:
218            self.loader.exec_module(self.twister_module)
219
220        assert str(sys_exit.value) == '0'
221
222        # Check whether we have something as basic as zephyr.elf file
223        with tarfile.open(package_path, "r") as tar:
224            assert any([path.endswith('zephyr.elf') for path in tar.getnames()])
225
226        # Delete everything but for the package
227        for clean_up in os.listdir(os.path.join(out_path)):
228            if not clean_up.endswith(package_name):
229                clean_up_path = os.path.join(out_path, clean_up)
230                if os.path.isfile(clean_up_path):
231                    os.remove(clean_up_path)
232                else:
233                    shutil.rmtree(os.path.join(out_path, clean_up))
234
235        # Unpack the package
236        with tarfile.open(package_path, "r") as tar:
237            tar.extractall(path=out_path)
238
239        # Why does package.py put files inside the out_path folder?
240        # It forces us to move files up one directory after extraction.
241        file_names = os.listdir(os.path.join(out_path, os.path.basename(out_path)))
242        for file_name in file_names:
243            shutil.move(os.path.join(out_path, os.path.basename(out_path), file_name), out_path)
244
245        args = ['-i', '--outdir', out_path, '-T', path] + \
246               ['--test-only'] + \
247               [val for pair in zip(
248                   ['-p'] * len(test_platforms), test_platforms
249               ) for val in pair]
250
251        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
252                pytest.raises(SystemExit) as sys_exit:
253            self.loader.exec_module(self.twister_module)
254
255        assert str(sys_exit.value) == '0'
256