1#!/usr/bin/env python3
2# Copyright (c) 2024 Intel Corporation
3#
4# SPDX-License-Identifier: Apache-2.0
5"""
6Blackbox tests for twister's command line functions
7"""
8import importlib
9import re
10import mock
11import os
12import pytest
13import sys
14import json
15
16# pylint: disable=duplicate-code
17# pylint: disable=no-name-in-module
18from conftest import TEST_DATA, ZEPHYR_BASE, testsuite_filename_mock, clear_log_in_test
19from twisterlib.testplan import TestPlan
20
21
22@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
23class TestCoverage:
24    TESTDATA_1 = [
25        (
26            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'),
27            ['qemu_x86'],
28            [
29                'coverage.log', 'coverage.json',
30                'coverage'
31            ],
32        ),
33    ]
34    TESTDATA_2 = [
35        (
36            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'),
37            ['qemu_x86'],
38            [
39                'GCOV_COVERAGE_DUMP_START', 'GCOV_COVERAGE_DUMP_END'
40            ],
41        ),
42    ]
43    TESTDATA_3 = [
44        (
45            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'),
46            ['qemu_x86'],
47            [
48                'coverage.log', 'coverage.json',
49                'coverage'
50            ],
51            r'{"files": \[\], "gcovr/format_version": ".*"}'
52        ),
53    ]
54    TESTDATA_4 = [
55        (
56            'gcovr',
57            [
58                'coverage.log', 'coverage.json',
59                'coverage', os.path.join('coverage','coverage.xml')
60            ],
61            'xml'
62        ),
63        (
64            'gcovr',
65            [
66                'coverage.log', 'coverage.json',
67                'coverage', os.path.join('coverage','coverage.sonarqube.xml')
68            ],
69            'sonarqube'
70        ),
71        (
72            'gcovr',
73            [
74                'coverage.log', 'coverage.json',
75                'coverage', os.path.join('coverage','coverage.txt')
76            ],
77            'txt'
78        ),
79        (
80            'gcovr',
81            [
82                'coverage.log', 'coverage.json',
83                'coverage', os.path.join('coverage','coverage.csv')
84            ],
85            'csv'
86        ),
87        (
88            'gcovr',
89            [
90                'coverage.log', 'coverage.json',
91                'coverage', os.path.join('coverage','coverage.coveralls.json')
92            ],
93            'coveralls'
94        ),
95        (
96            'gcovr',
97            [
98                'coverage.log', 'coverage.json',
99                'coverage', os.path.join('coverage','index.html')
100            ],
101            'html'
102        ),
103        (
104            'lcov',
105            [
106                'coverage.log', 'coverage.info',
107                'ztest.info', 'coverage',
108                os.path.join('coverage','index.html')
109            ],
110            'html'
111        ),
112        (
113            'lcov',
114            [
115                'coverage.log', 'coverage.info',
116                'ztest.info'
117            ],
118            'lcov'
119        ),
120    ]
121    TESTDATA_5 = [
122        (
123            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'),
124            ['qemu_x86'],
125            'gcovr',
126            'Running gcovr -r'
127        ),
128        (
129            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'),
130            ['qemu_x86'],
131            'lcov',
132            'Running lcov --gcov-tool'
133        )
134    ]
135    TESTDATA_6 = [
136        (
137            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'),
138            ['qemu_x86'],
139            ['The specified file does not exist.', r'\[Errno 13\] Permission denied:'],
140        )
141    ]
142    TESTDATA_7 = [
143        (
144            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'),
145            ['qemu_x86_64', 'qemu_x86'],
146            ['qemu_x86_64', 'qemu_x86', ['qemu_x86_64', 'qemu_x86']],
147        )
148    ]
149
150    @classmethod
151    def setup_class(cls):
152        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
153        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
154        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
155        cls.twister_module = importlib.util.module_from_spec(cls.spec)
156
157    @pytest.mark.parametrize(
158        'test_path, test_platforms, file_name',
159        TESTDATA_1,
160        ids=[
161            'coverage',
162        ]
163    )
164    def test_coverage(self, capfd, test_path, test_platforms, out_path, file_name):
165        args = ['-i','--outdir', out_path, '-T', test_path] + \
166               ['--coverage', '--coverage-tool', 'gcovr'] + \
167               [val for pair in zip(
168                   ['-p'] * len(test_platforms), test_platforms
169               ) for val in pair]
170
171        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
172                pytest.raises(SystemExit) as sys_exit:
173            self.loader.exec_module(self.twister_module)
174
175        out, err = capfd.readouterr()
176        sys.stdout.write(out)
177        sys.stderr.write(err)
178
179        assert str(sys_exit.value) == '0'
180
181        for f_name in file_name:
182            path = os.path.join(out_path, f_name)
183            assert os.path.exists(path), f'file not found {f_name}'
184
185    @pytest.mark.parametrize(
186        'test_path, test_platforms, expected',
187        TESTDATA_2,
188        ids=[
189            'enable_coverage',
190        ]
191    )
192    def test_enable_coverage(self, capfd, test_path, test_platforms, out_path, expected):
193        args = ['-i','--outdir', out_path, '-T', test_path] + \
194               ['--enable-coverage', '-vv', '-ll', 'DEBUG'] + \
195               [val for pair in zip(
196                   ['-p'] * len(test_platforms), test_platforms
197               ) for val in pair]
198
199        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
200                pytest.raises(SystemExit) as sys_exit:
201            self.loader.exec_module(self.twister_module)
202
203        out, err = capfd.readouterr()
204        sys.stdout.write(out)
205        sys.stderr.write(err)
206
207        assert str(sys_exit.value) == '0'
208
209        for line in expected:
210            match = re.search(line, err)
211            assert match, f'line not found: {line}'
212
213    @pytest.mark.parametrize(
214        'test_path, test_platforms, file_name, expected_content',
215        TESTDATA_3,
216        ids=[
217            'coverage_basedir',
218        ]
219    )
220    def test_coverage_basedir(self, capfd, test_path, test_platforms, out_path, file_name, expected_content):
221        base_dir = os.path.join(TEST_DATA, "test_dir")
222        if os.path.exists(base_dir):
223            os.rmdir(base_dir)
224        os.mkdir(base_dir)
225        args = ['--outdir', out_path,'-i', '-T', test_path] + \
226               ['--coverage', '--coverage-tool', 'gcovr',  '-v', '--coverage-basedir', base_dir] + \
227               [val for pair in zip(
228                   ['-p'] * len(test_platforms), test_platforms
229               ) for val in pair]
230
231        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
232                pytest.raises(SystemExit) as sys_exit:
233            self.loader.exec_module(self.twister_module)
234
235        out, err = capfd.readouterr()
236        sys.stdout.write(out)
237        sys.stderr.write(err)
238
239        assert str(sys_exit.value) == '0'
240
241        for f_name in file_name:
242            path = os.path.join(out_path, f_name)
243            assert os.path.exists(path), f'file not found {f_name}'
244            if f_name == 'coverage.json':
245                with open(path, "r") as json_file:
246                    json_content = json.load(json_file)
247                    pattern = re.compile(expected_content)
248                    assert pattern.match(json.dumps(json_content, sort_keys=True))
249        if os.path.exists(base_dir):
250            os.rmdir(base_dir)
251
252    @pytest.mark.parametrize(
253        'cov_tool, file_name, cov_format',
254        TESTDATA_4,
255        ids=[
256            'coverage_format gcovr xml',
257            'coverage_format gcovr sonarqube',
258            'coverage_format gcovr txt',
259            'coverage_format gcovr csv',
260            'coverage_format gcovr coveralls',
261            'coverage_format gcovr html',
262            'coverage_format lcov html',
263            'coverage_format lcov lcov',
264        ]
265    )
266    def test_coverage_format(self, capfd, out_path, cov_tool, file_name, cov_format):
267        test_path = os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2')
268        test_platforms = ['qemu_x86']
269        args = ['--outdir', out_path,'-i', '-T', test_path] + \
270               ['--coverage', '--coverage-tool', cov_tool, '--coverage-formats', cov_format, '-v'] + \
271               [val for pair in zip(
272                   ['-p'] * len(test_platforms), test_platforms
273               ) for val in pair]
274
275        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
276                pytest.raises(SystemExit) as sys_exit:
277            self.loader.exec_module(self.twister_module)
278
279        out, err = capfd.readouterr()
280        sys.stdout.write(out)
281        sys.stderr.write(err)
282
283        assert str(sys_exit.value) == '0'
284
285        for f_name in file_name:
286            path = os.path.join(out_path, f_name)
287            assert os.path.exists(path), f'file not found {f_name}, probably format {cov_format} not work properly'
288
289
290
291    @pytest.mark.parametrize(
292        'test_path, test_platforms, cov_tool, expected_content',
293        TESTDATA_5,
294        ids=[
295            'coverage_tool gcovr',
296            'coverage_tool lcov'
297        ]
298    )
299    def test_coverage_tool(self, capfd, caplog, test_path, test_platforms, out_path, cov_tool, expected_content):
300        args = ['--outdir', out_path,'-i', '-T', test_path] + \
301            ['--coverage', '--coverage-tool', cov_tool,  '-v'] + \
302               [val for pair in zip(
303                   ['-p'] * len(test_platforms), test_platforms
304               ) for val in pair]
305
306        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
307                pytest.raises(SystemExit) as sys_exit:
308            self.loader.exec_module(self.twister_module)
309
310        out, err = capfd.readouterr()
311        sys.stdout.write(out)
312        sys.stderr.write(err)
313
314        assert str(sys_exit.value) == '0'
315
316        assert re.search(expected_content, caplog.text), f'{cov_tool} line not found'
317
318    @pytest.mark.parametrize(
319        'test_path, test_platforms, expected_content',
320        TESTDATA_6,
321        ids=[
322            'missing tool'
323        ]
324    )
325    def test_gcov_tool(self, capfd, test_path, test_platforms, out_path, expected_content):
326        args = ['--outdir', out_path, '-i', '-T', test_path] + \
327            ['--coverage', '--gcov-tool', TEST_DATA, '-v'] + \
328               [val for pair in zip(
329                   ['-p'] * len(test_platforms), test_platforms
330               ) for val in pair]
331
332        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
333                pytest.raises(SystemExit) as sys_exit:
334            self.loader.exec_module(self.twister_module)
335
336        out, err = capfd.readouterr()
337        sys.stdout.write(out)
338        sys.stderr.write(err)
339
340        assert str(sys_exit.value) == '1'
341        for line in expected_content:
342            result = re.search(line, err)
343            assert result, f'missing information in log: {line}'
344
345    @pytest.mark.parametrize(
346        'test_path, test_platforms, cov_platform',
347        TESTDATA_7,
348        ids=[
349            'coverage platform'
350        ]
351    )
352    def test_coverage_platform(self, capfd, test_path, test_platforms, out_path, cov_platform):
353        def search_cov():
354            pattern = r'TOTAL\s+(\d+)'
355            coverage_file_path = os.path.join(out_path, 'coverage', 'coverage.txt')
356            with open(coverage_file_path, 'r') as file:
357                data = file.read()
358            match = re.search(pattern, data)
359            if match:
360                total = int(match.group(1))
361                return total
362            else:
363                print('Error, pattern not found')
364
365        run = []
366        for element in cov_platform:
367            args = ['--outdir', out_path, '-i', '-T', test_path] + \
368               ['--coverage', '--coverage-formats', 'txt', '-v'] + \
369               [val for pair in zip(
370                   ['-p'] * len(test_platforms), test_platforms
371               ) for val in pair]
372
373            if isinstance(element, list):
374                for nested_element in element:
375                    args += ['--coverage-platform', nested_element]
376            else:
377                args += ['--coverage-platform', element]
378
379            with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
380                    pytest.raises(SystemExit) as sys_exit:
381                self.loader.exec_module(self.twister_module)
382
383            assert str(sys_exit.value) == '0'
384
385            run += [search_cov()]
386
387            capfd.readouterr()
388            clear_log_in_test()
389
390        assert run[2] > run[0], 'Broader coverage platform selection did not result in broader coverage'
391        assert run[2] > run[1], 'Broader coverage platform selection did not result in broader coverage'
392