1#!/usr/bin/env python3
2# Copyright (c) 2024 Intel Corporation
3#
4# SPDX-License-Identifier: Apache-2.0
5"""
6Blackbox tests for twister's command line functions
7"""
8import importlib
9import re
10from unittest import mock
11import os
12import pytest
13import sys
14import json
15
16# pylint: disable=duplicate-code, disable=no-name-in-module
17from conftest import TEST_DATA, ZEPHYR_BASE, suite_filename_mock, clear_log_in_test
18from twisterlib.testplan import TestPlan
19
20
21@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', suite_filename_mock)
22class TestCoverage:
23    TESTDATA_1 = [
24        (
25            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'),
26            ['qemu_x86'],
27            [
28                'coverage.log', 'coverage.json',
29                'coverage'
30            ],
31        ),
32    ]
33    TESTDATA_2 = [
34        (
35            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'),
36            ['qemu_x86'],
37            [
38                'GCOV_COVERAGE_DUMP_START', 'GCOV_COVERAGE_DUMP_END'
39            ],
40        ),
41    ]
42    TESTDATA_3 = [
43        (
44            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'),
45            ['qemu_x86'],
46            [
47                'coverage.log', 'coverage.json',
48                'coverage'
49            ],
50            r'{"files": \[\], "gcovr/format_version": ".*"}'
51        ),
52    ]
53    TESTDATA_4 = [
54        (
55            'gcovr',
56            [
57                'coverage.log', 'coverage.json',
58                'coverage', os.path.join('coverage','coverage.xml')
59            ],
60            'xml'
61        ),
62        (
63            'gcovr',
64            [
65                'coverage.log', 'coverage.json',
66                'coverage', os.path.join('coverage','coverage.sonarqube.xml')
67            ],
68            'sonarqube'
69        ),
70        (
71            'gcovr',
72            [
73                'coverage.log', 'coverage.json',
74                'coverage', os.path.join('coverage','coverage.txt')
75            ],
76            'txt'
77        ),
78        (
79            'gcovr',
80            [
81                'coverage.log', 'coverage.json',
82                'coverage', os.path.join('coverage','coverage.csv')
83            ],
84            'csv'
85        ),
86        (
87            'gcovr',
88            [
89                'coverage.log', 'coverage.json',
90                'coverage', os.path.join('coverage','coverage.coveralls.json')
91            ],
92            'coveralls'
93        ),
94        (
95            'gcovr',
96            [
97                'coverage.log', 'coverage.json',
98                'coverage', os.path.join('coverage','index.html')
99            ],
100            'html'
101        ),
102        (
103            'lcov',
104            [
105                'coverage.log', 'coverage.info',
106                'ztest.info', 'coverage',
107                os.path.join('coverage','index.html')
108            ],
109            'html'
110        ),
111        (
112            'lcov',
113            [
114                'coverage.log', 'coverage.info',
115                'ztest.info'
116            ],
117            'lcov'
118        ),
119    ]
120    TESTDATA_5 = [
121        (
122            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'),
123            ['qemu_x86'],
124            'gcovr',
125            'Running: gcovr '
126        ),
127        (
128            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'),
129            ['qemu_x86'],
130            'lcov',
131            'Running lcov --gcov-tool'
132        )
133    ]
134    TESTDATA_6 = [
135        (
136            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'),
137            ['qemu_x86'],
138            ['GCOVR failed with '],
139        )
140    ]
141    TESTDATA_7 = [
142        (
143            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'),
144            ['qemu_x86_64', 'qemu_x86'],
145            ['qemu_x86_64', 'qemu_x86', ['qemu_x86_64', 'qemu_x86']],
146        )
147    ]
148
149    @classmethod
150    def setup_class(cls):
151        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
152        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
153        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
154        cls.twister_module = importlib.util.module_from_spec(cls.spec)
155
156    @pytest.mark.parametrize(
157        'test_path, test_platforms, file_name',
158        TESTDATA_1,
159        ids=[
160            'coverage',
161        ]
162    )
163    def test_coverage(self, capfd, test_path, test_platforms, out_path, file_name):
164        args = ['-i','--outdir', out_path, '-T', test_path] + \
165               ['--coverage', '--coverage-tool', 'gcovr'] + \
166               [val for pair in zip(
167                   ['-p'] * len(test_platforms), test_platforms
168               ) for val in pair]
169
170        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
171                pytest.raises(SystemExit) as sys_exit:
172            self.loader.exec_module(self.twister_module)
173
174        out, err = capfd.readouterr()
175        sys.stdout.write(out)
176        sys.stderr.write(err)
177
178        assert str(sys_exit.value) == '0'
179
180        for f_name in file_name:
181            path = os.path.join(out_path, f_name)
182            assert os.path.exists(path), f'file not found {f_name}'
183
184    @pytest.mark.parametrize(
185        'test_path, test_platforms, expected',
186        TESTDATA_2,
187        ids=[
188            'enable_coverage',
189        ]
190    )
191    def test_enable_coverage(self, capfd, test_path, test_platforms, out_path, expected):
192        args = ['-i','--outdir', out_path, '-T', test_path] + \
193               ['--enable-coverage', '-vv', '-ll', 'DEBUG'] + \
194               [val for pair in zip(
195                   ['-p'] * len(test_platforms), test_platforms
196               ) for val in pair]
197
198        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
199                pytest.raises(SystemExit) as sys_exit:
200            self.loader.exec_module(self.twister_module)
201
202        out, err = capfd.readouterr()
203        sys.stdout.write(out)
204        sys.stderr.write(err)
205
206        assert str(sys_exit.value) == '0'
207
208        for line in expected:
209            match = re.search(line, err)
210            assert match, f'line not found: {line}'
211
212    @pytest.mark.parametrize(
213        'test_path, test_platforms, file_name, expected_content',
214        TESTDATA_3,
215        ids=[
216            'coverage_basedir',
217        ]
218    )
219    def test_coverage_basedir(self, capfd, test_path, test_platforms, out_path, file_name, expected_content):
220        base_dir = os.path.join(TEST_DATA, "test_dir")
221        if os.path.exists(base_dir):
222            os.rmdir(base_dir)
223        os.mkdir(base_dir)
224        args = ['--outdir', out_path,'-i', '-T', test_path] + \
225               ['--coverage', '--coverage-tool', 'gcovr',  '-v', '--coverage-basedir', base_dir] + \
226               [val for pair in zip(
227                   ['-p'] * len(test_platforms), test_platforms
228               ) for val in pair]
229
230        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
231                pytest.raises(SystemExit) as sys_exit:
232            self.loader.exec_module(self.twister_module)
233
234        out, err = capfd.readouterr()
235        sys.stdout.write(out)
236        sys.stderr.write(err)
237
238        assert str(sys_exit.value) == '0'
239
240        for f_name in file_name:
241            path = os.path.join(out_path, f_name)
242            assert os.path.exists(path), f'file not found {f_name}'
243            if f_name == 'coverage.json':
244                with open(path, "r") as json_file:
245                    json_content = json.load(json_file)
246                    pattern = re.compile(expected_content)
247                    assert pattern.match(json.dumps(json_content, sort_keys=True))
248        if os.path.exists(base_dir):
249            os.rmdir(base_dir)
250
251    @pytest.mark.parametrize(
252        'cov_tool, file_name, cov_format',
253        TESTDATA_4,
254        ids=[
255            'coverage_format gcovr xml',
256            'coverage_format gcovr sonarqube',
257            'coverage_format gcovr txt',
258            'coverage_format gcovr csv',
259            'coverage_format gcovr coveralls',
260            'coverage_format gcovr html',
261            'coverage_format lcov html',
262            'coverage_format lcov lcov',
263        ]
264    )
265    def test_coverage_format(self, capfd, out_path, cov_tool, file_name, cov_format):
266        test_path = os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2')
267        test_platforms = ['qemu_x86']
268        args = ['--outdir', out_path,'-i', '-T', test_path] + \
269               ['--coverage', '--coverage-tool', cov_tool, '--coverage-formats', cov_format, '-v'] + \
270               [val for pair in zip(
271                   ['-p'] * len(test_platforms), test_platforms
272               ) for val in pair]
273
274        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
275                pytest.raises(SystemExit) as sys_exit:
276            self.loader.exec_module(self.twister_module)
277
278        out, err = capfd.readouterr()
279        sys.stdout.write(out)
280        sys.stderr.write(err)
281
282        assert str(sys_exit.value) == '0'
283
284        for f_name in file_name:
285            path = os.path.join(out_path, f_name)
286            assert os.path.exists(path), f'file not found {f_name}, probably format {cov_format} not work properly'
287
288
289
290    @pytest.mark.parametrize(
291        'test_path, test_platforms, cov_tool, expected_content',
292        TESTDATA_5,
293        ids=[
294            'coverage_tool gcovr',
295            'coverage_tool lcov'
296        ]
297    )
298    def test_coverage_tool(self, capfd, caplog, test_path, test_platforms, out_path, cov_tool, expected_content):
299        args = ['--outdir', out_path,'-i', '-T', test_path] + \
300            ['--coverage', '--coverage-tool', cov_tool,  '-v'] + \
301               [val for pair in zip(
302                   ['-p'] * len(test_platforms), test_platforms
303               ) for val in pair]
304
305        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
306                pytest.raises(SystemExit) as sys_exit:
307            self.loader.exec_module(self.twister_module)
308
309        out, err = capfd.readouterr()
310        sys.stdout.write(out)
311        sys.stderr.write(err)
312
313        assert str(sys_exit.value) == '0'
314
315        assert re.search(expected_content, caplog.text), f'{cov_tool} line not found'
316
317    @pytest.mark.parametrize(
318        'test_path, test_platforms, expected_content',
319        TESTDATA_6,
320        ids=[
321            'missing tool'
322        ]
323    )
324    def test_gcov_tool(self, capfd, test_path, test_platforms, out_path, expected_content):
325        args = ['--outdir', out_path, '-i', '-T', test_path] + \
326            ['--coverage', '--gcov-tool', TEST_DATA, '-v'] + \
327               [val for pair in zip(
328                   ['-p'] * len(test_platforms), test_platforms
329               ) for val in pair]
330
331        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
332                pytest.raises(SystemExit) as sys_exit:
333            self.loader.exec_module(self.twister_module)
334
335        out, err = capfd.readouterr()
336        sys.stdout.write(out)
337        sys.stderr.write(err)
338
339        assert str(sys_exit.value) == '1'
340        for line in expected_content:
341            result = re.search(line, err)
342            assert result, f'missing information in log: {line}'
343
344    @pytest.mark.parametrize(
345        'test_path, test_platforms, cov_platform',
346        TESTDATA_7,
347        ids=[
348            'coverage platform'
349        ]
350    )
351    def test_coverage_platform(self, capfd, test_path, test_platforms, out_path, cov_platform):
352        def search_cov():
353            pattern = r'TOTAL\s+(\d+)'
354            coverage_file_path = os.path.join(out_path, 'coverage', 'coverage.txt')
355            with open(coverage_file_path, 'r') as file:
356                data = file.read()
357            match = re.search(pattern, data)
358            if match:
359                total = int(match.group(1))
360                return total
361            else:
362                print('Error, pattern not found')
363
364        run = []
365        for element in cov_platform:
366            args = ['--outdir', out_path, '-i', '-T', test_path] + \
367               ['--coverage', '--coverage-formats', 'txt', '-v'] + \
368               [val for pair in zip(
369                   ['-p'] * len(test_platforms), test_platforms
370               ) for val in pair]
371
372            if isinstance(element, list):
373                for nested_element in element:
374                    args += ['--coverage-platform', nested_element]
375            else:
376                args += ['--coverage-platform', element]
377
378            with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
379                    pytest.raises(SystemExit) as sys_exit:
380                self.loader.exec_module(self.twister_module)
381
382            assert str(sys_exit.value) == '0'
383
384            run += [search_cov()]
385
386            capfd.readouterr()
387            clear_log_in_test()
388
389        assert run[2] > run[0], 'Broader coverage platform selection did not result in broader coverage'
390        assert run[2] > run[1], 'Broader coverage platform selection did not result in broader coverage'
391