1#!/usr/bin/env python3 2# Copyright (c) 2024 Intel Corporation 3# 4# SPDX-License-Identifier: Apache-2.0 5""" 6Blackbox tests for twister's command line functions 7""" 8import importlib 9import re 10from unittest import mock 11import os 12import pytest 13import sys 14import json 15 16# pylint: disable=duplicate-code, disable=no-name-in-module 17from conftest import TEST_DATA, ZEPHYR_BASE, suite_filename_mock, clear_log_in_test 18from twisterlib.testplan import TestPlan 19 20 21@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', suite_filename_mock) 22class TestCoverage: 23 TESTDATA_1 = [ 24 ( 25 os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'), 26 ['qemu_x86'], 27 [ 28 'coverage.log', 'coverage.json', 29 'coverage' 30 ], 31 ), 32 ] 33 TESTDATA_2 = [ 34 ( 35 os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'), 36 ['qemu_x86'], 37 [ 38 'GCOV_COVERAGE_DUMP_START', 'GCOV_COVERAGE_DUMP_END' 39 ], 40 ), 41 ] 42 TESTDATA_3 = [ 43 ( 44 os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'), 45 ['qemu_x86'], 46 [ 47 'coverage.log', 'coverage.json', 48 'coverage' 49 ], 50 r'{"files": \[\], "gcovr/format_version": ".*"}' 51 ), 52 ] 53 TESTDATA_4 = [ 54 ( 55 'gcovr', 56 [ 57 'coverage.log', 'coverage.json', 58 'coverage', os.path.join('coverage','coverage.xml') 59 ], 60 'xml' 61 ), 62 ( 63 'gcovr', 64 [ 65 'coverage.log', 'coverage.json', 66 'coverage', os.path.join('coverage','coverage.sonarqube.xml') 67 ], 68 'sonarqube' 69 ), 70 ( 71 'gcovr', 72 [ 73 'coverage.log', 'coverage.json', 74 'coverage', os.path.join('coverage','coverage.txt') 75 ], 76 'txt' 77 ), 78 ( 79 'gcovr', 80 [ 81 'coverage.log', 'coverage.json', 82 'coverage', os.path.join('coverage','coverage.csv') 83 ], 84 'csv' 85 ), 86 ( 87 'gcovr', 88 [ 89 'coverage.log', 'coverage.json', 90 'coverage', os.path.join('coverage','coverage.coveralls.json') 91 ], 92 'coveralls' 93 ), 94 ( 95 'gcovr', 96 [ 97 'coverage.log', 'coverage.json', 98 'coverage', os.path.join('coverage','index.html') 99 ], 100 'html' 101 ), 102 ( 103 'lcov', 104 [ 105 'coverage.log', 'coverage.info', 106 'ztest.info', 'coverage', 107 os.path.join('coverage','index.html') 108 ], 109 'html' 110 ), 111 ( 112 'lcov', 113 [ 114 'coverage.log', 'coverage.info', 115 'ztest.info' 116 ], 117 'lcov' 118 ), 119 ] 120 TESTDATA_5 = [ 121 ( 122 os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'), 123 ['qemu_x86'], 124 'gcovr', 125 'Running: gcovr ' 126 ), 127 ( 128 os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'), 129 ['qemu_x86'], 130 'lcov', 131 'Running lcov --gcov-tool' 132 ) 133 ] 134 TESTDATA_6 = [ 135 ( 136 os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'), 137 ['qemu_x86'], 138 ['GCOVR failed with '], 139 ) 140 ] 141 TESTDATA_7 = [ 142 ( 143 os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'), 144 ['qemu_x86_64', 'qemu_x86'], 145 ['qemu_x86_64', 'qemu_x86', ['qemu_x86_64', 'qemu_x86']], 146 ) 147 ] 148 149 @classmethod 150 def setup_class(cls): 151 apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister') 152 cls.loader = importlib.machinery.SourceFileLoader('__main__', apath) 153 cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader) 154 cls.twister_module = importlib.util.module_from_spec(cls.spec) 155 156 @pytest.mark.parametrize( 157 'test_path, test_platforms, file_name', 158 TESTDATA_1, 159 ids=[ 160 'coverage', 161 ] 162 ) 163 def test_coverage(self, capfd, test_path, test_platforms, out_path, file_name): 164 args = ['-i','--outdir', out_path, '-T', test_path] + \ 165 ['--coverage', '--coverage-tool', 'gcovr'] + \ 166 [val for pair in zip( 167 ['-p'] * len(test_platforms), test_platforms 168 ) for val in pair] 169 170 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 171 pytest.raises(SystemExit) as sys_exit: 172 self.loader.exec_module(self.twister_module) 173 174 out, err = capfd.readouterr() 175 sys.stdout.write(out) 176 sys.stderr.write(err) 177 178 assert str(sys_exit.value) == '0' 179 180 for f_name in file_name: 181 path = os.path.join(out_path, f_name) 182 assert os.path.exists(path), f'file not found {f_name}' 183 184 @pytest.mark.parametrize( 185 'test_path, test_platforms, expected', 186 TESTDATA_2, 187 ids=[ 188 'enable_coverage', 189 ] 190 ) 191 def test_enable_coverage(self, capfd, test_path, test_platforms, out_path, expected): 192 args = ['-i','--outdir', out_path, '-T', test_path] + \ 193 ['--enable-coverage', '-vv', '-ll', 'DEBUG'] + \ 194 [val for pair in zip( 195 ['-p'] * len(test_platforms), test_platforms 196 ) for val in pair] 197 198 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 199 pytest.raises(SystemExit) as sys_exit: 200 self.loader.exec_module(self.twister_module) 201 202 out, err = capfd.readouterr() 203 sys.stdout.write(out) 204 sys.stderr.write(err) 205 206 assert str(sys_exit.value) == '0' 207 208 for line in expected: 209 match = re.search(line, err) 210 assert match, f'line not found: {line}' 211 212 @pytest.mark.parametrize( 213 'test_path, test_platforms, file_name, expected_content', 214 TESTDATA_3, 215 ids=[ 216 'coverage_basedir', 217 ] 218 ) 219 def test_coverage_basedir(self, capfd, test_path, test_platforms, out_path, file_name, expected_content): 220 base_dir = os.path.join(TEST_DATA, "test_dir") 221 if os.path.exists(base_dir): 222 os.rmdir(base_dir) 223 os.mkdir(base_dir) 224 args = ['--outdir', out_path,'-i', '-T', test_path] + \ 225 ['--coverage', '--coverage-tool', 'gcovr', '-v', '--coverage-basedir', base_dir] + \ 226 [val for pair in zip( 227 ['-p'] * len(test_platforms), test_platforms 228 ) for val in pair] 229 230 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 231 pytest.raises(SystemExit) as sys_exit: 232 self.loader.exec_module(self.twister_module) 233 234 out, err = capfd.readouterr() 235 sys.stdout.write(out) 236 sys.stderr.write(err) 237 238 assert str(sys_exit.value) == '0' 239 240 for f_name in file_name: 241 path = os.path.join(out_path, f_name) 242 assert os.path.exists(path), f'file not found {f_name}' 243 if f_name == 'coverage.json': 244 with open(path, "r") as json_file: 245 json_content = json.load(json_file) 246 pattern = re.compile(expected_content) 247 assert pattern.match(json.dumps(json_content, sort_keys=True)) 248 if os.path.exists(base_dir): 249 os.rmdir(base_dir) 250 251 @pytest.mark.parametrize( 252 'cov_tool, file_name, cov_format', 253 TESTDATA_4, 254 ids=[ 255 'coverage_format gcovr xml', 256 'coverage_format gcovr sonarqube', 257 'coverage_format gcovr txt', 258 'coverage_format gcovr csv', 259 'coverage_format gcovr coveralls', 260 'coverage_format gcovr html', 261 'coverage_format lcov html', 262 'coverage_format lcov lcov', 263 ] 264 ) 265 def test_coverage_format(self, capfd, out_path, cov_tool, file_name, cov_format): 266 test_path = os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2') 267 test_platforms = ['qemu_x86'] 268 args = ['--outdir', out_path,'-i', '-T', test_path] + \ 269 ['--coverage', '--coverage-tool', cov_tool, '--coverage-formats', cov_format, '-v'] + \ 270 [val for pair in zip( 271 ['-p'] * len(test_platforms), test_platforms 272 ) for val in pair] 273 274 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 275 pytest.raises(SystemExit) as sys_exit: 276 self.loader.exec_module(self.twister_module) 277 278 out, err = capfd.readouterr() 279 sys.stdout.write(out) 280 sys.stderr.write(err) 281 282 assert str(sys_exit.value) == '0' 283 284 for f_name in file_name: 285 path = os.path.join(out_path, f_name) 286 assert os.path.exists(path), f'file not found {f_name}, probably format {cov_format} not work properly' 287 288 289 290 @pytest.mark.parametrize( 291 'test_path, test_platforms, cov_tool, expected_content', 292 TESTDATA_5, 293 ids=[ 294 'coverage_tool gcovr', 295 'coverage_tool lcov' 296 ] 297 ) 298 def test_coverage_tool(self, capfd, caplog, test_path, test_platforms, out_path, cov_tool, expected_content): 299 args = ['--outdir', out_path,'-i', '-T', test_path] + \ 300 ['--coverage', '--coverage-tool', cov_tool, '-v'] + \ 301 [val for pair in zip( 302 ['-p'] * len(test_platforms), test_platforms 303 ) for val in pair] 304 305 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 306 pytest.raises(SystemExit) as sys_exit: 307 self.loader.exec_module(self.twister_module) 308 309 out, err = capfd.readouterr() 310 sys.stdout.write(out) 311 sys.stderr.write(err) 312 313 assert str(sys_exit.value) == '0' 314 315 assert re.search(expected_content, caplog.text), f'{cov_tool} line not found' 316 317 @pytest.mark.parametrize( 318 'test_path, test_platforms, expected_content', 319 TESTDATA_6, 320 ids=[ 321 'missing tool' 322 ] 323 ) 324 def test_gcov_tool(self, capfd, test_path, test_platforms, out_path, expected_content): 325 args = ['--outdir', out_path, '-i', '-T', test_path] + \ 326 ['--coverage', '--gcov-tool', TEST_DATA, '-v'] + \ 327 [val for pair in zip( 328 ['-p'] * len(test_platforms), test_platforms 329 ) for val in pair] 330 331 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 332 pytest.raises(SystemExit) as sys_exit: 333 self.loader.exec_module(self.twister_module) 334 335 out, err = capfd.readouterr() 336 sys.stdout.write(out) 337 sys.stderr.write(err) 338 339 assert str(sys_exit.value) == '1' 340 for line in expected_content: 341 result = re.search(line, err) 342 assert result, f'missing information in log: {line}' 343 344 @pytest.mark.parametrize( 345 'test_path, test_platforms, cov_platform', 346 TESTDATA_7, 347 ids=[ 348 'coverage platform' 349 ] 350 ) 351 def test_coverage_platform(self, capfd, test_path, test_platforms, out_path, cov_platform): 352 def search_cov(): 353 pattern = r'TOTAL\s+(\d+)' 354 coverage_file_path = os.path.join(out_path, 'coverage', 'coverage.txt') 355 with open(coverage_file_path, 'r') as file: 356 data = file.read() 357 match = re.search(pattern, data) 358 if match: 359 total = int(match.group(1)) 360 return total 361 else: 362 print('Error, pattern not found') 363 364 run = [] 365 for element in cov_platform: 366 args = ['--outdir', out_path, '-i', '-T', test_path] + \ 367 ['--coverage', '--coverage-formats', 'txt', '-v'] + \ 368 [val for pair in zip( 369 ['-p'] * len(test_platforms), test_platforms 370 ) for val in pair] 371 372 if isinstance(element, list): 373 for nested_element in element: 374 args += ['--coverage-platform', nested_element] 375 else: 376 args += ['--coverage-platform', element] 377 378 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 379 pytest.raises(SystemExit) as sys_exit: 380 self.loader.exec_module(self.twister_module) 381 382 assert str(sys_exit.value) == '0' 383 384 run += [search_cov()] 385 386 capfd.readouterr() 387 clear_log_in_test() 388 389 assert run[2] > run[0], 'Broader coverage platform selection did not result in broader coverage' 390 assert run[2] > run[1], 'Broader coverage platform selection did not result in broader coverage' 391
