1#!/usr/bin/env python3
2# Copyright (c) 2023 Intel Corporation
3# Copyright (c) 2024 Arm Limited (or its affiliates). All rights reserved.
4#
5# SPDX-License-Identifier: Apache-2.0
6"""
7Tests for environment.py classes' methods
8"""
9
10import mock
11import os
12import pytest
13import shutil
14
15from contextlib import nullcontext
16
17import twisterlib.environment
18
19
20TESTDATA_1 = [
21    (
22        None,
23        None,
24        None,
25        ['--short-build-path', '-k'],
26        '--short-build-path requires Ninja to be enabled'
27    ),
28    (
29        'nt',
30        None,
31        None,
32        ['--device-serial-pty', 'dummy'],
33        '--device-serial-pty is not supported on Windows OS'
34    ),
35    (
36        None,
37        None,
38        None,
39        ['--west-runner=dummy'],
40        'west-runner requires west-flash to be enabled'
41    ),
42    (
43        None,
44        None,
45        None,
46        ['--west-flash=\"--board-id=dummy\"'],
47        'west-flash requires device-testing to be enabled'
48    ),
49    (
50        None,
51        {
52            'exist': [],
53            'missing': ['valgrind']
54        },
55        None,
56        ['--enable-valgrind'],
57        'valgrind enabled but valgrind executable not found'
58    ),
59    (
60        None,
61        None,
62        None,
63        [
64            '--device-testing',
65            '--device-serial',
66            'dummy',
67        ],
68        'When --device-testing is used with --device-serial' \
69        ' or --device-serial-pty, exactly one platform must' \
70        ' be specified'
71    ),
72    (
73        None,
74        None,
75        None,
76        [
77            '--device-testing',
78            '--device-serial',
79            'dummy',
80            '--platform',
81            'dummy_platform1',
82            '--platform',
83            'dummy_platform2'
84        ],
85        'When --device-testing is used with --device-serial' \
86        ' or --device-serial-pty, exactly one platform must' \
87        ' be specified'
88    ),
89# Note the underscore.
90    (
91        None,
92        None,
93        None,
94        ['--device-flash-with-test'],
95        '--device-flash-with-test requires --device_testing'
96    ),
97    (
98        None,
99        None,
100        None,
101        ['--shuffle-tests'],
102        '--shuffle-tests requires --subset'
103    ),
104    (
105        None,
106        None,
107        None,
108        ['--shuffle-tests-seed', '0'],
109        '--shuffle-tests-seed requires --shuffle-tests'
110    ),
111    (
112        None,
113        None,
114        None,
115        ['/dummy/unrecognised/arg'],
116        'Unrecognized arguments found: \'/dummy/unrecognised/arg\'.' \
117        ' Use -- to delineate extra arguments for test binary' \
118        ' or pass -h for help.'
119    ),
120    (
121        None,
122        None,
123        True,
124        [],
125        'By default Twister should work without pytest-twister-harness' \
126        ' plugin being installed, so please, uninstall it by' \
127        ' `pip uninstall pytest-twister-harness` and' \
128        ' `git clean -dxf scripts/pylib/pytest-twister-harness`.'
129    ),
130]
131
132
133@pytest.mark.parametrize(
134    'os_name, which_dict, pytest_plugin, args, expected_error',
135    TESTDATA_1,
136    ids=[
137        'short build path without ninja',
138        'device-serial-pty on Windows',
139        'west runner without west flash',
140        'west-flash without device-testing',
141        'valgrind without executable',
142        'device serial without platform',
143        'device serial with multiple platforms',
144        'device flash with test without device testing',
145        'shuffle-tests without subset',
146        'shuffle-tests-seed without shuffle-tests',
147        'unrecognised argument',
148        'pytest-twister-harness installed'
149    ]
150)
151def test_parse_arguments_errors(
152    caplog,
153    os_name,
154    which_dict,
155    pytest_plugin,
156    args,
157    expected_error
158):
159    def mock_which(name):
160        if name in which_dict['missing']:
161            return False
162        elif name in which_dict['exist']:
163            return which_dict['path'][which_dict['exist']] \
164                if which_dict['path'][which_dict['exist']] \
165                else f'dummy/path/{name}'
166        else:
167            return f'dummy/path/{name}'
168
169    with mock.patch('sys.argv', ['twister'] + args):
170        parser = twisterlib.environment.add_parse_arguments()
171
172    if which_dict:
173        which_dict['path'] = {name: shutil.which(name) \
174            for name in which_dict['exist']}
175        which_mock = mock.Mock(side_effect=mock_which)
176
177    with mock.patch('os.name', os_name) \
178            if os_name is not None else nullcontext(), \
179         mock.patch('shutil.which', which_mock) \
180            if which_dict else nullcontext(), \
181         mock.patch('twisterlib.environment' \
182                    '.PYTEST_PLUGIN_INSTALLED', pytest_plugin) \
183            if pytest_plugin is not None else nullcontext():
184        with pytest.raises(SystemExit) as exit_info:
185            twisterlib.environment.parse_arguments(parser, args)
186
187    assert exit_info.value.code == 1
188    assert expected_error in ' '.join(caplog.text.split())
189
190
191def test_parse_arguments_errors_size():
192    """`options.size` is not an error, rather a different functionality."""
193
194    args = ['--size', 'dummy.elf']
195
196    with mock.patch('sys.argv', ['twister'] + args):
197        parser = twisterlib.environment.add_parse_arguments()
198
199    mock_calc_parent = mock.Mock()
200    mock_calc_parent.child = mock.Mock(return_value=mock.Mock())
201
202    def mock_calc(*args, **kwargs):
203        return mock_calc_parent.child(args, kwargs)
204
205    with mock.patch('twisterlib.size_calc.SizeCalculator', mock_calc):
206        with pytest.raises(SystemExit) as exit_info:
207            twisterlib.environment.parse_arguments(parser, args)
208
209    assert exit_info.value.code == 0
210
211    mock_calc_parent.child.assert_has_calls([mock.call(('dummy.elf', []), {})])
212    mock_calc_parent.child().size_report.assert_has_calls([mock.call()])
213
214
215def test_parse_arguments_warnings(caplog):
216    args = ['--allow-installed-plugin']
217
218    with mock.patch('sys.argv', ['twister'] + args):
219        parser = twisterlib.environment.add_parse_arguments()
220
221    with mock.patch('twisterlib.environment.PYTEST_PLUGIN_INSTALLED', True):
222        twisterlib.environment.parse_arguments(parser, args)
223
224    assert 'You work with installed version of' \
225           ' pytest-twister-harness plugin.' in ' '.join(caplog.text.split())
226
227
228TESTDATA_2 = [
229    (['--enable-size-report']),
230    (['--compare-report', 'dummy']),
231]
232
233
234@pytest.mark.parametrize(
235    'additional_args',
236    TESTDATA_2,
237    ids=['show footprint', 'compare report']
238)
239def test_parse_arguments(zephyr_base, additional_args):
240    args = ['--coverage', '--platform', 'dummy_platform'] + \
241           additional_args + ['--', 'dummy_extra_1', 'dummy_extra_2']
242
243    with mock.patch('sys.argv', ['twister'] + args):
244        parser = twisterlib.environment.add_parse_arguments()
245
246    options = twisterlib.environment.parse_arguments(parser, args)
247
248    assert os.path.join(zephyr_base, 'tests') in options.testsuite_root
249    assert os.path.join(zephyr_base, 'samples') in options.testsuite_root
250
251    assert options.enable_size_report
252
253    assert options.enable_coverage
254
255    assert options.coverage_platform == ['dummy_platform']
256
257    assert options.extra_test_args == ['dummy_extra_1', 'dummy_extra_2']
258
259
260TESTDATA_3 = [
261    (
262        mock.Mock(
263            ninja=True,
264            board_root=['dummy1', 'dummy2'],
265            testsuite_root=[
266                os.path.join('dummy', 'path', "tests"),
267                os.path.join('dummy', 'path', "samples")
268            ],
269            outdir='dummy_abspath',
270        ),
271        mock.Mock(
272            generator_cmd='ninja',
273            generator='Ninja',
274            test_roots=[
275                os.path.join('dummy', 'path', "tests"),
276                os.path.join('dummy', 'path', "samples")
277            ],
278            board_roots=['dummy1', 'dummy2'],
279            outdir='dummy_abspath',
280        )
281    ),
282    (
283        mock.Mock(
284            ninja=False,
285            board_root='dummy0',
286            testsuite_root=[
287                os.path.join('dummy', 'path', "tests"),
288                os.path.join('dummy', 'path', "samples")
289            ],
290            outdir='dummy_abspath',
291        ),
292        mock.Mock(
293            generator_cmd='make',
294            generator='Unix Makefiles',
295            test_roots=[
296                os.path.join('dummy', 'path', "tests"),
297                os.path.join('dummy', 'path', "samples")
298            ],
299            board_roots=['dummy0'],
300            outdir='dummy_abspath',
301        )
302    ),
303]
304
305
306@pytest.mark.parametrize(
307    'options, expected_env',
308    TESTDATA_3,
309    ids=[
310        'ninja',
311        'make'
312    ]
313)
314def test_twisterenv_init(options, expected_env):
315    original_abspath = os.path.abspath
316
317    def mocked_abspath(path):
318        if path == 'dummy_abspath':
319            return 'dummy_abspath'
320        elif isinstance(path, mock.Mock):
321            return None
322        else:
323            return original_abspath(path)
324
325    with mock.patch('os.path.abspath', side_effect=mocked_abspath):
326        twister_env = twisterlib.environment.TwisterEnv(options=options)
327
328    assert twister_env.generator_cmd == expected_env.generator_cmd
329    assert twister_env.generator == expected_env.generator
330
331    assert twister_env.test_roots == expected_env.test_roots
332
333    assert twister_env.board_roots == expected_env.board_roots
334    assert twister_env.outdir == expected_env.outdir
335
336
337def test_twisterenv_discover():
338    options = mock.Mock(
339        ninja=True
340    )
341
342    original_abspath = os.path.abspath
343
344    def mocked_abspath(path):
345        if path == 'dummy_abspath':
346            return 'dummy_abspath'
347        elif isinstance(path, mock.Mock):
348            return None
349        else:
350            return original_abspath(path)
351
352    with mock.patch('os.path.abspath', side_effect=mocked_abspath):
353        twister_env = twisterlib.environment.TwisterEnv(options=options)
354
355    mock_datetime = mock.Mock(
356        now=mock.Mock(
357            return_value=mock.Mock(
358                isoformat=mock.Mock(return_value='dummy_time')
359            )
360        )
361    )
362
363    with mock.patch.object(
364            twisterlib.environment.TwisterEnv,
365            'check_zephyr_version',
366            mock.Mock()) as mock_czv, \
367         mock.patch.object(
368            twisterlib.environment.TwisterEnv,
369            'get_toolchain',
370            mock.Mock()) as mock_gt, \
371         mock.patch('twisterlib.environment.datetime', mock_datetime):
372        twister_env.discover()
373
374    mock_czv.assert_called_once()
375    mock_gt.assert_called_once()
376    assert twister_env.run_date == 'dummy_time'
377
378
379TESTDATA_4 = [
380    (
381        mock.Mock(returncode=0, stdout='dummy stdout version'),
382        mock.Mock(returncode=0, stdout='dummy stdout date'),
383        ['Zephyr version: dummy stdout version'],
384        'dummy stdout version',
385        'dummy stdout date'
386    ),
387    (
388        mock.Mock(returncode=0, stdout=''),
389        mock.Mock(returncode=0, stdout='dummy stdout date'),
390        ['Could not determine version'],
391        'Unknown',
392        'dummy stdout date'
393    ),
394    (
395        mock.Mock(returncode=1, stdout='dummy stdout version'),
396        mock.Mock(returncode=0, stdout='dummy stdout date'),
397        ['Could not determine version'],
398        'Unknown',
399        'dummy stdout date'
400    ),
401    (
402        OSError,
403        mock.Mock(returncode=1),
404        ['Could not determine version'],
405        'Unknown',
406        'Unknown'
407    ),
408]
409
410
411@pytest.mark.parametrize(
412    'git_describe_return, git_show_return, expected_logs,' \
413    ' expected_version, expected_commit_date',
414    TESTDATA_4,
415    ids=[
416        'valid',
417        'no zephyr version on describe',
418        'error on git describe',
419        'execution error on git describe',
420    ]
421)
422def test_twisterenv_check_zephyr_version(
423    caplog,
424    git_describe_return,
425    git_show_return,
426    expected_logs,
427    expected_version,
428    expected_commit_date
429):
430    def mock_run(command, *args, **kwargs):
431        if all([keyword in command for keyword in ['git', 'describe']]):
432            if isinstance(git_describe_return, type) and \
433               issubclass(git_describe_return, Exception):
434                raise git_describe_return()
435            return git_describe_return
436        if all([keyword in command for keyword in ['git', 'show']]):
437            if isinstance(git_show_return, type) and \
438               issubclass(git_show_return, Exception):
439                raise git_show_return()
440            return git_show_return
441
442    options = mock.Mock(
443        ninja=True
444    )
445
446    original_abspath = os.path.abspath
447
448    def mocked_abspath(path):
449        if path == 'dummy_abspath':
450            return 'dummy_abspath'
451        elif isinstance(path, mock.Mock):
452            return None
453        else:
454            return original_abspath(path)
455
456    with mock.patch('os.path.abspath', side_effect=mocked_abspath):
457        twister_env = twisterlib.environment.TwisterEnv(options=options)
458
459    with mock.patch('subprocess.run', mock.Mock(side_effect=mock_run)):
460        twister_env.check_zephyr_version()
461    print(expected_logs)
462    print(caplog.text)
463    assert twister_env.version == expected_version
464    assert twister_env.commit_date == expected_commit_date
465    assert all([expected_log in caplog.text for expected_log in expected_logs])
466
467
468TESTDATA_5 = [
469    (
470        False,
471        None,
472        None,
473        'Unable to find `cmake` in path',
474        None
475    ),
476    (
477        True,
478        0,
479        b'somedummy\x1B[123-@d1770',
480        'Finished running dummy/script/path',
481        {
482            'returncode': 0,
483            'msg': 'Finished running dummy/script/path',
484            'stdout': 'somedummyd1770',
485        }
486    ),
487    (
488        True,
489        1,
490        b'another\x1B_dummy',
491        'CMake script failure: dummy/script/path',
492        {
493            'returncode': 1,
494            'returnmsg': 'anotherdummy'
495        }
496    ),
497]
498
499
500@pytest.mark.parametrize(
501    'find_cmake, return_code, out, expected_log, expected_result',
502    TESTDATA_5,
503    ids=[
504        'cmake not found',
505        'regex sanitation 1',
506        'regex sanitation 2'
507    ]
508)
509def test_twisterenv_run_cmake_script(
510    caplog,
511    find_cmake,
512    return_code,
513    out,
514    expected_log,
515    expected_result
516):
517    def mock_which(name, *args, **kwargs):
518        return 'dummy/cmake/path' if find_cmake else None
519
520    def mock_popen(command, *args, **kwargs):
521        return mock.Mock(
522            pid=0,
523            returncode=return_code,
524            communicate=mock.Mock(
525                return_value=(out, '')
526            )
527        )
528
529    args = ['dummy/script/path', 'var1=val1']
530
531    with mock.patch('shutil.which', mock_which), \
532         mock.patch('subprocess.Popen', mock.Mock(side_effect=mock_popen)), \
533         pytest.raises(Exception) \
534            if not find_cmake else nullcontext() as exception:
535        results = twisterlib.environment.TwisterEnv.run_cmake_script(args)
536
537    assert 'Running cmake script dummy/script/path' in caplog.text
538
539    assert expected_log in caplog.text
540
541    if exception is not None:
542        return
543
544    assert expected_result.items() <= results.items()
545
546
547TESTDATA_6 = [
548    (
549        {
550            'returncode': 0,
551            'stdout': '{\"ZEPHYR_TOOLCHAIN_VARIANT\": \"dummy toolchain\"}'
552        },
553        None,
554        'Using \'dummy toolchain\' toolchain.'
555    ),
556    (
557        {'returncode': 1},
558        2,
559        None
560    ),
561]
562
563
564@pytest.mark.parametrize(
565    'script_result, exit_value, expected_log',
566    TESTDATA_6,
567    ids=['valid', 'error']
568)
569def test_get_toolchain(caplog, script_result, exit_value, expected_log):
570    options = mock.Mock(
571        ninja=True
572    )
573
574    original_abspath = os.path.abspath
575
576    def mocked_abspath(path):
577        if path == 'dummy_abspath':
578            return 'dummy_abspath'
579        elif isinstance(path, mock.Mock):
580            return None
581        else:
582            return original_abspath(path)
583
584    with mock.patch('os.path.abspath', side_effect=mocked_abspath):
585        twister_env = twisterlib.environment.TwisterEnv(options=options)
586
587    with mock.patch.object(
588            twisterlib.environment.TwisterEnv,
589            'run_cmake_script',
590            mock.Mock(return_value=script_result)), \
591         pytest.raises(SystemExit) \
592            if exit_value is not None else nullcontext() as exit_info:
593        twister_env.get_toolchain()
594
595    if exit_info is not None:
596        assert exit_info.value.code == exit_value
597    else:
598        assert expected_log in caplog.text
599