1#!/usr/bin/env python3
2# Copyright (c) 2024 Intel Corporation
3#
4# SPDX-License-Identifier: Apache-2.0
5"""
6Blackbox tests for twister's command line functions - simple does-error-out or not tests
7"""
8
9import importlib
10import mock
11import os
12import pytest
13import sys
14import re
15
16# pylint: disable=no-name-in-module
17from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock
18from twisterlib.testplan import TestPlan
19from twisterlib.error import TwisterRuntimeError
20
21
22class TestError:
23    TESTDATA_1 = [
24        (
25            os.path.join(TEST_DATA, 'tests', 'dummy'),
26            os.path.join('scripts', 'tests', 'twister_blackbox', 'test_data', 'tests',
27                         'dummy', 'agnostic', 'group1', 'subgroup1',
28                         'dummy.agnostic.group1.subgroup1'),
29            SystemExit
30        ),
31        (
32            None,
33            'dummy.agnostic.group1.subgroup1',
34            TwisterRuntimeError
35        ),
36        (
37            os.path.join(TEST_DATA, 'tests', 'dummy'),
38            'dummy.agnostic.group1.subgroup1',
39            SystemExit
40        )
41    ]
42    TESTDATA_2 = [
43        (
44            '',
45            r'always_overflow.dummy SKIPPED \(RAM overflow\)'
46        ),
47        (
48            '--overflow-as-errors',
49            r'always_overflow.dummy ERROR Build failure \(build\)'
50        )
51    ]
52
53    @classmethod
54    def setup_class(cls):
55        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
56        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
57        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
58        cls.twister_module = importlib.util.module_from_spec(cls.spec)
59
60    @classmethod
61    def teardown_class(cls):
62        pass
63
64    @pytest.mark.parametrize(
65        'testroot, test, expected_exception',
66        TESTDATA_1,
67        ids=['valid', 'invalid', 'valid']
68    )
69    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
70    def test_test(self, out_path, testroot, test, expected_exception):
71        test_platforms = ['qemu_x86', 'intel_adl_crb']
72        args = []
73        if testroot:
74            args = ['-T', testroot]
75        args += ['-i', '--outdir', out_path, '--test', test, '-y'] + \
76               [val for pair in zip(
77                   ['-p'] * len(test_platforms), test_platforms
78               ) for val in pair]
79
80        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
81                pytest.raises(expected_exception) as exc:
82            self.loader.exec_module(self.twister_module)
83
84        if expected_exception == SystemExit:
85            assert str(exc.value) == '0'
86        assert True
87
88    @pytest.mark.parametrize(
89        'switch, expected',
90        TESTDATA_2,
91        ids=[
92            'overflow skip',
93            'overflow error',
94        ],
95    )
96
97    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
98    def test_overflow_as_errors(self, capfd, out_path, switch, expected):
99        path = os.path.join(TEST_DATA, 'tests', 'qemu_overflow')
100        test_platforms = ['qemu_x86']
101        args = ['--outdir', out_path, '-T', path, '-vv'] + \
102               ['--build-only'] + \
103               [val for pair in zip(
104                   ['-p'] * len(test_platforms), test_platforms
105               ) for val in pair]
106        if switch:
107            args += [switch]
108
109        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
110            pytest.raises(SystemExit) as sys_exit:
111            self.loader.exec_module(self.twister_module)
112
113        out, err = capfd.readouterr()
114        sys.stdout.write(out)
115        sys.stderr.write(err)
116
117        print(args)
118        if switch:
119            assert str(sys_exit.value) == '1'
120        else:
121            assert str(sys_exit.value) == '0'
122
123        assert re.search(expected, err)
124