1#!/usr/bin/env python3
2# Copyright (c) 2024 Intel Corporation
3#
4# SPDX-License-Identifier: Apache-2.0
5"""
6Blackbox tests for twister's command line functions related to the shuffling of the test order.
7"""
8
9import importlib
10import mock
11import os
12import pytest
13import sys
14import json
15
16# pylint: disable=no-name-in-module
17from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock
18from twisterlib.testplan import TestPlan
19
20
21class TestShuffle:
22    @classmethod
23    def setup_class(cls):
24        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
25        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
26        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
27        cls.twister_module = importlib.util.module_from_spec(cls.spec)
28
29    @classmethod
30    def teardown_class(cls):
31        pass
32
33    @pytest.mark.parametrize(
34        'seed, ratio, expected_order',
35        [
36            ('123', '1/2', ['dummy.device.group', 'dummy.agnostic.group1.subgroup2']),
37            ('123', '2/2', ['dummy.agnostic.group2', 'dummy.agnostic.group1.subgroup1']),
38            ('321', '1/2', ['dummy.agnostic.group2', 'dummy.agnostic.group1.subgroup2']),
39            ('321', '2/2', ['dummy.device.group', 'dummy.agnostic.group1.subgroup1']),
40            ('123', '1/3', ['dummy.device.group', 'dummy.agnostic.group1.subgroup2']),
41            ('123', '2/3', ['dummy.agnostic.group2']),
42            ('123', '3/3', ['dummy.agnostic.group1.subgroup1']),
43            ('321', '1/3', ['dummy.agnostic.group2', 'dummy.agnostic.group1.subgroup2']),
44            ('321', '2/3', ['dummy.device.group']),
45            ('321', '3/3', ['dummy.agnostic.group1.subgroup1'])
46        ],
47        ids=['first half, 123', 'second half, 123', 'first half, 321', 'second half, 321',
48             'first third, 123', 'middle third, 123', 'last third, 123',
49             'first third, 321', 'middle third, 321', 'last third, 321'
50]
51    )
52    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
53    def test_shuffle_tests(self, out_path, seed, ratio, expected_order):
54        test_platforms = ['qemu_x86', 'intel_adl_crb']
55        path = os.path.join(TEST_DATA, 'tests', 'dummy')
56        args = ['-i', '--outdir', out_path, '-T', path, '-y'] + \
57               ['--shuffle-tests', '--shuffle-tests-seed', seed] + \
58               ['--subset', ratio] + \
59               [val for pair in zip(
60                   ['-p'] * len(test_platforms), test_platforms
61               ) for val in pair]
62
63        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
64                pytest.raises(SystemExit) as sys_exit:
65            self.loader.exec_module(self.twister_module)
66
67        with open(os.path.join(out_path, 'testplan.json')) as f:
68            j = json.load(f)
69
70        testsuites = [os.path.basename(ts['name']) for ts in j['testsuites']]
71
72        assert testsuites == expected_order
73
74        assert str(sys_exit.value) == '0'
75