#!/usr/bin/env python3 # Copyright (c) 2020-2024 Intel Corporation # # SPDX-License-Identifier: Apache-2.0 ''' This test file contains testsuites for testsuite.py module of twister ''' import sys import os import mock import pytest from contextlib import nullcontext ZEPHYR_BASE = os.getenv("ZEPHYR_BASE") sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister")) from twisterlib.statuses import TwisterStatus from twisterlib.testplan import TestPlan, change_skip_to_error_if_integration from twisterlib.testinstance import TestInstance from twisterlib.testsuite import TestSuite from twisterlib.platform import Platform from twisterlib.quarantine import Quarantine from twisterlib.error import TwisterRuntimeError def test_testplan_add_testsuites_short(class_testplan): """ Testing add_testcase function of Testsuite class in twister """ # Test 1: Check the list of testsuites after calling add testsuites function is as expected class_testplan.SAMPLE_FILENAME = 'test_sample_app.yaml' class_testplan.TESTSUITE_FILENAME = 'test_data.yaml' class_testplan.add_testsuites() tests_rel_dir = 'scripts/tests/twister/test_data/testsuites/tests/' expected_testsuites = ['test_b.check_1', 'test_b.check_2', 'test_c.check_1', 'test_c.check_2', 'test_a.check_1', 'test_a.check_2', 'test_d.check_1', 'test_e.check_1', 'sample_test.app', 'test_config.main'] testsuite_list = [] for key in sorted(class_testplan.testsuites.keys()): testsuite_list.append(os.path.basename(os.path.normpath(key))) assert sorted(testsuite_list) == sorted(expected_testsuites) # Test 2 : Assert Testcase name is expected & all testsuites values are testcase class objects suite = class_testplan.testsuites.get(tests_rel_dir + 'test_a/test_a.check_1') assert suite.name == tests_rel_dir + 'test_a/test_a.check_1' assert all(isinstance(n, TestSuite) for n in class_testplan.testsuites.values()) @pytest.mark.parametrize("board_root_dir", [("board_config_file_not_exist"), ("board_config")]) def test_add_configurations_short(test_data, class_env, board_root_dir): """ Testing add_configurations function of TestPlan class in Twister Test : Asserting on default platforms list """ class_env.board_roots = [os.path.abspath(test_data + board_root_dir)] plan = TestPlan(class_env) plan.parse_configuration(config_file=class_env.test_config) if board_root_dir == "board_config": plan.add_configurations() print(sorted(plan.default_platforms)) assert sorted(plan.default_platforms) == sorted(['demo_board_1/unit_testing', 'demo_board_3/unit_testing']) elif board_root_dir == "board_config_file_not_exist": plan.add_configurations() assert sorted(plan.default_platforms) != sorted(['demo_board_1']) def test_get_all_testsuites_short(class_testplan, all_testsuites_dict): """ Testing get_all_testsuites function of TestPlan class in Twister """ plan = class_testplan plan.testsuites = all_testsuites_dict expected_tests = ['sample_test.app', 'test_a.check_1.1a', 'test_a.check_1.1c', 'test_a.check_1.2a', 'test_a.check_1.2b', 'test_a.check_1.Unit_1c', 'test_a.check_1.unit_1a', 'test_a.check_1.unit_1b', 'test_a.check_2.1a', 'test_a.check_2.1c', 'test_a.check_2.2a', 'test_a.check_2.2b', 'test_a.check_2.Unit_1c', 'test_a.check_2.unit_1a', 'test_a.check_2.unit_1b', 'test_b.check_1', 'test_b.check_2', 'test_c.check_1', 'test_c.check_2', 'test_d.check_1.unit_1a', 'test_d.check_1.unit_1b', 'test_e.check_1.feature5.1a', 'test_e.check_1.feature5.1b', 'test_config.main'] assert sorted(plan.get_all_tests()) == sorted(expected_tests) def test_get_platforms_short(class_testplan, platforms_list): """ Testing get_platforms function of TestPlan class in Twister """ plan = class_testplan plan.platforms = platforms_list platform = plan.get_platform("demo_board_1") assert isinstance(platform, Platform) assert platform.name == "demo_board_1/unit_testing" TESTDATA_PART1 = [ ("toolchain_allow", ['gcc'], None, None, "Not in testsuite toolchain allow list"), ("platform_allow", ['demo_board_1/unit_testing'], None, None, "Not in testsuite platform allow list"), ("toolchain_exclude", ['zephyr'], None, None, "In test case toolchain exclude"), ("platform_exclude", ['demo_board_2'], None, None, "In test case platform exclude"), ("arch_exclude", ['x86'], None, None, "In test case arch exclude"), ("arch_allow", ['arm'], None, None, "Not in test case arch allow list"), ("skip", True, None, None, "Skip filter"), ("tags", set(['sensor', 'bluetooth']), "ignore_tags", ['bluetooth'], "Excluded tags per platform (exclude_tags)"), ("min_flash", "2024", "flash", "1024", "Not enough FLASH"), ("min_ram", "500", "ram", "256", "Not enough RAM"), ("None", "None", "env", ['BSIM_OUT_PATH', 'demo_env'], "Environment (BSIM_OUT_PATH, demo_env) not satisfied"), ("build_on_all", True, None, None, "Platform is excluded on command line."), ("build_on_all", True, "level", "foobar", "Unknown test level 'foobar'"), (None, None, "supported_toolchains", ['gcc', 'xcc', 'xt-clang'], "Not supported by the toolchain"), ] @pytest.mark.parametrize("tc_attribute, tc_value, plat_attribute, plat_value, expected_discards", TESTDATA_PART1) def test_apply_filters_part1(class_testplan, all_testsuites_dict, platforms_list, tc_attribute, tc_value, plat_attribute, plat_value, expected_discards): """ Testing apply_filters function of TestPlan class in Twister Part 1: Response of apply_filters function have appropriate values according to the filters """ plan = class_testplan if tc_attribute is None and plat_attribute is None: plan.apply_filters() plan.platforms = platforms_list plan.platform_names = [p.name for p in platforms_list] plan.testsuites = all_testsuites_dict for plat in plan.platforms: if plat_attribute == "ignore_tags": plat.ignore_tags = plat_value if plat_attribute == "flash": plat.flash = plat_value if plat_attribute == "ram": plat.ram = plat_value if plat_attribute == "env": plat.env = plat_value plat.env_satisfied = False if plat_attribute == "supported_toolchains": plat.supported_toolchains = plat_value for _, testcase in plan.testsuites.items(): if tc_attribute == "toolchain_allow": testcase.toolchain_allow = tc_value if tc_attribute == "platform_allow": testcase.platform_allow = tc_value if tc_attribute == "toolchain_exclude": testcase.toolchain_exclude = tc_value if tc_attribute == "platform_exclude": testcase.platform_exclude = tc_value if tc_attribute == "arch_exclude": testcase.arch_exclude = tc_value if tc_attribute == "arch_allow": testcase.arch_allow = tc_value if tc_attribute == "skip": testcase.skip = tc_value if tc_attribute == "tags": testcase.tags = tc_value if tc_attribute == "min_flash": testcase.min_flash = tc_value if tc_attribute == "min_ram": testcase.min_ram = tc_value if plat_attribute == "level": plan.options.level = plat_value if tc_attribute == "build_on_all": for _, testcase in plan.testsuites.items(): testcase.build_on_all = tc_value plan.apply_filters(exclude_platform=['demo_board_1']) elif plat_attribute == "supported_toolchains": plan.apply_filters(force_toolchain=False, exclude_platform=['demo_board_1'], platform=['demo_board_2/unit_testing']) elif tc_attribute is None and plat_attribute is None: plan.apply_filters() else: plan.apply_filters(exclude_platform=['demo_board_1'], platform=['demo_board_2/unit_testing']) filtered_instances = list(filter(lambda item: item.status == TwisterStatus.FILTER, plan.instances.values())) for d in filtered_instances: assert d.reason == expected_discards TESTDATA_PART2 = [ ("runnable", "True", "Not runnable on device"), ("exclude_tag", ['test_a'], "Command line testsuite exclude filter"), ("run_individual_tests", ['scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1'], "TestSuite name filter"), ("arch", ['arm_test'], "Command line testsuite arch filter"), ("tag", ['test_d'], "Command line testsuite tag filter") ] @pytest.mark.parametrize("extra_filter, extra_filter_value, expected_discards", TESTDATA_PART2) def test_apply_filters_part2(class_testplan, all_testsuites_dict, platforms_list, extra_filter, extra_filter_value, expected_discards): """ Testing apply_filters function of TestPlan class in Twister Part 2 : Response of apply_filters function (discard dictionary) have appropriate values according to the filters """ class_testplan.platforms = platforms_list class_testplan.platform_names = [p.name for p in platforms_list] class_testplan.testsuites = all_testsuites_dict kwargs = { extra_filter : extra_filter_value, "exclude_platform" : [ 'demo_board_1' ], "platform" : [ 'demo_board_2' ] } class_testplan.apply_filters(**kwargs) filtered_instances = list(filter(lambda item: item.status == TwisterStatus.FILTER, class_testplan.instances.values())) for d in filtered_instances: assert d.reason == expected_discards TESTDATA_PART3 = [ (20, 20, -1, 0), (-2, -1, 10, 20), (0, 0, 0, 0) ] @pytest.mark.parametrize("tc_min_flash, plat_flash, tc_min_ram, plat_ram", TESTDATA_PART3) def test_apply_filters_part3(class_testplan, all_testsuites_dict, platforms_list, tc_min_flash, plat_flash, tc_min_ram, plat_ram): """ Testing apply_filters function of TestPlan class in Twister Part 3 : Testing edge cases for ram and flash values of platforms & testsuites """ class_testplan.platforms = platforms_list class_testplan.platform_names = [p.name for p in platforms_list] class_testplan.testsuites = all_testsuites_dict for plat in class_testplan.platforms: plat.flash = plat_flash plat.ram = plat_ram for _, testcase in class_testplan.testsuites.items(): testcase.min_ram = tc_min_ram testcase.min_flash = tc_min_flash class_testplan.apply_filters(exclude_platform=['demo_board_1'], platform=['demo_board_2']) filtered_instances = list(filter(lambda item: item.status == TwisterStatus.FILTER, class_testplan.instances.values())) assert not filtered_instances def test_add_instances_short(tmp_path, class_env, all_testsuites_dict, platforms_list): """ Testing add_instances() function of TestPlan class in Twister Test 1: instances dictionary keys have expected values (Platform Name + Testcase Name) Test 2: Values of 'instances' dictionary in Testsuite class are an instance of 'TestInstance' class Test 3: Values of 'instances' dictionary have expected values. """ class_env.outdir = tmp_path plan = TestPlan(class_env) plan.platforms = platforms_list platform = plan.get_platform("demo_board_2") instance_list = [] for _, testcase in all_testsuites_dict.items(): instance = TestInstance(testcase, platform, class_env.outdir) instance_list.append(instance) plan.add_instances(instance_list) assert list(plan.instances.keys()) == \ [platform.name + '/' + s for s in list(all_testsuites_dict.keys())] assert all(isinstance(n, TestInstance) for n in list(plan.instances.values())) assert list(plan.instances.values()) == instance_list QUARANTINE_BASIC = { 'demo_board_1/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3', 'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3' } QUARANTINE_WITH_REGEXP = { 'demo_board_2/unit_testing/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'a2 and c2 on x86', 'demo_board_1/unit_testing/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d', 'demo_board_3/unit_testing/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d', 'demo_board_2/unit_testing/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d', 'demo_board_2/unit_testing/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'a2 and c2 on x86' } QUARANTINE_PLATFORM = { 'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'all on board_3', 'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'all on board_3', 'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all on board_3', 'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_1' : 'all on board_3', 'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_2' : 'all on board_3', 'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_1' : 'all on board_3', 'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'all on board_3', 'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_e/test_e.check_1' : 'all on board_3', 'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_config/test_config.main' : 'all on board_3' } QUARANTINE_MULTIFILES = { **QUARANTINE_BASIC, **QUARANTINE_WITH_REGEXP } @pytest.mark.parametrize( ("quarantine_files, quarantine_verify, expected_val"), [ (['basic.yaml'], False, QUARANTINE_BASIC), (['with_regexp.yaml'], False, QUARANTINE_WITH_REGEXP), (['with_regexp.yaml'], True, QUARANTINE_WITH_REGEXP), (['platform.yaml'], False, QUARANTINE_PLATFORM), (['basic.yaml', 'with_regexp.yaml'], False, QUARANTINE_MULTIFILES), (['empty.yaml'], False, {}) ], ids=[ 'basic', 'with_regexp', 'quarantine_verify', 'platform', 'multifiles', 'empty' ]) def test_quarantine_short(class_testplan, platforms_list, test_data, quarantine_files, quarantine_verify, expected_val): """ Testing quarantine feature in Twister """ class_testplan.options.all = True class_testplan.platforms = platforms_list class_testplan.platform_names = [p.name for p in platforms_list] class_testplan.TESTSUITE_FILENAME = 'test_data.yaml' class_testplan.add_testsuites() quarantine_list = [ os.path.join(test_data, 'quarantines', quarantine_file) for quarantine_file in quarantine_files ] class_testplan.quarantine = Quarantine(quarantine_list) class_testplan.options.quarantine_verify = quarantine_verify class_testplan.apply_filters() for testname, instance in class_testplan.instances.items(): if quarantine_verify: if testname in expected_val: assert instance.status == TwisterStatus.NONE else: assert instance.status == TwisterStatus.FILTER assert instance.reason == "Not under quarantine" else: if testname in expected_val: assert instance.status == TwisterStatus.FILTER assert instance.reason == "Quarantine: " + expected_val[testname] else: assert instance.status == TwisterStatus.NONE TESTDATA_PART4 = [ (os.path.join('test_d', 'test_d.check_1'), ['dummy'], None, 'Snippet not supported'), (os.path.join('test_c', 'test_c.check_1'), ['cdc-acm-console'], 0, None), (os.path.join('test_d', 'test_d.check_1'), ['dummy', 'cdc-acm-console'], 2, 'Snippet not supported'), ] @pytest.mark.parametrize( 'testpath, required_snippets, expected_filtered_len, expected_filtered_reason', TESTDATA_PART4, ids=['app', 'global', 'multiple'] ) def test_required_snippets_short( class_testplan, all_testsuites_dict, platforms_list, testpath, required_snippets, expected_filtered_len, expected_filtered_reason ): """ Testing required_snippets function of TestPlan class in Twister """ plan = class_testplan testpath = os.path.join('scripts', 'tests', 'twister', 'test_data', 'testsuites', 'tests', testpath) testsuite = class_testplan.testsuites.get(testpath) plan.platforms = platforms_list print(platforms_list) plan.platform_names = [p.name for p in platforms_list] plan.testsuites = {testpath: testsuite} for _, testcase in plan.testsuites.items(): testcase.exclude_platform = [] testcase.required_snippets = required_snippets testcase.build_on_all = True plan.apply_filters() filtered_instances = list( filter(lambda item: item.status == TwisterStatus.FILTER, plan.instances.values()) ) if expected_filtered_len is not None: assert len(filtered_instances) == expected_filtered_len if expected_filtered_reason is not None: for d in filtered_instances: assert d.reason == expected_filtered_reason def test_testplan_get_level(): testplan = TestPlan(env=mock.Mock()) lvl1 = mock.Mock() lvl1.name = 'a lvl' lvl2 = mock.Mock() lvl2.name = 'a lvl' lvl3 = mock.Mock() lvl3.name = 'other lvl' testplan.levels.append(lvl1) testplan.levels.append(lvl2) testplan.levels.append(lvl3) name = 'a lvl' res = testplan.get_level(name) assert res == lvl1 res = testplan.get_level(name) assert res == lvl1 lvl_missed = mock.Mock() lvl_missed.name = 'missed lvl' res = testplan.get_level('missed_lvl') assert res is None testplan.levels.remove(lvl1) testplan.levels.remove(lvl2) res = testplan.get_level(name) assert res is None TESTDATA_1 = [ ('', {}), ( """\ levels: - name: lvl1 adds: - sc1 - sc2 inherits: [] - name: lvl2 adds: - sc1-1 - sc1-2 inherits: [lvl1] """, { 'lvl1': ['sc1', 'sc2'], 'lvl2': ['sc1-1', 'sc1-2', 'sc1', 'sc2'] } ), ] @pytest.mark.parametrize( 'config_yaml, expected_scenarios', TESTDATA_1, ids=['no config', 'valid config'] ) def test_testplan_parse_configuration(tmp_path, config_yaml, expected_scenarios): testplan = TestPlan(env=mock.Mock()) testplan.scenarios = ['sc1', 'sc1-1', 'sc1-2', 'sc2'] tmp_config_file = tmp_path / 'config_file.yaml' if config_yaml: tmp_config_file.write_text(config_yaml) with pytest.raises(TwisterRuntimeError) if not config_yaml else nullcontext(): testplan.parse_configuration(tmp_config_file) if not testplan.levels: assert expected_scenarios == {} for level in testplan.levels: assert sorted(level.scenarios) == sorted(expected_scenarios[level.name]) TESTDATA_2 = [ ([], [], False), (['ts1.tc3'], [], True), (['ts2.tc2'], ['- ts2'], False), ] @pytest.mark.parametrize( 'sub_tests, expected_outs, expect_error', TESTDATA_2, ids=['no subtests', 'subtests not found', 'valid subtests'] ) def test_testplan_find_subtests( capfd, sub_tests, expected_outs, expect_error ): testplan = TestPlan(env=mock.Mock()) testplan.options = mock.Mock(sub_test=sub_tests) testplan.run_individual_testsuite = [] testplan.testsuites = { 'ts1': mock.Mock( testcases=[ mock.Mock(), mock.Mock(), ] ), 'ts2': mock.Mock( testcases=[ mock.Mock(), mock.Mock(), mock.Mock(), ] ) } testplan.testsuites['ts1'].name = 'ts1' testplan.testsuites['ts1'].testcases[0].name = 'ts1.tc1' testplan.testsuites['ts1'].testcases[1].name = 'ts1.tc2' testplan.testsuites['ts2'].name = 'ts2' testplan.testsuites['ts2'].testcases[0].name = 'ts2.tc1' testplan.testsuites['ts2'].testcases[1].name = 'ts2.tc2' testplan.testsuites['ts2'].testcases[2].name = 'ts2.tc3' with pytest.raises(TwisterRuntimeError) if expect_error else nullcontext(): testplan.find_subtests() out, err = capfd.readouterr() sys.stdout.write(out) sys.stdout.write(err) assert all([printout in out for printout in expected_outs]) TESTDATA_3 = [ (0, 0, [], False, [], TwisterRuntimeError, []), (1, 1, [], False, [], TwisterRuntimeError, []), (1, 0, [], True, [], TwisterRuntimeError, ['No quarantine list given to be verified']), # (1, 0, ['qfile.yaml'], False, ['# empty'], None, ['Quarantine file qfile.yaml is empty']), (1, 0, ['qfile.yaml'], False, ['- platforms:\n - demo_board_3\n comment: "board_3"'], None, []), ] @pytest.mark.parametrize( 'added_testsuite_count, load_errors, ql, qv, ql_data, exception, expected_logs', TESTDATA_3, ids=['no tests', 'load errors', 'quarantine verify without quarantine list', # 'empty quarantine file', 'valid quarantine file'] ) def test_testplan_discover( tmp_path, caplog, added_testsuite_count, load_errors, ql, qv, ql_data, exception, expected_logs ): for qf, data in zip(ql, ql_data): tmp_qf = tmp_path / qf tmp_qf.write_text(data) testplan = TestPlan(env=mock.Mock()) testplan.options = mock.Mock( test='ts1', quarantine_list=[tmp_path / qf for qf in ql], quarantine_verify=qv, ) testplan.testsuites = { 'ts1': mock.Mock(id=1), 'ts2': mock.Mock(id=2), } testplan.run_individual_testsuite = 'ts0' testplan.load_errors = load_errors testplan.add_testsuites = mock.Mock(return_value=added_testsuite_count) testplan.find_subtests = mock.Mock() testplan.report_duplicates = mock.Mock() testplan.parse_configuration = mock.Mock() testplan.add_configurations = mock.Mock() with pytest.raises(exception) if exception else nullcontext(): testplan.discover() testplan.add_testsuites.assert_called_once_with(testsuite_filter='ts1') assert all([log in caplog.text for log in expected_logs]) TESTDATA_4 = [ (None, None, None, None, '00', TwisterRuntimeError, [], []), (None, True, None, None, '6/4', TwisterRuntimeError, set(['t-p3', 't-p4', 't-p1', 't-p2']), []), (None, None, 'load_tests.json', None, '0/4', TwisterRuntimeError, set(['lt-p1', 'lt-p3', 'lt-p4', 'lt-p2']), []), ('suffix', None, None, True, '2/4', None, set(['ts-p4', 'ts-p2', 'ts-p1', 'ts-p3']), [2, 4]), ] @pytest.mark.parametrize( 'report_suffix, only_failed, load_tests, test_only, subset,' \ ' exception, expected_selected_platforms, expected_generate_subset_args', TESTDATA_4, ids=['apply_filters only', 'only failed', 'load tests', 'test only'] ) def test_testplan_load( tmp_path, report_suffix, only_failed, load_tests, test_only, subset, exception, expected_selected_platforms, expected_generate_subset_args ): twister_json = """\ { "testsuites": [ { "name": "ts1", "platform": "t-p1", "testcases": [] }, { "name": "ts1", "platform": "t-p2", "testcases": [] }, { "name": "ts2", "platform": "t-p3", "testcases": [] }, { "name": "ts2", "platform": "t-p4", "testcases": [] } ] } """ twister_file = tmp_path / 'twister.json' twister_file.write_text(twister_json) twister_suffix_json = """\ { "testsuites": [ { "name": "ts1", "platform": "ts-p1", "testcases": [] }, { "name": "ts1", "platform": "ts-p2", "testcases": [] }, { "name": "ts2", "platform": "ts-p3", "testcases": [] }, { "name": "ts2", "platform": "ts-p4", "testcases": [] } ] } """ twister_suffix_file = tmp_path / 'twister_suffix.json' twister_suffix_file.write_text(twister_suffix_json) load_tests_json = """\ { "testsuites": [ { "name": "ts1", "platform": "lt-p1", "testcases": [] }, { "name": "ts1", "platform": "lt-p2", "testcases": [] }, { "name": "ts2", "platform": "lt-p3", \"testcases": [] }, { "name": "ts2", "platform": "lt-p4", "testcases": [] } ] } """ load_tests_file = tmp_path / 'load_tests.json' load_tests_file.write_text(load_tests_json) testplan = TestPlan(env=mock.Mock(outdir=tmp_path)) testplan.testsuites = { 'ts1': mock.Mock(testcases=[], extra_configs=[]), 'ts2': mock.Mock(testcases=[], extra_configs=[]), } testplan.testsuites['ts1'].name = 'ts1' testplan.testsuites['ts2'].name = 'ts2' testplan.options = mock.Mock( report_summary=None, outdir=tmp_path, report_suffix=report_suffix, only_failed=only_failed, load_tests=tmp_path / load_tests if load_tests else None, test_only=test_only, exclude_platform=['t-p0', 't-p1', 'ts-p0', 'ts-p1', 'lt-p0', 'lt-p1'], platform=['t-p1', 't-p2', 't-p3', 't-p4', 'ts-p1', 'ts-p2', 'ts-p3', 'ts-p4', 'lt-p1', 'lt-p2', 'lt-p3', 'lt-p4'], subset=subset ) testplan.platforms=[mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock()] testplan.platforms[0].name = 't-p1' testplan.platforms[1].name = 't-p2' testplan.platforms[2].name = 't-p3' testplan.platforms[3].name = 't-p4' testplan.platforms[4].name = 'ts-p1' testplan.platforms[5].name = 'ts-p2' testplan.platforms[6].name = 'ts-p3' testplan.platforms[7].name = 'ts-p4' testplan.platforms[8].name = 'lt-p1' testplan.platforms[9].name = 'lt-p2' testplan.platforms[10].name = 'lt-p3' testplan.platforms[11].name = 'lt-p4' testplan.platforms[0].aliases = ['t-p1'] testplan.platforms[1].aliases = ['t-p2'] testplan.platforms[2].aliases = ['t-p3'] testplan.platforms[3].aliases = ['t-p4'] testplan.platforms[4].aliases = ['ts-p1'] testplan.platforms[5].aliases = ['ts-p2'] testplan.platforms[6].aliases = ['ts-p3'] testplan.platforms[7].aliases = ['ts-p4'] testplan.platforms[8].aliases = ['lt-p1'] testplan.platforms[9].aliases = ['lt-p2'] testplan.platforms[10].aliases = ['lt-p3'] testplan.platforms[11].aliases = ['lt-p4'] testplan.platforms[0].normalized_name = 't-p1' testplan.platforms[1].normalized_name = 't-p2' testplan.platforms[2].normalized_name = 't-p3' testplan.platforms[3].normalized_name = 't-p4' testplan.platforms[4].normalized_name = 'ts-p1' testplan.platforms[5].normalized_name = 'ts-p2' testplan.platforms[6].normalized_name = 'ts-p3' testplan.platforms[7].normalized_name = 'ts-p4' testplan.platforms[8].normalized_name = 'lt-p1' testplan.platforms[9].normalized_name = 'lt-p2' testplan.platforms[10].normalized_name = 'lt-p3' testplan.platforms[11].normalized_name = 'lt-p4' testplan.generate_subset = mock.Mock() testplan.apply_filters = mock.Mock() with mock.patch('twisterlib.testinstance.TestInstance.create_overlay', mock.Mock()), \ mock.patch('twisterlib.testinstance.TestInstance.check_runnable', return_value=True), \ pytest.raises(exception) if exception else nullcontext(): testplan.load() assert testplan.selected_platforms == expected_selected_platforms if expected_generate_subset_args: testplan.generate_subset.assert_called_once_with(*expected_generate_subset_args) else: testplan.generate_subset.assert_not_called() TESTDATA_5 = [ (False, False, None, 1, 2, ['plat1/testA', 'plat1/testB', 'plat1/testC', 'plat3/testA', 'plat3/testB', 'plat3/testC']), (False, False, None, 1, 5, ['plat1/testA', 'plat3/testA', 'plat3/testB', 'plat3/testC']), (False, False, None, 2, 2, ['plat2/testA', 'plat2/testB']), (True, False, None, 1, 2, ['plat1/testA', 'plat2/testA', 'plat1/testB', 'plat3/testA', 'plat3/testB', 'plat3/testC']), (True, False, None, 2, 2, ['plat2/testB', 'plat1/testC']), (True, True, 123, 1, 2, ['plat2/testA', 'plat2/testB', 'plat1/testC', 'plat3/testB', 'plat3/testA', 'plat3/testC']), (True, True, 123, 2, 2, ['plat1/testB', 'plat1/testA']), ] @pytest.mark.parametrize( 'device_testing, shuffle, seed, subset, sets, expected_subset', TESTDATA_5, ids=['subset 1', 'subset 1 out of 5', 'subset 2', 'device testing, subset 1', 'device testing, subset 2', 'device testing, shuffle with seed, subset 1', 'device testing, shuffle with seed, subset 2'] ) def test_testplan_generate_subset( device_testing, shuffle, seed, subset, sets, expected_subset ): testplan = TestPlan(env=mock.Mock()) testplan.options = mock.Mock( device_testing=device_testing, shuffle_tests=shuffle, shuffle_tests_seed=seed ) testplan.instances = { 'plat1/testA': mock.Mock(status=TwisterStatus.NONE), 'plat1/testB': mock.Mock(status=TwisterStatus.NONE), 'plat1/testC': mock.Mock(status=TwisterStatus.NONE), 'plat2/testA': mock.Mock(status=TwisterStatus.NONE), 'plat2/testB': mock.Mock(status=TwisterStatus.NONE), 'plat3/testA': mock.Mock(status=TwisterStatus.SKIP), 'plat3/testB': mock.Mock(status=TwisterStatus.SKIP), 'plat3/testC': mock.Mock(status=TwisterStatus.ERROR), } testplan.generate_subset(subset, sets) assert [instance for instance in testplan.instances.keys()] == \ expected_subset def test_testplan_handle_modules(): testplan = TestPlan(env=mock.Mock()) modules = [mock.Mock(meta={'name': 'name1'}), mock.Mock(meta={'name': 'name2'})] with mock.patch('twisterlib.testplan.parse_modules', return_value=modules): testplan.handle_modules() assert testplan.modules == ['name1', 'name2'] TESTDATA_6 = [ (True, False, False, 0, 'report_test_tree'), (True, True, False, 0, 'report_test_tree'), (True, False, True, 0, 'report_test_tree'), (True, True, True, 0, 'report_test_tree'), (False, True, False, 0, 'report_test_list'), (False, True, True, 0, 'report_test_list'), (False, False, True, 0, 'report_tag_list'), (False, False, False, 1, None), ] @pytest.mark.parametrize( 'test_tree, list_tests, list_tags, expected_res, expected_method', TESTDATA_6, ids=['test tree', 'test tree + test list', 'test tree + tag list', 'test tree + test list + tag list', 'test list', 'test list + tag list', 'tag list', 'no report'] ) def test_testplan_report( test_tree, list_tests, list_tags, expected_res, expected_method ): testplan = TestPlan(env=mock.Mock()) testplan.report_test_tree = mock.Mock() testplan.report_test_list = mock.Mock() testplan.report_tag_list = mock.Mock() testplan.options = mock.Mock( test_tree=test_tree, list_tests=list_tests, list_tags=list_tags, ) res = testplan.report() assert res == expected_res methods = ['report_test_tree', 'report_test_list', 'report_tag_list'] if expected_method: methods.remove(expected_method) getattr(testplan, expected_method).assert_called_once() for method in methods: getattr(testplan, method).assert_not_called() TESTDATA_7 = [ ( [ mock.Mock( yamlfile='a.yaml', scenarios=['scenario1', 'scenario2'] ), mock.Mock( yamlfile='b.yaml', scenarios=['scenario1'] ) ], TwisterRuntimeError, 'Duplicated test scenarios found:\n' \ '- scenario1 found in:\n' \ ' - a.yaml\n' \ ' - b.yaml\n', [] ), ( [ mock.Mock( yamlfile='a.yaml', scenarios=['scenario.a.1', 'scenario.a.2'] ), mock.Mock( yamlfile='b.yaml', scenarios=['scenario.b.1'] ) ], None, None, ['No duplicates found.'] ), ] @pytest.mark.parametrize( 'testsuites, expected_error, error_msg, expected_logs', TESTDATA_7, ids=['a duplicate', 'no duplicates'] ) def test_testplan_report_duplicates( capfd, caplog, testsuites, expected_error, error_msg, expected_logs ): def mock_get(name): return list(filter(lambda x: name in x.scenarios, testsuites)) testplan = TestPlan(env=mock.Mock()) testplan.scenarios = [scenario for testsuite in testsuites \ for scenario in testsuite.scenarios] testplan.get_testsuite = mock.Mock(side_effect=mock_get) with pytest.raises(expected_error) if expected_error is not None else \ nullcontext() as err: testplan.report_duplicates() if expected_error: assert str(err._excinfo[1]) == error_msg assert all([log in caplog.text for log in expected_logs]) def test_testplan_report_tag_list(capfd): testplan = TestPlan(env=mock.Mock()) testplan.testsuites = { 'testsuite0': mock.Mock(tags=set(['tag1', 'tag2'])), 'testsuite1': mock.Mock(tags=set(['tag1', 'tag2', 'tag3'])), 'testsuite2': mock.Mock(tags=set(['tag1', 'tag3'])), 'testsuite3': mock.Mock(tags=set(['tag'])) } testplan.report_tag_list() out,err = capfd.readouterr() sys.stdout.write(out) sys.stderr.write(err) assert '- tag' in out assert '- tag1' in out assert '- tag2' in out assert '- tag3' in out def test_testplan_report_test_tree(capfd): testplan = TestPlan(env=mock.Mock()) testplan.get_tests_list = mock.Mock( return_value=['1.dummy.case.1', '1.dummy.case.2', '2.dummy.case.1', '2.dummy.case.2', '3.dummy.case.1', '3.dummy.case.2', '4.dummy.case.1', '4.dummy.case.2', '5.dummy.case.1', '5.dummy.case.2', 'sample.group1.case1', 'sample.group1.case2', 'sample.group2.case', 'sample.group3.case1', 'sample.group3.case2', 'sample.group3.case3'] ) testplan.report_test_tree() out,err = capfd.readouterr() sys.stdout.write(out) sys.stderr.write(err) expected = """ Testsuite ├── Samples │ ├── group1 │ │ ├── sample.group1.case1 │ │ └── sample.group1.case2 │ ├── group2 │ │ └── sample.group2.case │ └── group3 │ ├── sample.group3.case1 │ ├── sample.group3.case2 │ └── sample.group3.case3 └── Tests ├── 1 │ └── dummy │ ├── 1.dummy.case.1 │ └── 1.dummy.case.2 ├── 2 │ └── dummy │ ├── 2.dummy.case.1 │ └── 2.dummy.case.2 ├── 3 │ └── dummy │ ├── 3.dummy.case.1 │ └── 3.dummy.case.2 ├── 4 │ └── dummy │ ├── 4.dummy.case.1 │ └── 4.dummy.case.2 └── 5 └── dummy ├── 5.dummy.case.1 └── 5.dummy.case.2 """ expected = expected[1:] assert expected in out def test_testplan_report_test_list(capfd): testplan = TestPlan(env=mock.Mock()) testplan.get_tests_list = mock.Mock( return_value=['4.dummy.case.1', '4.dummy.case.2', '3.dummy.case.2', '2.dummy.case.2', '1.dummy.case.1', '1.dummy.case.2', '3.dummy.case.1', '2.dummy.case.1', '5.dummy.case.1', '5.dummy.case.2'] ) testplan.report_test_list() out,err = capfd.readouterr() sys.stdout.write(out) sys.stderr.write(err) assert ' - 1.dummy.case.1\n' \ ' - 1.dummy.case.2\n' \ ' - 2.dummy.case.1\n' \ ' - 2.dummy.case.2\n' \ ' - 3.dummy.case.1\n' \ ' - 3.dummy.case.2\n' \ ' - 4.dummy.case.1\n' \ ' - 4.dummy.case.2\n' \ ' - 5.dummy.case.1\n' \ ' - 5.dummy.case.2\n' \ '10 total.' in out def test_testplan_info(capfd): TestPlan.info('dummy text') out, err = capfd.readouterr() sys.stdout.write(out) sys.stderr.write(err) assert 'dummy text\n' in out TESTDATA_8 = [ (False, ['p1e2/unit_testing', 'p2/unit_testing', 'p3/unit_testing'], ['p2/unit_testing', 'p3/unit_testing']), (True, ['p1e2/unit_testing', 'p2/unit_testing', 'p3/unit_testing'], ['p3/unit_testing']), ] @pytest.mark.parametrize( 'override_default_platforms, expected_platform_names, expected_defaults', TESTDATA_8, ids=['no override defaults', 'override defaults'] ) def test_testplan_add_configurations( tmp_path, override_default_platforms, expected_platform_names, expected_defaults ): # tmp_path # └ boards <- board root # ├ zephyr # │ ├ p1 # │ | ├ p1e1.yaml # │ | └ p1e2.yaml # │ └ p2 # │ ├ p2.yaml # │ └ p2-1.yaml <- duplicate # │ └ p2-2.yaml <- load error # └ arm # └ p3 # ├ p3.yaml # └ p3_B.conf tmp_soc_root_dir = tmp_path / 'soc' tmp_soc_root_dir.mkdir() tmp_vend1_dir = tmp_soc_root_dir / 'zephyr' tmp_vend1_dir.mkdir() tmp_soc1_dir = tmp_vend1_dir / 's1' tmp_soc1_dir.mkdir() soc1_yaml = """\ family: - name: zephyr series: - name: zephyr_testing socs: - name: unit_testing """ soc1_yamlfile = tmp_soc1_dir / 'soc.yml' soc1_yamlfile.write_text(soc1_yaml) tmp_board_root_dir = tmp_path / 'boards' tmp_board_root_dir.mkdir() tmp_vend1_dir = tmp_board_root_dir / 'zephyr' tmp_vend1_dir.mkdir() tmp_p1_dir = tmp_vend1_dir / 'p1' tmp_p1_dir.mkdir() p1e1_bs_yaml = """\ boards: - name: p1e1 vendor: zephyr socs: - name: unit_testing - name: p1e2 vendor: zephyr socs: - name: unit_testing """ p1e1_yamlfile = tmp_p1_dir / 'board.yml' p1e1_yamlfile.write_text(p1e1_bs_yaml) p1e1_yaml = """\ identifier: p1e1 name: Platform 1 Edition 1 type: native arch: x86 vendor: zephyr toolchain: - zephyr twister: False """ p1e1_yamlfile = tmp_p1_dir / 'p1e1.yaml' p1e1_yamlfile.write_text(p1e1_yaml) p1e2_yaml = """\ identifier: p1e2 name: Platform 1 Edition 2 type: native arch: x86 vendor: zephyr toolchain: - zephyr """ p1e2_yamlfile = tmp_p1_dir / 'p1e2.yaml' p1e2_yamlfile.write_text(p1e2_yaml) tmp_p2_dir = tmp_vend1_dir / 'p2' tmp_p2_dir.mkdir() p2_bs_yaml = """\ boards: - name: p2 vendor: zephyr socs: - name: unit_testing - name: p2_2 vendor: zephyr socs: - name: unit_testing """ p2_yamlfile = tmp_p2_dir / 'board.yml' p2_yamlfile.write_text(p2_bs_yaml) p2_yaml = """\ identifier: p2/unit_testing name: Platform 2 type: sim arch: x86 vendor: vendor2 toolchain: - zephyr testing: default: True """ p2_yamlfile = tmp_p2_dir / 'p2.yaml' p2_yamlfile.write_text(p2_yaml) p2_2_yaml = """\ testing: ć#@%!#!#^#@%@:1.0 identifier: p2_2 name: Platform 2 2 type: sim arch: x86 vendor: vendor2 toolchain: - zephyr """ p2_2_yamlfile = tmp_p2_dir / 'p2-2.yaml' p2_2_yamlfile.write_text(p2_2_yaml) tmp_vend2_dir = tmp_board_root_dir / 'arm' tmp_vend2_dir.mkdir() tmp_p3_dir = tmp_vend2_dir / 'p3' tmp_p3_dir.mkdir() p3_bs_yaml = """\ boards: - name: p3 vendor: zephyr socs: - name: unit_testing """ p3_yamlfile = tmp_p3_dir / 'board.yml' p3_yamlfile.write_text(p3_bs_yaml) p3_yaml = """\ identifier: p3 name: Platform 3 type: unit arch: arm vendor: vendor3 toolchain: - zephyr testing: default: True """ p3_yamlfile = tmp_p3_dir / 'p3.yaml' p3_yamlfile.write_text(p3_yaml) env = mock.Mock(board_roots=[tmp_board_root_dir],soc_roots=[tmp_path], arch_roots=[tmp_path]) testplan = TestPlan(env=env) testplan.test_config = { 'platforms': { 'override_default_platforms': override_default_platforms, 'default_platforms': ['p3', 'p1e1'] } } testplan.add_configurations() if expected_defaults is not None: print(expected_defaults) print(testplan.default_platforms) assert sorted(expected_defaults) == sorted(testplan.default_platforms) if expected_platform_names is not None: print(expected_platform_names) print(testplan.platform_names) platform_names = [p.name for p in testplan.platforms] assert sorted(expected_platform_names) == sorted(platform_names) def test_testplan_get_all_tests(): testplan = TestPlan(env=mock.Mock()) tc1 = mock.Mock() tc1.name = 'tc1' tc2 = mock.Mock() tc2.name = 'tc2' tc3 = mock.Mock() tc3.name = 'tc3' tc4 = mock.Mock() tc4.name = 'tc4' tc5 = mock.Mock() tc5.name = 'tc5' ts1 = mock.Mock(testcases=[tc1, tc2]) ts2 = mock.Mock(testcases=[tc3, tc4, tc5]) testplan.testsuites = { 'ts1': ts1, 'ts2': ts2 } res = testplan.get_all_tests() assert sorted(res) == ['tc1', 'tc2', 'tc3', 'tc4', 'tc5'] TESTDATA_9 = [ ([], False, True, 11, 1), ([], False, False, 7, 2), ([], True, False, 9, 1), ([], True, True, 9, 1), ([], True, False, 9, 1), (['good_test/dummy.common.1', 'good_test/dummy.common.2', 'good_test/dummy.common.3'], False, True, 3, 1), (['good_test/dummy.common.1', 'good_test/dummy.common.2', 'duplicate_test/dummy.common.1', 'duplicate_test/dummy.common.2'], False, True, 4, 1), (['dummy.common.1', 'dummy.common.2'], False, False, 2, 1), (['good_test/dummy.common.1', 'good_test/dummy.common.2', 'good_test/dummy.common.3'], True, True, 0, 1), ] @pytest.mark.parametrize( 'testsuite_filter, use_alt_root, detailed_id, expected_suite_count, expected_errors', TESTDATA_9, ids=[ 'no testsuite filter, detailed id', 'no testsuite filter, short id', 'no testsuite filter, alt root, detailed id', 'no filter, alt root, detailed id', 'no filter, alt root, short id', 'testsuite filter', 'testsuite filter and valid duplicate', 'testsuite filter, short id and duplicate', 'testsuite filter, alt root', ] ) def test_testplan_add_testsuites(tmp_path, testsuite_filter, use_alt_root, detailed_id, expected_errors, expected_suite_count): # tmp_path # ├ tests <- test root # │ ├ good_test # │ │ └ testcase.yaml # │ ├ wrong_test # │ │ └ testcase.yaml # │ ├ good_sample # │ │ └ sample.yaml # │ ├ duplicate_test # │ │ └ testcase.yaml # │ └ others # │ └ other.txt # └ other_tests <- alternate test root # └ good_test # └ testcase.yaml tmp_test_root_dir = tmp_path / 'tests' tmp_test_root_dir.mkdir() tmp_good_test_dir = tmp_test_root_dir / 'good_test' tmp_good_test_dir.mkdir() testcase_yaml_1 = """\ tests: dummy.common.1: build_on_all: true dummy.common.2: build_on_all: true dummy.common.3: build_on_all: true dummy.special: build_on_all: false """ testfile_1 = tmp_good_test_dir / 'testcase.yaml' testfile_1.write_text(testcase_yaml_1) tmp_bad_test_dir = tmp_test_root_dir / 'wrong_test' tmp_bad_test_dir.mkdir() testcase_yaml_2 = """\ tests: wrong: yaml: {]} """ testfile_2 = tmp_bad_test_dir / 'testcase.yaml' testfile_2.write_text(testcase_yaml_2) tmp_good_sample_dir = tmp_test_root_dir / 'good_sample' tmp_good_sample_dir.mkdir() samplecase_yaml_1 = """\ tests: sample.dummy.common.1: tags: - samples sample.dummy.common.2: tags: - samples sample.dummy.special.1: tags: - samples """ samplefile_1 = tmp_good_sample_dir / 'sample.yaml' samplefile_1.write_text(samplecase_yaml_1) tmp_duplicate_test_dir = tmp_test_root_dir / 'duplicate_test' tmp_duplicate_test_dir.mkdir() # The duplicate needs to have the same number of tests as these configurations # can be read either with duplicate_test first, or good_test first, so number # of selected tests needs to be the same in both situations. testcase_yaml_4 = """\ tests: dummy.common.1: build_on_all: true dummy.common.2: build_on_all: true dummy.common.3: build_on_all: true dummy.special: build_on_all: false """ testfile_4 = tmp_duplicate_test_dir / 'testcase.yaml' testfile_4.write_text(testcase_yaml_4) tmp_other_dir = tmp_test_root_dir / 'others' tmp_other_dir.mkdir() _ = tmp_other_dir / 'other.txt' tmp_alt_test_root_dir = tmp_path / 'other_tests' tmp_alt_test_root_dir.mkdir() tmp_alt_good_test_dir = tmp_alt_test_root_dir / 'good_test' tmp_alt_good_test_dir.mkdir() testcase_yaml_3 = """\ tests: dummy.alt.1: build_on_all: true dummy.alt.2: build_on_all: true """ testfile_3 = tmp_alt_good_test_dir / 'testcase.yaml' testfile_3.write_text(testcase_yaml_3) env = mock.Mock( test_roots=[tmp_test_root_dir], options=mock.Mock(detailed_test_id=detailed_id), alt_config_root=[tmp_alt_test_root_dir] if use_alt_root else [] ) testplan = TestPlan(env=env) res = testplan.add_testsuites(testsuite_filter) assert res == expected_suite_count assert testplan.load_errors == expected_errors def test_testplan_str(): testplan = TestPlan(env=mock.Mock()) testplan.name = 'my name' res = testplan.__str__() assert res == 'my name' TESTDATA_10 = [ ('a platform', True), ('other platform', False), ] @pytest.mark.parametrize( 'name, expect_found', TESTDATA_10, ids=['platform exists', 'no platform'] ) def test_testplan_get_platform(name, expect_found): testplan = TestPlan(env=mock.Mock()) p1 = mock.Mock() p1.name = 'some platform' p1.aliases = [p1.name] p2 = mock.Mock() p2.name = 'a platform' p2.aliases = [p2.name] testplan.platforms = [p1, p2] res = testplan.get_platform(name) if expect_found: assert res.name == name else: assert res is None TESTDATA_11 = [ (True, 'runnable'), (False, 'buildable'), ] @pytest.mark.parametrize( 'device_testing, expected_tfilter', TESTDATA_11, ids=['device testing', 'no device testing'] ) def test_testplan_load_from_file(caplog, device_testing, expected_tfilter): def get_platform(name): p = mock.Mock() p.name = name p.normalized_name = name return p ts1tc1 = mock.Mock() ts1tc1.name = 'TS1.tc1' ts1 = mock.Mock(testcases=[ts1tc1]) ts1.name = 'TestSuite 1' ts2 = mock.Mock(testcases=[]) ts2.name = 'TestSuite 2' ts3tc1 = mock.Mock() ts3tc1.name = 'TS3.tc1' ts3tc2 = mock.Mock() ts3tc2.name = 'TS3.tc2' ts3 = mock.Mock(testcases=[ts3tc1, ts3tc2]) ts3.name = 'TestSuite 3' ts4tc1 = mock.Mock() ts4tc1.name = 'TS4.tc1' ts4 = mock.Mock(testcases=[ts4tc1]) ts4.name = 'TestSuite 4' ts5 = mock.Mock(testcases=[]) ts5.name = 'TestSuite 5' testplan = TestPlan(env=mock.Mock(outdir=os.path.join('out', 'dir'))) testplan.options = mock.Mock(device_testing=device_testing, test_only=True, report_summary=None) testplan.testsuites = { 'TestSuite 1': ts1, 'TestSuite 2': ts2, 'TestSuite 3': ts3, 'TestSuite 4': ts4, 'TestSuite 5': ts5 } testplan.get_platform = mock.Mock(side_effect=get_platform) testplan_data = """\ { "testsuites": [ { "name": "TestSuite 1", "platform": "Platform 1", "run_id": 1, "execution_time": 60.00, "used_ram": 4096, "available_ram": 12278, "used_rom": 1024, "available_rom": 1047552, "status": "passed", "reason": "OK", "testcases": [ { "identifier": "TS1.tc1", "status": "passed", "reason": "passed", "execution_time": 60.00, "log": "" } ] }, { "name": "TestSuite 2", "platform": "Platform 1" }, { "name": "TestSuite 3", "platform": "Platform 1", "run_id": 1, "execution_time": 360.00, "used_ram": 4096, "available_ram": 12278, "used_rom": 1024, "available_rom": 1047552, "status": "error", "reason": "File Not Found Error", "testcases": [ { "identifier": "TS3.tc1", "status": "error", "reason": "File Not Found Error.", "execution_time": 360.00, "log": "[ERROR]: File 'dummy.yaml' not found!\\nClosing..." }, { "identifier": "TS3.tc2" } ] }, { "name": "TestSuite 4", "platform": "Platform 1", "execution_time": 360.00, "used_ram": 4096, "available_ram": 12278, "used_rom": 1024, "available_rom": 1047552, "status": "skipped", "reason": "Not in requested test list.", "testcases": [ { "identifier": "TS4.tc1", "status": "skipped", "reason": "Not in requested test list.", "execution_time": 360.00, "log": "[INFO] Parsing..." }, { "identifier": "TS3.tc2" } ] }, { "name": "TestSuite 5", "platform": "Platform 2" } ] } """ filter_platform = ['Platform 1'] check_runnable_mock = mock.Mock(return_value=True) with mock.patch('builtins.open', mock.mock_open(read_data=testplan_data)), \ mock.patch('twisterlib.testinstance.TestInstance.check_runnable', check_runnable_mock), \ mock.patch('twisterlib.testinstance.TestInstance.create_overlay', mock.Mock()): testplan.load_from_file('dummy.yaml', filter_platform) expected_instances = { 'Platform 1/TestSuite 1': { 'metrics': { 'handler_time': 60.0, 'used_ram': 4096, 'used_rom': 1024, 'available_ram': 12278, 'available_rom': 1047552 }, 'retries': 0, 'testcases': { 'TS1.tc1': { 'status': TwisterStatus.PASS, 'reason': 'passed', 'duration': 60.0, 'output': '' } } }, 'Platform 1/TestSuite 2': { 'metrics': { 'handler_time': 0, 'used_ram': 0, 'used_rom': 0, 'available_ram': 0, 'available_rom': 0 }, 'retries': 0, 'testcases': [] }, 'Platform 1/TestSuite 3': { 'metrics': { 'handler_time': 360.0, 'used_ram': 4096, 'used_rom': 1024, 'available_ram': 12278, 'available_rom': 1047552 }, 'retries': 1, 'testcases': { 'TS3.tc1': { 'status': TwisterStatus.ERROR, 'reason': None, 'duration': 360.0, 'output': '[ERROR]: File \'dummy.yaml\' not found!\nClosing...' }, 'TS3.tc2': { 'status': TwisterStatus.NONE, 'reason': None, 'duration': 0, 'output': '' } } }, 'Platform 1/TestSuite 4': { 'metrics': { 'handler_time': 360.0, 'used_ram': 4096, 'used_rom': 1024, 'available_ram': 12278, 'available_rom': 1047552 }, 'retries': 0, 'testcases': { 'TS4.tc1': { 'status': TwisterStatus.SKIP, 'reason': 'Not in requested test list.', 'duration': 360.0, 'output': '[INFO] Parsing...' } } }, } for n, i in testplan.instances.items(): assert expected_instances[n]['metrics'] == i.metrics assert expected_instances[n]['retries'] == i.retries for t in i.testcases: assert expected_instances[n]['testcases'][str(t)]['status'] == t.status assert expected_instances[n]['testcases'][str(t)]['reason'] == t.reason assert expected_instances[n]['testcases'][str(t)]['duration'] == t.duration assert expected_instances[n]['testcases'][str(t)]['output'] == t.output check_runnable_mock.assert_called_with(mock.ANY, mock.ANY) expected_logs = [ 'loading TestSuite 1...', 'loading TestSuite 2...', 'loading TestSuite 3...', 'loading TestSuite 4...', ] assert all([log in caplog.text for log in expected_logs]) def test_testplan_add_instances(): testplan = TestPlan(env=mock.Mock()) instance1 = mock.Mock() instance1.name = 'instance 1' instance2 = mock.Mock() instance2.name = 'instance 2' instance_list = [instance1, instance2] testplan.add_instances(instance_list) assert testplan.instances == { 'instance 1': instance1, 'instance 2': instance2, } def test_testplan_get_testcase(): testplan = TestPlan(env=mock.Mock()) testplan.testsuites = { 'test1.suite0': mock.Mock(testcases=[mock.Mock(), mock.Mock()]), 'test1.suite1': mock.Mock(testcases=[mock.Mock(), mock.Mock()]), 'test1.suite2': mock.Mock(testcases=[mock.Mock(), mock.Mock()]), 'test1.suite3': mock.Mock(testcases=[]) } testplan.testsuites['test1.suite0'].testcases[0].name = 'test1.suite0.case0' testplan.testsuites['test1.suite0'].testcases[1].name = 'test1.suite0.case1' # testplan.testsuites['test1.suite1'].testcases[0].name = 'test1.suite1.case0' testplan.testsuites['test1.suite1'].testcases[1].name = 'test1.suite1.case0' # in suite duplicate # testplan.testsuites['test1.suite2'].testcases[0].name = 'test1.suite2.case0' testplan.testsuites['test1.suite2'].testcases[1].name = 'test1.suite1.case0' # out suite duplicate id = 'test1.suite1.case0' res = testplan.get_testcase(id) assert len(res) == 3 assert testplan.testsuites['test1.suite1'] in res assert testplan.testsuites['test1.suite2'] in res def test_testplan_verify_platforms_existence(caplog): testplan = TestPlan(env=mock.Mock()) testplan.platform_names = ['a platform', 'other platform'] platform_names = ['other platform', 'some platform'] log_info = 'PLATFORM ERROR' with pytest.raises(SystemExit) as se: testplan.verify_platforms_existence(platform_names, log_info) assert str(se.value) == '2' assert 'PLATFORM ERROR - unrecognized platform - some platform' TESTDATA_12 = [ (True), (False) ] @pytest.mark.parametrize( 'exists', TESTDATA_12, ids=['links dir exists', 'links dir does not exist'] ) def test_testplan_create_build_dir_links(exists): outdir = os.path.join('out', 'dir') instances_linked = [] def mock_link(links_dir_path, instance): assert links_dir_path == os.path.join(outdir, 'twister_links') instances_linked.append(instance) instances = { 'inst0': mock.Mock(status=TwisterStatus.PASS), 'inst1': mock.Mock(status=TwisterStatus.SKIP), 'inst2': mock.Mock(status=TwisterStatus.ERROR), } expected_instances = [instances['inst0'], instances['inst2']] testplan = TestPlan(env=mock.Mock(outdir=outdir)) testplan._create_build_dir_link = mock.Mock(side_effect=mock_link) testplan.instances = instances with mock.patch('os.path.exists', return_value=exists), \ mock.patch('os.mkdir', mock.Mock()) as mkdir_mock: testplan.create_build_dir_links() if not exists: mkdir_mock.assert_called_once() assert expected_instances == instances_linked TESTDATA_13 = [ ('nt'), ('Linux') ] @pytest.mark.parametrize( 'os_name', TESTDATA_13, ) def test_testplan_create_build_dir_link(os_name): def mock_makedirs(path, exist_ok=False): assert exist_ok assert path == instance_build_dir def mock_symlink(source, target): assert source == instance_build_dir assert target == os.path.join('links', 'path', 'test_0') def mock_call(cmd, shell=False): assert shell assert cmd == ['mklink', '/J', os.path.join('links', 'path', 'test_0'), instance_build_dir] def mock_join(*paths): slash = "\\" if os.name == 'nt' else "/" return slash.join(paths) with mock.patch('os.name', os_name), \ mock.patch('os.symlink', side_effect=mock_symlink), \ mock.patch('os.makedirs', side_effect=mock_makedirs), \ mock.patch('subprocess.call', side_effect=mock_call), \ mock.patch('os.path.join', side_effect=mock_join): testplan = TestPlan(env=mock.Mock()) links_dir_path = os.path.join('links', 'path') instance_build_dir = os.path.join('some', 'far', 'off', 'build', 'dir') instance = mock.Mock(build_dir=instance_build_dir) testplan._create_build_dir_link(links_dir_path, instance) assert instance.build_dir == os.path.join('links', 'path', 'test_0') assert testplan.link_dir_counter == 1 TESTDATA_14 = [ ('bad platform', 'dummy reason', [], 'dummy status', 'dummy reason'), ('good platform', 'quarantined', [], TwisterStatus.ERROR, 'quarantined but is one of the integration platforms'), ('good platform', 'dummy reason', [{'type': 'command line filter'}], 'dummy status', 'dummy reason'), ('good platform', 'dummy reason', [{'type': 'Skip filter'}], 'dummy status', 'dummy reason'), ('good platform', 'dummy reason', [{'type': 'platform key filter'}], 'dummy status', 'dummy reason'), ('good platform', 'dummy reason', [{'type': 'Toolchain filter'}], 'dummy status', 'dummy reason'), ('good platform', 'dummy reason', [{'type': 'Module filter'}], 'dummy status', 'dummy reason'), ('good platform', 'dummy reason', [{'type': 'testsuite filter'}], TwisterStatus.ERROR, 'dummy reason but is one of the integration platforms'), ] @pytest.mark.parametrize( 'platform_name, reason, filters,' \ ' expected_status, expected_reason', TESTDATA_14, ids=['wrong platform', 'quarantined', 'command line filtered', 'skip filtered', 'platform key filtered', 'toolchain filtered', 'module filtered', 'skip to error change'] ) def test_change_skip_to_error_if_integration( platform_name, reason, filters, expected_status, expected_reason ): options = mock.Mock() platform = mock.Mock() platform.name = platform_name testsuite = mock.Mock(integration_platforms=['good platform', 'a platform']) instance = mock.Mock( testsuite=testsuite, platform=platform, filters=filters, status='dummy status', reason=reason ) change_skip_to_error_if_integration(options, instance) assert instance.status == expected_status assert instance.reason == expected_reason