Lines Matching +full:bsim +full:- +full:test +full:- +full:results
3 # Copyright (c) 2018-2024 Intel Corporation
5 # SPDX-License-Identifier: Apache-2.0
68 Most of the stats are at test instance level
69 Except that case statistics are for cases of ALL test instances
71 total = yaml test scenarios * applicable platforms
74 completed = done - filtered_static
77 pass rate = passed / (total - filtered_configs)
78 case pass rate = passed_cases / (cases - filtered_cases - skipped_cases)
112 # initialized to number of test instances
147 length = int(log10(-n))+2
151 selected_cases = self.cases - self.filtered_cases
152 selected_configs = self.done - self.filtered_static - self.filtered_runtime
157 Node(f"Total test suites: {self.total}", parent=root)
158 processed_suites = Node(f"Processed test suites: {self.done}", parent=root)
160 f"Filtered test suites: {self.filtered_configs}",
163 Node(f"Filtered test suites (static): {self.filtered_static}", parent=filtered_suites)
164 Node(f"Filtered test suites (at runtime): {self.filtered_runtime}", parent=filtered_suites)
165 selected_suites = Node(f"Selected test suites: {selected_configs}", parent=processed_suites)
166 Node(f"Skipped test suites: {self.skipped}", parent=selected_suites)
167 Node(f"Passed test suites: {self.passed}", parent=selected_suites)
168 Node(f"Built only test suites: {self.notrun}", parent=selected_suites)
169 Node(f"Failed test suites: {self.failed}", parent=selected_suites)
170 Node(f"Errors in test suites: {self.error}", parent=selected_suites)
172 total_cases = Node(f"Total test cases: {self.cases}", parent=root)
173 Node(f"Filtered test cases: {self.filtered_cases}", parent=total_cases)
174 selected_cases_node = Node(f"Selected test cases: {selected_cases}", parent=total_cases)
175 Node(f"Passed test cases: {self.passed_cases}", parent=selected_cases_node)
176 Node(f"Skipped test cases: {self.skipped_cases}", parent=selected_cases_node)
177 Node(f"Built only test cases: {self.notrun_cases}", parent=selected_cases_node)
178 Node(f"Blocked test cases: {self.blocked_cases}", parent=selected_cases_node)
179 Node(f"Failed test cases: {self.failed_cases}", parent=selected_cases_node)
181 f"Errors in test cases: {self.error_cases}",
187 "The following test case statuses should not appear in a proper execution",
191 Node(f"Statusless test cases: {self.none_cases}", parent=error_cases_node)
193 Node(f"Test cases only started: {self.started_cases}", parent=error_cases_node)
507 config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
508 dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
562 duration = time.time() - start_time
573 self.instance.add_missing_case_status(TwisterStatus.NOTRUN, "Test was built only")
610 logger.debug(f"Test skipped due to {overflow_found[0]} Overflow")
634 gen_edt_args = "--edtlib-Werror"
645 f'-B{self.build_dir}',
646 f'-DTC_RUNID={self.instance.run_id}',
647 f'-DTC_NAME={self.instance.testsuite.name}',
648 f'-D{warning_command}={warnings_as_errors}',
649 f'-DEXTRA_GEN_EDT_ARGS={gen_edt_args}',
650 f'-G{self.env.generator}',
651 f'-DPython3_EXECUTABLE={pathlib.Path(sys.executable).as_posix()}'
654 if self.instance.testsuite.harness == 'bsim':
656 '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
657 '-DCONFIG_ASSERT=y',
658 '-DCONFIG_COVERAGE=y'
667 f'-DMODULES={",".join(filter_stages)}',
668 f'-P{canonical_zephyr_base}/cmake/package_helper.cmake',
674 f'-S{canonical_zephyr_base}/share/sysbuild',
675 f'-DAPP_DIR={self.source_dir}'
679 f'-S{self.source_dir}'
685 cmake_opts = [f'-DBOARD={self.platform.name}']
690 '-DSNIPPET={}'.format(';'.join(self.instance.testsuite.required_snippets))
719 duration = time.time() - start_time
761 self.log = "config-twister.log"
826 # so --west-flash must be passed.
832 logger.warning("Sysbuild test will be skipped. West must be used for flashing.")
876 def trace(self) -> bool:
882 logger.info(f"{filename:-^100}")
899 logger.info(f"{filename:-^100}")
942 task = dict({'op': op, 'test': self.instance}, **additionals)
951 def process(self, pipeline, done, message, lock, results): argument
965 # Here we check the dt/kconfig filter results coming from running cmake
970 results.filtered_runtime_increment()
998 # Here we check the runtime filter results coming from running cmake
1003 results.filtered_runtime_increment()
1020 logger.debug(f"build test: {self.instance.name}")
1030 results.skipped_increment()
1043 if self.instance.testsuite.harness in ['ztest', 'test']:
1045 f"Determine test cases for test instance: {self.instance.name}"
1048 self.determine_testcases(results)
1101 logger.debug(f"run test: {self.instance.name}")
1125 # Report results and output progress to screen
1130 self.report_out(results)
1188 def determine_testcases(self, results): argument
1189 logger.debug(f"Determine test cases for test suite: {self.instance.testsuite.id}")
1202 # because only new ztest fx exposes test functions precisely.
1211 # The 2nd capture group is new ztest unit test name.
1225 f"Test instance {self.instance.name} already has {len(self.instance.testcases)} cases."
1264 # below ones are needed to make --test-only work as well
1306 def _get_artifact_allow_list_for_domain(self, domain: str) -> list[str]:
1308 Return a list of files needed to test a given domain.
1320 def _get_binaries(self) -> list[str]:
1334 # Get binaries for a single-domain build
1336 # Get binaries in the case of a multiple-domain build
1351 def _get_binaries_from_runners(self, domain='') -> list[str]:
1355 multiple-domain builds by passing in one domain at a time.
1447 def _add_instance_testcases_to_status_counts(instance, results, decrement=False): argument
1448 increment_value = -1 if decrement else 1
1452 results.passed_cases_increment(increment_value)
1454 results.notrun_cases_increment(increment_value)
1456 results.blocked_cases_increment(increment_value)
1458 results.skipped_cases_increment(increment_value)
1460 results.filtered_cases_increment(increment_value)
1462 results.error_cases_increment(increment_value)
1464 results.failed_cases_increment(increment_value)
1469 results.none_cases_increment(increment_value)
1471 f' test case {tc.name}.')
1472 results.warnings_increment(1)
1474 results.started_cases_increment(increment_value)
1476 f' test case {tc.name}.')
1477 results.warnings_increment(1)
1481 f' test case {tc.name}.'
1483 results.warnings_increment(1)
1486 def report_out(self, results): argument
1487 total_to_do = results.total - results.filtered_static
1489 results.done_increment()
1491 if results.iteration == 1:
1492 results.cases_increment(len(instance.testcases))
1494 self._add_instance_testcases_to_status_counts(instance, results)
1502 results.error_increment()
1504 results.failed_increment()
1515 results.skipped_increment()
1517 results.filtered_configs_increment()
1519 results.passed_increment()
1521 results.notrun_increment()
1547 f"{results.done - results.filtered_static:>{total_tests_width}}/{total_to_do}"
1565 (float(results.done - results.filtered_static) / total_to_do) * 100
1568 unfiltered = results.done - results.filtered_static
1575 f"{TwisterStatus.get_color(TwisterStatus.NOTRUN)}{results.notrun:>4}{Fore.RESET}"
1579 if results.filtered_configs > 0
1583 f"{filtered_section_color}{results.filtered_configs:>4}{Fore.RESET}"
1586 TwisterStatus.get_color(TwisterStatus.FAIL) if results.failed > 0 else Fore.RESET
1589 f"{failed_section_color}{results.failed:>4}{Fore.RESET}"
1592 TwisterStatus.get_color(TwisterStatus.ERROR) if results.error > 0 else Fore.RESET
1595 f"{error_section_color}{results.error:>4}{Fore.RESET}"
1598 f"INFO - Total complete: {complete_section}"
1614 args_expanded = ["-D{}".format(a.replace('"', '\"')) for a in config_options]
1638 args_expanded.extend(["-D{}".format(a.replace('"', '\"')) for a in cmake_extra_args])
1639 args_expanded.extend(["-D{}".format(a.replace('"', '')) for a in args])
1677 build_result = self.run_build(['--build', self.build_dir])
1728 build_result = self.run_build(['--build', self.build_dir, "--target", "footprint"])
1763 def __init__(self, instances, suites, env=None) -> None:
1771 self.results = None
1782 self.results = ExecutionCounter(total=len(self.instances))
1811 self.results.iteration_increment()
1813 if self.results.iteration > 1:
1814 logger.info(f"{self.results.iteration} Iteration:")
1816 self.results.done = self.results.total - self.results.failed
1817 self.results.failed = 0
1819 self.results.error = 0
1820 self.results.done -= self.results.error
1822 self.results.done = self.results.filtered_static
1840 if self.results.error and self.options.retry_build_errors:
1843 retries = retries - 1
1844 if retries == 0 or ( self.results.failed == 0 and not retry_errors):
1852 test instance never enter the pipeline. While some pipeline output needs
1857 self.results.filtered_static_increment()
1858 self.results.filtered_configs_increment()
1859 self.results.filtered_cases_increment(len(instance.testsuite.testcases))
1860 self.results.cases_increment(len(instance.testsuite.testcases))
1862 self.results.error_increment()
1866 f"{len(self.suites)} test scenarios ({len(self.instances)} configurations) selected,"
1867 f" {self.results.filtered_configs} configurations filtered"
1868 f" ({self.results.filtered_static} by static filter,"
1869 f" {self.results.filtered_configs - self.results.filtered_static} at runtime)."
1898 if self.results.iteration > 1:
1901 self.results,
1914 pipeline.put({"op": "run", "test": instance})
1916 pipeline.put({"op": "filter", "test": instance})
1920 pipeline.put({"op": "build", "test": instance})
1922 pipeline.put({"op": "cmake", "test": instance})
1925 def pipeline_mgr(self, pipeline, done_queue, lock, results): argument
1935 instance = task['test']
1938 pb.process(pipeline, done_queue, task, lock, results)
1953 instance = task['test']
1956 pb.process(pipeline, done_queue, task, lock, results)
1977 p = Process(target=self.pipeline_mgr, args=(pipeline, done, lock, self.results, ))
1997 """Analyze filter expressions from test yaml