Lines Matching full:instance
68 Most of the stats are at test instance level
518 self.instance = None
563 self.instance.build_time += duration
571 if not self.instance.run:
572 self.instance.status = TwisterStatus.NOTRUN
573 self.instance.add_missing_case_status(TwisterStatus.NOTRUN, "Test was built only")
575 self.instance.status = TwisterStatus.PASS
611 self.instance.status = TwisterStatus.SKIP
612 self.instance.reason = f"{overflow_found[0]} overflow"
613 change_skip_to_error_if_integration(self.options, self.instance)
615 self.instance.status = TwisterStatus.SKIP
616 self.instance.reason = "imgtool overflow"
617 change_skip_to_error_if_integration(self.options, self.instance)
619 self.instance.status = TwisterStatus.ERROR
620 self.instance.reason = "Build failure"
640 if self.instance.sysbuild:
646 f'-DTC_RUNID={self.instance.run_id}',
647 f'-DTC_NAME={self.instance.testsuite.name}',
654 if self.instance.testsuite.harness == 'bsim':
671 if self.instance.sysbuild and not filter_stages:
688 if self.instance.testsuite.required_snippets:
690 '-DSNIPPET={}'.format(';'.join(self.instance.testsuite.required_snippets))
720 self.instance.build_time += duration
734 self.instance.status = TwisterStatus.ERROR
735 self.instance.reason = "CMake build failure"
737 for tc in self.instance.testcases:
738 tc.status = self.instance.status
770 if self.instance.sysbuild and not filter_stages:
775 self.instance.domains = domains
828 self.instance.sysbuild
859 def __init__(self, instance: TestInstance, env: TwisterEnv, jobserver, **kwargs):
861 instance.testsuite,
862 instance.platform,
863 instance.testsuite.source_dir,
864 instance.build_dir,
869 self.instance = instance
902 for tc in self.instance.testcases:
915 build_dir = self.instance.build_dir
923 if os.path.exists(v_log) and "Valgrind" in self.instance.reason:
942 task = dict({'op': op, 'test': self.instance}, **additionals)
957 self.instance.setup_handler(self.env)
961 ret = self.cmake(filter_stages=self.instance.filter_stages)
962 if self.instance.status in [TwisterStatus.FAIL, TwisterStatus.ERROR]:
966 if self.instance.name in ret['filter'] and ret['filter'][self.instance.name]:
967 logger.debug(f"filtering {self.instance.name}")
968 self.instance.status = TwisterStatus.FILTER
969 self.instance.reason = "runtime filter"
971 self.instance.add_missing_case_status(TwisterStatus.FILTER)
977 self.instance.status = TwisterStatus.ERROR
979 self.instance.reason = reason
980 self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
989 if self.instance.status in [TwisterStatus.FAIL, TwisterStatus.ERROR]:
992 if self.instance.status == TwisterStatus.NONE:
993 logger.debug(f"CMake only: PASS {self.instance.name}")
994 self.instance.status = TwisterStatus.NOTRUN
995 self.instance.add_missing_case_status(TwisterStatus.NOTRUN, 'CMake only')
999 if self.instance.name in ret['filter'] and ret['filter'][self.instance.name]:
1000 logger.debug(f"filtering {self.instance.name}")
1001 self.instance.status = TwisterStatus.FILTER
1002 self.instance.reason = "runtime filter"
1004 self.instance.add_missing_case_status(TwisterStatus.FILTER)
1010 self.instance.status = TwisterStatus.ERROR
1012 self.instance.reason = reason
1013 self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
1020 logger.debug(f"build test: {self.instance.name}")
1023 self.instance.status = TwisterStatus.ERROR
1024 self.instance.reason = "Build Failure"
1029 if self.instance.status == TwisterStatus.SKIP:
1031 self.instance.add_missing_case_status(
1033 self.instance.reason
1037 self.instance.add_missing_case_status(
1039 self.instance.reason
1043 if self.instance.testsuite.harness in ['ztest', 'test']:
1045 f"Determine test cases for test instance: {self.instance.name}"
1052 self.instance.status = TwisterStatus.ERROR
1053 self.instance.reason = str(e)
1059 self.instance.status = TwisterStatus.ERROR
1061 self.instance.reason = reason
1062 self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
1069 ret = self.gather_metrics(self.instance)
1071 self.instance.status = TwisterStatus.ERROR
1072 self.instance.reason = "Build Failure at gather_metrics."
1074 elif self.instance.run and self.instance.handler.ready:
1077 if self.instance.status == TwisterStatus.NOTRUN:
1079 f"(run:{self.instance.run},"
1080 f" handler.ready:{self.instance.handler.ready})"
1082 logger.debug(f"Instance {self.instance.name} can't run {run_conditions}")
1083 self.instance.add_missing_case_status(
1090 self.instance.status = TwisterStatus.ERROR
1092 self.instance.reason = reason
1093 self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
1101 logger.debug(f"run test: {self.instance.name}")
1103 logger.debug(f"run status: {self.instance.name} {self.instance.status}")
1106 self.instance.handler.thread = None
1107 self.instance.handler.duts = None
1111 "status": self.instance.status,
1112 "reason": self.instance.reason
1116 self.instance.status = TwisterStatus.ERROR
1118 self.instance.reason = reason
1119 self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
1129 done.put(self.instance)
1137 self.instance.status in [TwisterStatus.PASS, TwisterStatus.NOTRUN]:
1145 self.instance.status = TwisterStatus.ERROR
1147 self.instance.reason = reason
1148 self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
1161 or (mode == "all" and self.instance.reason != "CMake build failure")
1166 self.instance.status = TwisterStatus.ERROR
1168 self.instance.reason = reason
1169 self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
1189 logger.debug(f"Determine test cases for test suite: {self.instance.testsuite.id}")
1194 elf_file = self.instance.get_elf_file()
1213 if new_ztest_suite not in self.instance.testsuite.ztest_suite_names:
1216 f"not present in: {self.instance.testsuite.ztest_suite_names}"
1219 testcase_id = self.instance.compose_case_name(
1225 f"Test instance {self.instance.name} already has {len(self.instance.testcases)} cases."
1231 for tc in self.instance.testcases
1233 self.instance.testcases.clear()
1234 self.instance.testsuite.testcases.clear()
1237 testcase = self.instance.add_testcase(name=testcase_id)
1238 self.instance.testsuite.add_testcase(name=testcase_id)
1245 f"'{self.instance.testsuite.source_dir_rel}' "
1254 logger.debug(f"Cleaning up {self.instance.build_dir}")
1276 allow = [os.path.join(self.instance.build_dir, file) for file in allow]
1278 for dirpath, dirnames, filenames in os.walk(self.instance.build_dir, topdown=False):
1292 logger.debug(f"Cleaning up for Device Testing {self.instance.build_dir}")
1297 if self.instance.sysbuild:
1299 for domain in self.instance.domains.get_domains():
1323 self.instance.build_dir), basing on information from platform.binaries
1329 platform = self.instance.platform
1337 if self.instance.sysbuild:
1338 for domain in self.instance.domains.get_domains():
1354 self.instance.build_dir) from runners.yaml file. May be used for
1358 runners_file_path: str = os.path.join(self.instance.build_dir,
1395 directory for those files is f"{self.instance.build_dir}/zephyr"
1397 runners_dir_path: str = os.path.join(self.instance.build_dir, 'zephyr')
1432 file_path = os.path.join(self.instance.build_dir, file_path)
1447 def _add_instance_testcases_to_status_counts(instance, results, decrement=False): argument
1449 for tc in instance.testcases:
1470 logger.warning(f'A None status detected in instance {instance.name},'
1475 logger.warning(f'A started status detected in instance {instance.name},'
1480 f'An unknown status "{tc.status}" detected in instance {instance.name},'
1490 instance = self.instance
1492 results.cases_increment(len(instance.testcases))
1494 self._add_instance_testcases_to_status_counts(instance, results)
1497 f'{TwisterStatus.get_color(instance.status)}{str.upper(instance.status)}{Fore.RESET}'
1500 if instance.status in [TwisterStatus.ERROR, TwisterStatus.FAIL]:
1501 if instance.status == TwisterStatus.ERROR:
1506 status += " " + instance.reason
1509 f"{instance.platform.name:<25} {instance.testsuite.name:<50}"
1510 f" {status}: {instance.reason}"
1514 elif instance.status == TwisterStatus.SKIP:
1516 elif instance.status == TwisterStatus.FILTER:
1518 elif instance.status == TwisterStatus.PASS:
1520 elif instance.status == TwisterStatus.NOTRUN:
1523 logger.debug(f"Unknown status = {instance.status}")
1529 elif instance.status in [TwisterStatus.SKIP, TwisterStatus.FILTER]:
1530 more_info = instance.reason
1532 if instance.handler.ready and instance.run:
1533 more_info = instance.handler.type_str
1534 htime = instance.execution_time
1535 if instance.dut:
1536 more_info += f": {instance.dut},"
1542 if ( instance.status in [TwisterStatus.ERROR, TwisterStatus.FAIL]
1543 and hasattr(self.instance.handler, 'seed')
1544 and self.instance.handler.seed is not None ):
1548 f" {instance.platform.name:<25} {instance.testsuite.name:<50}"
1553 for tc in self.instance.testcases:
1559 if instance.status in [TwisterStatus.ERROR, TwisterStatus.FAIL]:
1650 if self.instance.platform.arch == cond_args[1]:
1653 if self.instance.platform.name == cond_args[1]:
1656 if self.instance.platform.simulation == cond_args[1]:
1666 self.instance.handler,
1671 self.instance.build_dir,
1676 harness = HarnessImporter.get_harness(self.instance.testsuite.harness.capitalize())
1680 harness.instance = self.instance
1683 self.instance.status = TwisterStatus.ERROR
1684 self.instance.reason = str(error)
1685 logger.error(self.instance.reason)
1691 instance = self.instance
1693 if instance.handler.ready:
1694 logger.debug(f"Reset instance status from '{instance.status}' to None before run.")
1695 instance.status = TwisterStatus.NONE
1697 if instance.handler.type_str == "device":
1698 instance.handler.duts = self.duts
1700 if(self.options.seed is not None and instance.platform.name.startswith("native_")):
1704 instance.handler.seed = self.options.seed
1706 if self.options.extra_test_args and instance.platform.arch == "posix":
1707 instance.handler.extra_test_args = self.options.extra_test_args
1709 harness = HarnessImporter.get_harness(instance.testsuite.harness.capitalize())
1711 harness.configure(instance)
1713 instance.status = TwisterStatus.ERROR
1714 instance.reason = str(error)
1715 logger.error(instance.reason)
1719 harness.pytest_run(instance.handler.get_test_timeout())
1721 instance.handler.handle(harness)
1725 def gather_metrics(self, instance: TestInstance):
1730 self.calc_size(instance=instance, from_buildlog=self.options.footprint_from_buildlog)
1732 instance.metrics["used_ram"] = 0
1733 instance.metrics["used_rom"] = 0
1734 instance.metrics["available_rom"] = 0
1735 instance.metrics["available_ram"] = 0
1736 instance.metrics["unrecognized"] = []
1740 def calc_size(instance: TestInstance, from_buildlog: bool):
1741 if instance.status not in [TwisterStatus.ERROR, TwisterStatus.FAIL, TwisterStatus.SKIP]:
1742 if instance.platform.type not in ["native", "qemu", "unit"]:
1743 generate_warning = bool(instance.platform.type == "mcu")
1744 size_calc = instance.calculate_sizes(
1748 instance.metrics["used_ram"] = size_calc.get_used_ram()
1749 instance.metrics["used_rom"] = size_calc.get_used_rom()
1750 instance.metrics["available_rom"] = size_calc.get_available_rom()
1751 instance.metrics["available_ram"] = size_calc.get_available_ram()
1752 instance.metrics["unrecognized"] = size_calc.unrecognized_sections()
1754 instance.metrics["used_ram"] = 0
1755 instance.metrics["used_rom"] = 0
1756 instance.metrics["available_rom"] = 0
1757 instance.metrics["available_ram"] = 0
1758 instance.metrics["unrecognized"] = []
1759 instance.metrics["handler_time"] = instance.execution_time
1852 test instance never enter the pipeline. While some pipeline output needs
1855 for instance in self.instances.values():
1856 if instance.status == TwisterStatus.FILTER and instance.reason != 'runtime filter':
1859 self.results.filtered_cases_increment(len(instance.testsuite.testcases))
1860 self.results.cases_increment(len(instance.testsuite.testcases))
1861 elif instance.status == TwisterStatus.ERROR:
1879 for instance in self.instances.values():
1881 instance.run = False
1892 if instance.status not in no_retry_statuses:
1893 logger.debug(f"adding {instance.name}")
1894 if instance.status != TwisterStatus.NONE:
1895 instance.retries += 1
1896 instance.status = TwisterStatus.NONE
1900 instance,
1906 instance.filter_stages = []
1907 if instance.testsuite.filter:
1908 instance.filter_stages = self.get_cmake_filter_stages(
1909 instance.testsuite.filter,
1913 if test_only and instance.run:
1914 pipeline.put({"op": "run", "test": instance})
1915 elif instance.filter_stages and "full" not in instance.filter_stages:
1916 pipeline.put({"op": "filter", "test": instance})
1918 cache_file = os.path.join(instance.build_dir, "CMakeCache.txt")
1920 pipeline.put({"op": "build", "test": instance})
1922 pipeline.put({"op": "cmake", "test": instance})
1935 instance = task['test']
1936 pb = ProjectBuilder(instance, self.env, self.jobserver)
1940 pb.instance.status in [TwisterStatus.FAIL, TwisterStatus.ERROR]:
1953 instance = task['test']
1954 pb = ProjectBuilder(instance, self.env, self.jobserver)
1958 pb.instance.status in [TwisterStatus.FAIL, TwisterStatus.ERROR]: