1# pylint: disable=C0301,C0103,C0111 2from __future__ import print_function 3from sys import platform 4import os 5import sys 6import socket 7import fnmatch 8import subprocess 9import psutil 10import shutil 11import tempfile 12import uuid 13import re 14from collections import OrderedDict, defaultdict 15from time import monotonic, sleep 16from typing import List, Dict, Tuple, Set 17from argparse import Namespace 18 19import robot, robot.result, robot.running 20from robot.libraries.BuiltIn import BuiltIn 21from robot.libraries.DateTime import Time 22 23import xml.etree.ElementTree as ET 24 25from tests_engine import TestResult 26 27this_path = os.path.abspath(os.path.dirname(__file__)) 28 29 30class Timeout: 31 def __init__(self, value: str): 32 self.seconds = Time(value).seconds 33 self.value = value 34 35 def __repr__(self): 36 return f"{self.value} ({self.seconds}s)" 37 38 39def install_cli_arguments(parser): 40 group = parser.add_mutually_exclusive_group() 41 42 group.add_argument("--robot-framework-remote-server-full-directory", 43 dest="remote_server_full_directory", 44 action="store", 45 help="Full location of robot framework remote server binary.") 46 47 group.add_argument("--robot-framework-remote-server-directory-prefix", 48 dest="remote_server_directory_prefix", 49 action="store", 50 default=os.path.join(this_path, '../output/bin'), 51 help="Directory of robot framework remote server binary. This is concatenated with current configuration to create full path.") 52 53 parser.add_argument("--robot-framework-remote-server-name", 54 dest="remote_server_name", 55 action="store", 56 default="Renode.exe", 57 help="Name of robot framework remote server binary.") 58 59 parser.add_argument("--robot-framework-remote-server-port", "-P", 60 dest="remote_server_port", 61 action="store", 62 default=0, 63 type=int, 64 help="Port of robot framework remote server binary. Use '0' to automatically select any unused private port.") 65 66 parser.add_argument("--enable-xwt", 67 dest="enable_xwt", 68 action="store_true", 69 default=False, 70 help="Enables support for XWT.") 71 72 parser.add_argument("--show-log", 73 dest="show_log", 74 action="store_true", 75 default=False, 76 help="Display log messages in console (might corrupt robot summary output).") 77 78 parser.add_argument("--keep-renode-output", 79 dest="keep_renode_output", 80 action="store_true", 81 default=False, 82 help=" ".join([ 83 "Redirect Renode stdout and stderr to log files.", 84 "Only non-empty log files are kept (i.e. up to 2 per suite).", 85 "This is separate from the usual logs generated by RobotFramework.", 86 "Implies --show-log (output is redirected and does not appear in console).", 87 ])) 88 89 parser.add_argument("--verbose", 90 dest="verbose", 91 action="store_true", 92 default=False, 93 help="Print verbose info from Robot Framework.") 94 95 parser.add_argument("--hot-spot", 96 dest="hotspot", 97 action="store", 98 default=None, 99 help="Test given hot spot action.") 100 101 parser.add_argument("--variable", 102 dest="variables", 103 action="append", 104 default=None, 105 help="Variable to pass to Robot.") 106 107 parser.add_argument("--css-file", 108 dest="css_file", 109 action="store", 110 default=os.path.join(this_path, '../lib/resources/styles/robot.css'), 111 help="Custom CSS style for the result files.") 112 113 parser.add_argument("--debug-on-error", 114 dest="debug_on_error", 115 action="store_true", 116 default=False, 117 help="Enables the Renode User Interface when test fails.") 118 119 parser.add_argument("--cleanup-timeout", 120 dest="cleanup_timeout", 121 action="store", 122 default=3, 123 type=int, 124 help="Robot frontend process cleanup timeout.") 125 126 parser.add_argument("--listener", 127 action="append", 128 help="Path to additional progress listener (can be provided many times).") 129 130 parser.add_argument("--renode-config", 131 dest="renode_config", 132 action="store", 133 default=None, 134 help="Path to the Renode config file.") 135 136 parser.add_argument("--kill-stale-renode-instances", 137 dest="autokill_renode", 138 action="store_true", 139 default=False, 140 help="Automatically kill stale Renode instances without asking.") 141 142 parser.add_argument("--gather-execution-metrics", 143 dest="execution_metrics", 144 action="store_true", 145 default=False, 146 help="Gather execution metrics for each suite.") 147 148 parser.add_argument("--test-timeout", 149 dest="timeout", 150 action="store", 151 default=None, 152 type=Timeout, 153 help=" ".join([ 154 "Default test case timeout after which Renode keywords will be interrupted.", 155 "It's parsed by Robot Framework's DateTime library so all its time formats are supported.", 156 ])) 157 158 159def verify_cli_arguments(options): 160 # port is not available on Windows 161 if platform != "win32": 162 if options.port == str(options.remote_server_port): 163 print('Port {} is reserved for Robot Framework remote server and cannot be used for remote debugging.'.format(options.remote_server_port)) 164 sys.exit(1) 165 if options.port is not None and options.jobs != 1: 166 print("Debug port cannot be used in parallel runs") 167 sys.exit(1) 168 169 if options.css_file: 170 if not os.path.isabs(options.css_file): 171 options.css_file = os.path.join(this_path, options.css_file) 172 173 if not os.path.isfile(options.css_file): 174 print("Unable to find provided CSS file: {0}.".format(options.css_file)) 175 sys.exit(1) 176 177 if options.remote_server_port != 0 and options.jobs != 1: 178 print("Parallel execution and fixed Robot port number options cannot be used together") 179 sys.exit(1) 180 181 182def is_process_running(pid): 183 if not psutil.pid_exists(pid): 184 return False 185 proc = psutil.Process(pid) 186 # docs note: is_running() will return True also if the process is a zombie (p.status() == psutil.STATUS_ZOMBIE) 187 return proc.is_running() and proc.status() != psutil.STATUS_ZOMBIE 188 189 190def is_port_available(port, autokill): 191 port_handle = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 192 available = False 193 try: 194 port_handle.bind(("localhost", port)) 195 port_handle.close() 196 available = True 197 except: 198 available = can_be_freed_by_killing_other_job(port, autokill) 199 return available 200 201 202def can_be_freed_by_killing_other_job(port, autokill): 203 if not sys.stdin.isatty(): 204 return 205 try: 206 for proc in [psutil.Process(pid) for pid in psutil.pids()]: 207 if '--robot-server-port' in proc.cmdline() and str(port) in proc.cmdline(): 208 if not is_process_running(proc.pid): 209 # process is zombie 210 continue 211 212 if autokill: 213 result = 'y' 214 else: 215 print('It seems that Renode process (pid {}, name {}) is currently running on port {}'.format(proc.pid, proc.name(), port)) 216 result = input('Do you want me to kill it? [y/N] ') 217 218 if result in ['Y', 'y']: 219 proc.kill() 220 return True 221 break 222 except Exception: 223 # do nothing here 224 pass 225 return False 226 227 228class KeywordsFinder(robot.model.SuiteVisitor): 229 def __init__(self, keyword): 230 self.keyword = keyword 231 self.occurences = 0 232 self.arguments = [] 233 234 235 def visit_keyword(self, keyword): 236 if keyword.name == self.keyword: 237 self.occurences += 1 238 arguments = keyword.args 239 self.arguments.append(arguments) 240 241 242 def got_results(self): 243 return self.occurences > 0 244 245 246class TestsFinder(robot.model.SuiteVisitor): 247 def __init__(self, keyword): 248 self.keyword = keyword 249 self.tests_matching = [] 250 self.tests_not_matching = [] 251 252 253 def isMatching(self, test): 254 finder = KeywordsFinder(self.keyword) 255 test.visit(finder) 256 return finder.got_results() 257 258 259 def visit_test(self, test): 260 if self.isMatching(test): 261 self.tests_matching.append(test) 262 else: 263 self.tests_not_matching.append(test) 264 265 266class RobotTestSuite(object): 267 after_timeout_message_suffix = ' '.join([ 268 "Failed on Renode restarted after timeout,", 269 "will be retried if `-N/--retry` option was used." 270 ]) 271 instances_count = 0 272 robot_frontend_process = None 273 renode_pid = -1 # It's not always robot_frontend_process.pid, e.g., with `--run-gdb` option. 274 hotspot_action = ['None', 'Pause', 'Serialize'] 275 # Used to share the port between all suites when running sequentially 276 remote_server_port = -1 277 retry_test_regex = re.compile(r"\[RETRY\] (PASS|FAIL) on (\d+)\. retry\.") 278 retry_suite_regex = re.compile(r"|".join(( 279 r"\[Errno \d+\] Connection refused", 280 r"Connection to remote server broken: \[WinError \d+\]", 281 r"Connecting remote server at [^ ]+ failed", 282 "Getting keyword names from library 'Remote' failed", 283 after_timeout_message_suffix, 284 ))) 285 timeout_expected_tag = 'timeout_expected' 286 287 def __init__(self, path): 288 self.path = path 289 self._dependencies_met = set() 290 self.remote_server_directory = None 291 # Subset of RobotTestSuite.log_files which are "owned" by the running instance 292 self.suite_log_files = None 293 294 self.tests_with_hotspots = [] 295 self.tests_without_hotspots = [] 296 297 298 def check(self, options, number_of_runs): 299 # Checking if there are no other jobs is moved to `prepare` as it is now possible to skip used ports 300 pass 301 302 303 def get_output_dir(self, options, iteration_index, suite_retry_index): 304 return os.path.join( 305 options.results_directory, 306 f"iteration{iteration_index}" if options.iteration_count > 1 else "", 307 f"retry{suite_retry_index}" if options.retry_count > 1 else "", 308 ) 309 310 311 def prepare(self, options): 312 RobotTestSuite.instances_count += 1 313 314 hotSpotTestFinder = TestsFinder(keyword="Handle Hot Spot") 315 suiteBuilder = robot.running.builder.TestSuiteBuilder() 316 suite = suiteBuilder.build(self.path) 317 suite.visit(hotSpotTestFinder) 318 319 self.tests_with_hotspots = [test.name for test in hotSpotTestFinder.tests_matching] 320 self.tests_without_hotspots = [test.name for test in hotSpotTestFinder.tests_not_matching] 321 322 # In parallel runs, Renode is started for each suite. 323 # The same is done in sequential runs with --keep-renode-output. 324 # see: run 325 if options.jobs == 1 and not options.keep_renode_output: 326 if not RobotTestSuite._is_frontend_running(): 327 RobotTestSuite.robot_frontend_process = self._run_remote_server(options) 328 # Save port to reuse when running sequentially 329 RobotTestSuite.remote_server_port = self.remote_server_port 330 else: 331 # Restore port allocated by a previous suite 332 self.remote_server_port = RobotTestSuite.remote_server_port 333 334 335 @classmethod 336 def _is_frontend_running(cls): 337 return cls.robot_frontend_process is not None and is_process_running(cls.robot_frontend_process.pid) 338 339 340 def _run_remote_server(self, options, iteration_index=1, suite_retry_index=0, remote_server_port=None): 341 # Let's reset PID and check it's set before returning to prevent keeping old PID. 342 self.renode_pid = -1 343 344 if options.runner == 'dotnet': 345 remote_server_name = "Renode.dll" 346 else: 347 remote_server_name = options.remote_server_name 348 349 self.remote_server_directory = options.remote_server_full_directory 350 remote_server_binary = os.path.join(self.remote_server_directory, remote_server_name) 351 352 if not os.path.isfile(remote_server_binary): 353 print("Robot framework remote server binary not found: '{}'! Did you forget to build?".format(remote_server_binary)) 354 sys.exit(1) 355 356 if remote_server_port is None: 357 remote_server_port = options.remote_server_port 358 359 if remote_server_port != 0 and not is_port_available(remote_server_port, options.autokill_renode): 360 print("The selected port {} is not available".format(remote_server_port)) 361 sys.exit(1) 362 363 command = [remote_server_binary, '--robot-server-port', str(remote_server_port)] 364 if not options.show_log and not options.keep_renode_output: 365 command.append('--hide-log') 366 if not options.enable_xwt: 367 command.append('--disable-gui') 368 if options.debug_on_error: 369 command.append('--robot-debug-on-error') 370 if options.keep_temps: 371 command.append('--keep-temporary-files') 372 if options.renode_config: 373 command.append('--config') 374 command.append(options.renode_config) 375 376 if options.runner == 'mono': 377 command.insert(0, 'mono') 378 if options.port is not None: 379 if options.suspend: 380 print('Waiting for a debugger at port: {}'.format(options.port)) 381 command.insert(1, '--debug') 382 command.insert(2, '--debugger-agent=transport=dt_socket,server=y,suspend={0},address=127.0.0.1:{1}'.format('y' if options.suspend else 'n', options.port)) 383 elif options.debug_mode: 384 command.insert(1, '--debug') 385 options.exclude.append('skip_mono') 386 elif options.runner == 'dotnet': 387 command.insert(0, 'dotnet') 388 options.exclude.append('skip_dotnet') 389 390 renode_command = command 391 392 # if we started GDB, wait for the user to start Renode as a child process 393 if options.run_gdb: 394 command = ['gdb', '-nx', '-ex', 'handle SIGXCPU SIG33 SIG35 SIG36 SIGPWR nostop noprint', '--args'] + command 395 p = psutil.Popen(command, cwd=self.remote_server_directory, bufsize=1) 396 397 if options.keep_renode_output: 398 print("Note: --keep-renode-output is not supported when using --run-gdb") 399 400 print("Waiting for Renode process to start") 401 while True: 402 # We strip argv[0] because if we pass just `mono` to GDB it will resolve 403 # it to a full path to mono on the PATH, for example /bin/mono 404 renode_child = next((c for c in p.children() if c.cmdline()[1:] == renode_command[1:]), None) 405 if renode_child: 406 break 407 sleep(0.5) 408 self.renode_pid = renode_child.pid 409 elif options.perf_output_path: 410 pid_file_uuid = uuid.uuid4() 411 pid_filename = f'pid_file_{pid_file_uuid}' 412 413 command = ['perf', 'record', '-q', '-g', '-F', 'max'] + command + ['--pid-file', pid_filename] 414 415 perf_stdout_stderr_file_name = "perf_stdout_stderr" 416 417 if options.keep_renode_output: 418 print("Note: --keep-renode-output is not supported when using --perf-output-path") 419 420 print(f"WARNING: perf stdout and stderr is being redirected to {perf_stdout_stderr_file_name}") 421 422 perf_stdout_stderr_file = open(perf_stdout_stderr_file_name, "w") 423 p = subprocess.Popen(command, cwd=self.remote_server_directory, bufsize=1, stdout=perf_stdout_stderr_file, stderr=perf_stdout_stderr_file) 424 425 pid_file_path = os.path.join(self.remote_server_directory, pid_filename) 426 perf_renode_timeout = 10 427 428 while not os.path.exists(pid_file_path) and perf_renode_timeout > 0: 429 sleep(0.5) 430 perf_renode_timeout -= 1 431 432 if perf_renode_timeout <= 0: 433 raise RuntimeError("Renode pid file could not be found, can't attach perf") 434 435 with open(pid_file_path, 'r') as pid_file: 436 self.renode_pid = pid_file.read() 437 else: 438 # Start Renode 439 if options.keep_renode_output: 440 output_dir = self.get_output_dir(options, iteration_index, suite_retry_index) 441 logs_dir = os.path.join(output_dir, 'logs') 442 os.makedirs(logs_dir, exist_ok=True) 443 file_name = os.path.splitext(os.path.basename(self.path))[0] 444 suite_name = RobotTestSuite._create_suite_name(file_name, None) 445 fout = open(os.path.join(logs_dir, f"{suite_name}.renode_stdout.log"), "wb", buffering=0) 446 ferr = open(os.path.join(logs_dir, f"{suite_name}.renode_stderr.log"), "wb", buffering=0) 447 p = subprocess.Popen(command, cwd=self.remote_server_directory, bufsize=1, stdout=fout, stderr=ferr) 448 self.renode_pid = p.pid 449 else: 450 p = subprocess.Popen(command, cwd=self.remote_server_directory, bufsize=1) 451 self.renode_pid = p.pid 452 453 timeout_s = 180 454 countdown = float(timeout_s) 455 temp_dir = tempfile.gettempdir() 456 renode_port_file = os.path.join(temp_dir, f'renode-{self.renode_pid}', 'robot_port') 457 while countdown > 0: 458 try: 459 with open(renode_port_file) as f: 460 port_num = f.readline() 461 if port_num == '': 462 continue 463 self.remote_server_port = int(port_num) 464 break 465 except: 466 sleep(0.5) 467 countdown -= 0.5 468 else: 469 self._close_remote_server(p, options) 470 raise TimeoutError(f"Couldn't access port file for Renode instance pid {self.renode_pid}; timed out after {timeout_s}s") 471 472 # If a certain port was expected, let's make sure Renode uses it. 473 if remote_server_port and remote_server_port != self.remote_server_port: 474 self._close_remote_server(p, options) 475 raise RuntimeError(f"Renode was expected to use port {remote_server_port} but {self.remote_server_port} port is used instead!") 476 477 assert self.renode_pid != -1, "Renode PID has to be set before returning" 478 return p 479 480 def __move_perf_data(self, options): 481 perf_data_path = os.path.join(self.remote_server_directory, "perf.data") 482 483 if not perf_data_path: 484 raise RuntimeError("perf.data file was not generated succesfully") 485 486 if not os.path.isdir(options.perf_output_path): 487 raise RuntimeError(f"{options.perf_output_path} is not a valid directory path") 488 489 shutil.move(perf_data_path, options.perf_output_path) 490 491 def _close_remote_server(self, proc, options, cleanup_timeout_override=None, silent=False): 492 if proc: 493 if not silent: 494 print('Closing Renode pid {}'.format(proc.pid)) 495 496 # Let's prevent using these after the server is closed. 497 self.robot_frontend_process = None 498 self.renode_pid = -1 499 500 try: 501 process = psutil.Process(proc.pid) 502 os.kill(proc.pid, 2) 503 if cleanup_timeout_override is not None: 504 cleanup_timeout = cleanup_timeout_override 505 else: 506 cleanup_timeout = options.cleanup_timeout 507 process.wait(timeout=cleanup_timeout) 508 509 if options.perf_output_path: 510 self.__move_perf_data(options) 511 except psutil.TimeoutExpired: 512 process.kill() 513 process.wait() 514 except psutil.NoSuchProcess: 515 #evidently closed by other means 516 pass 517 518 if options.perf_output_path and proc.stdout: 519 proc.stdout.close() 520 521 # None of the previously provided states are available after closing the server. 522 self._dependencies_met = set() 523 524 @classmethod 525 def _has_renode_crashed(cls, test: ET.Element) -> bool: 526 # only finds immediate children - required because `status` 527 # nodes are also present lower in the tree for example 528 # for every keyword but we only need the status of the test 529 status: ET.Element = test.find('status') 530 if status.text is not None and cls.retry_suite_regex.search(status.text): 531 return True 532 else: 533 return any(cls.retry_suite_regex.search(msg.text) for msg in test.iter("msg")) 534 535 536 def run(self, options, run_id=0, iteration_index=1, suite_retry_index=0): 537 if self.path.endswith('renode-keywords.robot'): 538 print('Ignoring helper file: {}'.format(self.path)) 539 return True 540 541 # The list is cleared only on the first run attempt in each iteration so 542 # that tests that time out aren't retried in the given iteration but are 543 # started as usual in subsequent iterations. 544 if suite_retry_index == 0: 545 self.tests_with_unexpected_timeouts = [] 546 547 # in non-parallel runs there is only one Renode process for all runs, 548 # unless --keep-renode-output is enabled, in which case a new process 549 # is spawned for every suite to ensure logs are separate files. 550 # see: prepare 551 if options.jobs != 1 or options.keep_renode_output: 552 # Parallel groups run in separate processes so these aren't really 553 # shared, they're only needed to restart Renode in timeout handler. 554 RobotTestSuite.robot_frontend_process = self._run_remote_server(options, iteration_index, suite_retry_index) 555 RobotTestSuite.remote_server_port = self.remote_server_port 556 557 print(f'Running suite on Renode pid {self.renode_pid} using port {self.remote_server_port}: {self.path}') 558 559 result = None 560 def get_result(): 561 return result if result is not None else TestResult(True, None) 562 563 start_timestamp = monotonic() 564 565 if any(self.tests_without_hotspots): 566 result = get_result().ok and self._run_inner(options.fixture, 567 None, 568 self.tests_without_hotspots, 569 options, 570 iteration_index, 571 suite_retry_index) 572 if any(self.tests_with_hotspots): 573 for hotspot in RobotTestSuite.hotspot_action: 574 if options.hotspot and options.hotspot != hotspot: 575 continue 576 result = get_result().ok and self._run_inner(options.fixture, 577 hotspot, 578 self.tests_with_hotspots, 579 options, 580 iteration_index, 581 suite_retry_index) 582 583 end_timestamp = monotonic() 584 585 if result is None: 586 print(f'No tests executed for suite {self.path}', flush=True) 587 else: 588 status = 'finished successfully' if result.ok else 'failed' 589 exec_time = round(end_timestamp - start_timestamp, 2) 590 print(f'Suite {self.path} {status} in {exec_time} seconds.', flush=True) 591 592 if options.jobs != 1 or options.keep_renode_output: 593 self._close_remote_server(RobotTestSuite.robot_frontend_process, options) 594 595 # make sure renode is still alive when a non-parallel run depends on it 596 if options.jobs == 1 and not options.keep_renode_output: 597 if not self._is_frontend_running(): 598 print("Renode has unexpectedly died when running sequentially! Trying to respawn before continuing...") 599 RobotTestSuite.robot_frontend_process = self._run_remote_server(options, iteration_index, suite_retry_index) 600 # Save port to reuse when running sequentially 601 RobotTestSuite.remote_server_port = self.remote_server_port 602 603 return get_result() 604 605 606 def _get_dependencies(self, test_case): 607 608 suiteBuilder = robot.running.builder.TestSuiteBuilder() 609 suite = suiteBuilder.build(self.path) 610 test = next(t for t in suite.tests if hasattr(t, 'name') and t.name == test_case) 611 requirements = [s.args[0] for s in test.body if hasattr(s, 'name') and s.name == 'Requires'] 612 if len(requirements) == 0: 613 return set() 614 if len(requirements) > 1: 615 raise Exception('Too many requirements for a single test. At most one is allowed.') 616 providers = [t for t in suite.tests if any(hasattr(s, 'name') and s.name == 'Provides' and s.args[0] == requirements[0] for s in t.body)] 617 if len(providers) > 1: 618 raise Exception('Too many providers for state {0} found: {1}'.format(requirements[0], ', '.join(providers.name))) 619 if len(providers) == 0: 620 raise Exception('No provider for state {0} found'.format(requirements[0])) 621 res = self._get_dependencies(providers[0].name) 622 res.add(providers[0].name) 623 return res 624 625 626 def cleanup(self, options): 627 assert hasattr(RobotTestSuite, "log_files"), "tests_engine.py did not assign RobotTestSuite.log_files" 628 RobotTestSuite.instances_count -= 1 629 if RobotTestSuite.instances_count == 0: 630 self._close_remote_server(RobotTestSuite.robot_frontend_process, options) 631 print("Aggregating all robot results") 632 grouped_log_files = self.group_log_paths(RobotTestSuite.log_files) 633 for iteration in range(1, options.iteration_count + 1): 634 for retry in range(options.retry_count): 635 output_dir = self.get_output_dir(options, iteration, retry) 636 log_files = grouped_log_files[(iteration, retry)] 637 638 # An output_dir can be missing for suite retries that were never "used" 639 if not os.path.isdir(output_dir) or not log_files: 640 continue 641 642 robot.rebot( 643 *log_files, 644 processemptysuite=True, 645 name='Test Suite', 646 loglevel="TRACE:INFO", 647 outputdir=output_dir, 648 output='robot_output.xml' 649 ) 650 for file in set(log_files): 651 os.remove(file) 652 if options.css_file: 653 with open(options.css_file) as style: 654 style_content = style.read() 655 for report_name in ("report.html", "log.html"): 656 with open(os.path.join(output_dir, report_name), "a") as report: 657 report.write("<style media=\"all\" type=\"text/css\">") 658 report.write(style_content) 659 report.write("</style>") 660 661 662 if options.keep_renode_output: 663 logs_pattern = re.compile(r"(?P<suite_name>.*)\.renode_std(out|err)\.log") 664 for dirpath, _, fnames in os.walk(options.results_directory): 665 if os.path.basename(dirpath.rstrip("/")) != "logs": 666 continue 667 668 failed_suites = self.find_suites_with_fails(os.path.dirname(dirpath)) 669 for fname in fnames: 670 fpath = os.path.join(dirpath, fname) 671 m = logs_pattern.match(fname) 672 if m: 673 # Remove empty logs 674 if os.path.getsize(fpath) == 0: 675 os.remove(fpath) 676 continue 677 678 if options.save_logs == "onfail": 679 # Remove logs which weren't failures 680 suite_name = m.group("suite_name") 681 if suite_name not in failed_suites: 682 os.remove(fpath) 683 684 # If the logs directory is empty, delete it 685 try: 686 os.rmdir(dirpath) 687 except OSError: 688 pass 689 690 691 def should_retry_suite(self, options, iteration_index, suite_retry_index): 692 tree = None 693 assert self.suite_log_files is not None, "The suite has not yet been run." 694 output_dir = self.get_output_dir(options, iteration_index, suite_retry_index) 695 for log_file in self.suite_log_files: 696 try: 697 tree = ET.parse(os.path.join(output_dir, log_file)) 698 except FileNotFoundError as e: 699 raise e 700 701 root = tree.getroot() 702 for suite in root.iter('suite'): 703 if not suite.get('source', False): 704 continue # it is a tag used to group other suites without meaning on its own 705 706 # Always retry if our Setup failed. 707 for kw in suite.iter('kw'): 708 if kw.get('name') == 'Setup' and kw.get('library') == 'renode-keywords': 709 if kw.find('status').get('status') != 'PASS': 710 print('Renode Setup failure detected!') 711 return True 712 else: 713 break 714 715 # Look for regular expressions signifying a crash. 716 # Suite Setup and Suite Teardown aren't checked here cause they're in the `kw` tags. 717 for test in suite.iter('test'): 718 if self._has_renode_crashed(test): 719 return True 720 721 return False 722 723 724 @staticmethod 725 def _create_suite_name(test_name, hotspot): 726 return test_name + (' [HotSpot action: {0}]'.format(hotspot) if hotspot else '') 727 728 729 def _run_dependencies(self, test_cases_names, options, iteration_index=1, suite_retry_index=0): 730 test_cases_names.difference_update(self._dependencies_met) 731 if not any(test_cases_names): 732 return True 733 self._dependencies_met.update(test_cases_names) 734 return self._run_inner(None, None, test_cases_names, options, iteration_index, suite_retry_index) 735 736 737 def _run_inner(self, fixture, hotspot, test_cases_names, options, iteration_index=1, suite_retry_index=0): 738 file_name = os.path.splitext(os.path.basename(self.path))[0] 739 suite_name = RobotTestSuite._create_suite_name(file_name, hotspot) 740 741 output_dir = self.get_output_dir(options, iteration_index, suite_retry_index) 742 variables = [ 743 'SKIP_RUNNING_SERVER:True', 744 'DIRECTORY:{}'.format(self.remote_server_directory), 745 'PORT_NUMBER:{}'.format(self.remote_server_port), 746 'RESULTS_DIRECTORY:{}'.format(output_dir), 747 ] 748 if hotspot: 749 variables.append('HOTSPOT_ACTION:' + hotspot) 750 if options.debug_mode: 751 variables.append('CONFIGURATION:Debug') 752 if options.debug_on_error: 753 variables.append('HOLD_ON_ERROR:True') 754 if options.execution_metrics: 755 variables.append('CREATE_EXECUTION_METRICS:True') 756 if options.save_logs == "always": 757 variables.append('SAVE_LOGS_WHEN:Always') 758 if options.runner == 'dotnet': 759 variables.append('BINARY_NAME:Renode.dll') 760 variables.append('RENODE_PID:{}'.format(self.renode_pid)) 761 variables.append('NET_PLATFORM:True') 762 else: 763 options.exclude.append('profiling') 764 765 if options.variables: 766 variables += options.variables 767 768 test_cases = [(test_name, '{0}.{1}'.format(suite_name, test_name)) for test_name in test_cases_names] 769 if fixture: 770 test_cases = [x for x in test_cases if fnmatch.fnmatch(x[1], '*' + fixture + '*')] 771 if len(test_cases) == 0: 772 return None 773 deps = set() 774 for test_name in (t[0] for t in test_cases): 775 deps.update(self._get_dependencies(test_name)) 776 if not self._run_dependencies(deps, options, iteration_index, suite_retry_index): 777 return False 778 779 # Listeners are called in the exact order as in `listeners` list for both `start_test` and `end_test`. 780 output_formatter = 'robot_output_formatter_verbose.py' if options.verbose else 'robot_output_formatter.py' 781 listeners = [ 782 os.path.join(this_path, f'retry_and_timeout_listener.py:{options.retry_count}'), 783 # Has to be the last one to print final state, message etc. after all the changes made by other listeners. 784 os.path.join(this_path, output_formatter), 785 ] 786 if options.listener: 787 listeners += options.listener 788 789 metadata = {"HotSpot_Action": hotspot if hotspot else '-'} 790 log_file = os.path.join(output_dir, 'results-{0}{1}.robot.xml'.format(file_name, '_' + hotspot if hotspot else '')) 791 792 keywords_path = os.path.abspath(os.path.join(this_path, "renode-keywords.robot")) 793 keywords_path = keywords_path.replace(os.path.sep, "/") # Robot wants forward slashes even on Windows 794 # This variable is provided for compatibility with Robot files that use Resource ${RENODEKEYWORDS} 795 variables.append('RENODEKEYWORDS:{}'.format(keywords_path)) 796 tools_path = os.path.join(os.path.dirname(this_path), "tools") 797 tools_path = tools_path.replace(os.path.sep, "/") 798 variables.append('RENODETOOLS:{}'.format(tools_path)) 799 suite_builder = robot.running.builder.TestSuiteBuilder() 800 suite = suite_builder.build(self.path) 801 suite.resource.imports.create(type="Resource", name=keywords_path) 802 803 suite.configure(include_tags=options.include, exclude_tags=options.exclude, 804 include_tests=[t[1] for t in test_cases], metadata=metadata, 805 name=suite_name, empty_suite_ok=True) 806 807 # Provide default values for {Suite,Test}{Setup,Teardown} 808 if not suite.setup: 809 suite.setup.config(name="Setup") 810 if not suite.teardown: 811 suite.teardown.config(name="Teardown") 812 813 for test in suite.tests: 814 if not test.setup: 815 test.setup.config(name="Reset Emulation") 816 if not test.teardown: 817 test.teardown.config(name="Test Teardown") 818 819 # Let's just fail tests which previously unexpectedly timed out. 820 if test.name in self.tests_with_unexpected_timeouts: 821 test.config(setup=None, teardown=None) 822 test.body.clear() 823 test.body.create_keyword('Fail', ["Test timed out in a previous run and won't be retried."]) 824 825 # This tag tells `RetryFailed` max retries for this test. 826 if 'test:retry' in test.tags: 827 test.tags.remove('test:retry') 828 test.tags.add('test:retry(0)') 829 830 # Timeout tests with `self.timeout_expected_tag` will be set as passed in the listener 831 # during timeout handling. Their timeout won't be influenced by the global timeout option. 832 if self.timeout_expected_tag in test.tags: 833 if not test.timeout: 834 print(f"!!!!! Test with a `{self.timeout_expected_tag}` tag must have `[Timeout]` set: {test.longname}") 835 sys.exit(1) 836 elif options.timeout: 837 # Timeout from tags is used if it's shorter than the global timeout. 838 if not test.timeout or Time(test.timeout).seconds >= options.timeout.seconds: 839 test.timeout = options.timeout.value 840 841 # Timeout handler is used in `retry_and_timeout_listener.py` and, to be able to call it, 842 # `self` is smuggled in suite's `parent` which is typically None for the main suite. 843 # Listener grabs it on suite start and resets to original value. 844 suite.parent = (self, suite.parent) 845 self.timeout_handler = self._create_timeout_handler(options, iteration_index, suite_retry_index) 846 847 result = suite.run(console='none', listener=listeners, exitonfailure=options.stop_on_error, output=log_file, log=None, loglevel='TRACE', report=None, variable=variables, skiponfailure=['non_critical', 'skipped']) 848 849 self.suite_log_files = [] 850 file_name = os.path.splitext(os.path.basename(self.path))[0] 851 output_dir = self.get_output_dir(options, iteration_index, suite_retry_index) 852 if any(self.tests_without_hotspots): 853 log_file = os.path.join(output_dir, 'results-{0}.robot.xml'.format(file_name)) 854 if os.path.isfile(log_file): 855 self.suite_log_files.append(log_file) 856 if any(self.tests_with_hotspots): 857 for hotspot in RobotTestSuite.hotspot_action: 858 if options.hotspot and options.hotspot != hotspot: 859 continue 860 log_file = os.path.join(output_dir, 'results-{0}{1}.robot.xml'.format(file_name, '_' + hotspot if hotspot else '')) 861 if os.path.isfile(log_file): 862 self.suite_log_files.append(log_file) 863 864 if options.runner == "mono": 865 self.copy_mono_logs(options, iteration_index, suite_retry_index) 866 867 return TestResult(result.return_code == 0, self.suite_log_files) 868 869 def _create_timeout_handler(self, options, iteration_index, suite_retry_index): 870 def _timeout_handler(test: robot.running.TestCase, result: robot.result.TestCase): 871 if self.timeout_expected_tag in test.tags: 872 message_start = '----- Test timed out, as expected,' 873 else: 874 # Let's make the first message stand out if the timeout wasn't expected. 875 message_start = '!!!!! Test timed out' 876 877 # Tests with unexpected timeouts won't be retried. 878 self.tests_with_unexpected_timeouts = test.name 879 print(f"{message_start} after {Time(test.timeout).seconds}s: {test.parent.name}.{test.name}") 880 print(f"----- Skipped flushing emulation log and saving state due to the timeout, restarting Renode...") 881 882 self._close_remote_server(RobotTestSuite.robot_frontend_process, options, cleanup_timeout_override=0, silent=True) 883 RobotTestSuite.robot_frontend_process = self._run_remote_server(options, iteration_index, suite_retry_index, self.remote_server_port) 884 885 # It's typically used in suite setup (renode-keywords.robot:Setup) but we don't need to 886 # call full setup which imports library etc. We only need to resend settings to Renode. 887 BuiltIn().run_keyword("Setup Renode") 888 print(f"----- ...done, running remaining tests on Renode pid {self.renode_pid} using the same port {self.remote_server_port}") 889 return _timeout_handler 890 891 892 def copy_mono_logs(self, options: Namespace, iteration_index: int, suite_retry_index: int) -> None: 893 """Copies 'mono_crash.*.json' files into the suite's logs directory. 894 895 These files are occasionally created when mono crashes. There are also 'mono_crash.*.blob' 896 files, but they contain heavier memory dumps and have questionable usefulness.""" 897 output_dir = self.get_output_dir(options, iteration_index, suite_retry_index) 898 logs_dir = os.path.join(output_dir, "logs") 899 for dirpath, dirnames, fnames in os.walk(os.getcwd()): 900 # Do not descend into "logs" directories, to prevent later invocations from 901 # stealing files already moved by earlier invocations 902 logs_indices = [x for x in range(len(dirnames)) if dirnames[x] == "logs"] 903 logs_indices.sort(reverse=True) 904 for logs_idx in logs_indices: 905 del dirnames[logs_idx] 906 907 for fname in filter(lambda x: x.startswith("mono_crash.") and x.endswith(".json"), fnames): 908 os.makedirs(logs_dir, exist_ok=True) 909 src_fpath = os.path.join(dirpath, fname) 910 dest_fpath = os.path.join(logs_dir, fname) 911 print(f"Moving mono_crash file: '{src_fpath}' -> '{dest_fpath}'") 912 os.rename(src_fpath, dest_fpath) 913 914 915 def tests_failed_due_to_renode_crash(self) -> bool: 916 # Return false if the test has not yet run 917 if self.suite_log_files is None: 918 return 919 for file in self.suite_log_files: 920 try: 921 tree = ET.parse(file) 922 except FileNotFoundError: 923 continue 924 925 root = tree.getroot() 926 927 for suite in root.iter('suite'): 928 if not suite.get('source', False): 929 continue # it is a tag used to group other suites without meaning on its own 930 for test in suite.iter('test'): 931 # do not check skipped tests 932 if test.find("./tags/[tag='skipped']"): 933 continue 934 935 # only finds immediate children - required because `status` 936 # nodes are also present lower in the tree for example 937 # for every keyword but we only need the status of the test 938 status = test.find('status') 939 if status.attrib["status"] != "FAIL": 940 continue # passed tests should not be checked for crashes 941 942 # check whether renode crashed during this test 943 if self._has_renode_crashed(test): 944 return True 945 946 return False 947 948 949 @staticmethod 950 def find_failed_tests(path, file="robot_output.xml"): 951 ret = {'mandatory': set(), 'non_critical': set()} 952 953 # Aggregate failed tests from all report files (can be multiple if iterations or retries were used) 954 for dirpath, _, fnames in os.walk(path): 955 for fname in filter(lambda x: x == file, fnames): 956 tree = ET.parse(os.path.join(dirpath, fname)) 957 root = tree.getroot() 958 for suite in root.iter('suite'): 959 if not suite.get('source', False): 960 continue # it is a tag used to group other suites without meaning on its own 961 for test in suite.iter('test'): 962 status = test.find('status') # only finds immediate children - important requirement 963 if status.attrib['status'] == 'FAIL': 964 test_name = test.attrib['name'] 965 suite_name = suite.attrib['name'] 966 if suite_name == "Test Suite": 967 # If rebot is invoked with only 1 suite, it renames that suite to Test Suite 968 # instead of wrapping in a new top-level Test Suite. A workaround is to extract 969 # the suite name from the *.robot file name. 970 suite_name = os.path.basename(suite.attrib["source"]).rsplit(".", 1)[0] 971 if test.find("./tags/[tag='skipped']"): 972 continue # skipped test should not be classified as fail 973 if test.find("./tags/[tag='non_critical']"): 974 ret['non_critical'].add(f"{suite_name}.{test_name}") 975 else: 976 ret['mandatory'].add(f"{suite_name}.{test_name}") 977 978 if not ret['mandatory'] and not ret['non_critical']: 979 return None 980 return ret 981 982 983 @classmethod 984 def find_suites_with_fails(cls, path, file="robot_output.xml"): 985 """Finds suites which contain at least one test case failure. 986 987 A suite may be successful and still contain failures, e.g. if the --retry option 988 was used and a test passed on a later attempt.""" 989 ret = set() 990 991 for dirpath, _, fnames in os.walk(path): 992 for fname in filter(lambda x: x == file, fnames): 993 tree = ET.parse(os.path.join(dirpath, fname)) 994 root = tree.getroot() 995 for suite in root.iter('suite'): 996 if not suite.get('source', False): 997 continue # it is a tag used to group other suites without meaning on its own 998 999 suite_name = suite.attrib['name'] 1000 if suite_name == "Test Suite": 1001 # If rebot is invoked with only 1 suite, it renames that suite to Test Suite 1002 # instead of wrapping in a new top-level Test Suite. A workaround is to extract 1003 # the suite name from the *.robot file name. 1004 suite_name = os.path.basename(suite.attrib["source"]).rsplit(".", 1)[0] 1005 1006 for test in suite.iter('test'): 1007 if test.find("./tags/[tag='skipped']"): 1008 continue # skipped test should not be classified as fail 1009 status = test.find('status') # only finds immediate children - important requirement 1010 if status.attrib["status"] == "FAIL": 1011 ret.add(suite_name) 1012 break 1013 if status.text is not None and cls.retry_test_regex.search(status.text): 1014 # Retried test cases still count as fails 1015 ret.add(suite_name) 1016 break 1017 1018 return ret 1019 1020 1021 @staticmethod 1022 def group_log_paths(paths: List[str]) -> Dict[Tuple[int, int], Set[str]]: 1023 """Breaks a list of log paths into subsets grouped by (iteration, suite_retry) pairs.""" 1024 re_path_indices_patterns = ( 1025 re.compile(r"\biteration(?P<iteration>\d+)/retry(?P<suite_retry>\d+)/"), 1026 re.compile(r"\bretry(?P<suite_retry>\d+)/"), 1027 re.compile(r"\biteration(?P<iteration>\d+)/"), 1028 ) 1029 ret = defaultdict(lambda: set()) 1030 for path in paths: 1031 iteration = 1 1032 suite_retry = 0 1033 for pattern in re_path_indices_patterns: 1034 match = pattern.search(path) 1035 if match is None: 1036 continue 1037 try: 1038 iteration = int(match.group("iteration")) 1039 except IndexError: 1040 pass 1041 try: 1042 suite_retry = int(match.group("suite_retry")) 1043 except IndexError: 1044 pass 1045 ret[(iteration, suite_retry)].add(path) 1046 return ret 1047 1048 1049 @classmethod 1050 def find_rerun_tests(cls, path): 1051 1052 def analyze_xml(label, retry_dir, file="robot_output.xml"): 1053 try: 1054 tree = ET.parse(os.path.join(retry_dir, file)) 1055 except FileNotFoundError: 1056 return 1057 root = tree.getroot() 1058 for suite in root.iter('suite'): 1059 if not suite.get('source', False): 1060 continue # it is a tag used to group other suites without meaning on its own 1061 suite_name = suite.attrib['name'] 1062 if suite_name == "Test Suite": 1063 # If rebot is invoked with only 1 suite, it renames that suite to Test Suite 1064 # instead of wrapping in a new top-level Test Suite. A workaround is to extract 1065 # the suite name from the *.robot file name. 1066 suite_name = os.path.basename(suite.attrib["source"]).rsplit(".", 1)[0] 1067 for test in suite.iter('test'): 1068 test_name = test.attrib['name'] 1069 tags = [] 1070 if test.find("./tags/[tag='skipped']"): 1071 continue # skipped test should not be classified as fail 1072 if test.find("./tags/[tag='non_critical']"): 1073 tags.append("non_critical") 1074 status = test.find('status') # only finds immediate children - important requirement 1075 m = cls.retry_test_regex.search(status.text) if status.text is not None else None 1076 1077 # Check whether renode crashed during this test 1078 has_renode_crashed = cls._has_renode_crashed(test) 1079 1080 status_str = status.attrib["status"] 1081 nth = (1 + int(m.group(2))) if m else 1 1082 key = f"{suite_name}.{test_name}" 1083 if key not in data: 1084 data[key] = [] 1085 data[key].append({ 1086 "label": label, # e.g. "retry0", "retry1", "iteration1/retry2", ... 1087 "status": status_str, # e.g. "PASS", "FAIL", "SKIP", ... 1088 "nth": nth, # The number of test case attempts that led to the above status 1089 "tags": tags, # e.g. ["non_critical"], [], ... 1090 "crash": has_renode_crashed, 1091 }) 1092 1093 def analyze_iteration(iteration_dir): 1094 iteration_dirname = os.path.basename(iteration_dir) 1095 report_fpath = os.path.join(iteration_dir, "robot_output.xml") 1096 if os.path.isfile(report_fpath): 1097 analyze_xml(iteration_dirname, iteration_dir) 1098 return 1099 i = -1 1100 while True: 1101 i += 1 1102 retry_dirpath = os.path.join(iteration_dir, f"retry{i}") 1103 if os.path.isdir(retry_dirpath): 1104 analyze_xml(os.path.join(iteration_dirname, f"retry{i}"), retry_dirpath) 1105 continue 1106 break 1107 1108 data = OrderedDict() 1109 i = -1 1110 while True: 1111 i += 1 1112 iteration_dirpath = os.path.join(path, f"iteration{i + 1}") 1113 retry_dirpath = os.path.join(path, f"retry{i}") 1114 if os.path.isdir(iteration_dirpath): 1115 analyze_iteration(iteration_dirpath) 1116 continue 1117 elif os.path.isdir(retry_dirpath): 1118 analyze_xml(f"retry{i}", retry_dirpath) 1119 continue 1120 break 1121 1122 return data 1123
