1# pylint: disable=C0301,C0103,C0111
2from __future__ import print_function
3from sys import platform
4import os
5import signal
6import psutil
7import subprocess
8from time import monotonic
9from typing import Dict, List
10
11import xml.etree.ElementTree as ET
12import glob
13
14from tests_engine import TestResult
15
16this_path = os.path.abspath(os.path.dirname(__file__))
17
18def install_cli_arguments(parser):
19    parser.add_argument("--properties-file", action="store", help="Location of properties file.")
20    parser.add_argument("--skip-building", action="store_true", help="Do not build tests before run.")
21    parser.add_argument("--force-net-framework-version", action="store", dest="framework_ver_override", help="Override target .NET Framework version when building tests.")
22
23class NUnitTestSuite(object):
24    nunit_path = os.path.join(this_path, './../lib/resources/tools/nunit3/nunit3-console.exe')
25
26    def __init__(self, path):
27        #super(NUnitTestSuite, self).__init__(path)
28        self.path = path
29
30    def check(self, options, number_of_runs): #API requires this method
31        pass
32
33
34    def get_output_dir(self, options, iteration_index, suite_retry_index):
35        # Unused mechanism, this exists to keep a uniform interface with
36        # robot_tests_provider.py.
37        return options.results_directory
38
39
40    # NOTE: if we switch to using msbuild on all platforms, we can get rid of this function and only use the '-' prefix
41    def build_params(self, *params):
42        def __decorate_build_param(p):
43            if self.builder == 'xbuild':
44                return '/' + p
45            else:
46                return '-' + p
47
48        ret = []
49        for i in params:
50            ret += [__decorate_build_param(i)]
51        return ret
52
53    def prepare(self, options):
54        if not options.skip_building:
55            print("Building {0}".format(self.path))
56            if options.runner == 'dotnet':
57                self.builder = 'dotnet'
58                params = ['build', '--verbosity', 'quiet', '--configuration', options.configuration, '/p:NET=true']
59            else:
60                if platform == "win32":
61                    self.builder = 'MSBuild.exe'
62                else:
63                    self.builder = 'xbuild'
64                params = self.build_params('p:PropertiesLocation={0}'.format(options.properties_file), 'p:OutputPath={0}'.format(options.results_directory), 'nologo', 'verbosity:quiet', 'p:OutputDir=tests_output', 'p:Configuration={0}'.format(options.configuration))
65                if options.framework_ver_override:
66                    params += self.build_params(f'p:TargetFrameworkVersion=v{options.framework_ver_override}')
67            result = subprocess.call([self.builder, *params, self.path])
68            if result != 0:
69                print("Building project `{}` failed with error code: {}".format(self.path, result))
70                return result
71        else:
72            print('Skipping the build')
73
74        return 0
75
76    def _cleanup_dangling(self, process, proc_name, test_agent_name):
77        for proc in psutil.process_iter(['pid', 'name', 'cmdline']):
78            if proc_name in (proc.info['name'] or ''):
79                flat_cmdline = ' '.join(proc.info['cmdline'] or [])
80                if test_agent_name in flat_cmdline and '--pid={}'.format(process.pid) in flat_cmdline:
81                    # let's kill it
82                    print('KILLING A DANGLING {} test process {}'.format(test_agent_name, proc.info['pid']))
83                    os.kill(proc.info['pid'], signal.SIGTERM)
84
85    def run(self, options, run_id, iteration_index=1, suite_retry_index=0):
86        # The iteration_index and suite_retry_index arguments are not implemented.
87        # They exist for the sake of a uniform interface with robot_tests_provider.
88        print('Running ' + self.path)
89
90        project_file = os.path.split(self.path)[1]
91        output_file = os.path.join(options.results_directory, 'results-{}.xml'.format(project_file))
92
93        if options.runner == 'dotnet':
94            print('Using native dotnet test runner -' + self.path, flush=True)
95            # we don't build here - we had problems with concurrently occurring builds when copying files to one output directory
96            # so we run test with --no-build and build tests in previous stage
97            args = ['dotnet', 'test', "--no-build", "--logger", "console;verbosity=detailed", "--logger", "trx;LogFileName={}".format(output_file), '--configuration', options.configuration, self.path]
98        else:
99            args = [NUnitTestSuite.nunit_path, '--domain=None', '--noheader', '--labels=Before', '--result={}'.format(output_file), project_file.replace("csproj", "dll")]
100
101        # Unfortunately, debugging like this won't work on .NET, see: https://github.com/dotnet/sdk/issues/4994
102        # The easiest workaround is to set VSTEST_HOST_DEBUG=1 in your environment
103        if options.stop_on_error:
104            args.append('--stoponerror')
105        if (platform.startswith("linux") or platform == "darwin") and options.runner != 'dotnet':
106            args.insert(0, 'mono')
107            if options.port is not None:
108                if options.suspend:
109                    print('Waiting for a debugger at port: {}'.format(options.port))
110                args.insert(1, '--debug')
111                args.insert(2, '--debugger-agent=transport=dt_socket,server=y,suspend={0},address=127.0.0.1:{1}'.format('y' if options.suspend else 'n', options.port))
112            elif options.debug_mode:
113                args.insert(1, '--debug')
114
115        where_conditions = []
116        if options.fixture:
117            if options.runner == 'dotnet':
118                where_conditions.append(options.fixture)
119            else:
120                where_conditions.append('test =~ .*{}.*'.format(options.fixture))
121
122        cat = 'TestCategory' if options.runner == 'dotnet' else 'cat'
123        equals = '=' if options.runner == 'dotnet' else '=='
124        if options.exclude:
125            for category in options.exclude:
126                where_conditions.append('{} != {}'.format(cat, category))
127        if options.include:
128            for category in options.include:
129                where_conditions.append('{} {} {}'.format(cat, equals, category))
130
131        if where_conditions:
132            if options.runner == 'dotnet':
133                args.append('--filter')
134                args.append(' & '.join('({})'.format(x) for x in where_conditions))
135            else:
136                args.append('--where= ' + ' and '.join(['({})'.format(x) for x in where_conditions]))
137
138        if options.run_gdb:
139            args = ['gdb', '-ex', 'handle SIGXCPU SIG33 SIG35 SIG36 SIGPWR nostop noprint', '--args'] + args
140
141        startTimestamp = monotonic()
142        if options.runner == 'dotnet':
143            args += ['--', 'NUnit.DisplayName=FullName']
144            process = subprocess.Popen(args)
145            print('dotnet test runner PID is {}'.format(process.pid), flush=True)
146        else:
147            process = subprocess.Popen(args, cwd=options.results_directory)
148            print('NUnit3 runner PID is {}'.format(process.pid), flush=True)
149
150        process.wait()
151        if options.runner == 'dotnet':
152            self._cleanup_dangling(process, 'dotnet', 'dotnet test')
153        else:
154            self._cleanup_dangling(process, 'mono', 'nunit-agent.exe')
155
156        result = process.returncode == 0
157        endTimestamp = monotonic()
158        print('Suite ' + self.path + (' finished successfully!' if result else ' failed!') + ' in ' + str(round(endTimestamp - startTimestamp, 2)) + ' seconds.', flush=True)
159        return TestResult(result, [output_file])
160
161    def cleanup(self, options):
162        pass
163
164
165    def should_retry_suite(self, options, iteration_index, suite_retry_index):
166        # Unused mechanism, this exists to keep a uniform interface with
167        # robot_tests_provider.py.
168        return False
169
170
171    def tests_failed_due_to_renode_crash(self) -> bool:
172        # Unused mechanism, this exists to keep a uniform interface with
173        # robot_tests_provider.py.
174        return False
175
176
177    @staticmethod
178    def find_failed_tests(path, files_pattern='*.csproj.xml'):
179        test_files = glob.glob(os.path.join(path, files_pattern))
180        ret = {'mandatory': []}
181        for test_file in test_files:
182            tree = ET.parse(test_file)
183            root = tree.getroot()
184
185            # we analyze both types of output files (nunit and dotnet test) to avoid passing options as parameter
186            # the cost should be negligible in the context of compiling and running test suites
187
188            # nunit runner
189            for test in root.iter('test-case'):
190                if test.attrib['result'] == 'Failed':
191                    ret['mandatory'].append(test.attrib['fullname'])
192
193            # dotnet runner
194            xmlns="http://microsoft.com/schemas/VisualStudio/TeamTest/2010"
195            for test in root.iter(f"{{{xmlns}}}UnitTestResult"):
196                if test.attrib['outcome'] == 'Failed':
197                    ret['mandatory'].append(test.attrib['testName'])
198
199        if not ret['mandatory']:
200            return None
201        return ret
202
203
204    @staticmethod
205    def find_rerun_tests(path):
206        # Unused mechanism, this exists to keep a uniform interface with
207        # robot_tests_provider.py.
208        return None
209