1#!/usr/bin/env python
2#
3# Copyright (c) 2016, The OpenThread Authors.
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are met:
8# 1. Redistributions of source code must retain the above copyright
9#    notice, this list of conditions and the following disclaimer.
10# 2. Redistributions in binary form must reproduce the above copyright
11#    notice, this list of conditions and the following disclaimer in the
12#    documentation and/or other materials provided with the distribution.
13# 3. Neither the name of the copyright holder nor the
14#    names of its contributors may be used to endorse or promote products
15#    derived from this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
21# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27# POSSIBILITY OF SUCH DAMAGE.
28#
29
30import ConfigParser
31import argparse
32import fnmatch
33import logging
34import json
35import os
36import sys
37import time
38import unittest
39from collections import OrderedDict
40
41from autothreadharness.harness_case import HarnessCase
42from autothreadharness.open_thread_controller import OpenThreadController
43from autothreadharness import settings
44
45logging.basicConfig(level=logging.INFO)
46
47logger = logging.getLogger()
48"""Logger: The global logger"""
49
50logger.setLevel(logging.INFO)
51
52RESUME_SCRIPT_PATH = "%appdata%\\Microsoft\\Windows\\Start Menu\\Programs\\" "Startup\\continue_harness.bat"
53
54
55class SimpleTestResult(unittest.TestResult):
56
57    executions = 0
58
59    def __init__(self, path, auto_reboot_args=None, keep_explorer=False, add_all_devices=False):
60        """Record test results in json file
61
62        Args:
63            path (str): File path to record the results
64            auto_reboot (bool): Whether reboot when harness die
65        """
66        super(SimpleTestResult, self).__init__()
67        self.path = path
68        self.auto_reboot_args = auto_reboot_args
69        self.result = json.load(open(self.path, 'r'))
70        self.log_handler = None
71        self.started = None
72        self.keep_explorer = keep_explorer
73        self.add_all_devices = add_all_devices
74        SimpleTestResult.executions += 1
75        logger.info('Initial state is %s', json.dumps(self.result, indent=2))
76
77    def startTest(self, test):
78        logger.info(
79            '\n========================================\n%s\n========================================',
80            test.__class__.__name__,
81        )
82
83        test.add_all_devices = self.add_all_devices
84        # create start up script if auto reboot enabled
85        if self.auto_reboot_args:
86            test.auto_reboot = True
87            os.system('echo %s > "%s"' %
88                      (' '.join(self.auto_reboot_args + ['-c', test.__class__.__name__]), RESUME_SCRIPT_PATH))
89
90        # record start timestamp
91        self.started = time.strftime('%Y-%m-%dT%H:%M:%S')
92
93        os.system('mkdir %s' % test.result_dir)
94        self.log_handler = logging.FileHandler('%s\\auto-%s.log' % (test.result_dir, time.strftime('%Y%m%d%H%M%S')))
95        self.log_handler.setLevel(logging.DEBUG)
96        self.log_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
97        logger.addHandler(self.log_handler)
98
99    def add_result(self, test, passed, error=None):
100        """Record test result into json file
101
102        Args:
103            test (TestCase): The test just run
104            passed (bool): Whether the case is passed
105        """
106        fails = self.result.get(test.__class__.__name__, {}).get('fails', 0)
107        if passed is False:
108            fails += 1
109        self.result[str(test.__class__.__name__)] = {
110            'started': self.started,
111            'stopped': time.strftime('%Y-%m-%dT%H:%M:%S'),
112            'passed': passed,
113            'fails': fails,
114            'error': error,
115            'executions': SimpleTestResult.executions,
116        }
117        if self.auto_reboot_args:
118            os.system('del "%s"' % RESUME_SCRIPT_PATH)
119
120        json.dump(OrderedDict(sorted(self.result.items(), key=lambda t: t[0])), open(self.path, 'w'), indent=2)
121
122        # save logs
123        logger.removeHandler(self.log_handler)
124        self.log_handler.close()
125        self.log_handler = None
126        time.sleep(2)
127
128        # close explorers
129        if not self.keep_explorer:
130            os.system('taskkill /f /im explorer.exe && start explorer.exe')
131
132    def addSuccess(self, test):
133        logger.info('case[%s] pass', test.__class__.__name__)
134        super(SimpleTestResult, self).addSuccess(test)
135        self.add_result(test, True)
136
137    def addFailure(self, test, err):
138        logger.warning('case[%s] fail', test.__class__.__name__)
139        super(SimpleTestResult, self).addFailure(test, err)
140        self.add_result(test, False)
141
142    def addError(self, test, err):
143        logger.error('case[%s] error', test.__class__.__name__, exc_info=err)
144
145        if err and err[0] is SystemExit:
146            if self.auto_reboot_args:
147                logger.warning('rebooting..')
148                os.system('shutdown /r /t 1')
149            else:
150                logger.warning('exiting..')
151            sys.exit(1)
152
153        super(SimpleTestResult, self).addError(test, err)
154        self.add_result(test, None, str(err[1]))
155
156
157def list_devices(names=None, continue_from=None, **kwargs):
158    """List devices in settings file and print versions"""
159
160    if not names:
161        names = [device for device, _type in settings.GOLDEN_DEVICES if _type == 'OpenThread']
162
163    if continue_from:
164        continue_from = names.index(continue_from)
165    else:
166        continue_from = 0
167
168    for port in names[continue_from:]:
169        try:
170            with OpenThreadController(port) as otc:
171                print('%s: %s' % (port, otc.version))
172        except BaseException:
173            logger.exception('failed to get version of %s' % port)
174
175
176def discover(
177    names=None,
178    pattern=['*.py'],
179    skip='efp',
180    dry_run=False,
181    denylist=None,
182    name_greps=None,
183    manual_reset=False,
184    delete_history=False,
185    max_devices=0,
186    continue_from=None,
187    result_file='./result.json',
188    auto_reboot=False,
189    keep_explorer=False,
190    add_all_devices=False,
191):
192    """Discover all test cases and skip those passed
193
194    Args:
195        pattern (str): Pattern to match case modules, refer python's unittest
196                       documentation for more details
197        skip (str): types cases to skip
198    """
199    if not os.path.exists(settings.OUTPUT_PATH):
200        os.mkdir(settings.OUTPUT_PATH)
201
202    if delete_history:
203        os.system('del history.json')
204
205    if denylist:
206        try:
207            excludes = [line.strip('\n') for line in open(denylist, 'r').readlines() if not line.startswith('#')]
208        except BaseException:
209            logger.exception('Failed to open test case denylist file')
210            raise
211    else:
212        excludes = []
213
214    log = None
215    if os.path.isfile(result_file):
216        try:
217            log = json.load(open(result_file, 'r'))
218        except BaseException:
219            logger.exception('Failed to open result file')
220
221    if not log:
222        log = {}
223        json.dump(log, open(result_file, 'w'), indent=2)
224
225    new_th = False
226    harness_info = ConfigParser.ConfigParser()
227    harness_info.read('%s\\info.ini' % settings.HARNESS_HOME)
228    if harness_info.has_option('Thread_Harness_Info', 'Version') and harness_info.has_option(
229            'Thread_Harness_Info', 'Mode'):
230        harness_version = harness_info.get('Thread_Harness_Info', 'Version').rsplit(' ', 1)[1]
231        harness_mode = harness_info.get('Thread_Harness_Info', 'Mode')
232
233        if harness_mode == 'External' and harness_version > '1.4.0':
234            new_th = True
235
236        if harness_mode == 'Internal' and harness_version > '49.4':
237            new_th = True
238
239    suite = unittest.TestSuite()
240    if new_th:
241        discovered = unittest.defaultTestLoader.discover('cases', pattern)
242    else:
243        discovered = unittest.defaultTestLoader.discover('cases_R140', pattern)
244
245    if names and continue_from:
246        names = names[names.index(continue_from):]
247
248    for s1 in discovered:
249        for s2 in s1:
250            for case in s2:
251                if case.__class__ is HarnessCase:
252                    continue
253                case_name = str(case.__class__.__name__)
254
255                # grep name
256                if name_greps and not any(fnmatch.fnmatch(case_name, name_grep) for name_grep in name_greps):
257                    logger.info('case[%s] skipped by name greps', case_name)
258                    continue
259
260                # allowlist
261                if len(names) and case_name not in names:
262                    logger.info('case[%s] skipped', case_name)
263                    continue
264
265                # skip cases
266                if case_name in log:
267                    if ((log[case_name]['passed'] and ('p' in skip)) or
268                        (log[case_name]['passed'] is False and ('f' in skip)) or (log[case_name]['passed'] is None and
269                                                                                  ('e' in skip))):
270                        logger.warning('case[%s] skipped for its status[%s]', case_name, log[case_name]['passed'])
271                        continue
272
273                # continue from
274                if continue_from:
275                    if continue_from != case_name:
276                        logger.warning('case[%s] skipped for continue from[%s]', case_name, continue_from)
277                        continue
278                    else:
279                        continue_from = None
280
281                # denylist
282                if case_name in excludes:
283                    logger.warning('case[%s] skipped for denylist', case_name)
284                    continue
285
286                # max devices
287                if max_devices and case.golden_devices_required > max_devices:
288                    logger.warning('case[%s] skipped for exceeding max golden devices allowed[%d]', case_name,
289                                   max_devices)
290                    continue
291
292                suite.addTest(case)
293                logger.info('case[%s] added', case_name)
294
295    if auto_reboot:
296        argv = []
297        argv.append('"%s"' % os.sep.join([os.getcwd(), 'start.bat']))
298        argv.extend(['-p', pattern])
299        argv.extend(['-k', skip])
300        argv.extend(['-o', result_file])
301        argv.append('-a')
302
303        if manual_reset:
304            argv.append('-m')
305
306        if delete_history:
307            argv.append('-d')
308
309        auto_reboot_args = argv + names
310    else:
311        auto_reboot_args = None
312        if os.path.isfile(RESUME_SCRIPT_PATH):
313            os.system('del "%s"' % RESUME_SCRIPT_PATH)
314
315    # manual reset
316    if manual_reset:
317        settings.PDU_CONTROLLER_TYPE = 'MANUAL_PDU_CONTROLLER'
318        settings.PDU_CONTROLLER_OPEN_PARAMS = {}
319        settings.PDU_CONTROLLER_REBOOT_PARAMS = {}
320
321    result = SimpleTestResult(result_file, auto_reboot_args, keep_explorer, add_all_devices)
322    for case in suite:
323        logger.info(case.__class__.__name__)
324
325    if dry_run:
326        return
327
328    suite.run(result)
329    return result
330
331
332def main():
333    parser = argparse.ArgumentParser(description='Thread harness test case runner')
334    parser.add_argument('--auto-reboot',
335                        '-a',
336                        action='store_true',
337                        default=False,
338                        help='restart system when harness service die')
339    parser.add_argument('names',
340                        metavar='NAME',
341                        type=str,
342                        nargs='*',
343                        default=None,
344                        help='test case name, omit to test all')
345    parser.add_argument('--denylist',
346                        '-b',
347                        metavar='DENYLIST_FILE',
348                        type=str,
349                        help='file to list test cases to skip',
350                        default=None)
351    parser.add_argument('--continue-from', '-c', type=str, default=None, help='first case to test')
352    parser.add_argument('--delete-history', '-d', action='store_true', default=False, help='clear history on startup')
353    parser.add_argument('--keep-explorer',
354                        '-e',
355                        action='store_true',
356                        default=False,
357                        help='do not restart explorer.exe at the end')
358    parser.add_argument('--name-greps', '-g', action='append', default=None, help='grep case by names')
359    parser.add_argument('--list-file', '-i', type=str, default=None, help='file to list cases names to test')
360    parser.add_argument(
361        '--skip',
362        '-k',
363        metavar='SKIP',
364        type=str,
365        help='type of results to skip. e for error, f for fail, p for pass.',
366        default='',
367    )
368    parser.add_argument('--list-devices', '-l', action='store_true', default=False, help='list devices')
369    parser.add_argument('--manual-reset', '-m', action='store_true', default=False, help='reset devices manually')
370    parser.add_argument('--dry-run', '-n', action='store_true', default=False, help='just show what to run')
371    parser.add_argument(
372        '--result-file',
373        '-o',
374        type=str,
375        default=settings.OUTPUT_PATH + '\\result.json',
376        help='file to store and read current status',
377    )
378    parser.add_argument('--pattern',
379                        '-p',
380                        metavar='PATTERN',
381                        type=str,
382                        help='file name pattern, default to "*.py"',
383                        default='*.py')
384    parser.add_argument('--rerun-fails', '-r', type=int, default=0, help='number of times to rerun failed test cases')
385    parser.add_argument('--add-all-devices',
386                        '-t',
387                        action='store_true',
388                        default=False,
389                        help='add all devices to the test bed')
390    parser.add_argument('--max-devices', '-u', type=int, default=0, help='max golden devices allowed')
391
392    args = vars(parser.parse_args())
393
394    if args['list_file']:
395        try:
396            names = [line.strip('\n') for line in open(args['list_file'], 'r').readlines() if not line.startswith('#')]
397        except BaseException:
398            logger.exception('Failed to open test case list file')
399            raise
400        else:
401            args['names'] = args['names'] + names
402
403    args.pop('list_file')
404
405    if args.pop('list_devices', False):
406        list_devices(**args)
407        return
408
409    rerun_fails = args.pop('rerun_fails')
410    result = discover(**args)
411
412    if rerun_fails > 0:
413        for i in range(rerun_fails):
414            failed_names = {name for name in result.result if result.result[name]['passed'] is False}
415            if not failed_names:
416                break
417            logger.info('Rerunning failed test cases')
418            logger.info('Rerun #{}:'.format(i + 1))
419            result = discover(
420                names=failed_names,
421                pattern=args['pattern'],
422                skip='',
423                result_file=args['result_file'],
424                auto_reboot=args['auto_reboot'],
425                keep_explorer=args['keep_explorer'],
426                add_all_devices=args['add_all_devices'],
427            )
428
429
430if __name__ == '__main__':
431    main()
432