1#!/usr/bin/env python3
2#
3# Remote test case executor
4# Copyright (c) 2016, Tieto Corporation
5#
6# This software may be distributed under the terms of the BSD license.
7# See README for more details.
8
9import os
10import re
11import sys
12import time
13import traceback
14import getopt
15from datetime import datetime
16from random import shuffle
17
18import logging
19logger = logging.getLogger()
20
21scriptsdir = os.path.dirname(os.path.realpath(sys.modules[__name__].__file__))
22sys.path.append(os.path.join(scriptsdir, '..', '..', 'wpaspy'))
23sys.path.append(os.path.join(scriptsdir, '..', 'hwsim'))
24
25import wpaspy
26import config
27from test_devices import show_devices
28from test_devices import check_devices
29from rutils import TestSkip
30from utils import HwsimSkip
31from hwsim_wrapper import run_hwsim_test
32
33def usage():
34    print("USAGE: " + sys.argv[0] + " -t devices")
35    print("USAGE: " + sys.argv[0] + " -t check_devices")
36    print("USAGE: " + sys.argv[0] + " -d <dut_name> -t <all|sanity|tests_to_run> [-r <ref_name>] [-c <cfg_file.py>] [-m <all|monitor_name>] [-h hwsim_tests] [-f hwsim_modules][-R][-T][-P][-S][-v]")
37    print("USAGE: " + sys.argv[0])
38
39def get_devices(devices, duts, refs, monitors):
40    for dut in duts:
41        config.get_device(devices, dut, lock=True)
42    for ref in refs:
43        config.get_device(devices, ref, lock=True)
44    for monitor in monitors:
45        if monitor == "all":
46            continue
47        if monitor in duts:
48            continue
49        if monitor in refs:
50            continue
51        config.get_device(devices, monitor, lock=True)
52
53def put_devices(devices, duts, refs, monitors):
54    for dut in duts:
55        config.put_device(devices, dut)
56    for ref in refs:
57        config.put_device(devices, ref)
58    for monitor in monitors:
59        if monitor == "all":
60            continue
61        if monitor in duts:
62            continue
63        if monitor in refs:
64            continue
65        config.put_device(devices, monitor)
66
67def main():
68    duts = []
69    refs = []
70    monitors = []
71    filter_keys = []
72    requested_tests = ["help"]
73    requested_hwsim_tests = []
74    hwsim_tests = []
75    requested_modules = []
76    modules_tests = []
77    cfg_file = "cfg.py"
78    log_dir = "./logs/"
79    verbose = False
80    trace = False
81    restart = False
82    perf = False
83    shuffle_tests = False
84
85    # parse input parameters
86    try:
87        opts, args = getopt.getopt(sys.argv[1:], "d:f:r:t:l:k:c:m:h:vRPTS",
88                                   ["dut=", "modules=", "ref=", "tests=",
89                                    "log-dir=",
90                                    "cfg=", "key=", "monitor=", "hwsim="])
91    except getopt.GetoptError as err:
92        print(err)
93        usage()
94        sys.exit(2)
95
96    for option, argument in opts:
97        if option == "-v":
98            verbose = True
99        elif option == "-R":
100            restart = True
101        elif option == "-T":
102            trace = True
103        elif option == "-P":
104            perf = True
105        elif option == "-S":
106            shuffle_tests = True
107        elif option in ("-d", "--dut"):
108            duts.append(argument)
109        elif option in ("-r", "--ref"):
110            refs.append(argument)
111        elif option in ("-t", "--tests"):
112            requested_tests = re.split('; | |, ', argument)
113        elif option in ("-l", "--log-dir"):
114            log_dir = argument
115        elif option in ("-k", "--key"):
116            filter_keys.append(argument)
117        elif option in ("-m", "--monitor"):
118            monitors.append(argument)
119        elif option in ("-c", "--cfg"):
120            cfg_file = argument
121        elif option in ("-h", "--hwsim"):
122            requested_hwsim_tests = re.split('; | |, ', argument)
123        elif option in ("-f", "--modules"):
124            requested_modules = re.split('; | |, ', argument)
125        else:
126            assert False, "unhandled option"
127
128    # get env configuration
129    setup_params = config.get_setup_params(cfg_file)
130    devices = config.get_devices(cfg_file)
131
132    # put logs in log_dir
133    symlink = os.path.join(log_dir, "current");
134    if os.path.exists(symlink):
135        os.unlink(symlink)
136    log_dir = os.path.join(log_dir, time.strftime("%Y_%m_%d_%H_%M_%S"))
137    if not os.path.exists(log_dir):
138        os.makedirs(log_dir)
139    os.symlink(os.path.join("../", log_dir), symlink)
140
141    # setup restart/trace/perf request
142    setup_params['local_log_dir'] = log_dir
143    setup_params['restart_device'] = restart
144    setup_params['trace'] = trace
145    setup_params['perf'] = perf
146
147    # configure logger
148    logger.setLevel(logging.DEBUG)
149
150    stdout_handler = logging.StreamHandler()
151    stdout_handler.setLevel(logging.WARNING)
152    if verbose:
153        stdout_handler.setLevel(logging.DEBUG)
154    logger.addHandler(stdout_handler)
155
156    formatter = logging.Formatter('%(asctime)s - %(message)s')
157    file_name = os.path.join(log_dir, 'run-tests.log')
158    log_handler = logging.FileHandler(file_name)
159    log_handler.setLevel(logging.DEBUG)
160    log_handler.setFormatter(formatter)
161    logger.addHandler(log_handler)
162
163    # import available tests
164    tests = []
165    failed = []
166    test_modules = []
167    files = os.listdir(scriptsdir)
168    for t in files:
169        m = re.match(r'(test_.*)\.py$', t)
170        if m:
171            mod = __import__(m.group(1))
172            test_modules.append(mod.__name__.replace('test_', '', 1))
173            for key, val in mod.__dict__.items():
174                if key.startswith("test_"):
175                    tests.append(val)
176    test_names = list(set([t.__name__.replace('test_', '', 1) for t in tests]))
177
178    # import test_*
179    files = os.listdir("../hwsim/")
180    for t in files:
181        m = re.match(r'(test_.*)\.py$', t)
182        if m:
183            mod = __import__(m.group(1))
184            test_modules.append(mod.__name__.replace('test_', '', 1))
185            for key, val in mod.__dict__.items():
186                if key.startswith("test_"):
187                    hwsim_tests.append(val)
188
189    # setup hwsim tests
190    hwsim_tests_to_run = []
191    if len(requested_hwsim_tests) > 0:
192        # apply filters
193        for filter_key in filter_keys:
194            filtered_tests = []
195            for hwsim_test in hwsim_tests:
196                if re.search(filter_key, hwsim_test.__name__):
197                    filtered_tests.append(hwsim_test)
198            hwsim_tests = filtered_tests
199
200        # setup hwsim_test we should run
201        if requested_hwsim_tests[0] == "all":
202            hwsim_tests_to_run = hwsim_tests
203        elif requested_hwsim_tests[0] == "remote":
204            hwsim_tests_to_run = [t for t in hwsim_tests
205                                  if hasattr(t, "remote_compatible") and
206                                     t.remote_compatible]
207        else:
208            for test in requested_hwsim_tests:
209                t = None
210                for tt in hwsim_tests:
211                    name = tt.__name__.replace('test_', '', 1)
212                    if name == test:
213                        t = tt
214                        break
215                if not t:
216                    logger.warning("hwsim test case: " + test + " NOT-FOUND")
217                    continue
218                hwsim_tests_to_run.append(t)
219
220    # import test_* from modules
221    files = os.listdir("../hwsim/")
222    for t in files:
223        m = re.match(r'(test_.*)\.py$', t)
224        if m:
225            mod = __import__(m.group(1))
226            if mod.__name__.replace('test_', '', 1) not in requested_modules:
227                continue
228            for key, val in mod.__dict__.items():
229                if key.startswith("test_"):
230                    modules_tests.append(val)
231
232    if len(requested_modules) > 0:
233        requested_hwsim_tests = modules_tests
234        hwsim_tests_to_run = modules_tests
235
236    # sort the list
237    test_names.sort()
238    tests.sort(key=lambda t: t.__name__)
239
240    # print help
241    if requested_tests[0] == "help" and len(requested_hwsim_tests) == 0:
242        usage()
243        print("\nAvailable Devices:")
244        for device in devices:
245            print("\t", device['name'])
246        print("\nAvailable tests:")
247        for test in test_names:
248            print("\t", test)
249        print("\nAvailable hwsim tests:")
250        for hwsim_test in hwsim_tests:
251            print("\t", hwsim_test.__name__.replace('test_', '', 1))
252        return
253
254    # show/check devices
255    if requested_tests[0] == "devices":
256        show_devices(devices, setup_params)
257        return
258
259    # apply filters
260    for filter_key in filter_keys:
261        filtered_tests = []
262        for test in tests:
263            if re.search(filter_key, test.__name__):
264                filtered_tests.append(test)
265        tests = filtered_tests
266
267    # setup test we should run
268    tests_to_run = []
269    if requested_tests[0] == "all":
270        tests_to_run = tests
271    if requested_tests[0] == "help":
272        pass
273    elif requested_tests[0] == "sanity":
274        for test in tests:
275            if test.__name__.startswith("test_sanity_"):
276                tests_to_run.append(test)
277    else:
278        for test in requested_tests:
279            t = None
280            for tt in tests:
281                name = tt.__name__.replace('test_', '', 1)
282                if name == test:
283                    t = tt
284                    break
285            if not t:
286                logger.warning("test case: " + test + " NOT-FOUND")
287                continue
288            tests_to_run.append(t)
289
290    if shuffle_tests:
291        shuffle(tests_to_run)
292        shuffle(hwsim_tests_to_run)
293
294    # lock devices
295    try:
296        get_devices(devices, duts, refs, monitors)
297    except Exception as e:
298        logger.warning("get devices failed: " + str(e))
299        logger.info(traceback.format_exc())
300        put_devices(devices, duts, refs, monitors)
301        return
302    except:
303        logger.warning("get devices failed")
304        logger.info(traceback.format_exc())
305        put_devices(devices, duts, refs, monitors)
306        return
307
308    # now run test cases
309    for dut in duts:
310        if len(requested_hwsim_tests) > 0:
311            logger.warning("DUT (apdev): " + str(dut))
312        else:
313            logger.warning("DUT: " + str(dut))
314    for ref in refs:
315        if len(requested_hwsim_tests) > 0:
316            logger.warning("REF   (dev): " + str(ref))
317        else:
318            logger.warning("REF: " + str(ref))
319    for monitor in monitors:
320        logger.warning("MON: " + str(monitor))
321
322    # run check_devices at beginning
323    logger.warning("RUN check_devices")
324    try:
325        check_devices(devices, setup_params, refs, duts, monitors)
326    except Exception as e:
327        logger.warning("FAILED: " + str(e))
328        logger.info(traceback.format_exc())
329        put_devices(devices, duts, refs, monitors)
330        return
331    except:
332        logger.warning("FAILED")
333        logger.info(traceback.format_exc())
334        put_devices(devices, duts, refs, monitors)
335        return
336    logger.warning("PASS")
337
338    test_no = 1
339    for test in tests_to_run:
340        try:
341            start = datetime.now()
342            setup_params['tc_name'] = test.__name__.replace('test_', '', 1)
343            logger.warning("START - " + setup_params['tc_name'] + " (" + str(test_no) + "/" + str(len(tests_to_run)) + ")")
344            if test.__doc__:
345                logger.info("Test: " + test.__doc__)
346
347            # run tc
348            res = test(devices, setup_params, refs, duts, monitors)
349
350            end = datetime.now()
351            logger.warning("PASS (" + res + ") - " + str((end - start).total_seconds()) + "s")
352        except KeyboardInterrupt:
353            put_devices(devices, duts, refs, monitors)
354            raise
355        except TestSkip as e:
356            end = datetime.now()
357            logger.warning("SKIP (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
358        except Exception as e:
359            end = datetime.now()
360            logger.warning("FAILED (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
361            logger.info(traceback.format_exc())
362            failed.append(test.__name__.replace('test_', '', 1))
363        except:
364            end = datetime.now()
365            logger.warning("FAILED - " + str((end - start).total_seconds()) + "s")
366            logger.info(traceback.format_exc())
367            failed.append(test.__name__.replace('test_', '', 1))
368        test_no += 1
369
370    test_no = 1
371    for hwsim_test in hwsim_tests_to_run:
372        try:
373            start = datetime.now()
374            setup_params['tc_name'] = hwsim_test.__name__.replace('test_', '', 1)
375            logger.warning("START - " + setup_params['tc_name'] + " (" + str(test_no) + "/" + str(len(hwsim_tests_to_run)) + ")")
376            res = run_hwsim_test(devices, setup_params, refs, duts, monitors, hwsim_test)
377            end = datetime.now()
378            logger.warning("PASS (" + res + ") - " + str((end - start).total_seconds()) + "s")
379        except KeyboardInterrupt:
380            put_devices(devices, duts, refs, monitors)
381            raise
382        except HwsimSkip as e:
383            end = datetime.now()
384            logger.warning("SKIP (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
385            failed.append(hwsim_test.__name__.replace('test_', '', 1))
386        except Exception as e:
387            end = datetime.now()
388            logger.warning("FAILED (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
389            logger.info(traceback.format_exc())
390            failed.append(hwsim_test.__name__.replace('test_', '', 1))
391        except:
392            end = datetime.now()
393            logger.warning("FAILED - " + str((end - start).total_seconds()) + "s")
394            logger.info(traceback.format_exc())
395            failed.append(hwsim_test.__name__.replace('test_', '', 1))
396        test_no += 1
397
398    # unlock devices
399    put_devices(devices, duts, refs, monitors)
400
401    if len(failed) > 0:
402        logger.warning("Failed test cases:")
403        for test in failed:
404            logger.warning("\t" + test)
405
406
407if __name__ == "__main__":
408        main()
409