1#!/usr/bin/env python3
2#
3#  Copyright (c) 2020, The OpenThread Authors.
4#  All rights reserved.
5#
6#  Redistribution and use in source and binary forms, with or without
7#  modification, are permitted provided that the following conditions are met:
8#  1. Redistributions of source code must retain the above copyright
9#     notice, this list of conditions and the following disclaimer.
10#  2. Redistributions in binary form must reproduce the above copyright
11#     notice, this list of conditions and the following disclaimer in the
12#     documentation and/or other materials provided with the distribution.
13#  3. Neither the name of the copyright holder nor the
14#     names of its contributors may be used to endorse or promote products
15#     derived from this software without specific prior written permission.
16#
17#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20#  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
21#  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22#  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23#  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24#  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25#  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26#  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27#  POSSIBILITY OF SUCH DAMAGE.
28#
29import logging
30import multiprocessing
31import os
32import queue
33import subprocess
34import traceback
35from collections import Counter
36from typing import List
37
38import config
39
40THREAD_VERSION = os.getenv('THREAD_VERSION')
41VIRTUAL_TIME = int(os.getenv('VIRTUAL_TIME', '1'))
42MAX_JOBS = int(os.getenv('MAX_JOBS', (multiprocessing.cpu_count() * 2 if VIRTUAL_TIME else 10)))
43
44_BACKBONE_TESTS_DIR = 'tests/scripts/thread-cert/backbone'
45
46_COLOR_PASS = '\033[0;32m'
47_COLOR_FAIL = '\033[0;31m'
48_COLOR_NONE = '\033[0m'
49
50logging.basicConfig(level=logging.DEBUG,
51                    format='File "%(pathname)s", line %(lineno)d, in %(funcName)s\n'
52                    '%(asctime)s - %(levelname)s - %(message)s')
53
54
55def bash(cmd: str, check=True, stdout=None):
56    subprocess.run(cmd, shell=True, check=check, stdout=stdout)
57
58
59def run_cert(job_id: int, port_offset: int, script: str, run_directory: str):
60    if not os.access(script, os.X_OK):
61        logging.warning('Skip test %s, not executable', script)
62        return
63
64    try:
65        test_name = os.path.splitext(os.path.basename(script))[0] + '_' + str(job_id)
66        logfile = f'{run_directory}/{test_name}.log' if run_directory else f'{test_name}.log'
67        env = os.environ.copy()
68        env['PORT_OFFSET'] = str(port_offset)
69        env['TEST_NAME'] = test_name
70        env['PYTHONPATH'] = os.path.dirname(os.path.abspath(__file__))
71
72        try:
73            print(f'Running {test_name}')
74            with open(logfile, 'wt') as output:
75                abs_script = os.path.abspath(script)
76                subprocess.check_call(abs_script,
77                                      stdout=output,
78                                      stderr=output,
79                                      stdin=subprocess.DEVNULL,
80                                      cwd=run_directory,
81                                      env=env)
82        except subprocess.CalledProcessError:
83            bash(f'cat {logfile} 1>&2')
84            logging.error("Run test %s failed, please check the log file: %s", test_name, logfile)
85            raise
86
87    except Exception:
88        traceback.print_exc()
89        raise
90
91
92pool = multiprocessing.Pool(processes=MAX_JOBS)
93
94
95def cleanup_backbone_env():
96    logging.info("Cleaning up Backbone testing environment ...")
97    bash('pkill socat 2>/dev/null || true')
98    bash('pkill dumpcap 2>/dev/null || true')
99    bash(f'docker rm -f $(docker ps -a -q -f "name=otbr_") 2>/dev/null || true')
100    bash(f'docker network rm $(docker network ls -q -f "name=backbone") 2>/dev/null || true')
101
102
103def setup_backbone_env():
104    if THREAD_VERSION == '1.1':
105        raise RuntimeError('Backbone tests do not work with THREAD_VERSION=1.1')
106
107    if VIRTUAL_TIME:
108        raise RuntimeError('Backbone tests only work with VIRTUAL_TIME=0')
109
110    bash(f'docker image inspect {config.OTBR_DOCKER_IMAGE} >/dev/null')
111
112
113def parse_args():
114    import argparse
115    parser = argparse.ArgumentParser(description='Process some integers.')
116    parser.add_argument('--multiply', type=int, default=1, help='run each test for multiple times')
117    parser.add_argument('--run-directory', type=str, default=None, help='run each test in the specified directory')
118    parser.add_argument("scripts", nargs='+', type=str, help='specify Backbone test scripts')
119
120    args = parser.parse_args()
121    logging.info("Max jobs: %d", MAX_JOBS)
122    logging.info("Run directory: %s", args.run_directory or '.')
123    logging.info("Multiply: %d", args.multiply)
124    logging.info("Test scripts: %d", len(args.scripts))
125    return args
126
127
128def check_has_backbone_tests(scripts):
129    for script in scripts:
130        relpath = os.path.relpath(script, _BACKBONE_TESTS_DIR)
131        if not relpath.startswith('..'):
132            return True
133
134    return False
135
136
137class PortOffsetPool:
138
139    def __init__(self, size: int):
140        self._size = size
141        self._pool = queue.Queue(maxsize=size)
142        for port_offset in range(0, size):
143            self.release(port_offset)
144
145    def allocate(self) -> int:
146        return self._pool.get()
147
148    def release(self, port_offset: int):
149        assert 0 <= port_offset < self._size, port_offset
150        self._pool.put_nowait(port_offset)
151
152
153def run_tests(scripts: List[str], multiply: int = 1, run_directory: str = None):
154    script_fail_count = Counter()
155    script_succ_count = Counter()
156
157    # Run each script for multiple times
158    script_ids = [(script, i) for script in scripts for i in range(multiply)]
159    port_offset_pool = PortOffsetPool(MAX_JOBS)
160
161    def error_callback(port_offset, script, err):
162        port_offset_pool.release(port_offset)
163
164        script_fail_count[script] += 1
165        if script_succ_count[script] + script_fail_count[script] == multiply:
166            color = _COLOR_PASS if script_fail_count[script] == 0 else _COLOR_FAIL
167            print(f'{color}PASS {script_succ_count[script]} FAIL {script_fail_count[script]}{_COLOR_NONE} {script}')
168
169    def pass_callback(port_offset, script):
170        port_offset_pool.release(port_offset)
171
172        script_succ_count[script] += 1
173        if script_succ_count[script] + script_fail_count[script] == multiply:
174            color = _COLOR_PASS if script_fail_count[script] == 0 else _COLOR_FAIL
175            print(f'{color}PASS {script_succ_count[script]} FAIL {script_fail_count[script]}{_COLOR_NONE} {script}')
176
177    for script, i in script_ids:
178        port_offset = port_offset_pool.allocate()
179        pool.apply_async(
180            run_cert, [i, port_offset, script, run_directory],
181            callback=lambda ret, port_offset=port_offset, script=script: pass_callback(port_offset, script),
182            error_callback=lambda err, port_offset=port_offset, script=script: error_callback(
183                port_offset, script, err))
184
185    pool.close()
186    pool.join()
187    return sum(script_fail_count.values())
188
189
190def main():
191    args = parse_args()
192
193    has_backbone_tests = check_has_backbone_tests(args.scripts)
194    logging.info('Has Backbone tests: %s', has_backbone_tests)
195
196    if has_backbone_tests:
197        cleanup_backbone_env()
198        setup_backbone_env()
199
200    try:
201        fail_count = run_tests(args.scripts, args.multiply, args.run_directory)
202        exit(fail_count)
203    finally:
204        if has_backbone_tests:
205            cleanup_backbone_env()
206
207
208if __name__ == '__main__':
209    main()
210