1#!/usr/bin/env python3
2#
3#  Copyright (c) 2020, The OpenThread Authors.
4#  All rights reserved.
5#
6#  Redistribution and use in source and binary forms, with or without
7#  modification, are permitted provided that the following conditions are met:
8#  1. Redistributions of source code must retain the above copyright
9#     notice, this list of conditions and the following disclaimer.
10#  2. Redistributions in binary form must reproduce the above copyright
11#     notice, this list of conditions and the following disclaimer in the
12#     documentation and/or other materials provided with the distribution.
13#  3. Neither the name of the copyright holder nor the
14#     names of its contributors may be used to endorse or promote products
15#     derived from this software without specific prior written permission.
16#
17#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20#  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
21#  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22#  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23#  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24#  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25#  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26#  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27#  POSSIBILITY OF SUCH DAMAGE.
28#
29import logging
30import multiprocessing
31import os
32import queue
33import subprocess
34import time
35import traceback
36from collections import Counter
37from typing import List
38
39import config
40
41THREAD_VERSION = os.getenv('THREAD_VERSION')
42VIRTUAL_TIME = int(os.getenv('VIRTUAL_TIME', '1'))
43MAX_JOBS = int(os.getenv('MAX_JOBS', (multiprocessing.cpu_count() * 2 if VIRTUAL_TIME else 10)))
44
45_BACKBONE_TESTS_DIR = 'tests/scripts/thread-cert/backbone'
46
47_COLOR_PASS = '\033[0;32m'
48_COLOR_FAIL = '\033[0;31m'
49_COLOR_NONE = '\033[0m'
50
51logging.basicConfig(level=logging.DEBUG,
52                    format='File "%(pathname)s", line %(lineno)d, in %(funcName)s\n'
53                    '%(asctime)s - %(levelname)s - %(message)s')
54
55
56def bash(cmd: str, check=True, stdout=None):
57    subprocess.run(cmd, shell=True, check=check, stdout=stdout)
58
59
60def run_cert(job_id: int, port_offset: int, script: str, run_directory: str):
61    if not os.access(script, os.X_OK):
62        logging.warning('Skip test %s, not executable', script)
63        return
64
65    try:
66        test_name = os.path.splitext(os.path.basename(script))[0] + '_' + str(job_id)
67        logfile = f'{run_directory}/{test_name}.log' if run_directory else f'{test_name}.log'
68        env = os.environ.copy()
69        env['PORT_OFFSET'] = str(port_offset)
70        env['TEST_NAME'] = test_name
71        env['PYTHONPATH'] = os.path.dirname(os.path.abspath(__file__))
72
73        try:
74            print(f'Running PORT_OFFSET={port_offset} {test_name}')
75            with open(logfile, 'wt') as output:
76                abs_script = os.path.abspath(script)
77                subprocess.check_call(abs_script,
78                                      stdout=output,
79                                      stderr=output,
80                                      stdin=subprocess.DEVNULL,
81                                      cwd=run_directory,
82                                      env=env)
83        except subprocess.CalledProcessError:
84            bash(f'cat {logfile} 1>&2')
85            logging.error("Run test %s failed, please check the log file: %s", test_name, logfile)
86            raise
87
88    except Exception:
89        traceback.print_exc()
90        raise
91
92
93pool = multiprocessing.Pool(processes=MAX_JOBS)
94
95
96def cleanup_backbone_env():
97    logging.info("Cleaning up Backbone testing environment ...")
98    bash('pkill socat 2>/dev/null || true')
99    bash('pkill dumpcap 2>/dev/null || true')
100    bash(f'docker rm -f $(docker ps -a -q -f "name=otbr_") 2>/dev/null || true')
101    bash(f'docker network rm $(docker network ls -q -f "name=backbone") 2>/dev/null || true')
102
103
104def setup_backbone_env():
105    if THREAD_VERSION == '1.1':
106        raise RuntimeError('Backbone tests do not work with THREAD_VERSION=1.1')
107
108    if VIRTUAL_TIME:
109        raise RuntimeError('Backbone tests only work with VIRTUAL_TIME=0')
110
111    bash(f'docker image inspect {config.OTBR_DOCKER_IMAGE} >/dev/null')
112
113
114def parse_args():
115    import argparse
116    parser = argparse.ArgumentParser(description='Process some integers.')
117    parser.add_argument('--multiply', type=int, default=1, help='run each test for multiple times')
118    parser.add_argument('--run-directory', type=str, default=None, help='run each test in the specified directory')
119    parser.add_argument("scripts", nargs='+', type=str, help='specify Backbone test scripts')
120
121    args = parser.parse_args()
122    logging.info("Max jobs: %d", MAX_JOBS)
123    logging.info("Run directory: %s", args.run_directory or '.')
124    logging.info("Multiply: %d", args.multiply)
125    logging.info("Test scripts: %d", len(args.scripts))
126    return args
127
128
129def check_has_backbone_tests(scripts):
130    for script in scripts:
131        relpath = os.path.relpath(script, _BACKBONE_TESTS_DIR)
132        if not relpath.startswith('..'):
133            return True
134
135    return False
136
137
138class PortOffsetPool:
139
140    def __init__(self, size: int):
141        self._size = size
142        self._pool = queue.Queue(maxsize=size)
143        for port_offset in range(0, size):
144            self.release(port_offset)
145
146    def allocate(self) -> int:
147        return self._pool.get()
148
149    def release(self, port_offset: int):
150        assert 0 <= port_offset < self._size, port_offset
151        self._pool.put_nowait(port_offset)
152
153
154def run_tests(scripts: List[str], multiply: int = 1, run_directory: str = None):
155    script_fail_count = Counter()
156    script_succ_count = Counter()
157
158    # Run each script for multiple times
159    script_ids = [(script, i) for script in scripts for i in range(multiply)]
160    port_offset_pool = PortOffsetPool(MAX_JOBS)
161
162    def error_callback(port_offset, script, err, start_time):
163        port_offset_pool.release(port_offset)
164
165        elapsed_time = round(time.time() - start_time)
166        script_fail_count[script] += 1
167        if script_succ_count[script] + script_fail_count[script] == multiply:
168            color = _COLOR_PASS if script_fail_count[script] == 0 else _COLOR_FAIL
169            print(
170                f'{color}PASS {script_succ_count[script]} FAIL {script_fail_count[script]}{_COLOR_NONE} {script} in {elapsed_time}s'
171            )
172
173    def pass_callback(port_offset, script, start_time):
174        port_offset_pool.release(port_offset)
175
176        elapsed_time = round(time.time() - start_time)
177        script_succ_count[script] += 1
178        if script_succ_count[script] + script_fail_count[script] == multiply:
179            color = _COLOR_PASS if script_fail_count[script] == 0 else _COLOR_FAIL
180            print(
181                f'{color}PASS {script_succ_count[script]} FAIL {script_fail_count[script]}{_COLOR_NONE} {script} in {elapsed_time}s'
182            )
183
184    for script, i in script_ids:
185        port_offset = port_offset_pool.allocate()
186        start_time = time.time()
187        pool.apply_async(run_cert, [i, port_offset, script, run_directory],
188                         callback=lambda ret, port_offset=port_offset, script=script, start_time=start_time:
189                         pass_callback(port_offset, script, start_time),
190                         error_callback=lambda err, port_offset=port_offset, script=script, start_time=start_time:
191                         error_callback(port_offset, script, err, start_time))
192
193    pool.close()
194    pool.join()
195    return sum(script_fail_count.values())
196
197
198def main():
199    args = parse_args()
200
201    has_backbone_tests = check_has_backbone_tests(args.scripts)
202    logging.info('Has Backbone tests: %s', has_backbone_tests)
203
204    if has_backbone_tests:
205        cleanup_backbone_env()
206        setup_backbone_env()
207
208    try:
209        fail_count = run_tests(args.scripts, args.multiply, args.run_directory)
210        exit(fail_count)
211    finally:
212        if has_backbone_tests:
213            cleanup_backbone_env()
214
215
216if __name__ == '__main__':
217    main()
218