1#!/usr/bin/env python3
2#
3#  Copyright (c) 2020, The OpenThread Authors.
4#  All rights reserved.
5#
6#  Redistribution and use in source and binary forms, with or without
7#  modification, are permitted provided that the following conditions are met:
8#  1. Redistributions of source code must retain the above copyright
9#     notice, this list of conditions and the following disclaimer.
10#  2. Redistributions in binary form must reproduce the above copyright
11#     notice, this list of conditions and the following disclaimer in the
12#     documentation and/or other materials provided with the distribution.
13#  3. Neither the name of the copyright holder nor the
14#     names of its contributors may be used to endorse or promote products
15#     derived from this software without specific prior written permission.
16#
17#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20#  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
21#  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22#  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23#  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24#  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25#  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26#  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27#  POSSIBILITY OF SUCH DAMAGE.
28#
29import logging
30import multiprocessing
31import os
32import queue
33import subprocess
34import traceback
35from collections import Counter
36from typing import List
37
38import config
39
40THREAD_VERSION = os.getenv('THREAD_VERSION')
41VIRTUAL_TIME = int(os.getenv('VIRTUAL_TIME', '1'))
42MAX_JOBS = int(os.getenv('MAX_JOBS', (multiprocessing.cpu_count() * 2 if VIRTUAL_TIME else 10)))
43
44_BACKBONE_TESTS_DIR = 'tests/scripts/thread-cert/backbone'
45
46_COLOR_PASS = '\033[0;32m'
47_COLOR_FAIL = '\033[0;31m'
48_COLOR_NONE = '\033[0m'
49
50logging.basicConfig(level=logging.DEBUG,
51                    format='File "%(pathname)s", line %(lineno)d, in %(funcName)s\n'
52                    '%(asctime)s - %(levelname)s - %(message)s')
53
54
55def bash(cmd: str, check=True, stdout=None):
56    subprocess.run(cmd, shell=True, check=check, stdout=stdout)
57
58
59def run_cert(port_offset: int, script: str):
60    try:
61        test_name = os.path.splitext(os.path.basename(script))[0] + '_' + str(port_offset)
62        logfile = test_name + '.log'
63        env = os.environ.copy()
64        env['PORT_OFFSET'] = str(port_offset)
65        env['TEST_NAME'] = test_name
66
67        try:
68            with open(logfile, 'wt') as output:
69                subprocess.check_call(["python3", script],
70                                      stdout=output,
71                                      stderr=output,
72                                      stdin=subprocess.DEVNULL,
73                                      env=env)
74        except subprocess.CalledProcessError:
75            bash(f'cat {logfile} 1>&2')
76            logging.error("Run test %s failed, please check the log file: %s", test_name, logfile)
77            raise
78
79    except Exception:
80        traceback.print_exc()
81        raise
82
83
84pool = multiprocessing.Pool(processes=MAX_JOBS)
85
86
87def cleanup_backbone_env():
88    logging.info("Cleaning up Backbone testing environment ...")
89    bash('pkill socat 2>/dev/null || true')
90    bash('pkill dumpcap 2>/dev/null || true')
91    bash(f'docker rm -f $(docker ps -a -q -f "name=otbr_") 2>/dev/null || true')
92    bash(f'docker network rm $(docker network ls -q -f "name=backbone") 2>/dev/null || true')
93
94
95def setup_backbone_env():
96    bash('sudo modprobe ip6table_filter')
97
98    if THREAD_VERSION != '1.2':
99        raise RuntimeError('Backbone tests only work with THREAD_VERSION=1.2')
100
101    if VIRTUAL_TIME:
102        raise RuntimeError('Backbone tests only work with VIRTUAL_TIME=0')
103
104    bash(f'docker image inspect {config.OTBR_DOCKER_IMAGE} >/dev/null')
105
106
107def parse_args():
108    import argparse
109    parser = argparse.ArgumentParser(description='Process some integers.')
110    parser.add_argument('--multiply', type=int, default=1, help='run each test for multiple times')
111    parser.add_argument("scripts", nargs='+', type=str, help='specify Backbone test scripts')
112
113    args = parser.parse_args()
114    logging.info("Max jobs: %d", MAX_JOBS)
115    logging.info("Multiply: %d", args.multiply)
116    logging.info("Test scripts: %d", len(args.scripts))
117    return args
118
119
120def check_has_backbone_tests(scripts):
121    for script in scripts:
122        relpath = os.path.relpath(script, _BACKBONE_TESTS_DIR)
123        if not relpath.startswith('..'):
124            return True
125
126    return False
127
128
129class PortOffsetPool:
130
131    def __init__(self, size: int):
132        self._size = size
133        self._pool = queue.Queue(maxsize=size)
134        for port_offset in range(0, size):
135            self.release(port_offset)
136
137    def allocate(self) -> int:
138        return self._pool.get()
139
140    def release(self, port_offset: int):
141        assert 0 <= port_offset < self._size, port_offset
142        self._pool.put_nowait(port_offset)
143
144
145def run_tests(scripts: List[str], multiply: int = 1):
146    script_fail_count = Counter()
147    script_succ_count = Counter()
148
149    # Run each script for multiple times
150    scripts = [script for script in scripts for _ in range(multiply)]
151    port_offset_pool = PortOffsetPool(MAX_JOBS)
152
153    def error_callback(port_offset, script, err):
154        port_offset_pool.release(port_offset)
155
156        script_fail_count[script] += 1
157        if script_succ_count[script] + script_fail_count[script] == multiply:
158            color = _COLOR_PASS if script_fail_count[script] == 0 else _COLOR_FAIL
159            print(f'{color}PASS {script_succ_count[script]} FAIL {script_fail_count[script]}{_COLOR_NONE} {script}')
160
161    def pass_callback(port_offset, script):
162        port_offset_pool.release(port_offset)
163
164        script_succ_count[script] += 1
165        if script_succ_count[script] + script_fail_count[script] == multiply:
166            color = _COLOR_PASS if script_fail_count[script] == 0 else _COLOR_FAIL
167            print(f'{color}PASS {script_succ_count[script]} FAIL {script_fail_count[script]}{_COLOR_NONE} {script}')
168
169    for i, script in enumerate(scripts):
170        port_offset = port_offset_pool.allocate()
171        pool.apply_async(
172            run_cert, [port_offset, script],
173            callback=lambda ret, port_offset=port_offset, script=script: pass_callback(port_offset, script),
174            error_callback=lambda err, port_offset=port_offset, script=script: error_callback(
175                port_offset, script, err))
176
177    pool.close()
178    pool.join()
179    return sum(script_fail_count.values())
180
181
182def main():
183    args = parse_args()
184
185    has_backbone_tests = check_has_backbone_tests(args.scripts)
186    logging.info('Has Backbone tests: %s', has_backbone_tests)
187
188    if has_backbone_tests:
189        cleanup_backbone_env()
190        setup_backbone_env()
191
192    try:
193        fail_count = run_tests(args.scripts, args.multiply)
194        exit(fail_count)
195    finally:
196        if has_backbone_tests:
197            cleanup_backbone_env()
198
199
200if __name__ == '__main__':
201    main()
202