1#!/usr/bin/env python
2#
3# Copyright 2018-2019 Espressif Systems (Shanghai) PTE LTD
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17from __future__ import print_function, unicode_literals
18
19import errno
20import filecmp
21import os
22import pty
23import socket
24import subprocess
25import sys
26import tempfile
27import threading
28import time
29from builtins import object
30from io import open
31
32XTENSA_ARGS = '--toolchain-prefix xtensa-esp32-elf-'
33RISCV_ARGS = '--decode-panic backtrace --target esp32c3 --toolchain-prefix riscv32-esp-elf-'
34
35test_list = (
36    # Add new tests here. All files should be placed in IN_DIR. Columns are
37    # Input file            Filter string                                File with expected output   Timeout    ELF file        Extra args
38    ('in1.txt',             '',                                          'in1f1.txt',                60,    'dummy_xtensa.elf',       XTENSA_ARGS),
39    ('in1.txt',             '*:V',                                       'in1f1.txt',                60,    'dummy_xtensa.elf',       XTENSA_ARGS),
40    ('in1.txt',             'hello_world',                               'in1f2.txt',                60,    'dummy_xtensa.elf',       XTENSA_ARGS),
41    ('in1.txt',             '*:N',                                       'in1f3.txt',                60,    'dummy_xtensa.elf',       XTENSA_ARGS),
42    ('in2.txt',             'boot mdf_device_handle:I mesh:E vfs:I',     'in2f1.txt',               420,    'dummy_xtensa.elf',       XTENSA_ARGS),
43    ('in2.txt',             'vfs',                                       'in2f2.txt',               420,    'dummy_xtensa.elf',       XTENSA_ARGS),
44    ('core1.txt',           '',                                          'core1_out.txt',            60,    'dummy_xtensa.elf',       XTENSA_ARGS),
45    ('riscv_panic1.txt',    '',                                          'riscv_panic1_out.txt',     60,    'dummy_riscv.elf',        RISCV_ARGS),
46)
47
48IN_DIR = 'tests/'       # tests are in this directory
49OUT_DIR = 'outputs/'    # test results are written to this directory (kept only for debugging purposes)
50ERR_OUT = 'monitor_error_output.'
51IDF_MONITOR_WAPPER = 'idf_monitor_wrapper.py'
52SERIAL_ALIVE_FILE = '/tmp/serial_alive'  # the existence of this file signalize that idf_monitor is ready to receive
53
54# connection related to communicating with idf_monitor through sockets
55HOST = 'localhost'
56# blocking socket operations are used with timeout:
57SOCKET_TIMEOUT = 30
58# the test is restarted after failure (idf_monitor has to be killed):
59RETRIES_PER_TEST = 2
60
61
62def monitor_timeout(process):
63    if process.poll() is None:
64        # idf_monitor_wrapper is still running
65        try:
66            process.kill()
67            print('\tidf_monitor_wrapper was killed because it did not finish in time.')
68        except OSError as e:
69            if e.errno == errno.ESRCH:
70                # ignores a possible race condition which can occur when the process exits between poll() and kill()
71                pass
72            else:
73                raise
74
75
76class TestRunner(object):
77    def __enter__(self):
78        self.serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
79        self.serversocket.bind((HOST, 0))
80        self.port = self.serversocket.getsockname()[1]
81        self.serversocket.listen(5)
82        return self
83
84    def __exit__(self, type, value, traceback):
85        try:
86            self.serversocket.shutdown(socket.SHUT_RDWR)
87            self.serversocket.close()
88            print('Socket was closed successfully')
89        except (OSError, socket.error):
90            pass
91
92    def accept_connection(self):
93        """ returns a socket for sending the input for idf_monitor which must be closed before calling this again. """
94        (clientsocket, address) = self.serversocket.accept()
95        # exception will be thrown here if the idf_monitor didn't connect in time
96        return clientsocket
97
98
99def test_iteration(runner, test):
100    try:
101        # Make sure that the file doesn't exist. It will be recreated by idf_monitor_wrapper.py
102        os.remove(SERIAL_ALIVE_FILE)
103    except OSError:
104        pass
105    print('\nRunning test on {} with filter "{}" and expecting {}'.format(test[0], test[1], test[2]))
106    try:
107        with open(OUT_DIR + test[2], 'w', encoding='utf-8') as o_f, \
108                tempfile.NamedTemporaryFile(dir=OUT_DIR, prefix=ERR_OUT, mode='w', delete=False) as e_f:
109            monitor_cmd = [sys.executable, IDF_MONITOR_WAPPER,
110                           '--port', 'socket://{}:{}?logging=debug'.format(HOST, runner.port),
111                           '--print_filter', test[1],
112                           '--serial_alive_file', SERIAL_ALIVE_FILE,
113                           '--elf-file', test[4]]
114            monitor_cmd += test[5].split()
115            (master_fd, slave_fd) = pty.openpty()
116            print('\t', ' '.join(monitor_cmd), sep='')
117            print('\tstdout="{}" stderr="{}" stdin="{}"'.format(o_f.name, e_f.name, os.ttyname(slave_fd)))
118            print('\tMonitor timeout: {} seconds'.format(test[3]))
119            start = time.time()
120            # the server socket is alive so idf_monitor can start now
121            proc = subprocess.Popen(monitor_cmd, stdin=slave_fd, stdout=o_f, stderr=e_f, close_fds=True, bufsize=0)
122            # - idf_monitor's stdin needs to be connected to some pseudo-tty in docker image even when it is not
123            #   used at all
124            # - setting bufsize is needed because the default value is different on Python 2 and 3
125            # - the default close_fds is also different on Python 2 and 3
126        monitor_watchdog = threading.Timer(test[3], monitor_timeout, [proc])
127        monitor_watchdog.start()
128        client = runner.accept_connection()
129        # The connection is ready but idf_monitor cannot yet receive data (the serial reader thread is not running).
130        # This seems to happen on Ubuntu 16.04 LTS and is not related to the version of Python or pyserial.
131        # Updating to Ubuntu 18.04 LTS also helps but here, a workaround is used: A wrapper is used for IDF monitor
132        # which checks the serial reader thread and creates a file when it is running.
133        while not os.path.isfile(SERIAL_ALIVE_FILE) and proc.poll() is None:
134            print('\tSerial reader is not ready. Do a sleep...')
135            time.sleep(1)
136        # Only now can we send the inputs:
137        with open(IN_DIR + test[0], 'rb') as f:
138            print('\tSending {} to the socket'.format(f.name))
139            for chunk in iter(lambda: f.read(1024), b''):
140                client.sendall(chunk)
141        idf_exit_sequence = b'\x1d\n'
142        print('\tSending <exit> to the socket')
143        client.sendall(idf_exit_sequence)
144        close_end_time = start + 0.75 * test[3]  # time when the process is close to be killed
145        while True:
146            ret = proc.poll()
147            if ret is not None:
148                break
149            if time.time() > close_end_time:
150                # The process isn't finished yet so we are starting to send additional exit sequences because maybe
151                # the other end didn't received it.
152                print('\tSending additional <exit> to the socket')
153                client.sendall(idf_exit_sequence)
154            time.sleep(1)
155        end = time.time()
156        print('\tidf_monitor exited after {:.2f} seconds'.format(end - start))
157        if ret < 0:
158            raise RuntimeError('idf_monitor was terminated by signal {}'.format(-ret))
159        # idf_monitor needs to end before the socket is closed in order to exit without an exception.
160    finally:
161        if monitor_watchdog:
162            monitor_watchdog.cancel()
163        os.close(slave_fd)
164        os.close(master_fd)
165        if client:
166            client.close()
167        print('\tThe client was closed successfully')
168    f1 = IN_DIR + test[2]
169    f2 = OUT_DIR + test[2]
170    print('\tdiff {} {}'.format(f1, f2))
171    if filecmp.cmp(f1, f2, shallow=False):
172        print('\tTest has passed')
173    else:
174        raise RuntimeError('The contents of the files are different. Please examine the artifacts.')
175
176
177def main():
178    gstart = time.time()
179    if not os.path.exists(OUT_DIR):
180        os.mkdir(OUT_DIR)
181
182    socket.setdefaulttimeout(SOCKET_TIMEOUT)
183
184    for test in test_list:
185        for i in range(RETRIES_PER_TEST):
186            with TestRunner() as runner:
187                # Each test (and each retry) is run with a different port (and server socket). This is done for
188                # the CI run where retry with a different socket is necessary to pass the test. According to the
189                # experiments, retry with the same port (and server socket) is not sufficient.
190                try:
191                    test_iteration(runner, test)
192                    # no more retries if test_iteration exited without an exception
193                    break
194                except Exception as e:
195                    if i < RETRIES_PER_TEST - 1:
196                        print('Test has failed with exception:', e)
197                        print('Another attempt will be made.')
198                    else:
199                        raise
200
201    gend = time.time()
202    print('Execution took {:.2f} seconds\n'.format(gend - gstart))
203
204
205if __name__ == '__main__':
206    main()
207