1# Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http:#www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15""" Interface for test cases. """
16import functools
17import os
18import socket
19import time
20from datetime import datetime
21
22import junit_xml
23
24from . import DUT, App, Env, Utility
25from .Utility import format_case_id
26
27
28class TestCaseFailed(AssertionError):
29    def __init__(self, *cases):
30        """
31        Raise this exception if one or more test cases fail in a 'normal' way (ie the test runs but fails, no unexpected exceptions)
32
33        This will avoid dumping the Python stack trace, because the assumption is the junit error info and full job log already has
34        enough information for a developer to debug.
35
36        'cases' argument is the names of one or more test cases
37        """
38        message = 'Test case{} failed: {}'.format('s' if len(cases) > 1 else '', ', '.join(str(c) for c in cases))
39        super(TestCaseFailed, self).__init__(self, message)
40
41
42class DefaultEnvConfig(object):
43    """
44    default test configs. There're 3 places to set configs, priority is (high -> low):
45
46    1. overwrite set by caller of test method
47    2. values set by test_method decorator
48    3. default env config get from this class
49    """
50    DEFAULT_CONFIG = {
51        'app': App.BaseApp,
52        'dut': DUT.BaseDUT,
53        'env_tag': 'default',
54        'env_config_file': None,
55        'test_suite_name': None,
56    }
57
58    @classmethod
59    def set_default_config(cls, **kwargs):
60        """
61        :param kwargs: configs need to be updated
62        :return: None
63        """
64        cls.DEFAULT_CONFIG.update(kwargs)
65
66    @classmethod
67    def get_default_config(cls):
68        """
69        :return: current default config
70        """
71        return cls.DEFAULT_CONFIG.copy()
72
73
74set_default_config = DefaultEnvConfig.set_default_config
75get_default_config = DefaultEnvConfig.get_default_config
76
77
78MANDATORY_INFO = {
79    'execution_time': 1,
80    'env_tag': 'default',
81    'category': 'function',
82    'ignore': False,
83}
84
85
86class JunitReport(object):
87    # wrapper for junit test report
88    # TODO: JunitReport methods are not thread safe (although not likely to be used this way).
89
90    JUNIT_FILE_NAME = 'XUNIT_RESULT.xml'
91    JUNIT_DEFAULT_TEST_SUITE = 'test-suite'
92    JUNIT_TEST_SUITE = junit_xml.TestSuite(JUNIT_DEFAULT_TEST_SUITE,
93                                           hostname=socket.gethostname(),
94                                           timestamp=datetime.utcnow().isoformat())
95    JUNIT_CURRENT_TEST_CASE = None
96    _TEST_CASE_CREATED_TS = 0
97
98    @classmethod
99    def output_report(cls, junit_file_path):
100        """ Output current test result to file. """
101        with open(os.path.join(junit_file_path, cls.JUNIT_FILE_NAME), 'w') as f:
102            junit_xml.to_xml_report_file(f, [cls.JUNIT_TEST_SUITE], prettyprint=False)
103
104    @classmethod
105    def get_current_test_case(cls):
106        """
107        By default, the test framework will handle junit test report automatically.
108        While some test case might want to update some info to test report.
109        They can use this method to get current test case created by test framework.
110
111        :return: current junit test case instance created by ``JunitTestReport.create_test_case``
112        """
113        return cls.JUNIT_CURRENT_TEST_CASE
114
115    @classmethod
116    def test_case_finish(cls, test_case):
117        """
118        Append the test case to test suite so it can be output to file.
119        Execution time will be automatically updated (compared to ``create_test_case``).
120        """
121        test_case.elapsed_sec = time.time() - cls._TEST_CASE_CREATED_TS
122        cls.JUNIT_TEST_SUITE.test_cases.append(test_case)
123
124    @classmethod
125    def create_test_case(cls, name):
126        """
127        Extend ``junit_xml.TestCase`` with:
128
129        1. save create test case so it can be get by ``get_current_test_case``
130        2. log create timestamp, so ``elapsed_sec`` can be auto updated in ``test_case_finish``.
131
132        :param name: test case name
133        :return: instance of ``junit_xml.TestCase``
134        """
135        # set stdout to empty string, so we can always append string to stdout.
136        # It won't affect output logic. If stdout is empty, it won't be put to report.
137        test_case = junit_xml.TestCase(name, stdout='')
138        cls.JUNIT_CURRENT_TEST_CASE = test_case
139        cls._TEST_CASE_CREATED_TS = time.time()
140        return test_case
141
142    @classmethod
143    def update_performance(cls, performance_items):
144        """
145        Update performance results to ``stdout`` of current test case.
146
147        :param performance_items: a list of performance items. each performance item is a key-value pair.
148        """
149        assert cls.JUNIT_CURRENT_TEST_CASE
150
151        for item in performance_items:
152            cls.JUNIT_CURRENT_TEST_CASE.stdout += '[Performance][{}]: {}\n'.format(item[0], item[1])
153
154
155def test_method(**kwargs):
156    """
157    decorator for test case function.
158    The following keyword arguments are pre-defined.
159    Any other keyword arguments will be regarded as filter for the test case,
160    able to access them by ``case_info`` attribute of test method.
161
162    :keyword app: class for test app. see :doc:`App <App>` for details
163    :keyword dut: class for current dut. see :doc:`DUT <DUT>` for details
164    :keyword env_tag: name for test environment, used to select configs from config file
165    :keyword env_config_file: test env config file. usually will not set this keyword when define case
166    :keyword test_suite_name: test suite name, used for generating log folder name and adding xunit format test result.
167                              usually will not set this keyword when define case
168    :keyword junit_report_by_case: By default the test fw will handle junit report generation.
169                                   In some cases, one test function might test many test cases.
170                                   If this flag is set, test case can update junit report by its own.
171    """
172    def test(test_func):
173
174        case_info = MANDATORY_INFO.copy()
175        case_info['name'] = case_info['ID'] = test_func.__name__
176        case_info['junit_report_by_case'] = False
177        case_info.update(kwargs)
178
179        @functools.wraps(test_func)
180        def handle_test(extra_data=None, **overwrite):
181            """
182            create env, run test and record test results
183
184            :param extra_data: extra data that runner or main passed to test case
185            :param overwrite: args that runner or main want to overwrite
186            :return: None
187            """
188            # create env instance
189            env_config = DefaultEnvConfig.get_default_config()
190            for key in kwargs:
191                if key in env_config:
192                    env_config[key] = kwargs[key]
193
194            env_config.update(overwrite)
195            env_inst = Env.Env(**env_config)
196
197            # prepare for xunit test results
198            junit_file_path = env_inst.app_cls.get_log_folder(env_config['test_suite_name'])
199            junit_test_case = JunitReport.create_test_case(format_case_id(case_info['ID'],
200                                                                          target=env_inst.default_dut_cls.TARGET))
201            result = False
202            unexpected_error = False
203            try:
204                Utility.console_log('starting running test: ' + test_func.__name__, color='green')
205                # execute test function
206                test_func(env_inst, extra_data)
207                # if finish without exception, test result is True
208                result = True
209            except TestCaseFailed as e:
210                junit_test_case.add_failure_info(str(e))
211            except Exception as e:
212                Utility.handle_unexpected_exception(junit_test_case, e)
213                unexpected_error = True
214            finally:
215                # do close all DUTs, if result is False then print DUT debug info
216                close_errors = env_inst.close(dut_debug=(not result))
217                # We have a hook in DUT close, allow DUT to raise error to fail test case.
218                # For example, we don't allow DUT exception (reset) during test execution.
219                # We don't want to implement in exception detection in test function logic,
220                # as we need to add it to every test case.
221                # We can implement it in DUT receive thread,
222                # and raise exception in DUT close to fail test case if reset detected.
223                if close_errors:
224                    for error in close_errors:
225                        junit_test_case.add_failure_info('env close error: {}'.format(error))
226                    result = False
227                if not case_info['junit_report_by_case'] or unexpected_error:
228                    JunitReport.test_case_finish(junit_test_case)
229
230            # end case and output result
231            JunitReport.output_report(junit_file_path)
232
233            if result:
234                Utility.console_log('Test Succeed: ' + test_func.__name__, color='green')
235            else:
236                Utility.console_log(('Test Fail: ' + test_func.__name__), color='red')
237            return result
238
239        handle_test.case_info = case_info
240        handle_test.test_method = True
241        return handle_test
242    return test
243