1#!/usr/bin/env python3
2#
3# Licensed to the Apache Software Foundation (ASF) under one
4# or more contributor license agreements. See the NOTICE file
5# distributed with this work for additional information
6# regarding copyright ownership. The ASF licenses this file
7# to you under the Apache License, Version 2.0 (the
8# "License"); you may not use this file except in compliance
9# with the License. You may obtain a copy of the License at
10#
11#   http://www.apache.org/licenses/LICENSE-2.0
12#
13# Unless required by applicable law or agreed to in writing,
14# software distributed under the License is distributed on an
15# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16# KIND, either express or implied. See the License for the
17# specific language governing permissions and limitations
18# under the License.
19#
20
21#
22# Apache Thrift - integration (cross) test suite
23#
24# tests different server-client, protocol and transport combinations
25#
26# This script requires python 3.x due to the improvements in
27# subprocess management that are needed for reliability.
28#
29
30from __future__ import print_function
31from itertools import chain
32import json
33import logging
34import multiprocessing
35import argparse
36import os
37import sys
38
39import crossrunner
40from crossrunner.compat import path_join
41
42# 3.3 introduced subprocess timeouts on waiting for child
43req_version = (3, 3)
44cur_version = sys.version_info
45assert (cur_version >= req_version), "Python 3.3 or later is required for proper operation."
46
47
48ROOT_DIR = os.path.dirname(os.path.realpath(os.path.dirname(__file__)))
49TEST_DIR_RELATIVE = 'test'
50TEST_DIR = path_join(ROOT_DIR, TEST_DIR_RELATIVE)
51FEATURE_DIR_RELATIVE = path_join(TEST_DIR_RELATIVE, 'features')
52CONFIG_FILE = 'tests.json'
53
54
55def run_cross_tests(server_match, client_match, jobs, skip_known_failures, only_known_failures, retry_count, regex):
56    logger = multiprocessing.get_logger()
57    logger.debug('Collecting tests')
58    with open(path_join(TEST_DIR, CONFIG_FILE), 'r') as fp:
59        j = json.load(fp)
60    tests = crossrunner.collect_cross_tests(j, server_match, client_match, regex)
61    if not tests:
62        print('No test found that matches the criteria', file=sys.stderr)
63        print('  servers: %s' % server_match, file=sys.stderr)
64        print('  clients: %s' % client_match, file=sys.stderr)
65        return False
66    if only_known_failures:
67        logger.debug('Only running known failures')
68        known = crossrunner.load_known_failures(TEST_DIR)
69        tests = list(filter(lambda t: crossrunner.test_name(**t) in known, tests))
70    if skip_known_failures:
71        logger.debug('Skipping known failures')
72        known = crossrunner.load_known_failures(TEST_DIR)
73        tests = list(filter(lambda t: crossrunner.test_name(**t) not in known, tests))
74
75    dispatcher = crossrunner.TestDispatcher(TEST_DIR, ROOT_DIR, TEST_DIR_RELATIVE, jobs)
76    logger.debug('Executing %d tests' % len(tests))
77    try:
78        for r in [dispatcher.dispatch(test, retry_count) for test in tests]:
79            r.wait()
80        logger.debug('Waiting for completion')
81        return dispatcher.wait()
82    except (KeyboardInterrupt, SystemExit):
83        logger.debug('Interrupted, shutting down')
84        dispatcher.terminate()
85        return False
86
87
88def run_feature_tests(server_match, feature_match, jobs, skip_known_failures, only_known_failures, retry_count, regex):
89    basedir = path_join(ROOT_DIR, FEATURE_DIR_RELATIVE)
90    logger = multiprocessing.get_logger()
91    logger.debug('Collecting tests')
92    with open(path_join(TEST_DIR, CONFIG_FILE), 'r') as fp:
93        j = json.load(fp)
94    with open(path_join(basedir, CONFIG_FILE), 'r') as fp:
95        j2 = json.load(fp)
96    tests = crossrunner.collect_feature_tests(j, j2, server_match, feature_match, regex)
97    if not tests:
98        print('No test found that matches the criteria', file=sys.stderr)
99        print('  servers: %s' % server_match, file=sys.stderr)
100        print('  features: %s' % feature_match, file=sys.stderr)
101        return False
102    if only_known_failures:
103        logger.debug('Only running known failures')
104        known = crossrunner.load_known_failures(basedir)
105        tests = list(filter(lambda t: crossrunner.test_name(**t) in known, tests))
106    if skip_known_failures:
107        logger.debug('Skipping known failures')
108        known = crossrunner.load_known_failures(basedir)
109        tests = list(filter(lambda t: crossrunner.test_name(**t) not in known, tests))
110
111    dispatcher = crossrunner.TestDispatcher(TEST_DIR, ROOT_DIR, FEATURE_DIR_RELATIVE, jobs)
112    logger.debug('Executing %d tests' % len(tests))
113    try:
114        for r in [dispatcher.dispatch(test, retry_count) for test in tests]:
115            r.wait()
116        logger.debug('Waiting for completion')
117        return dispatcher.wait()
118    except (KeyboardInterrupt, SystemExit):
119        logger.debug('Interrupted, shutting down')
120        dispatcher.terminate()
121        return False
122
123
124def default_concurrency():
125    try:
126        return int(os.environ.get('THRIFT_CROSSTEST_CONCURRENCY'))
127    except (TypeError, ValueError):
128        # Since much time is spent sleeping, use many threads
129        return int(multiprocessing.cpu_count() * 1.25) + 1
130
131
132def main(argv):
133    parser = argparse.ArgumentParser()
134    parser.add_argument('--server', default='', nargs='*',
135                        help='list of servers to test')
136    parser.add_argument('--client', default='', nargs='*',
137                        help='list of clients to test')
138    parser.add_argument('-F', '--features', nargs='*', default=None,
139                        help='run server feature tests instead of cross language tests')
140    parser.add_argument('-R', '--regex', help='test name pattern to run')
141    parser.add_argument('-o', '--only-known_failures', action='store_true', dest='only_known_failures',
142                        help='only execute tests that are known to fail')
143    parser.add_argument('-s', '--skip-known-failures', action='store_true', dest='skip_known_failures',
144                        help='do not execute tests that are known to fail')
145    parser.add_argument('-r', '--retry-count', type=int,
146                        default=0, help='maximum retry on failure')
147    parser.add_argument('-j', '--jobs', type=int,
148                        default=default_concurrency(),
149                        help='number of concurrent test executions')
150
151    g = parser.add_argument_group(title='Advanced')
152    g.add_argument('-v', '--verbose', action='store_const',
153                   dest='log_level', const=logging.DEBUG, default=logging.WARNING,
154                   help='show debug output for test runner')
155    g.add_argument('-P', '--print-expected-failures', choices=['merge', 'overwrite'],
156                   dest='print_failures',
157                   help="generate expected failures based on last result and print to stdout")
158    g.add_argument('-U', '--update-expected-failures', choices=['merge', 'overwrite'],
159                   dest='update_failures',
160                   help="generate expected failures based on last result and save to default file location")
161    options = parser.parse_args(argv)
162
163    logger = multiprocessing.log_to_stderr()
164    logger.setLevel(options.log_level)
165
166    if options.features is not None and options.client:
167        print('Cannot specify both --features and --client ', file=sys.stderr)
168        return 1
169
170    # Allow multiple args separated with ',' for backward compatibility
171    server_match = list(chain(*[x.split(',') for x in options.server]))
172    client_match = list(chain(*[x.split(',') for x in options.client]))
173
174    if options.update_failures or options.print_failures:
175        dire = path_join(ROOT_DIR, FEATURE_DIR_RELATIVE) if options.features is not None else TEST_DIR
176        res = crossrunner.generate_known_failures(
177            dire, options.update_failures == 'overwrite',
178            options.update_failures, options.print_failures)
179    elif options.features is not None:
180        features = options.features or ['.*']
181        res = run_feature_tests(server_match, features, options.jobs,
182                                options.skip_known_failures, options.only_known_failures,
183                                options.retry_count, options.regex)
184    else:
185        res = run_cross_tests(server_match, client_match, options.jobs,
186                              options.skip_known_failures, options.only_known_failures,
187                              options.retry_count, options.regex)
188    return 0 if res else 1
189
190
191if __name__ == '__main__':
192    sys.exit(main(sys.argv[1:]))
193