1#!/usr/bin/env python3
2#
3# Script to compile and runs tests.
4#
5# Example:
6# ./scripts/test.py runners/test_runner -b
7#
8# Copyright (c) 2022, The littlefs authors.
9# SPDX-License-Identifier: BSD-3-Clause
10#
11
12import collections as co
13import csv
14import errno
15import glob
16import itertools as it
17import math as m
18import os
19import pty
20import re
21import shlex
22import shutil
23import signal
24import subprocess as sp
25import threading as th
26import time
27import toml
28
29
30RUNNER_PATH = './runners/test_runner'
31HEADER_PATH = 'runners/test_runner.h'
32
33GDB_PATH = ['gdb']
34VALGRIND_PATH = ['valgrind']
35PERF_SCRIPT = ['./scripts/perf.py']
36
37
38def openio(path, mode='r', buffering=-1):
39    # allow '-' for stdin/stdout
40    if path == '-':
41        if mode == 'r':
42            return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering)
43        else:
44            return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering)
45    else:
46        return open(path, mode, buffering)
47
48class TestCase:
49    # create a TestCase object from a config
50    def __init__(self, config, args={}):
51        self.name = config.pop('name')
52        self.path = config.pop('path')
53        self.suite = config.pop('suite')
54        self.lineno = config.pop('lineno', None)
55        self.if_ = config.pop('if', None)
56        if isinstance(self.if_, bool):
57            self.if_ = 'true' if self.if_ else 'false'
58        self.code = config.pop('code')
59        self.code_lineno = config.pop('code_lineno', None)
60        self.in_ = config.pop('in',
61            config.pop('suite_in', None))
62
63        self.reentrant = config.pop('reentrant',
64            config.pop('suite_reentrant', False))
65
66        # figure out defines and build possible permutations
67        self.defines = set()
68        self.permutations = []
69
70        # defines can be a dict or a list or dicts
71        suite_defines = config.pop('suite_defines', {})
72        if not isinstance(suite_defines, list):
73            suite_defines = [suite_defines]
74        defines = config.pop('defines', {})
75        if not isinstance(defines, list):
76            defines = [defines]
77
78        def csplit(v):
79            # split commas but only outside of parens
80            parens = 0
81            i_ = 0
82            for i in range(len(v)):
83                if v[i] == ',' and parens == 0:
84                    yield v[i_:i]
85                    i_ = i+1
86                elif v[i] in '([{':
87                    parens += 1
88                elif v[i] in '}])':
89                    parens -= 1
90            if v[i_:].strip():
91                yield v[i_:]
92
93        def parse_define(v):
94            # a define entry can be a list
95            if isinstance(v, list):
96                for v_ in v:
97                    yield from parse_define(v_)
98            # or a string
99            elif isinstance(v, str):
100                # which can be comma-separated values, with optional
101                # range statements. This matches the runtime define parser in
102                # the runner itself.
103                for v_ in csplit(v):
104                    m = re.search(r'\brange\b\s*\('
105                        '(?P<start>[^,\s]*)'
106                        '\s*(?:,\s*(?P<stop>[^,\s]*)'
107                        '\s*(?:,\s*(?P<step>[^,\s]*)\s*)?)?\)',
108                        v_)
109                    if m:
110                        start = (int(m.group('start'), 0)
111                            if m.group('start') else 0)
112                        stop = (int(m.group('stop'), 0)
113                            if m.group('stop') else None)
114                        step = (int(m.group('step'), 0)
115                            if m.group('step') else 1)
116                        if m.lastindex <= 1:
117                            start, stop = 0, start
118                        for x in range(start, stop, step):
119                            yield from parse_define('%s(%d)%s' % (
120                                v_[:m.start()], x, v_[m.end():]))
121                    else:
122                        yield v_
123            # or a literal value
124            elif isinstance(v, bool):
125                yield 'true' if v else 'false'
126            else:
127                yield v
128
129        # build possible permutations
130        for suite_defines_ in suite_defines:
131            self.defines |= suite_defines_.keys()
132            for defines_ in defines:
133                self.defines |= defines_.keys()
134                self.permutations.extend(dict(perm) for perm in it.product(*(
135                    [(k, v) for v in parse_define(vs)]
136                    for k, vs in sorted((suite_defines_ | defines_).items()))))
137
138        for k in config.keys():
139            print('%swarning:%s in %s, found unused key %r' % (
140                '\x1b[01;33m' if args['color'] else '',
141                '\x1b[m' if args['color'] else '',
142                self.name,
143                k),
144                file=sys.stderr)
145
146
147class TestSuite:
148    # create a TestSuite object from a toml file
149    def __init__(self, path, args={}):
150        self.path = path
151        self.name = os.path.basename(path)
152        if self.name.endswith('.toml'):
153            self.name = self.name[:-len('.toml')]
154
155        # load toml file and parse test cases
156        with open(self.path) as f:
157            # load tests
158            config = toml.load(f)
159
160            # find line numbers
161            f.seek(0)
162            case_linenos = []
163            code_linenos = []
164            for i, line in enumerate(f):
165                match = re.match(
166                    '(?P<case>\[\s*cases\s*\.\s*(?P<name>\w+)\s*\])'
167                        '|' '(?P<code>code\s*=)',
168                    line)
169                if match and match.group('case'):
170                    case_linenos.append((i+1, match.group('name')))
171                elif match and match.group('code'):
172                    code_linenos.append(i+2)
173
174            # sort in case toml parsing did not retain order
175            case_linenos.sort()
176
177            cases = config.pop('cases')
178            for (lineno, name), (nlineno, _) in it.zip_longest(
179                    case_linenos, case_linenos[1:],
180                    fillvalue=(float('inf'), None)):
181                code_lineno = min(
182                    (l for l in code_linenos if l >= lineno and l < nlineno),
183                    default=None)
184                cases[name]['lineno'] = lineno
185                cases[name]['code_lineno'] = code_lineno
186
187            self.if_ = config.pop('if', None)
188            if isinstance(self.if_, bool):
189                self.if_ = 'true' if self.if_ else 'false'
190
191            self.code = config.pop('code', None)
192            self.code_lineno = min(
193                (l for l in code_linenos
194                    if not case_linenos or l < case_linenos[0][0]),
195                default=None)
196
197            # a couple of these we just forward to all cases
198            defines = config.pop('defines', {})
199            in_ = config.pop('in', None)
200            reentrant = config.pop('reentrant', False)
201
202            self.cases = []
203            for name, case in sorted(cases.items(),
204                    key=lambda c: c[1].get('lineno')):
205                self.cases.append(TestCase(config={
206                    'name': name,
207                    'path': path + (':%d' % case['lineno']
208                        if 'lineno' in case else ''),
209                    'suite': self.name,
210                    'suite_defines': defines,
211                    'suite_in': in_,
212                    'suite_reentrant': reentrant,
213                    **case},
214                    args=args))
215
216            # combine per-case defines
217            self.defines = set.union(*(
218                set(case.defines) for case in self.cases))
219
220            # combine other per-case things
221            self.reentrant = any(case.reentrant for case in self.cases)
222
223        for k in config.keys():
224            print('%swarning:%s in %s, found unused key %r' % (
225                '\x1b[01;33m' if args['color'] else '',
226                '\x1b[m' if args['color'] else '',
227                self.name,
228                k),
229                file=sys.stderr)
230
231
232
233def compile(test_paths, **args):
234    # find .toml files
235    paths = []
236    for path in test_paths:
237        if os.path.isdir(path):
238            path = path + '/*.toml'
239
240        for path in glob.glob(path):
241            paths.append(path)
242
243    if not paths:
244        print('no test suites found in %r?' % test_paths)
245        sys.exit(-1)
246
247    # load the suites
248    suites = [TestSuite(path, args) for path in paths]
249    suites.sort(key=lambda s: s.name)
250
251    # check for name conflicts, these will cause ambiguity problems later
252    # when running tests
253    seen = {}
254    for suite in suites:
255        if suite.name in seen:
256            print('%swarning:%s conflicting suite %r, %s and %s' % (
257                '\x1b[01;33m' if args['color'] else '',
258                '\x1b[m' if args['color'] else '',
259                suite.name,
260                suite.path,
261                seen[suite.name].path),
262                file=sys.stderr)
263        seen[suite.name] = suite
264
265        for case in suite.cases:
266            # only allow conflicts if a case and its suite share a name
267            if case.name in seen and not (
268                    isinstance(seen[case.name], TestSuite)
269                    and seen[case.name].cases == [case]):
270                print('%swarning:%s conflicting case %r, %s and %s' % (
271                    '\x1b[01;33m' if args['color'] else '',
272                    '\x1b[m' if args['color'] else '',
273                    case.name,
274                    case.path,
275                    seen[case.name].path),
276                    file=sys.stderr)
277            seen[case.name] = case
278
279    # we can only compile one test suite at a time
280    if not args.get('source'):
281        if len(suites) > 1:
282            print('more than one test suite for compilation? (%r)' % test_paths)
283            sys.exit(-1)
284
285        suite = suites[0]
286
287    # write generated test source
288    if 'output' in args:
289        with openio(args['output'], 'w') as f:
290            _write = f.write
291            def write(s):
292                f.lineno += s.count('\n')
293                _write(s)
294            def writeln(s=''):
295                f.lineno += s.count('\n') + 1
296                _write(s)
297                _write('\n')
298            f.lineno = 1
299            f.write = write
300            f.writeln = writeln
301
302            f.writeln("// Generated by %s:" % sys.argv[0])
303            f.writeln("//")
304            f.writeln("// %s" % ' '.join(sys.argv))
305            f.writeln("//")
306            f.writeln()
307
308            # include test_runner.h in every generated file
309            f.writeln("#include \"%s\"" % args['include'])
310            f.writeln()
311
312            # write out generated functions, this can end up in different
313            # files depending on the "in" attribute
314            #
315            # note it's up to the specific generated file to declare
316            # the test defines
317            def write_case_functions(f, suite, case):
318                    # create case define functions
319                    if case.defines:
320                        # deduplicate defines by value to try to reduce the
321                        # number of functions we generate
322                        define_cbs = {}
323                        for i, defines in enumerate(case.permutations):
324                            for k, v in sorted(defines.items()):
325                                if v not in define_cbs:
326                                    name = ('__test__%s__%s__%d'
327                                        % (case.name, k, i))
328                                    define_cbs[v] = name
329                                    f.writeln('intmax_t %s('
330                                        '__attribute__((unused)) '
331                                        'void *data) {' % name)
332                                    f.writeln(4*' '+'return %s;' % v)
333                                    f.writeln('}')
334                                    f.writeln()
335                        f.writeln('const test_define_t '
336                            '__test__%s__defines[]['
337                            'TEST_IMPLICIT_DEFINE_COUNT+%d] = {'
338                            % (case.name, len(suite.defines)))
339                        for defines in case.permutations:
340                            f.writeln(4*' '+'{')
341                            for k, v in sorted(defines.items()):
342                                f.writeln(8*' '+'[%-24s] = {%s, NULL},' % (
343                                    k+'_i', define_cbs[v]))
344                            f.writeln(4*' '+'},')
345                        f.writeln('};')
346                        f.writeln()
347
348                    # create case filter function
349                    if suite.if_ is not None or case.if_ is not None:
350                        f.writeln('bool __test__%s__filter(void) {'
351                            % (case.name))
352                        f.writeln(4*' '+'return %s;'
353                            % ' && '.join('(%s)' % if_
354                                for if_ in [suite.if_, case.if_]
355                                if if_ is not None))
356                        f.writeln('}')
357                        f.writeln()
358
359                    # create case run function
360                    f.writeln('void __test__%s__run('
361                        '__attribute__((unused)) struct lfs_config *cfg) {'
362                        % (case.name))
363                    f.writeln(4*' '+'// test case %s' % case.name)
364                    if case.code_lineno is not None:
365                        f.writeln(4*' '+'#line %d "%s"'
366                            % (case.code_lineno, suite.path))
367                    f.write(case.code)
368                    if case.code_lineno is not None:
369                        f.writeln(4*' '+'#line %d "%s"'
370                            % (f.lineno+1, args['output']))
371                    f.writeln('}')
372                    f.writeln()
373
374            if not args.get('source'):
375                if suite.code is not None:
376                    if suite.code_lineno is not None:
377                        f.writeln('#line %d "%s"'
378                            % (suite.code_lineno, suite.path))
379                    f.write(suite.code)
380                    if suite.code_lineno is not None:
381                        f.writeln('#line %d "%s"'
382                            % (f.lineno+1, args['output']))
383                    f.writeln()
384
385                if suite.defines:
386                    for i, define in enumerate(sorted(suite.defines)):
387                        f.writeln('#ifndef %s' % define)
388                        f.writeln('#define %-24s '
389                            'TEST_IMPLICIT_DEFINE_COUNT+%d' % (define+'_i', i))
390                        f.writeln('#define %-24s '
391                            'TEST_DEFINE(%s)' % (define, define+'_i'))
392                        f.writeln('#endif')
393                    f.writeln()
394
395                # create case functions
396                for case in suite.cases:
397                    if case.in_ is None:
398                        write_case_functions(f, suite, case)
399                    else:
400                        if case.defines:
401                            f.writeln('extern const test_define_t '
402                                '__test__%s__defines[]['
403                                'TEST_IMPLICIT_DEFINE_COUNT+%d];'
404                                % (case.name, len(suite.defines)))
405                        if suite.if_ is not None or case.if_ is not None:
406                            f.writeln('extern bool __test__%s__filter('
407                                'void);'
408                                % (case.name))
409                        f.writeln('extern void __test__%s__run('
410                            'struct lfs_config *cfg);'
411                            % (case.name))
412                        f.writeln()
413
414                # create suite struct
415                #
416                # note we place this in the custom test_suites section with
417                # minimum alignment, otherwise GCC ups the alignment to
418                # 32-bytes for some reason
419                f.writeln('__attribute__((section("_test_suites"), '
420                    'aligned(1)))')
421                f.writeln('const struct test_suite __test__%s__suite = {'
422                    % suite.name)
423                f.writeln(4*' '+'.name = "%s",' % suite.name)
424                f.writeln(4*' '+'.path = "%s",' % suite.path)
425                f.writeln(4*' '+'.flags = %s,'
426                    % (' | '.join(filter(None, [
427                        'TEST_REENTRANT' if suite.reentrant else None]))
428                        or 0))
429                if suite.defines:
430                    # create suite define names
431                    f.writeln(4*' '+'.define_names = (const char *const['
432                        'TEST_IMPLICIT_DEFINE_COUNT+%d]){' % (
433                        len(suite.defines)))
434                    for k in sorted(suite.defines):
435                        f.writeln(8*' '+'[%-24s] = "%s",' % (k+'_i', k))
436                    f.writeln(4*' '+'},')
437                    f.writeln(4*' '+'.define_count = '
438                        'TEST_IMPLICIT_DEFINE_COUNT+%d,' % len(suite.defines))
439                f.writeln(4*' '+'.cases = (const struct test_case[]){')
440                for case in suite.cases:
441                    # create case structs
442                    f.writeln(8*' '+'{')
443                    f.writeln(12*' '+'.name = "%s",' % case.name)
444                    f.writeln(12*' '+'.path = "%s",' % case.path)
445                    f.writeln(12*' '+'.flags = %s,'
446                        % (' | '.join(filter(None, [
447                            'TEST_REENTRANT' if case.reentrant else None]))
448                            or 0))
449                    f.writeln(12*' '+'.permutations = %d,'
450                        % len(case.permutations))
451                    if case.defines:
452                        f.writeln(12*' '+'.defines '
453                            '= (const test_define_t*)__test__%s__defines,'
454                            % (case.name))
455                    if suite.if_ is not None or case.if_ is not None:
456                        f.writeln(12*' '+'.filter = __test__%s__filter,'
457                            % (case.name))
458                    f.writeln(12*' '+'.run = __test__%s__run,'
459                        % (case.name))
460                    f.writeln(8*' '+'},')
461                f.writeln(4*' '+'},')
462                f.writeln(4*' '+'.case_count = %d,' % len(suite.cases))
463                f.writeln('};')
464                f.writeln()
465
466            else:
467                # copy source
468                f.writeln('#line 1 "%s"' % args['source'])
469                with open(args['source']) as sf:
470                    shutil.copyfileobj(sf, f)
471                f.writeln()
472
473                # write any internal tests
474                for suite in suites:
475                    for case in suite.cases:
476                        if (case.in_ is not None
477                                and os.path.normpath(case.in_)
478                                    == os.path.normpath(args['source'])):
479                            # write defines, but note we need to undef any
480                            # new defines since we're in someone else's file
481                            if suite.defines:
482                                for i, define in enumerate(
483                                        sorted(suite.defines)):
484                                    f.writeln('#ifndef %s' % define)
485                                    f.writeln('#define %-24s '
486                                        'TEST_IMPLICIT_DEFINE_COUNT+%d' % (
487                                        define+'_i', i))
488                                    f.writeln('#define %-24s '
489                                        'TEST_DEFINE(%s)' % (
490                                        define, define+'_i'))
491                                    f.writeln('#define '
492                                        '__TEST__%s__NEEDS_UNDEF' % (
493                                        define))
494                                    f.writeln('#endif')
495                                f.writeln()
496
497                            write_case_functions(f, suite, case)
498
499                            if suite.defines:
500                                for define in sorted(suite.defines):
501                                    f.writeln('#ifdef __TEST__%s__NEEDS_UNDEF'
502                                        % define)
503                                    f.writeln('#undef __TEST__%s__NEEDS_UNDEF'
504                                        % define)
505                                    f.writeln('#undef %s' % define)
506                                    f.writeln('#undef %s' % (define+'_i'))
507                                    f.writeln('#endif')
508                                f.writeln()
509
510def find_runner(runner, **args):
511    cmd = runner.copy()
512
513    # run under some external command?
514    if args.get('exec'):
515        cmd[:0] = args['exec']
516
517    # run under valgrind?
518    if args.get('valgrind'):
519        cmd[:0] = args['valgrind_path'] + [
520            '--leak-check=full',
521            '--track-origins=yes',
522            '--error-exitcode=4',
523            '-q']
524
525    # run under perf?
526    if args.get('perf'):
527        cmd[:0] = args['perf_script'] + list(filter(None, [
528            '-R',
529            '--perf-freq=%s' % args['perf_freq']
530                if args.get('perf_freq') else None,
531            '--perf-period=%s' % args['perf_period']
532                if args.get('perf_period') else None,
533            '--perf-events=%s' % args['perf_events']
534                if args.get('perf_events') else None,
535            '--perf-path=%s' % args['perf_path']
536                if args.get('perf_path') else None,
537            '-o%s' % args['perf']]))
538
539    # other context
540    if args.get('geometry'):
541        cmd.append('-G%s' % args['geometry'])
542    if args.get('powerloss'):
543        cmd.append('-P%s' % args['powerloss'])
544    if args.get('disk'):
545        cmd.append('-d%s' % args['disk'])
546    if args.get('trace'):
547        cmd.append('-t%s' % args['trace'])
548    if args.get('trace_backtrace'):
549        cmd.append('--trace-backtrace')
550    if args.get('trace_period'):
551        cmd.append('--trace-period=%s' % args['trace_period'])
552    if args.get('trace_freq'):
553        cmd.append('--trace-freq=%s' % args['trace_freq'])
554    if args.get('read_sleep'):
555        cmd.append('--read-sleep=%s' % args['read_sleep'])
556    if args.get('prog_sleep'):
557        cmd.append('--prog-sleep=%s' % args['prog_sleep'])
558    if args.get('erase_sleep'):
559        cmd.append('--erase-sleep=%s' % args['erase_sleep'])
560
561    # defines?
562    if args.get('define'):
563        for define in args.get('define'):
564            cmd.append('-D%s' % define)
565
566    return cmd
567
568def list_(runner, test_ids=[], **args):
569    cmd = find_runner(runner, **args) + test_ids
570    if args.get('summary'):          cmd.append('--summary')
571    if args.get('list_suites'):      cmd.append('--list-suites')
572    if args.get('list_cases'):       cmd.append('--list-cases')
573    if args.get('list_suite_paths'): cmd.append('--list-suite-paths')
574    if args.get('list_case_paths'):  cmd.append('--list-case-paths')
575    if args.get('list_defines'):     cmd.append('--list-defines')
576    if args.get('list_permutation_defines'):
577                                     cmd.append('--list-permutation-defines')
578    if args.get('list_implicit_defines'):
579                                     cmd.append('--list-implicit-defines')
580    if args.get('list_geometries'):  cmd.append('--list-geometries')
581    if args.get('list_powerlosses'): cmd.append('--list-powerlosses')
582
583    if args.get('verbose'):
584        print(' '.join(shlex.quote(c) for c in cmd))
585    return sp.call(cmd)
586
587
588def find_perms(runner_, ids=[], **args):
589    case_suites = {}
590    expected_case_perms = co.defaultdict(lambda: 0)
591    expected_perms = 0
592    total_perms = 0
593
594    # query cases from the runner
595    cmd = runner_ + ['--list-cases'] + ids
596    if args.get('verbose'):
597        print(' '.join(shlex.quote(c) for c in cmd))
598    proc = sp.Popen(cmd,
599        stdout=sp.PIPE,
600        stderr=sp.PIPE if not args.get('verbose') else None,
601        universal_newlines=True,
602        errors='replace',
603        close_fds=False)
604    pattern = re.compile(
605        '^(?P<case>[^\s]+)'
606            '\s+(?P<flags>[^\s]+)'
607            '\s+(?P<filtered>\d+)/(?P<perms>\d+)')
608    # skip the first line
609    for line in it.islice(proc.stdout, 1, None):
610        m = pattern.match(line)
611        if m:
612            filtered = int(m.group('filtered'))
613            perms = int(m.group('perms'))
614            expected_case_perms[m.group('case')] += filtered
615            expected_perms += filtered
616            total_perms += perms
617    proc.wait()
618    if proc.returncode != 0:
619        if not args.get('verbose'):
620            for line in proc.stderr:
621                sys.stdout.write(line)
622        sys.exit(-1)
623
624    # get which suite each case belongs to via paths
625    cmd = runner_ + ['--list-case-paths'] + ids
626    if args.get('verbose'):
627        print(' '.join(shlex.quote(c) for c in cmd))
628    proc = sp.Popen(cmd,
629        stdout=sp.PIPE,
630        stderr=sp.PIPE if not args.get('verbose') else None,
631        universal_newlines=True,
632        errors='replace',
633        close_fds=False)
634    pattern = re.compile(
635        '^(?P<case>[^\s]+)'
636            '\s+(?P<path>[^:]+):(?P<lineno>\d+)')
637    # skip the first line
638    for line in it.islice(proc.stdout, 1, None):
639        m = pattern.match(line)
640        if m:
641            path = m.group('path')
642            # strip path/suffix here
643            suite = os.path.basename(path)
644            if suite.endswith('.toml'):
645                suite = suite[:-len('.toml')]
646            case_suites[m.group('case')] = suite
647    proc.wait()
648    if proc.returncode != 0:
649        if not args.get('verbose'):
650            for line in proc.stderr:
651                sys.stdout.write(line)
652        sys.exit(-1)
653
654    # figure out expected suite perms
655    expected_suite_perms = co.defaultdict(lambda: 0)
656    for case, suite in case_suites.items():
657        expected_suite_perms[suite] += expected_case_perms[case]
658
659    return (
660        case_suites,
661        expected_suite_perms,
662        expected_case_perms,
663        expected_perms,
664        total_perms)
665
666def find_path(runner_, id, **args):
667    path = None
668    # query from runner
669    cmd = runner_ + ['--list-case-paths', id]
670    if args.get('verbose'):
671        print(' '.join(shlex.quote(c) for c in cmd))
672    proc = sp.Popen(cmd,
673        stdout=sp.PIPE,
674        stderr=sp.PIPE if not args.get('verbose') else None,
675        universal_newlines=True,
676        errors='replace',
677        close_fds=False)
678    pattern = re.compile(
679        '^(?P<case>[^\s]+)'
680            '\s+(?P<path>[^:]+):(?P<lineno>\d+)')
681    # skip the first line
682    for line in it.islice(proc.stdout, 1, None):
683        m = pattern.match(line)
684        if m and path is None:
685            path_ = m.group('path')
686            lineno = int(m.group('lineno'))
687            path = (path_, lineno)
688    proc.wait()
689    if proc.returncode != 0:
690        if not args.get('verbose'):
691            for line in proc.stderr:
692                sys.stdout.write(line)
693        sys.exit(-1)
694
695    return path
696
697def find_defines(runner_, id, **args):
698    # query permutation defines from runner
699    cmd = runner_ + ['--list-permutation-defines', id]
700    if args.get('verbose'):
701        print(' '.join(shlex.quote(c) for c in cmd))
702    proc = sp.Popen(cmd,
703        stdout=sp.PIPE,
704        stderr=sp.PIPE if not args.get('verbose') else None,
705        universal_newlines=True,
706        errors='replace',
707        close_fds=False)
708    defines = co.OrderedDict()
709    pattern = re.compile('^(?P<define>\w+)=(?P<value>.+)')
710    for line in proc.stdout:
711        m = pattern.match(line)
712        if m:
713            define = m.group('define')
714            value = m.group('value')
715            defines[define] = value
716    proc.wait()
717    if proc.returncode != 0:
718        if not args.get('verbose'):
719            for line in proc.stderr:
720                sys.stdout.write(line)
721        sys.exit(-1)
722
723    return defines
724
725
726# Thread-safe CSV writer
727class TestOutput:
728    def __init__(self, path, head=None, tail=None):
729        self.f = openio(path, 'w+', 1)
730        self.lock = th.Lock()
731        self.head = head or []
732        self.tail = tail or []
733        self.writer = csv.DictWriter(self.f, self.head + self.tail)
734        self.rows = []
735
736    def close(self):
737        self.f.close()
738
739    def __enter__(self):
740        return self
741
742    def __exit__(self, *_):
743        self.f.close()
744
745    def writerow(self, row):
746        with self.lock:
747            self.rows.append(row)
748            if all(k in self.head or k in self.tail for k in row.keys()):
749                # can simply append
750                self.writer.writerow(row)
751            else:
752                # need to rewrite the file
753                self.head.extend(row.keys() - (self.head + self.tail))
754                self.f.seek(0)
755                self.f.truncate()
756                self.writer = csv.DictWriter(self.f, self.head + self.tail)
757                self.writer.writeheader()
758                for row in self.rows:
759                    self.writer.writerow(row)
760
761# A test failure
762class TestFailure(Exception):
763    def __init__(self, id, returncode, stdout, assert_=None):
764        self.id = id
765        self.returncode = returncode
766        self.stdout = stdout
767        self.assert_ = assert_
768
769def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
770    # get expected suite/case/perm counts
771    (case_suites,
772        expected_suite_perms,
773        expected_case_perms,
774        expected_perms,
775        total_perms) = find_perms(runner_, ids, **args)
776
777    passed_suite_perms = co.defaultdict(lambda: 0)
778    passed_case_perms = co.defaultdict(lambda: 0)
779    passed_perms = 0
780    powerlosses = 0
781    failures = []
782    killed = False
783
784    pattern = re.compile('^(?:'
785            '(?P<op>running|finished|skipped|powerloss) '
786                '(?P<id>(?P<case>[^:]+)[^\s]*)'
787            '|' '(?P<path>[^:]+):(?P<lineno>\d+):(?P<op_>assert):'
788                ' *(?P<message>.*)'
789        ')$')
790    locals = th.local()
791    children = set()
792
793    def run_runner(runner_, ids=[]):
794        nonlocal passed_suite_perms
795        nonlocal passed_case_perms
796        nonlocal passed_perms
797        nonlocal powerlosses
798        nonlocal locals
799
800        # run the tests!
801        cmd = runner_ + ids
802        if args.get('verbose'):
803            print(' '.join(shlex.quote(c) for c in cmd))
804
805        mpty, spty = pty.openpty()
806        proc = sp.Popen(cmd, stdout=spty, stderr=spty, close_fds=False)
807        os.close(spty)
808        children.add(proc)
809        mpty = os.fdopen(mpty, 'r', 1)
810
811        last_id = None
812        last_stdout = co.deque(maxlen=args.get('context', 5) + 1)
813        last_assert = None
814        try:
815            while True:
816                # parse a line for state changes
817                try:
818                    line = mpty.readline()
819                except OSError as e:
820                    if e.errno != errno.EIO:
821                        raise
822                    break
823                if not line:
824                    break
825                last_stdout.append(line)
826                if stdout_:
827                    try:
828                        stdout_.write(line)
829                        stdout_.flush()
830                    except BrokenPipeError:
831                        pass
832
833                m = pattern.match(line)
834                if m:
835                    op = m.group('op') or m.group('op_')
836                    if op == 'running':
837                        locals.seen_perms += 1
838                        last_id = m.group('id')
839                        last_stdout.clear()
840                        last_assert = None
841                    elif op == 'powerloss':
842                        last_id = m.group('id')
843                        powerlosses += 1
844                    elif op == 'finished':
845                        case = m.group('case')
846                        suite = case_suites[case]
847                        passed_suite_perms[suite] += 1
848                        passed_case_perms[case] += 1
849                        passed_perms += 1
850                        if output_:
851                            # get defines and write to csv
852                            defines = find_defines(
853                                runner_, m.group('id'), **args)
854                            output_.writerow({
855                                'suite': suite,
856                                'case': case,
857                                'test_passed': '1/1',
858                                **defines})
859                    elif op == 'skipped':
860                        locals.seen_perms += 1
861                    elif op == 'assert':
862                        last_assert = (
863                            m.group('path'),
864                            int(m.group('lineno')),
865                            m.group('message'))
866                        # go ahead and kill the process, aborting takes a while
867                        if args.get('keep_going'):
868                            proc.kill()
869        except KeyboardInterrupt:
870            raise TestFailure(last_id, 1, list(last_stdout))
871        finally:
872            children.remove(proc)
873            mpty.close()
874
875        proc.wait()
876        if proc.returncode != 0:
877            raise TestFailure(
878                last_id,
879                proc.returncode,
880                list(last_stdout),
881                last_assert)
882
883    def run_job(runner_, ids=[], start=None, step=None):
884        nonlocal failures
885        nonlocal killed
886        nonlocal locals
887
888        start = start or 0
889        step = step or 1
890        while start < total_perms:
891            job_runner = runner_.copy()
892            if args.get('isolate') or args.get('valgrind'):
893                job_runner.append('-s%s,%s,%s' % (start, start+step, step))
894            else:
895                job_runner.append('-s%s,,%s' % (start, step))
896
897            try:
898                # run the tests
899                locals.seen_perms = 0
900                run_runner(job_runner, ids)
901                assert locals.seen_perms > 0
902                start += locals.seen_perms*step
903
904            except TestFailure as failure:
905                # keep track of failures
906                if output_:
907                    case, _ = failure.id.split(':', 1)
908                    suite = case_suites[case]
909                    # get defines and write to csv
910                    defines = find_defines(runner_, failure.id, **args)
911                    output_.writerow({
912                        'suite': suite,
913                        'case': case,
914                        'test_passed': '0/1',
915                        **defines})
916
917                # race condition for multiple failures?
918                if failures and not args.get('keep_going'):
919                    break
920
921                failures.append(failure)
922
923                if args.get('keep_going') and not killed:
924                    # resume after failed test
925                    assert locals.seen_perms > 0
926                    start += locals.seen_perms*step
927                    continue
928                else:
929                    # stop other tests
930                    killed = True
931                    for child in children.copy():
932                        child.kill()
933                    break
934
935
936    # parallel jobs?
937    runners = []
938    if 'jobs' in args:
939        for job in range(args['jobs']):
940            runners.append(th.Thread(
941                target=run_job, args=(runner_, ids, job, args['jobs']),
942                daemon=True))
943    else:
944        runners.append(th.Thread(
945            target=run_job, args=(runner_, ids, None, None),
946            daemon=True))
947
948    def print_update(done):
949        if not args.get('verbose') and (args['color'] or done):
950            sys.stdout.write('%s%srunning %s%s:%s %s%s' % (
951                '\r\x1b[K' if args['color'] else '',
952                '\x1b[?7l' if not done else '',
953                ('\x1b[32m' if not failures else '\x1b[31m')
954                    if args['color'] else '',
955                name,
956                '\x1b[m' if args['color'] else '',
957                ', '.join(filter(None, [
958                    '%d/%d suites' % (
959                        sum(passed_suite_perms[k] == v
960                            for k, v in expected_suite_perms.items()),
961                        len(expected_suite_perms))
962                        if (not args.get('by_suites')
963                            and not args.get('by_cases')) else None,
964                    '%d/%d cases' % (
965                        sum(passed_case_perms[k] == v
966                            for k, v in expected_case_perms.items()),
967                        len(expected_case_perms))
968                        if not args.get('by_cases') else None,
969                    '%d/%d perms' % (passed_perms, expected_perms),
970                    '%dpls!' % powerlosses
971                        if powerlosses else None,
972                    '%s%d/%d failures%s' % (
973                            '\x1b[31m' if args['color'] else '',
974                            len(failures),
975                            expected_perms,
976                            '\x1b[m' if args['color'] else '')
977                        if failures else None])),
978                '\x1b[?7h' if not done else '\n'))
979            sys.stdout.flush()
980
981    for r in runners:
982        r.start()
983
984    try:
985        while any(r.is_alive() for r in runners):
986            time.sleep(0.01)
987            print_update(False)
988    except KeyboardInterrupt:
989        # this is handled by the runner threads, we just
990        # need to not abort here
991        killed = True
992    finally:
993        print_update(True)
994
995    for r in runners:
996        r.join()
997
998    return (
999        expected_perms,
1000        passed_perms,
1001        powerlosses,
1002        failures,
1003        killed)
1004
1005
1006def run(runner, test_ids=[], **args):
1007    # query runner for tests
1008    runner_ = find_runner(runner, **args)
1009    print('using runner: %s' % ' '.join(shlex.quote(c) for c in runner_))
1010    (_,
1011        expected_suite_perms,
1012        expected_case_perms,
1013        expected_perms,
1014        total_perms) = find_perms(runner_, test_ids, **args)
1015    print('found %d suites, %d cases, %d/%d permutations' % (
1016        len(expected_suite_perms),
1017        len(expected_case_perms),
1018        expected_perms,
1019        total_perms))
1020    print()
1021
1022    # automatic job detection?
1023    if args.get('jobs') == 0:
1024        args['jobs'] = len(os.sched_getaffinity(0))
1025
1026    # truncate and open logs here so they aren't disconnected between tests
1027    stdout = None
1028    if args.get('stdout'):
1029        stdout = openio(args['stdout'], 'w', 1)
1030    trace = None
1031    if args.get('trace'):
1032        trace = openio(args['trace'], 'w', 1)
1033    output = None
1034    if args.get('output'):
1035        output = TestOutput(args['output'],
1036            ['suite', 'case'],
1037            ['test_passed'])
1038
1039    # measure runtime
1040    start = time.time()
1041
1042    # spawn runners
1043    expected = 0
1044    passed = 0
1045    powerlosses = 0
1046    failures = []
1047    for by in (test_ids if test_ids
1048            else expected_case_perms.keys() if args.get('by_cases')
1049            else expected_suite_perms.keys() if args.get('by_suites')
1050            else [None]):
1051        # spawn jobs for stage
1052        (expected_,
1053            passed_,
1054            powerlosses_,
1055            failures_,
1056            killed) = run_stage(
1057                by or 'tests',
1058                runner_,
1059                [by] if by is not None else [],
1060                stdout,
1061                trace,
1062                output,
1063                **args)
1064        # collect passes/failures
1065        expected += expected_
1066        passed += passed_
1067        powerlosses += powerlosses_
1068        failures.extend(failures_)
1069        if (failures and not args.get('keep_going')) or killed:
1070            break
1071
1072    stop = time.time()
1073
1074    if stdout:
1075        try:
1076            stdout.close()
1077        except BrokenPipeError:
1078            pass
1079    if trace:
1080        try:
1081            trace.close()
1082        except BrokenPipeError:
1083            pass
1084    if output:
1085        output.close()
1086
1087    # show summary
1088    print()
1089    print('%sdone:%s %s' % (
1090        ('\x1b[32m' if not failures else '\x1b[31m')
1091            if args['color'] else '',
1092        '\x1b[m' if args['color'] else '',
1093        ', '.join(filter(None, [
1094            '%d/%d passed' % (passed, expected),
1095            '%d/%d failed' % (len(failures), expected),
1096            '%dpls!' % powerlosses if powerlosses else None,
1097            'in %.2fs' % (stop-start)]))))
1098    print()
1099
1100    # print each failure
1101    for failure in failures:
1102        assert failure.id is not None, '%s broken? %r' % (
1103            ' '.join(shlex.quote(c) for c in runner_),
1104            failure)
1105
1106        # get some extra info from runner
1107        path, lineno = find_path(runner_, failure.id, **args)
1108        defines = find_defines(runner_, failure.id, **args)
1109
1110        # show summary of failure
1111        print('%s%s:%d:%sfailure:%s %s%s failed' % (
1112            '\x1b[01m' if args['color'] else '',
1113            path, lineno,
1114            '\x1b[01;31m' if args['color'] else '',
1115            '\x1b[m' if args['color'] else '',
1116            failure.id,
1117            ' (%s)' % ', '.join('%s=%s' % (k,v) for k,v in defines.items())
1118                if defines else ''))
1119
1120        if failure.stdout:
1121            stdout = failure.stdout
1122            if failure.assert_ is not None:
1123                stdout = stdout[:-1]
1124            for line in stdout[-args.get('context', 5):]:
1125                sys.stdout.write(line)
1126
1127        if failure.assert_ is not None:
1128            path, lineno, message = failure.assert_
1129            print('%s%s:%d:%sassert:%s %s' % (
1130                '\x1b[01m' if args['color'] else '',
1131                path, lineno,
1132                '\x1b[01;31m' if args['color'] else '',
1133                '\x1b[m' if args['color'] else '',
1134                message))
1135            with open(path) as f:
1136                line = next(it.islice(f, lineno-1, None)).strip('\n')
1137                print(line)
1138        print()
1139
1140    # drop into gdb?
1141    if failures and (args.get('gdb')
1142            or args.get('gdb_case')
1143            or args.get('gdb_main')
1144            or args.get('gdb_pl') is not None
1145            or args.get('gdb_pl_before')
1146            or args.get('gdb_pl_after')):
1147        failure = failures[0]
1148        cmd = runner_ + [failure.id]
1149
1150        if args.get('gdb_main'):
1151            # we don't really need the case breakpoint here, but it
1152            # can be helpful
1153            path, lineno = find_path(runner_, failure.id, **args)
1154            cmd[:0] = args['gdb_path'] + [
1155                '-ex', 'break main',
1156                '-ex', 'break %s:%d' % (path, lineno),
1157                '-ex', 'run',
1158                '--args']
1159        elif args.get('gdb_case'):
1160            path, lineno = find_path(runner_, failure.id, **args)
1161            cmd[:0] = args['gdb_path'] + [
1162                '-ex', 'break %s:%d' % (path, lineno),
1163                '-ex', 'run',
1164                '--args']
1165        elif args.get('gdb_pl') is not None:
1166            path, lineno = find_path(runner_, failure.id, **args)
1167            cmd[:0] = args['gdb_path'] + [
1168                '-ex', 'break %s:%d' % (path, lineno),
1169                '-ex', 'ignore 1 %d' % args['gdb_pl'],
1170                '-ex', 'run',
1171                '--args']
1172        elif args.get('gdb_pl_before'):
1173            # figure out how many powerlosses there were
1174            powerlosses = (
1175                sum(1 for _ in re.finditer('[0-9a-f]',
1176                    failure.id.split(':', 2)[-1]))
1177                if failure.id.count(':') >= 2 else 0)
1178            path, lineno = find_path(runner_, failure.id, **args)
1179            cmd[:0] = args['gdb_path'] + [
1180                '-ex', 'break %s:%d' % (path, lineno),
1181                '-ex', 'ignore 1 %d' % max(powerlosses-1, 0),
1182                '-ex', 'run',
1183                '--args']
1184        elif args.get('gdb_pl_after'):
1185            # figure out how many powerlosses there were
1186            powerlosses = (
1187                sum(1 for _ in re.finditer('[0-9a-f]',
1188                    failure.id.split(':', 2)[-1]))
1189                if failure.id.count(':') >= 2 else 0)
1190            path, lineno = find_path(runner_, failure.id, **args)
1191            cmd[:0] = args['gdb_path'] + [
1192                '-ex', 'break %s:%d' % (path, lineno),
1193                '-ex', 'ignore 1 %d' % powerlosses,
1194                '-ex', 'run',
1195                '--args']
1196        elif failure.assert_ is not None:
1197            cmd[:0] = args['gdb_path'] + [
1198                '-ex', 'run',
1199                '-ex', 'frame function raise',
1200                '-ex', 'up 2',
1201                '--args']
1202        else:
1203            cmd[:0] = args['gdb_path'] + [
1204                '-ex', 'run',
1205                '--args']
1206
1207        # exec gdb interactively
1208        if args.get('verbose'):
1209            print(' '.join(shlex.quote(c) for c in cmd))
1210        os.execvp(cmd[0], cmd)
1211
1212    return 1 if failures else 0
1213
1214
1215def main(**args):
1216    # figure out what color should be
1217    if args.get('color') == 'auto':
1218        args['color'] = sys.stdout.isatty()
1219    elif args.get('color') == 'always':
1220        args['color'] = True
1221    else:
1222        args['color'] = False
1223
1224    if args.get('compile'):
1225        return compile(**args)
1226    elif (args.get('summary')
1227            or args.get('list_suites')
1228            or args.get('list_cases')
1229            or args.get('list_suite_paths')
1230            or args.get('list_case_paths')
1231            or args.get('list_defines')
1232            or args.get('list_permutation_defines')
1233            or args.get('list_implicit_defines')
1234            or args.get('list_geometries')
1235            or args.get('list_powerlosses')):
1236        return list_(**args)
1237    else:
1238        return run(**args)
1239
1240
1241if __name__ == "__main__":
1242    import argparse
1243    import sys
1244    argparse.ArgumentParser._handle_conflict_ignore = lambda *_: None
1245    argparse._ArgumentGroup._handle_conflict_ignore = lambda *_: None
1246    parser = argparse.ArgumentParser(
1247        description="Build and run tests.",
1248        allow_abbrev=False,
1249        conflict_handler='ignore')
1250    parser.add_argument(
1251        '-v', '--verbose',
1252        action='store_true',
1253        help="Output commands that run behind the scenes.")
1254    parser.add_argument(
1255        '--color',
1256        choices=['never', 'always', 'auto'],
1257        default='auto',
1258        help="When to use terminal colors. Defaults to 'auto'.")
1259
1260    # test flags
1261    test_parser = parser.add_argument_group('test options')
1262    test_parser.add_argument(
1263        'runner',
1264        nargs='?',
1265        type=lambda x: x.split(),
1266        help="Test runner to use for testing. Defaults to %r." % RUNNER_PATH)
1267    test_parser.add_argument(
1268        'test_ids',
1269        nargs='*',
1270        help="Description of tests to run.")
1271    test_parser.add_argument(
1272        '-Y', '--summary',
1273        action='store_true',
1274        help="Show quick summary.")
1275    test_parser.add_argument(
1276        '-l', '--list-suites',
1277        action='store_true',
1278        help="List test suites.")
1279    test_parser.add_argument(
1280        '-L', '--list-cases',
1281        action='store_true',
1282        help="List test cases.")
1283    test_parser.add_argument(
1284        '--list-suite-paths',
1285        action='store_true',
1286        help="List the path for each test suite.")
1287    test_parser.add_argument(
1288        '--list-case-paths',
1289        action='store_true',
1290        help="List the path and line number for each test case.")
1291    test_parser.add_argument(
1292        '--list-defines',
1293        action='store_true',
1294        help="List all defines in this test-runner.")
1295    test_parser.add_argument(
1296        '--list-permutation-defines',
1297        action='store_true',
1298        help="List explicit defines in this test-runner.")
1299    test_parser.add_argument(
1300        '--list-implicit-defines',
1301        action='store_true',
1302        help="List implicit defines in this test-runner.")
1303    test_parser.add_argument(
1304        '--list-geometries',
1305        action='store_true',
1306        help="List the available disk geometries.")
1307    test_parser.add_argument(
1308        '--list-powerlosses',
1309        action='store_true',
1310        help="List the available power-loss scenarios.")
1311    test_parser.add_argument(
1312        '-D', '--define',
1313        action='append',
1314        help="Override a test define.")
1315    test_parser.add_argument(
1316        '-G', '--geometry',
1317        help="Comma-separated list of disk geometries to test.")
1318    test_parser.add_argument(
1319        '-P', '--powerloss',
1320        help="Comma-separated list of power-loss scenarios to test.")
1321    test_parser.add_argument(
1322        '-d', '--disk',
1323        help="Direct block device operations to this file.")
1324    test_parser.add_argument(
1325        '-t', '--trace',
1326        help="Direct trace output to this file.")
1327    test_parser.add_argument(
1328        '--trace-backtrace',
1329        action='store_true',
1330        help="Include a backtrace with every trace statement.")
1331    test_parser.add_argument(
1332        '--trace-period',
1333        help="Sample trace output at this period in cycles.")
1334    test_parser.add_argument(
1335        '--trace-freq',
1336        help="Sample trace output at this frequency in hz.")
1337    test_parser.add_argument(
1338        '-O', '--stdout',
1339        help="Direct stdout to this file. Note stderr is already merged here.")
1340    test_parser.add_argument(
1341        '-o', '--output',
1342        help="CSV file to store results.")
1343    test_parser.add_argument(
1344        '--read-sleep',
1345        help="Artificial read delay in seconds.")
1346    test_parser.add_argument(
1347        '--prog-sleep',
1348        help="Artificial prog delay in seconds.")
1349    test_parser.add_argument(
1350        '--erase-sleep',
1351        help="Artificial erase delay in seconds.")
1352    test_parser.add_argument(
1353        '-j', '--jobs',
1354        nargs='?',
1355        type=lambda x: int(x, 0),
1356        const=0,
1357        help="Number of parallel runners to run. 0 runs one runner per core.")
1358    test_parser.add_argument(
1359        '-k', '--keep-going',
1360        action='store_true',
1361        help="Don't stop on first error.")
1362    test_parser.add_argument(
1363        '-i', '--isolate',
1364        action='store_true',
1365        help="Run each test permutation in a separate process.")
1366    test_parser.add_argument(
1367        '-b', '--by-suites',
1368        action='store_true',
1369        help="Step through tests by suite.")
1370    test_parser.add_argument(
1371        '-B', '--by-cases',
1372        action='store_true',
1373        help="Step through tests by case.")
1374    test_parser.add_argument(
1375        '--context',
1376        type=lambda x: int(x, 0),
1377        default=5,
1378        help="Show this many lines of stdout on test failure. "
1379            "Defaults to 5.")
1380    test_parser.add_argument(
1381        '--gdb',
1382        action='store_true',
1383        help="Drop into gdb on test failure.")
1384    test_parser.add_argument(
1385        '--gdb-case',
1386        action='store_true',
1387        help="Drop into gdb on test failure but stop at the beginning "
1388            "of the failing test case.")
1389    test_parser.add_argument(
1390        '--gdb-main',
1391        action='store_true',
1392        help="Drop into gdb on test failure but stop at the beginning "
1393            "of main.")
1394    test_parser.add_argument(
1395        '--gdb-pl',
1396        type=lambda x: int(x, 0),
1397        help="Drop into gdb on this specific powerloss.")
1398    test_parser.add_argument(
1399        '--gdb-pl-before',
1400        action='store_true',
1401        help="Drop into gdb before the powerloss that caused the failure.")
1402    test_parser.add_argument(
1403        '--gdb-pl-after',
1404        action='store_true',
1405        help="Drop into gdb after the powerloss that caused the failure.")
1406    test_parser.add_argument(
1407        '--gdb-path',
1408        type=lambda x: x.split(),
1409        default=GDB_PATH,
1410        help="Path to the gdb executable, may include flags. "
1411            "Defaults to %r." % GDB_PATH)
1412    test_parser.add_argument(
1413        '--exec',
1414        type=lambda e: e.split(),
1415        help="Run under another executable.")
1416    test_parser.add_argument(
1417        '--valgrind',
1418        action='store_true',
1419        help="Run under Valgrind to find memory errors. Implicitly sets "
1420            "--isolate.")
1421    test_parser.add_argument(
1422        '--valgrind-path',
1423        type=lambda x: x.split(),
1424        default=VALGRIND_PATH,
1425        help="Path to the Valgrind executable, may include flags. "
1426            "Defaults to %r." % VALGRIND_PATH)
1427    test_parser.add_argument(
1428        '-p', '--perf',
1429        help="Run under Linux's perf to sample performance counters, writing "
1430            "samples to this file.")
1431    test_parser.add_argument(
1432        '--perf-freq',
1433        help="perf sampling frequency. This is passed directly to the perf "
1434            "script.")
1435    test_parser.add_argument(
1436        '--perf-period',
1437        help="perf sampling period. This is passed directly to the perf "
1438            "script.")
1439    test_parser.add_argument(
1440        '--perf-events',
1441        help="perf events to record. This is passed directly to the perf "
1442            "script.")
1443    test_parser.add_argument(
1444        '--perf-script',
1445        type=lambda x: x.split(),
1446        default=PERF_SCRIPT,
1447        help="Path to the perf script to use. Defaults to %r." % PERF_SCRIPT)
1448    test_parser.add_argument(
1449        '--perf-path',
1450        type=lambda x: x.split(),
1451        help="Path to the perf executable, may include flags. This is passed "
1452            "directly to the perf script")
1453
1454    # compilation flags
1455    comp_parser = parser.add_argument_group('compilation options')
1456    comp_parser.add_argument(
1457        'test_paths',
1458        nargs='*',
1459        help="Description of *.toml files to compile. May be a directory "
1460            "or a list of paths.")
1461    comp_parser.add_argument(
1462        '-c', '--compile',
1463        action='store_true',
1464        help="Compile a test suite or source file.")
1465    comp_parser.add_argument(
1466        '-s', '--source',
1467        help="Source file to compile, possibly injecting internal tests.")
1468    comp_parser.add_argument(
1469        '--include',
1470        default=HEADER_PATH,
1471        help="Inject this header file into every compiled test file. "
1472            "Defaults to %r." % HEADER_PATH)
1473    comp_parser.add_argument(
1474        '-o', '--output',
1475        help="Output file.")
1476
1477    # runner/test_paths overlap, so need to do some munging here
1478    args = parser.parse_intermixed_args()
1479    args.test_paths = [' '.join(args.runner or [])] + args.test_ids
1480    args.runner = args.runner or [RUNNER_PATH]
1481
1482    sys.exit(main(**{k: v
1483        for k, v in vars(args).items()
1484        if v is not None}))
1485