1#!/usr/bin/env python3 2 3# This script manages littlefs tests, which are configured with 4# .toml files stored in the tests directory. 5# 6 7import toml 8import glob 9import re 10import os 11import io 12import itertools as it 13import collections.abc as abc 14import subprocess as sp 15import base64 16import sys 17import copy 18import shlex 19import pty 20import errno 21import signal 22 23TEST_PATHS = 'tests' 24RULES = """ 25# add block devices to sources 26TESTSRC ?= $(SRC) $(wildcard bd/*.c) 27 28define FLATTEN 29%(path)s%%$(subst /,.,$(target)): $(target) 30 ./scripts/explode_asserts.py $$< -o $$@ 31endef 32$(foreach target,$(TESTSRC),$(eval $(FLATTEN))) 33 34-include %(path)s*.d 35.SECONDARY: 36 37%(path)s.test: %(path)s.test.o \\ 38 $(foreach t,$(subst /,.,$(TESTSRC:.c=.o)),%(path)s.$t) 39 $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ 40 41# needed in case builddir is different 42%(path)s%%.o: %(path)s%%.c 43 $(CC) -c -MMD $(CFLAGS) $< -o $@ 44""" 45COVERAGE_RULES = """ 46%(path)s.test: override CFLAGS += -fprofile-arcs -ftest-coverage 47 48# delete lingering coverage 49%(path)s.test: | %(path)s.info.clean 50.PHONY: %(path)s.info.clean 51%(path)s.info.clean: 52 rm -f %(path)s*.gcda 53 54# accumulate coverage info 55.PHONY: %(path)s.info 56%(path)s.info: 57 $(strip $(LCOV) -c \\ 58 $(addprefix -d ,$(wildcard %(path)s*.gcda)) \\ 59 --rc 'geninfo_adjust_src_path=$(shell pwd)' \\ 60 -o $@) 61 $(LCOV) -e $@ $(addprefix /,$(SRC)) -o $@ 62ifdef COVERAGETARGET 63 $(strip $(LCOV) -a $@ \\ 64 $(addprefix -a ,$(wildcard $(COVERAGETARGET))) \\ 65 -o $(COVERAGETARGET)) 66endif 67""" 68GLOBALS = """ 69//////////////// AUTOGENERATED TEST //////////////// 70#include "lfs.h" 71#include "bd/lfs_testbd.h" 72#include <stdio.h> 73extern const char *lfs_testbd_path; 74extern uint32_t lfs_testbd_cycles; 75""" 76DEFINES = { 77 'LFS_READ_SIZE': 16, 78 'LFS_PROG_SIZE': 'LFS_READ_SIZE', 79 'LFS_BLOCK_SIZE': 512, 80 'LFS_BLOCK_COUNT': 1024, 81 'LFS_BLOCK_CYCLES': -1, 82 'LFS_CACHE_SIZE': '(64 % LFS_PROG_SIZE == 0 ? 64 : LFS_PROG_SIZE)', 83 'LFS_LOOKAHEAD_SIZE': 16, 84 'LFS_ERASE_VALUE': 0xff, 85 'LFS_ERASE_CYCLES': 0, 86 'LFS_BADBLOCK_BEHAVIOR': 'LFS_TESTBD_BADBLOCK_PROGERROR', 87} 88PROLOGUE = """ 89 // prologue 90 __attribute__((unused)) lfs_t lfs; 91 __attribute__((unused)) lfs_testbd_t bd; 92 __attribute__((unused)) lfs_file_t file; 93 __attribute__((unused)) lfs_dir_t dir; 94 __attribute__((unused)) struct lfs_info info; 95 __attribute__((unused)) char path[1024]; 96 __attribute__((unused)) uint8_t buffer[1024]; 97 __attribute__((unused)) lfs_size_t size; 98 __attribute__((unused)) int err; 99 100 __attribute__((unused)) const struct lfs_config cfg = { 101 .context = &bd, 102 .read = lfs_testbd_read, 103 .prog = lfs_testbd_prog, 104 .erase = lfs_testbd_erase, 105 .sync = lfs_testbd_sync, 106 .read_size = LFS_READ_SIZE, 107 .prog_size = LFS_PROG_SIZE, 108 .block_size = LFS_BLOCK_SIZE, 109 .block_count = LFS_BLOCK_COUNT, 110 .block_cycles = LFS_BLOCK_CYCLES, 111 .cache_size = LFS_CACHE_SIZE, 112 .lookahead_size = LFS_LOOKAHEAD_SIZE, 113 }; 114 115 __attribute__((unused)) const struct lfs_testbd_config bdcfg = { 116 .erase_value = LFS_ERASE_VALUE, 117 .erase_cycles = LFS_ERASE_CYCLES, 118 .badblock_behavior = LFS_BADBLOCK_BEHAVIOR, 119 .power_cycles = lfs_testbd_cycles, 120 }; 121 122 lfs_testbd_createcfg(&cfg, lfs_testbd_path, &bdcfg) => 0; 123""" 124EPILOGUE = """ 125 // epilogue 126 lfs_testbd_destroy(&cfg) => 0; 127""" 128PASS = '\033[32m✓\033[0m' 129FAIL = '\033[31m✗\033[0m' 130 131class TestFailure(Exception): 132 def __init__(self, case, returncode=None, stdout=None, assert_=None): 133 self.case = case 134 self.returncode = returncode 135 self.stdout = stdout 136 self.assert_ = assert_ 137 138class TestCase: 139 def __init__(self, config, filter=filter, 140 suite=None, caseno=None, lineno=None, **_): 141 self.config = config 142 self.filter = filter 143 self.suite = suite 144 self.caseno = caseno 145 self.lineno = lineno 146 147 self.code = config['code'] 148 self.code_lineno = config['code_lineno'] 149 self.defines = config.get('define', {}) 150 self.if_ = config.get('if', None) 151 self.in_ = config.get('in', None) 152 153 self.result = None 154 155 def __str__(self): 156 if hasattr(self, 'permno'): 157 if any(k not in self.case.defines for k in self.defines): 158 return '%s#%d#%d (%s)' % ( 159 self.suite.name, self.caseno, self.permno, ', '.join( 160 '%s=%s' % (k, v) for k, v in self.defines.items() 161 if k not in self.case.defines)) 162 else: 163 return '%s#%d#%d' % ( 164 self.suite.name, self.caseno, self.permno) 165 else: 166 return '%s#%d' % ( 167 self.suite.name, self.caseno) 168 169 def permute(self, class_=None, defines={}, permno=None, **_): 170 ncase = (class_ or type(self))(self.config) 171 for k, v in self.__dict__.items(): 172 setattr(ncase, k, v) 173 ncase.case = self 174 ncase.perms = [ncase] 175 ncase.permno = permno 176 ncase.defines = defines 177 return ncase 178 179 def build(self, f, **_): 180 # prologue 181 for k, v in sorted(self.defines.items()): 182 if k not in self.suite.defines: 183 f.write('#define %s %s\n' % (k, v)) 184 185 f.write('void test_case%d(%s) {' % (self.caseno, ','.join( 186 '\n'+8*' '+'__attribute__((unused)) intmax_t %s' % k 187 for k in sorted(self.perms[0].defines) 188 if k not in self.defines))) 189 190 f.write(PROLOGUE) 191 f.write('\n') 192 f.write(4*' '+'// test case %d\n' % self.caseno) 193 f.write(4*' '+'#line %d "%s"\n' % (self.code_lineno, self.suite.path)) 194 195 # test case goes here 196 f.write(self.code) 197 198 # epilogue 199 f.write(EPILOGUE) 200 f.write('}\n') 201 202 for k, v in sorted(self.defines.items()): 203 if k not in self.suite.defines: 204 f.write('#undef %s\n' % k) 205 206 def shouldtest(self, **args): 207 if (self.filter is not None and 208 len(self.filter) >= 1 and 209 self.filter[0] != self.caseno): 210 return False 211 elif (self.filter is not None and 212 len(self.filter) >= 2 and 213 self.filter[1] != self.permno): 214 return False 215 elif args.get('no_internal') and self.in_ is not None: 216 return False 217 elif self.if_ is not None: 218 if_ = self.if_ 219 while True: 220 for k, v in sorted(self.defines.items(), 221 key=lambda x: len(x[0]), reverse=True): 222 if k in if_: 223 if_ = if_.replace(k, '(%s)' % v) 224 break 225 else: 226 break 227 if_ = ( 228 re.sub('(\&\&|\?)', ' and ', 229 re.sub('(\|\||:)', ' or ', 230 re.sub('!(?!=)', ' not ', if_)))) 231 return eval(if_) 232 else: 233 return True 234 235 def test(self, exec=[], persist=False, cycles=None, 236 gdb=False, failure=None, disk=None, **args): 237 # build command 238 cmd = exec + ['./%s.test' % self.suite.path, 239 repr(self.caseno), repr(self.permno)] 240 241 # persist disk or keep in RAM for speed? 242 if persist: 243 if not disk: 244 disk = self.suite.path + '.disk' 245 if persist != 'noerase': 246 try: 247 with open(disk, 'w') as f: 248 f.truncate(0) 249 if args.get('verbose'): 250 print('truncate --size=0', disk) 251 except FileNotFoundError: 252 pass 253 254 cmd.append(disk) 255 256 # simulate power-loss after n cycles? 257 if cycles: 258 cmd.append(str(cycles)) 259 260 # failed? drop into debugger? 261 if gdb and failure: 262 ncmd = ['gdb'] 263 if gdb == 'assert': 264 ncmd.extend(['-ex', 'r']) 265 if failure.assert_: 266 ncmd.extend(['-ex', 'up 2']) 267 elif gdb == 'main': 268 ncmd.extend([ 269 '-ex', 'b %s:%d' % (self.suite.path, self.code_lineno), 270 '-ex', 'r']) 271 ncmd.extend(['--args'] + cmd) 272 273 if args.get('verbose'): 274 print(' '.join(shlex.quote(c) for c in ncmd)) 275 signal.signal(signal.SIGINT, signal.SIG_IGN) 276 sys.exit(sp.call(ncmd)) 277 278 # run test case! 279 mpty, spty = pty.openpty() 280 if args.get('verbose'): 281 print(' '.join(shlex.quote(c) for c in cmd)) 282 proc = sp.Popen(cmd, stdout=spty, stderr=spty) 283 os.close(spty) 284 mpty = os.fdopen(mpty, 'r', 1) 285 stdout = [] 286 assert_ = None 287 try: 288 while True: 289 try: 290 line = mpty.readline() 291 except OSError as e: 292 if e.errno == errno.EIO: 293 break 294 raise 295 if not line: 296 break; 297 stdout.append(line) 298 if args.get('verbose'): 299 sys.stdout.write(line) 300 # intercept asserts 301 m = re.match( 302 '^{0}([^:]+):(\d+):(?:\d+:)?{0}{1}:{0}(.*)$' 303 .format('(?:\033\[[\d;]*.| )*', 'assert'), 304 line) 305 if m and assert_ is None: 306 try: 307 with open(m.group(1)) as f: 308 lineno = int(m.group(2)) 309 line = (next(it.islice(f, lineno-1, None)) 310 .strip('\n')) 311 assert_ = { 312 'path': m.group(1), 313 'line': line, 314 'lineno': lineno, 315 'message': m.group(3)} 316 except: 317 pass 318 except KeyboardInterrupt: 319 raise TestFailure(self, 1, stdout, None) 320 proc.wait() 321 322 # did we pass? 323 if proc.returncode != 0: 324 raise TestFailure(self, proc.returncode, stdout, assert_) 325 else: 326 return PASS 327 328class ValgrindTestCase(TestCase): 329 def __init__(self, config, **args): 330 self.leaky = config.get('leaky', False) 331 super().__init__(config, **args) 332 333 def shouldtest(self, **args): 334 return not self.leaky and super().shouldtest(**args) 335 336 def test(self, exec=[], **args): 337 verbose = args.get('verbose') 338 uninit = (self.defines.get('LFS_ERASE_VALUE', None) == -1) 339 exec = [ 340 'valgrind', 341 '--leak-check=full', 342 ] + (['--undef-value-errors=no'] if uninit else []) + [ 343 ] + (['--track-origins=yes'] if not uninit else []) + [ 344 '--error-exitcode=4', 345 '--error-limit=no', 346 ] + (['--num-callers=1'] if not verbose else []) + [ 347 '-q'] + exec 348 return super().test(exec=exec, **args) 349 350class ReentrantTestCase(TestCase): 351 def __init__(self, config, **args): 352 self.reentrant = config.get('reentrant', False) 353 super().__init__(config, **args) 354 355 def shouldtest(self, **args): 356 return self.reentrant and super().shouldtest(**args) 357 358 def test(self, persist=False, gdb=False, failure=None, **args): 359 for cycles in it.count(1): 360 # clear disk first? 361 if cycles == 1 and persist != 'noerase': 362 persist = 'erase' 363 else: 364 persist = 'noerase' 365 366 # exact cycle we should drop into debugger? 367 if gdb and failure and failure.cycleno == cycles: 368 return super().test(gdb=gdb, persist=persist, cycles=cycles, 369 failure=failure, **args) 370 371 # run tests, but kill the program after prog/erase has 372 # been hit n cycles. We exit with a special return code if the 373 # program has not finished, since this isn't a test failure. 374 try: 375 return super().test(persist=persist, cycles=cycles, **args) 376 except TestFailure as nfailure: 377 if nfailure.returncode == 33: 378 continue 379 else: 380 nfailure.cycleno = cycles 381 raise 382 383class TestSuite: 384 def __init__(self, path, classes=[TestCase], defines={}, 385 filter=None, **args): 386 self.name = os.path.basename(path) 387 if self.name.endswith('.toml'): 388 self.name = self.name[:-len('.toml')] 389 if args.get('build_dir'): 390 self.toml = path 391 self.path = args['build_dir'] + '/' + path 392 else: 393 self.toml = path 394 self.path = path 395 self.classes = classes 396 self.defines = defines.copy() 397 self.filter = filter 398 399 with open(self.toml) as f: 400 # load tests 401 config = toml.load(f) 402 403 # find line numbers 404 f.seek(0) 405 linenos = [] 406 code_linenos = [] 407 for i, line in enumerate(f): 408 if re.match(r'\[\[\s*case\s*\]\]', line): 409 linenos.append(i+1) 410 if re.match(r'code\s*=\s*(\'\'\'|""")', line): 411 code_linenos.append(i+2) 412 413 code_linenos.reverse() 414 415 # grab global config 416 for k, v in config.get('define', {}).items(): 417 if k not in self.defines: 418 self.defines[k] = v 419 self.code = config.get('code', None) 420 if self.code is not None: 421 self.code_lineno = code_linenos.pop() 422 423 # create initial test cases 424 self.cases = [] 425 for i, (case, lineno) in enumerate(zip(config['case'], linenos)): 426 # code lineno? 427 if 'code' in case: 428 case['code_lineno'] = code_linenos.pop() 429 # merge conditions if necessary 430 if 'if' in config and 'if' in case: 431 case['if'] = '(%s) && (%s)' % (config['if'], case['if']) 432 elif 'if' in config: 433 case['if'] = config['if'] 434 # initialize test case 435 self.cases.append(TestCase(case, filter=filter, 436 suite=self, caseno=i+1, lineno=lineno, **args)) 437 438 def __str__(self): 439 return self.name 440 441 def __lt__(self, other): 442 return self.name < other.name 443 444 def permute(self, **args): 445 for case in self.cases: 446 # lets find all parameterized definitions, in one of [args.D, 447 # suite.defines, case.defines, DEFINES]. Note that each of these 448 # can be either a dict of defines, or a list of dicts, expressing 449 # an initial set of permutations. 450 pending = [{}] 451 for inits in [self.defines, case.defines, DEFINES]: 452 if not isinstance(inits, list): 453 inits = [inits] 454 455 npending = [] 456 for init, pinit in it.product(inits, pending): 457 ninit = pinit.copy() 458 for k, v in init.items(): 459 if k not in ninit: 460 try: 461 ninit[k] = eval(v) 462 except: 463 ninit[k] = v 464 npending.append(ninit) 465 466 pending = npending 467 468 # expand permutations 469 pending = list(reversed(pending)) 470 expanded = [] 471 while pending: 472 perm = pending.pop() 473 for k, v in sorted(perm.items()): 474 if not isinstance(v, str) and isinstance(v, abc.Iterable): 475 for nv in reversed(v): 476 nperm = perm.copy() 477 nperm[k] = nv 478 pending.append(nperm) 479 break 480 else: 481 expanded.append(perm) 482 483 # generate permutations 484 case.perms = [] 485 for i, (class_, defines) in enumerate( 486 it.product(self.classes, expanded)): 487 case.perms.append(case.permute( 488 class_, defines, permno=i+1, **args)) 489 490 # also track non-unique defines 491 case.defines = {} 492 for k, v in case.perms[0].defines.items(): 493 if all(perm.defines[k] == v for perm in case.perms): 494 case.defines[k] = v 495 496 # track all perms and non-unique defines 497 self.perms = [] 498 for case in self.cases: 499 self.perms.extend(case.perms) 500 501 self.defines = {} 502 for k, v in self.perms[0].defines.items(): 503 if all(perm.defines.get(k, None) == v for perm in self.perms): 504 self.defines[k] = v 505 506 return self.perms 507 508 def build(self, **args): 509 # build test files 510 tf = open(self.path + '.test.tc', 'w') 511 tf.write(GLOBALS) 512 if self.code is not None: 513 tf.write('#line %d "%s"\n' % (self.code_lineno, self.path)) 514 tf.write(self.code) 515 516 tfs = {None: tf} 517 for case in self.cases: 518 if case.in_ not in tfs: 519 tfs[case.in_] = open(self.path+'.'+ 520 re.sub('(\.c)?$', '.tc', case.in_.replace('/', '.')), 'w') 521 tfs[case.in_].write('#line 1 "%s"\n' % case.in_) 522 with open(case.in_) as f: 523 for line in f: 524 tfs[case.in_].write(line) 525 tfs[case.in_].write('\n') 526 tfs[case.in_].write(GLOBALS) 527 528 tfs[case.in_].write('\n') 529 case.build(tfs[case.in_], **args) 530 531 tf.write('\n') 532 tf.write('const char *lfs_testbd_path;\n') 533 tf.write('uint32_t lfs_testbd_cycles;\n') 534 tf.write('int main(int argc, char **argv) {\n') 535 tf.write(4*' '+'int case_ = (argc > 1) ? atoi(argv[1]) : 0;\n') 536 tf.write(4*' '+'int perm = (argc > 2) ? atoi(argv[2]) : 0;\n') 537 tf.write(4*' '+'lfs_testbd_path = (argc > 3) ? argv[3] : NULL;\n') 538 tf.write(4*' '+'lfs_testbd_cycles = (argc > 4) ? atoi(argv[4]) : 0;\n') 539 for perm in self.perms: 540 # test declaration 541 tf.write(4*' '+'extern void test_case%d(%s);\n' % ( 542 perm.caseno, ', '.join( 543 'intmax_t %s' % k for k in sorted(perm.defines) 544 if k not in perm.case.defines))) 545 # test call 546 tf.write(4*' '+ 547 'if (argc < 3 || (case_ == %d && perm == %d)) {' 548 ' test_case%d(%s); ' 549 '}\n' % (perm.caseno, perm.permno, perm.caseno, ', '.join( 550 str(v) for k, v in sorted(perm.defines.items()) 551 if k not in perm.case.defines))) 552 tf.write('}\n') 553 554 for tf in tfs.values(): 555 tf.close() 556 557 # write makefiles 558 with open(self.path + '.mk', 'w') as mk: 559 mk.write(RULES.replace(4*' ', '\t') % dict(path=self.path)) 560 mk.write('\n') 561 562 # add coverage hooks? 563 if args.get('coverage'): 564 mk.write(COVERAGE_RULES.replace(4*' ', '\t') % dict( 565 path=self.path)) 566 mk.write('\n') 567 568 # add truly global defines globally 569 for k, v in sorted(self.defines.items()): 570 mk.write('%s.test: override CFLAGS += -D%s=%r\n' 571 % (self.path, k, v)) 572 573 for path in tfs: 574 if path is None: 575 mk.write('%s: %s | %s\n' % ( 576 self.path+'.test.c', 577 self.toml, 578 self.path+'.test.tc')) 579 else: 580 mk.write('%s: %s %s | %s\n' % ( 581 self.path+'.'+path.replace('/', '.'), 582 self.toml, 583 path, 584 self.path+'.'+re.sub('(\.c)?$', '.tc', 585 path.replace('/', '.')))) 586 mk.write('\t./scripts/explode_asserts.py $| -o $@\n') 587 588 self.makefile = self.path + '.mk' 589 self.target = self.path + '.test' 590 return self.makefile, self.target 591 592 def test(self, **args): 593 # run test suite! 594 if not args.get('verbose', True): 595 sys.stdout.write(self.name + ' ') 596 sys.stdout.flush() 597 for perm in self.perms: 598 if not perm.shouldtest(**args): 599 continue 600 601 try: 602 result = perm.test(**args) 603 except TestFailure as failure: 604 perm.result = failure 605 if not args.get('verbose', True): 606 sys.stdout.write(FAIL) 607 sys.stdout.flush() 608 if not args.get('keep_going'): 609 if not args.get('verbose', True): 610 sys.stdout.write('\n') 611 raise 612 else: 613 perm.result = PASS 614 if not args.get('verbose', True): 615 sys.stdout.write(PASS) 616 sys.stdout.flush() 617 618 if not args.get('verbose', True): 619 sys.stdout.write('\n') 620 621def main(**args): 622 # figure out explicit defines 623 defines = {} 624 for define in args['D']: 625 k, v, *_ = define.split('=', 2) + [''] 626 defines[k] = v 627 628 # and what class of TestCase to run 629 classes = [] 630 if args.get('normal'): 631 classes.append(TestCase) 632 if args.get('reentrant'): 633 classes.append(ReentrantTestCase) 634 if args.get('valgrind'): 635 classes.append(ValgrindTestCase) 636 if not classes: 637 classes = [TestCase] 638 639 suites = [] 640 for testpath in args['test_paths']: 641 # optionally specified test case/perm 642 testpath, *filter = testpath.split('#') 643 filter = [int(f) for f in filter] 644 645 # figure out the suite's toml file 646 if os.path.isdir(testpath): 647 testpath = testpath + '/*.toml' 648 elif os.path.isfile(testpath): 649 testpath = testpath 650 elif testpath.endswith('.toml'): 651 testpath = TEST_PATHS + '/' + testpath 652 else: 653 testpath = TEST_PATHS + '/' + testpath + '.toml' 654 655 # find tests 656 for path in glob.glob(testpath): 657 suites.append(TestSuite(path, classes, defines, filter, **args)) 658 659 # sort for reproducibility 660 suites = sorted(suites) 661 662 # generate permutations 663 for suite in suites: 664 suite.permute(**args) 665 666 # build tests in parallel 667 print('====== building ======') 668 makefiles = [] 669 targets = [] 670 for suite in suites: 671 makefile, target = suite.build(**args) 672 makefiles.append(makefile) 673 targets.append(target) 674 675 cmd = (['make', '-f', 'Makefile'] + 676 list(it.chain.from_iterable(['-f', m] for m in makefiles)) + 677 [target for target in targets]) 678 mpty, spty = pty.openpty() 679 if args.get('verbose'): 680 print(' '.join(shlex.quote(c) for c in cmd)) 681 proc = sp.Popen(cmd, stdout=spty, stderr=spty) 682 os.close(spty) 683 mpty = os.fdopen(mpty, 'r', 1) 684 stdout = [] 685 while True: 686 try: 687 line = mpty.readline() 688 except OSError as e: 689 if e.errno == errno.EIO: 690 break 691 raise 692 if not line: 693 break; 694 stdout.append(line) 695 if args.get('verbose'): 696 sys.stdout.write(line) 697 # intercept warnings 698 m = re.match( 699 '^{0}([^:]+):(\d+):(?:\d+:)?{0}{1}:{0}(.*)$' 700 .format('(?:\033\[[\d;]*.| )*', 'warning'), 701 line) 702 if m and not args.get('verbose'): 703 try: 704 with open(m.group(1)) as f: 705 lineno = int(m.group(2)) 706 line = next(it.islice(f, lineno-1, None)).strip('\n') 707 sys.stdout.write( 708 "\033[01m{path}:{lineno}:\033[01;35mwarning:\033[m " 709 "{message}\n{line}\n\n".format( 710 path=m.group(1), line=line, lineno=lineno, 711 message=m.group(3))) 712 except: 713 pass 714 proc.wait() 715 if proc.returncode != 0: 716 if not args.get('verbose'): 717 for line in stdout: 718 sys.stdout.write(line) 719 sys.exit(-1) 720 721 print('built %d test suites, %d test cases, %d permutations' % ( 722 len(suites), 723 sum(len(suite.cases) for suite in suites), 724 sum(len(suite.perms) for suite in suites))) 725 726 total = 0 727 for suite in suites: 728 for perm in suite.perms: 729 total += perm.shouldtest(**args) 730 if total != sum(len(suite.perms) for suite in suites): 731 print('filtered down to %d permutations' % total) 732 733 # only requested to build? 734 if args.get('build'): 735 return 0 736 737 print('====== testing ======') 738 try: 739 for suite in suites: 740 suite.test(**args) 741 except TestFailure: 742 pass 743 744 print('====== results ======') 745 passed = 0 746 failed = 0 747 for suite in suites: 748 for perm in suite.perms: 749 if perm.result == PASS: 750 passed += 1 751 elif isinstance(perm.result, TestFailure): 752 sys.stdout.write( 753 "\033[01m{path}:{lineno}:\033[01;31mfailure:\033[m " 754 "{perm} failed\n".format( 755 perm=perm, path=perm.suite.path, lineno=perm.lineno, 756 returncode=perm.result.returncode or 0)) 757 if perm.result.stdout: 758 if perm.result.assert_: 759 stdout = perm.result.stdout[:-1] 760 else: 761 stdout = perm.result.stdout 762 for line in stdout[-5:]: 763 sys.stdout.write(line) 764 if perm.result.assert_: 765 sys.stdout.write( 766 "\033[01m{path}:{lineno}:\033[01;31massert:\033[m " 767 "{message}\n{line}\n".format( 768 **perm.result.assert_)) 769 sys.stdout.write('\n') 770 failed += 1 771 772 if args.get('coverage'): 773 # collect coverage info 774 # why -j1? lcov doesn't work in parallel because of gcov limitations 775 cmd = (['make', '-j1', '-f', 'Makefile'] + 776 list(it.chain.from_iterable(['-f', m] for m in makefiles)) + 777 (['COVERAGETARGET=%s' % args['coverage']] 778 if isinstance(args['coverage'], str) else []) + 779 [suite.path + '.info' for suite in suites 780 if any(perm.result == PASS for perm in suite.perms)]) 781 if args.get('verbose'): 782 print(' '.join(shlex.quote(c) for c in cmd)) 783 proc = sp.Popen(cmd, 784 stdout=sp.PIPE if not args.get('verbose') else None, 785 stderr=sp.STDOUT if not args.get('verbose') else None, 786 universal_newlines=True) 787 stdout = [] 788 for line in proc.stdout: 789 stdout.append(line) 790 proc.wait() 791 if proc.returncode != 0: 792 if not args.get('verbose'): 793 for line in stdout: 794 sys.stdout.write(line) 795 sys.exit(-1) 796 797 if args.get('gdb'): 798 failure = None 799 for suite in suites: 800 for perm in suite.perms: 801 if isinstance(perm.result, TestFailure): 802 failure = perm.result 803 if failure is not None: 804 print('======= gdb ======') 805 # drop into gdb 806 failure.case.test(failure=failure, **args) 807 sys.exit(0) 808 809 print('tests passed %d/%d (%.1f%%)' % (passed, total, 810 100*(passed/total if total else 1.0))) 811 print('tests failed %d/%d (%.1f%%)' % (failed, total, 812 100*(failed/total if total else 1.0))) 813 return 1 if failed > 0 else 0 814 815if __name__ == "__main__": 816 import argparse 817 parser = argparse.ArgumentParser( 818 description="Run parameterized tests in various configurations.") 819 parser.add_argument('test_paths', nargs='*', default=[TEST_PATHS], 820 help="Description of test(s) to run. By default, this is all tests \ 821 found in the \"{0}\" directory. Here, you can specify a different \ 822 directory of tests, a specific file, a suite by name, and even \ 823 specific test cases and permutations. For example \ 824 \"test_dirs#1\" or \"{0}/test_dirs.toml#1#1\".".format(TEST_PATHS)) 825 parser.add_argument('-D', action='append', default=[], 826 help="Overriding parameter definitions.") 827 parser.add_argument('-v', '--verbose', action='store_true', 828 help="Output everything that is happening.") 829 parser.add_argument('-k', '--keep-going', action='store_true', 830 help="Run all tests instead of stopping on first error. Useful for CI.") 831 parser.add_argument('-p', '--persist', choices=['erase', 'noerase'], 832 nargs='?', const='erase', 833 help="Store disk image in a file.") 834 parser.add_argument('-b', '--build', action='store_true', 835 help="Only build the tests, do not execute.") 836 parser.add_argument('-g', '--gdb', choices=['init', 'main', 'assert'], 837 nargs='?', const='assert', 838 help="Drop into gdb on test failure.") 839 parser.add_argument('--no-internal', action='store_true', 840 help="Don't run tests that require internal knowledge.") 841 parser.add_argument('-n', '--normal', action='store_true', 842 help="Run tests normally.") 843 parser.add_argument('-r', '--reentrant', action='store_true', 844 help="Run reentrant tests with simulated power-loss.") 845 parser.add_argument('--valgrind', action='store_true', 846 help="Run non-leaky tests under valgrind to check for memory leaks.") 847 parser.add_argument('--exec', default=[], type=lambda e: e.split(), 848 help="Run tests with another executable prefixed on the command line.") 849 parser.add_argument('--disk', 850 help="Specify a file to use for persistent/reentrant tests.") 851 parser.add_argument('--coverage', type=lambda x: x if x else True, 852 nargs='?', const='', 853 help="Collect coverage information during testing. This uses lcov/gcov \ 854 to accumulate coverage information into *.info files. May also \ 855 a path to a *.info file to accumulate coverage info into.") 856 parser.add_argument('--build-dir', 857 help="Build relative to the specified directory instead of the \ 858 current directory.") 859 860 sys.exit(main(**vars(parser.parse_args()))) 861
