1#!/usr/bin/env python3 2 3# This script manages littlefs tests, which are configured with 4# .toml files stored in the tests directory. 5# 6 7import toml 8import glob 9import re 10import os 11import io 12import itertools as it 13import collections.abc as abc 14import subprocess as sp 15import base64 16import sys 17import copy 18import shlex 19import pty 20import errno 21import signal 22 23TESTDIR = 'tests' 24RULES = """ 25define FLATTEN 26tests/%$(subst /,.,$(target)): $(target) 27 ./scripts/explode_asserts.py $$< -o $$@ 28endef 29$(foreach target,$(SRC),$(eval $(FLATTEN))) 30 31-include tests/*.d 32 33.SECONDARY: 34%.test: %.test.o $(foreach f,$(subst /,.,$(SRC:.c=.o)),%.$f) 35 $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ 36""" 37GLOBALS = """ 38//////////////// AUTOGENERATED TEST //////////////// 39#include "lfs.h" 40#include "bd/lfs_testbd.h" 41#include <stdio.h> 42extern const char *lfs_testbd_path; 43extern uint32_t lfs_testbd_cycles; 44""" 45DEFINES = { 46 'LFS_READ_SIZE': 16, 47 'LFS_PROG_SIZE': 'LFS_READ_SIZE', 48 'LFS_BLOCK_SIZE': 512, 49 'LFS_BLOCK_COUNT': 1024, 50 'LFS_BLOCK_CYCLES': -1, 51 'LFS_CACHE_SIZE': '(64 % LFS_PROG_SIZE == 0 ? 64 : LFS_PROG_SIZE)', 52 'LFS_LOOKAHEAD_SIZE': 16, 53 'LFS_ERASE_VALUE': 0xff, 54 'LFS_ERASE_CYCLES': 0, 55 'LFS_BADBLOCK_BEHAVIOR': 'LFS_TESTBD_BADBLOCK_PROGERROR', 56} 57PROLOGUE = """ 58 // prologue 59 __attribute__((unused)) lfs_t lfs; 60 __attribute__((unused)) lfs_testbd_t bd; 61 __attribute__((unused)) lfs_file_t file; 62 __attribute__((unused)) lfs_dir_t dir; 63 __attribute__((unused)) struct lfs_info info; 64 __attribute__((unused)) char path[1024]; 65 __attribute__((unused)) uint8_t buffer[1024]; 66 __attribute__((unused)) lfs_size_t size; 67 __attribute__((unused)) int err; 68 69 __attribute__((unused)) const struct lfs_config cfg = { 70 .context = &bd, 71 .read = lfs_testbd_read, 72 .prog = lfs_testbd_prog, 73 .erase = lfs_testbd_erase, 74 .sync = lfs_testbd_sync, 75 .read_size = LFS_READ_SIZE, 76 .prog_size = LFS_PROG_SIZE, 77 .block_size = LFS_BLOCK_SIZE, 78 .block_count = LFS_BLOCK_COUNT, 79 .block_cycles = LFS_BLOCK_CYCLES, 80 .cache_size = LFS_CACHE_SIZE, 81 .lookahead_size = LFS_LOOKAHEAD_SIZE, 82 }; 83 84 __attribute__((unused)) const struct lfs_testbd_config bdcfg = { 85 .erase_value = LFS_ERASE_VALUE, 86 .erase_cycles = LFS_ERASE_CYCLES, 87 .badblock_behavior = LFS_BADBLOCK_BEHAVIOR, 88 .power_cycles = lfs_testbd_cycles, 89 }; 90 91 lfs_testbd_createcfg(&cfg, lfs_testbd_path, &bdcfg) => 0; 92""" 93EPILOGUE = """ 94 // epilogue 95 lfs_testbd_destroy(&cfg) => 0; 96""" 97PASS = '\033[32m✓\033[0m' 98FAIL = '\033[31m✗\033[0m' 99 100class TestFailure(Exception): 101 def __init__(self, case, returncode=None, stdout=None, assert_=None): 102 self.case = case 103 self.returncode = returncode 104 self.stdout = stdout 105 self.assert_ = assert_ 106 107class TestCase: 108 def __init__(self, config, filter=filter, 109 suite=None, caseno=None, lineno=None, **_): 110 self.config = config 111 self.filter = filter 112 self.suite = suite 113 self.caseno = caseno 114 self.lineno = lineno 115 116 self.code = config['code'] 117 self.code_lineno = config['code_lineno'] 118 self.defines = config.get('define', {}) 119 self.if_ = config.get('if', None) 120 self.in_ = config.get('in', None) 121 122 def __str__(self): 123 if hasattr(self, 'permno'): 124 if any(k not in self.case.defines for k in self.defines): 125 return '%s#%d#%d (%s)' % ( 126 self.suite.name, self.caseno, self.permno, ', '.join( 127 '%s=%s' % (k, v) for k, v in self.defines.items() 128 if k not in self.case.defines)) 129 else: 130 return '%s#%d#%d' % ( 131 self.suite.name, self.caseno, self.permno) 132 else: 133 return '%s#%d' % ( 134 self.suite.name, self.caseno) 135 136 def permute(self, class_=None, defines={}, permno=None, **_): 137 ncase = (class_ or type(self))(self.config) 138 for k, v in self.__dict__.items(): 139 setattr(ncase, k, v) 140 ncase.case = self 141 ncase.perms = [ncase] 142 ncase.permno = permno 143 ncase.defines = defines 144 return ncase 145 146 def build(self, f, **_): 147 # prologue 148 for k, v in sorted(self.defines.items()): 149 if k not in self.suite.defines: 150 f.write('#define %s %s\n' % (k, v)) 151 152 f.write('void test_case%d(%s) {' % (self.caseno, ','.join( 153 '\n'+8*' '+'__attribute__((unused)) intmax_t %s' % k 154 for k in sorted(self.perms[0].defines) 155 if k not in self.defines))) 156 157 f.write(PROLOGUE) 158 f.write('\n') 159 f.write(4*' '+'// test case %d\n' % self.caseno) 160 f.write(4*' '+'#line %d "%s"\n' % (self.code_lineno, self.suite.path)) 161 162 # test case goes here 163 f.write(self.code) 164 165 # epilogue 166 f.write(EPILOGUE) 167 f.write('}\n') 168 169 for k, v in sorted(self.defines.items()): 170 if k not in self.suite.defines: 171 f.write('#undef %s\n' % k) 172 173 def shouldtest(self, **args): 174 if (self.filter is not None and 175 len(self.filter) >= 1 and 176 self.filter[0] != self.caseno): 177 return False 178 elif (self.filter is not None and 179 len(self.filter) >= 2 and 180 self.filter[1] != self.permno): 181 return False 182 elif args.get('no_internal', False) and self.in_ is not None: 183 return False 184 elif self.if_ is not None: 185 if_ = self.if_ 186 while True: 187 for k, v in sorted(self.defines.items(), 188 key=lambda x: len(x[0]), reverse=True): 189 if k in if_: 190 if_ = if_.replace(k, '(%s)' % v) 191 break 192 else: 193 break 194 if_ = ( 195 re.sub('(\&\&|\?)', ' and ', 196 re.sub('(\|\||:)', ' or ', 197 re.sub('!(?!=)', ' not ', if_)))) 198 return eval(if_) 199 else: 200 return True 201 202 def test(self, exec=[], persist=False, cycles=None, 203 gdb=False, failure=None, disk=None, **args): 204 # build command 205 cmd = exec + ['./%s.test' % self.suite.path, 206 repr(self.caseno), repr(self.permno)] 207 208 # persist disk or keep in RAM for speed? 209 if persist: 210 if not disk: 211 disk = self.suite.path + '.disk' 212 if persist != 'noerase': 213 try: 214 with open(disk, 'w') as f: 215 f.truncate(0) 216 if args.get('verbose', False): 217 print('truncate --size=0', disk) 218 except FileNotFoundError: 219 pass 220 221 cmd.append(disk) 222 223 # simulate power-loss after n cycles? 224 if cycles: 225 cmd.append(str(cycles)) 226 227 # failed? drop into debugger? 228 if gdb and failure: 229 ncmd = ['gdb'] 230 if gdb == 'assert': 231 ncmd.extend(['-ex', 'r']) 232 if failure.assert_: 233 ncmd.extend(['-ex', 'up 2']) 234 elif gdb == 'main': 235 ncmd.extend([ 236 '-ex', 'b %s:%d' % (self.suite.path, self.code_lineno), 237 '-ex', 'r']) 238 ncmd.extend(['--args'] + cmd) 239 240 if args.get('verbose', False): 241 print(' '.join(shlex.quote(c) for c in ncmd)) 242 signal.signal(signal.SIGINT, signal.SIG_IGN) 243 sys.exit(sp.call(ncmd)) 244 245 # run test case! 246 mpty, spty = pty.openpty() 247 if args.get('verbose', False): 248 print(' '.join(shlex.quote(c) for c in cmd)) 249 proc = sp.Popen(cmd, stdout=spty, stderr=spty) 250 os.close(spty) 251 mpty = os.fdopen(mpty, 'r', 1) 252 stdout = [] 253 assert_ = None 254 try: 255 while True: 256 try: 257 line = mpty.readline() 258 except OSError as e: 259 if e.errno == errno.EIO: 260 break 261 raise 262 stdout.append(line) 263 if args.get('verbose', False): 264 sys.stdout.write(line) 265 # intercept asserts 266 m = re.match( 267 '^{0}([^:]+):(\d+):(?:\d+:)?{0}{1}:{0}(.*)$' 268 .format('(?:\033\[[\d;]*.| )*', 'assert'), 269 line) 270 if m and assert_ is None: 271 try: 272 with open(m.group(1)) as f: 273 lineno = int(m.group(2)) 274 line = (next(it.islice(f, lineno-1, None)) 275 .strip('\n')) 276 assert_ = { 277 'path': m.group(1), 278 'line': line, 279 'lineno': lineno, 280 'message': m.group(3)} 281 except: 282 pass 283 except KeyboardInterrupt: 284 raise TestFailure(self, 1, stdout, None) 285 proc.wait() 286 287 # did we pass? 288 if proc.returncode != 0: 289 raise TestFailure(self, proc.returncode, stdout, assert_) 290 else: 291 return PASS 292 293class ValgrindTestCase(TestCase): 294 def __init__(self, config, **args): 295 self.leaky = config.get('leaky', False) 296 super().__init__(config, **args) 297 298 def shouldtest(self, **args): 299 return not self.leaky and super().shouldtest(**args) 300 301 def test(self, exec=[], **args): 302 verbose = args.get('verbose', False) 303 uninit = (self.defines.get('LFS_ERASE_VALUE', None) == -1) 304 exec = [ 305 'valgrind', 306 '--leak-check=full', 307 ] + (['--undef-value-errors=no'] if uninit else []) + [ 308 ] + (['--track-origins=yes'] if not uninit else []) + [ 309 '--error-exitcode=4', 310 '--error-limit=no', 311 ] + (['--num-callers=1'] if not verbose else []) + [ 312 '-q'] + exec 313 return super().test(exec=exec, **args) 314 315class ReentrantTestCase(TestCase): 316 def __init__(self, config, **args): 317 self.reentrant = config.get('reentrant', False) 318 super().__init__(config, **args) 319 320 def shouldtest(self, **args): 321 return self.reentrant and super().shouldtest(**args) 322 323 def test(self, persist=False, gdb=False, failure=None, **args): 324 for cycles in it.count(1): 325 # clear disk first? 326 if cycles == 1 and persist != 'noerase': 327 persist = 'erase' 328 else: 329 persist = 'noerase' 330 331 # exact cycle we should drop into debugger? 332 if gdb and failure and failure.cycleno == cycles: 333 return super().test(gdb=gdb, persist=persist, cycles=cycles, 334 failure=failure, **args) 335 336 # run tests, but kill the program after prog/erase has 337 # been hit n cycles. We exit with a special return code if the 338 # program has not finished, since this isn't a test failure. 339 try: 340 return super().test(persist=persist, cycles=cycles, **args) 341 except TestFailure as nfailure: 342 if nfailure.returncode == 33: 343 continue 344 else: 345 nfailure.cycleno = cycles 346 raise 347 348class TestSuite: 349 def __init__(self, path, classes=[TestCase], defines={}, 350 filter=None, **args): 351 self.name = os.path.basename(path) 352 if self.name.endswith('.toml'): 353 self.name = self.name[:-len('.toml')] 354 self.path = path 355 self.classes = classes 356 self.defines = defines.copy() 357 self.filter = filter 358 359 with open(path) as f: 360 # load tests 361 config = toml.load(f) 362 363 # find line numbers 364 f.seek(0) 365 linenos = [] 366 code_linenos = [] 367 for i, line in enumerate(f): 368 if re.match(r'\[\[\s*case\s*\]\]', line): 369 linenos.append(i+1) 370 if re.match(r'code\s*=\s*(\'\'\'|""")', line): 371 code_linenos.append(i+2) 372 373 code_linenos.reverse() 374 375 # grab global config 376 for k, v in config.get('define', {}).items(): 377 if k not in self.defines: 378 self.defines[k] = v 379 self.code = config.get('code', None) 380 if self.code is not None: 381 self.code_lineno = code_linenos.pop() 382 383 # create initial test cases 384 self.cases = [] 385 for i, (case, lineno) in enumerate(zip(config['case'], linenos)): 386 # code lineno? 387 if 'code' in case: 388 case['code_lineno'] = code_linenos.pop() 389 # merge conditions if necessary 390 if 'if' in config and 'if' in case: 391 case['if'] = '(%s) && (%s)' % (config['if'], case['if']) 392 elif 'if' in config: 393 case['if'] = config['if'] 394 # initialize test case 395 self.cases.append(TestCase(case, filter=filter, 396 suite=self, caseno=i+1, lineno=lineno, **args)) 397 398 def __str__(self): 399 return self.name 400 401 def __lt__(self, other): 402 return self.name < other.name 403 404 def permute(self, **args): 405 for case in self.cases: 406 # lets find all parameterized definitions, in one of [args.D, 407 # suite.defines, case.defines, DEFINES]. Note that each of these 408 # can be either a dict of defines, or a list of dicts, expressing 409 # an initial set of permutations. 410 pending = [{}] 411 for inits in [self.defines, case.defines, DEFINES]: 412 if not isinstance(inits, list): 413 inits = [inits] 414 415 npending = [] 416 for init, pinit in it.product(inits, pending): 417 ninit = pinit.copy() 418 for k, v in init.items(): 419 if k not in ninit: 420 try: 421 ninit[k] = eval(v) 422 except: 423 ninit[k] = v 424 npending.append(ninit) 425 426 pending = npending 427 428 # expand permutations 429 pending = list(reversed(pending)) 430 expanded = [] 431 while pending: 432 perm = pending.pop() 433 for k, v in sorted(perm.items()): 434 if not isinstance(v, str) and isinstance(v, abc.Iterable): 435 for nv in reversed(v): 436 nperm = perm.copy() 437 nperm[k] = nv 438 pending.append(nperm) 439 break 440 else: 441 expanded.append(perm) 442 443 # generate permutations 444 case.perms = [] 445 for i, (class_, defines) in enumerate( 446 it.product(self.classes, expanded)): 447 case.perms.append(case.permute( 448 class_, defines, permno=i+1, **args)) 449 450 # also track non-unique defines 451 case.defines = {} 452 for k, v in case.perms[0].defines.items(): 453 if all(perm.defines[k] == v for perm in case.perms): 454 case.defines[k] = v 455 456 # track all perms and non-unique defines 457 self.perms = [] 458 for case in self.cases: 459 self.perms.extend(case.perms) 460 461 self.defines = {} 462 for k, v in self.perms[0].defines.items(): 463 if all(perm.defines.get(k, None) == v for perm in self.perms): 464 self.defines[k] = v 465 466 return self.perms 467 468 def build(self, **args): 469 # build test files 470 tf = open(self.path + '.test.c.t', 'w') 471 tf.write(GLOBALS) 472 if self.code is not None: 473 tf.write('#line %d "%s"\n' % (self.code_lineno, self.path)) 474 tf.write(self.code) 475 476 tfs = {None: tf} 477 for case in self.cases: 478 if case.in_ not in tfs: 479 tfs[case.in_] = open(self.path+'.'+ 480 case.in_.replace('/', '.')+'.t', 'w') 481 tfs[case.in_].write('#line 1 "%s"\n' % case.in_) 482 with open(case.in_) as f: 483 for line in f: 484 tfs[case.in_].write(line) 485 tfs[case.in_].write('\n') 486 tfs[case.in_].write(GLOBALS) 487 488 tfs[case.in_].write('\n') 489 case.build(tfs[case.in_], **args) 490 491 tf.write('\n') 492 tf.write('const char *lfs_testbd_path;\n') 493 tf.write('uint32_t lfs_testbd_cycles;\n') 494 tf.write('int main(int argc, char **argv) {\n') 495 tf.write(4*' '+'int case_ = (argc > 1) ? atoi(argv[1]) : 0;\n') 496 tf.write(4*' '+'int perm = (argc > 2) ? atoi(argv[2]) : 0;\n') 497 tf.write(4*' '+'lfs_testbd_path = (argc > 3) ? argv[3] : NULL;\n') 498 tf.write(4*' '+'lfs_testbd_cycles = (argc > 4) ? atoi(argv[4]) : 0;\n') 499 for perm in self.perms: 500 # test declaration 501 tf.write(4*' '+'extern void test_case%d(%s);\n' % ( 502 perm.caseno, ', '.join( 503 'intmax_t %s' % k for k in sorted(perm.defines) 504 if k not in perm.case.defines))) 505 # test call 506 tf.write(4*' '+ 507 'if (argc < 3 || (case_ == %d && perm == %d)) {' 508 ' test_case%d(%s); ' 509 '}\n' % (perm.caseno, perm.permno, perm.caseno, ', '.join( 510 str(v) for k, v in sorted(perm.defines.items()) 511 if k not in perm.case.defines))) 512 tf.write('}\n') 513 514 for tf in tfs.values(): 515 tf.close() 516 517 # write makefiles 518 with open(self.path + '.mk', 'w') as mk: 519 mk.write(RULES.replace(4*' ', '\t')) 520 mk.write('\n') 521 522 # add truely global defines globally 523 for k, v in sorted(self.defines.items()): 524 mk.write('%s: override CFLAGS += -D%s=%r\n' % ( 525 self.path+'.test', k, v)) 526 527 for path in tfs: 528 if path is None: 529 mk.write('%s: %s | %s\n' % ( 530 self.path+'.test.c', 531 self.path, 532 self.path+'.test.c.t')) 533 else: 534 mk.write('%s: %s %s | %s\n' % ( 535 self.path+'.'+path.replace('/', '.'), 536 self.path, path, 537 self.path+'.'+path.replace('/', '.')+'.t')) 538 mk.write('\t./scripts/explode_asserts.py $| -o $@\n') 539 540 self.makefile = self.path + '.mk' 541 self.target = self.path + '.test' 542 return self.makefile, self.target 543 544 def test(self, **args): 545 # run test suite! 546 if not args.get('verbose', True): 547 sys.stdout.write(self.name + ' ') 548 sys.stdout.flush() 549 for perm in self.perms: 550 if not perm.shouldtest(**args): 551 continue 552 553 try: 554 result = perm.test(**args) 555 except TestFailure as failure: 556 perm.result = failure 557 if not args.get('verbose', True): 558 sys.stdout.write(FAIL) 559 sys.stdout.flush() 560 if not args.get('keep_going', False): 561 if not args.get('verbose', True): 562 sys.stdout.write('\n') 563 raise 564 else: 565 perm.result = PASS 566 if not args.get('verbose', True): 567 sys.stdout.write(PASS) 568 sys.stdout.flush() 569 570 if not args.get('verbose', True): 571 sys.stdout.write('\n') 572 573def main(**args): 574 # figure out explicit defines 575 defines = {} 576 for define in args['D']: 577 k, v, *_ = define.split('=', 2) + [''] 578 defines[k] = v 579 580 # and what class of TestCase to run 581 classes = [] 582 if args.get('normal', False): 583 classes.append(TestCase) 584 if args.get('reentrant', False): 585 classes.append(ReentrantTestCase) 586 if args.get('valgrind', False): 587 classes.append(ValgrindTestCase) 588 if not classes: 589 classes = [TestCase] 590 591 suites = [] 592 for testpath in args['testpaths']: 593 # optionally specified test case/perm 594 testpath, *filter = testpath.split('#') 595 filter = [int(f) for f in filter] 596 597 # figure out the suite's toml file 598 if os.path.isdir(testpath): 599 testpath = testpath + '/test_*.toml' 600 elif os.path.isfile(testpath): 601 testpath = testpath 602 elif testpath.endswith('.toml'): 603 testpath = TESTDIR + '/' + testpath 604 else: 605 testpath = TESTDIR + '/' + testpath + '.toml' 606 607 # find tests 608 for path in glob.glob(testpath): 609 suites.append(TestSuite(path, classes, defines, filter, **args)) 610 611 # sort for reproducability 612 suites = sorted(suites) 613 614 # generate permutations 615 for suite in suites: 616 suite.permute(**args) 617 618 # build tests in parallel 619 print('====== building ======') 620 makefiles = [] 621 targets = [] 622 for suite in suites: 623 makefile, target = suite.build(**args) 624 makefiles.append(makefile) 625 targets.append(target) 626 627 cmd = (['make', '-f', 'Makefile'] + 628 list(it.chain.from_iterable(['-f', m] for m in makefiles)) + 629 [target for target in targets]) 630 mpty, spty = pty.openpty() 631 if args.get('verbose', False): 632 print(' '.join(shlex.quote(c) for c in cmd)) 633 proc = sp.Popen(cmd, stdout=spty, stderr=spty) 634 os.close(spty) 635 mpty = os.fdopen(mpty, 'r', 1) 636 stdout = [] 637 while True: 638 try: 639 line = mpty.readline() 640 except OSError as e: 641 if e.errno == errno.EIO: 642 break 643 raise 644 stdout.append(line) 645 if args.get('verbose', False): 646 sys.stdout.write(line) 647 # intercept warnings 648 m = re.match( 649 '^{0}([^:]+):(\d+):(?:\d+:)?{0}{1}:{0}(.*)$' 650 .format('(?:\033\[[\d;]*.| )*', 'warning'), 651 line) 652 if m and not args.get('verbose', False): 653 try: 654 with open(m.group(1)) as f: 655 lineno = int(m.group(2)) 656 line = next(it.islice(f, lineno-1, None)).strip('\n') 657 sys.stdout.write( 658 "\033[01m{path}:{lineno}:\033[01;35mwarning:\033[m " 659 "{message}\n{line}\n\n".format( 660 path=m.group(1), line=line, lineno=lineno, 661 message=m.group(3))) 662 except: 663 pass 664 proc.wait() 665 666 if proc.returncode != 0: 667 if not args.get('verbose', False): 668 for line in stdout: 669 sys.stdout.write(line) 670 sys.exit(-3) 671 672 print('built %d test suites, %d test cases, %d permutations' % ( 673 len(suites), 674 sum(len(suite.cases) for suite in suites), 675 sum(len(suite.perms) for suite in suites))) 676 677 filtered = 0 678 for suite in suites: 679 for perm in suite.perms: 680 filtered += perm.shouldtest(**args) 681 if filtered != sum(len(suite.perms) for suite in suites): 682 print('filtered down to %d permutations' % filtered) 683 684 # only requested to build? 685 if args.get('build', False): 686 return 0 687 688 print('====== testing ======') 689 try: 690 for suite in suites: 691 suite.test(**args) 692 except TestFailure: 693 pass 694 695 print('====== results ======') 696 passed = 0 697 failed = 0 698 for suite in suites: 699 for perm in suite.perms: 700 if not hasattr(perm, 'result'): 701 continue 702 703 if perm.result == PASS: 704 passed += 1 705 else: 706 sys.stdout.write( 707 "\033[01m{path}:{lineno}:\033[01;31mfailure:\033[m " 708 "{perm} failed with {returncode}\n".format( 709 perm=perm, path=perm.suite.path, lineno=perm.lineno, 710 returncode=perm.result.returncode or 0)) 711 if perm.result.stdout: 712 if perm.result.assert_: 713 stdout = perm.result.stdout[:-1] 714 else: 715 stdout = perm.result.stdout 716 for line in stdout[-5:]: 717 sys.stdout.write(line) 718 if perm.result.assert_: 719 sys.stdout.write( 720 "\033[01m{path}:{lineno}:\033[01;31massert:\033[m " 721 "{message}\n{line}\n".format( 722 **perm.result.assert_)) 723 sys.stdout.write('\n') 724 failed += 1 725 726 if args.get('gdb', False): 727 failure = None 728 for suite in suites: 729 for perm in suite.perms: 730 if getattr(perm, 'result', PASS) != PASS: 731 failure = perm.result 732 if failure is not None: 733 print('======= gdb ======') 734 # drop into gdb 735 failure.case.test(failure=failure, **args) 736 sys.exit(0) 737 738 print('tests passed: %d' % passed) 739 print('tests failed: %d' % failed) 740 return 1 if failed > 0 else 0 741 742if __name__ == "__main__": 743 import argparse 744 parser = argparse.ArgumentParser( 745 description="Run parameterized tests in various configurations.") 746 parser.add_argument('testpaths', nargs='*', default=[TESTDIR], 747 help="Description of test(s) to run. By default, this is all tests \ 748 found in the \"{0}\" directory. Here, you can specify a different \ 749 directory of tests, a specific file, a suite by name, and even a \ 750 specific test case by adding brackets. For example \ 751 \"test_dirs[0]\" or \"{0}/test_dirs.toml[0]\".".format(TESTDIR)) 752 parser.add_argument('-D', action='append', default=[], 753 help="Overriding parameter definitions.") 754 parser.add_argument('-v', '--verbose', action='store_true', 755 help="Output everything that is happening.") 756 parser.add_argument('-k', '--keep-going', action='store_true', 757 help="Run all tests instead of stopping on first error. Useful for CI.") 758 parser.add_argument('-p', '--persist', choices=['erase', 'noerase'], 759 nargs='?', const='erase', 760 help="Store disk image in a file.") 761 parser.add_argument('-b', '--build', action='store_true', 762 help="Only build the tests, do not execute.") 763 parser.add_argument('-g', '--gdb', choices=['init', 'main', 'assert'], 764 nargs='?', const='assert', 765 help="Drop into gdb on test failure.") 766 parser.add_argument('--no-internal', action='store_true', 767 help="Don't run tests that require internal knowledge.") 768 parser.add_argument('-n', '--normal', action='store_true', 769 help="Run tests normally.") 770 parser.add_argument('-r', '--reentrant', action='store_true', 771 help="Run reentrant tests with simulated power-loss.") 772 parser.add_argument('-V', '--valgrind', action='store_true', 773 help="Run non-leaky tests under valgrind to check for memory leaks.") 774 parser.add_argument('-e', '--exec', default=[], type=lambda e: e.split(' '), 775 help="Run tests with another executable prefixed on the command line.") 776 parser.add_argument('-d', '--disk', 777 help="Specify a file to use for persistent/reentrant tests.") 778 sys.exit(main(**vars(parser.parse_args()))) 779
