1# Copyright (c) 2018 Open Source Foundries Limited.
2#
3# SPDX-License-Identifier: Apache-2.0
4
5'''Common code used by commands which execute runners.
6'''
7
8import argparse
9import logging
10from os import close, getcwd, path, fspath
11from pathlib import Path
12from subprocess import CalledProcessError
13import sys
14import tempfile
15import textwrap
16import traceback
17
18from west import log
19from build_helpers import find_build_dir, is_zephyr_build, load_domains, \
20    FIND_BUILD_DIR_DESCRIPTION
21from west.commands import CommandError
22from west.configuration import config
23from runners.core import FileType
24import yaml
25
26from zephyr_ext_common import ZEPHYR_SCRIPTS
27
28# Runners depend on edtlib. Make sure the copy in the tree is
29# available to them before trying to import any.
30sys.path.insert(0, str(ZEPHYR_SCRIPTS / 'dts' / 'python-devicetree' / 'src'))
31
32from runners import get_runner_cls, ZephyrBinaryRunner, MissingProgram
33from runners.core import RunnerConfig
34import zcmake
35
36# Context-sensitive help indentation.
37# Don't change this, or output from argparse won't match up.
38INDENT = ' ' * 2
39
40if log.VERBOSE >= log.VERBOSE_NORMAL:
41    # Using level 1 allows sub-DEBUG levels of verbosity. The
42    # west.log module decides whether or not to actually print the
43    # message.
44    #
45    # https://docs.python.org/3.7/library/logging.html#logging-levels.
46    LOG_LEVEL = 1
47else:
48    LOG_LEVEL = logging.INFO
49
50def _banner(msg):
51    log.inf('-- ' + msg, colorize=True)
52
53class WestLogFormatter(logging.Formatter):
54
55    def __init__(self):
56        super().__init__(fmt='%(name)s: %(message)s')
57
58class WestLogHandler(logging.Handler):
59
60    def __init__(self, *args, **kwargs):
61        super().__init__(*args, **kwargs)
62        self.setFormatter(WestLogFormatter())
63        self.setLevel(LOG_LEVEL)
64
65    def emit(self, record):
66        fmt = self.format(record)
67        lvl = record.levelno
68        if lvl > logging.CRITICAL:
69            log.die(fmt)
70        elif lvl >= logging.ERROR:
71            log.err(fmt)
72        elif lvl >= logging.WARNING:
73            log.wrn(fmt)
74        elif lvl >= logging.INFO:
75            _banner(fmt)
76        elif lvl >= logging.DEBUG:
77            log.dbg(fmt)
78        else:
79            log.dbg(fmt, level=log.VERBOSE_EXTREME)
80
81def command_verb(command):
82    return "flash" if command.name == "flash" else "debug"
83
84def add_parser_common(command, parser_adder=None, parser=None):
85    if parser_adder is not None:
86        parser = parser_adder.add_parser(
87            command.name,
88            formatter_class=argparse.RawDescriptionHelpFormatter,
89            help=command.help,
90            description=command.description)
91
92    # Remember to update west-completion.bash if you add or remove
93    # flags
94
95    group = parser.add_argument_group('general options',
96                                      FIND_BUILD_DIR_DESCRIPTION)
97
98    group.add_argument('-d', '--build-dir', metavar='DIR',
99                       help='application build directory')
100    # still supported for backwards compatibility, but questionably
101    # useful now that we do everything with runners.yaml
102    group.add_argument('-c', '--cmake-cache', metavar='FILE',
103                       help=argparse.SUPPRESS)
104    group.add_argument('-r', '--runner',
105                       help='override default runner from --build-dir')
106    group.add_argument('--skip-rebuild', action='store_true',
107                       help='do not refresh cmake dependencies first')
108    group.add_argument('--domain', action='append',
109                       help='execute runner only for given domain')
110
111    group = parser.add_argument_group(
112        'runner configuration',
113        textwrap.dedent(f'''\
114        ===================================================================
115          IMPORTANT:
116          Individual runners support additional options not printed here.
117        ===================================================================
118
119        Run "west {command.name} --context" for runner-specific options.
120
121        If a build directory is found, --context also prints per-runner
122        settings found in that build directory's runners.yaml file.
123
124        Use "west {command.name} --context -r RUNNER" to limit output to a
125        specific RUNNER.
126
127        Some runner settings also can be overridden with options like
128        --hex-file. However, this depends on the runner: not all runners
129        respect --elf-file / --hex-file / --bin-file, nor use gdb or openocd,
130        etc.'''))
131    group.add_argument('-H', '--context', action='store_true',
132                       help='print runner- and build-specific help')
133    # Options used to override RunnerConfig values in runners.yaml.
134    # TODO: is this actually useful?
135    group.add_argument('--board-dir', metavar='DIR', help='board directory')
136    # FIXME: these are runner-specific and should be moved to where --context
137    # can find them instead.
138    group.add_argument('--gdb', help='path to GDB')
139    group.add_argument('--openocd', help='path to openocd')
140    group.add_argument(
141        '--openocd-search', metavar='DIR', action='append',
142        help='path to add to openocd search path, if applicable')
143
144    return parser
145
146def do_run_common(command, user_args, user_runner_args, domains=None):
147    # This is the main routine for all the "west flash", "west debug",
148    # etc. commands.
149
150    if user_args.context:
151        dump_context(command, user_args, user_runner_args)
152        return
153
154    build_dir = get_build_dir(user_args)
155    if not user_args.skip_rebuild:
156        rebuild(command, build_dir, user_args)
157
158    if domains is None:
159        if user_args.domain is None:
160            # No domains are passed down and no domains specified by the user.
161            # So default domain will be used.
162            domains = [load_domains(build_dir).get_default_domain()]
163        else:
164            # No domains are passed down, but user has specified domains to use.
165            # Get the user specified domains.
166            domains = load_domains(build_dir).get_domains(user_args.domain)
167
168    if len(domains) > 1 and len(user_runner_args) > 0:
169        log.wrn("Specifying runner options for multiple domains is experimental.\n"
170                "If problems are experienced, please specify a single domain "
171                "using '--domain <domain>'")
172
173    for d in domains:
174        do_run_common_image(command, user_args, user_runner_args, d.build_dir)
175
176def do_run_common_image(command, user_args, user_runner_args, build_dir=None):
177    command_name = command.name
178    if build_dir is None:
179        build_dir = get_build_dir(user_args)
180    cache = load_cmake_cache(build_dir, user_args)
181    board = cache['CACHED_BOARD']
182
183    # Load runners.yaml.
184    yaml_path = runners_yaml_path(build_dir, board)
185    runners_yaml = load_runners_yaml(yaml_path)
186
187    # Get a concrete ZephyrBinaryRunner subclass to use based on
188    # runners.yaml and command line arguments.
189    runner_cls = use_runner_cls(command, board, user_args, runners_yaml,
190                                cache)
191    runner_name = runner_cls.name()
192
193    # Set up runner logging to delegate to west.log commands.
194    logger = logging.getLogger('runners')
195    logger.setLevel(LOG_LEVEL)
196    if not logger.hasHandlers():
197        # Only add a runners log handler if none has been added already.
198        logger.addHandler(WestLogHandler())
199
200    # If the user passed -- to force the parent argument parser to stop
201    # parsing, it will show up here, and needs to be filtered out.
202    runner_args = [arg for arg in user_runner_args if arg != '--']
203
204    # Arguments in this order to allow specific to override general:
205    #
206    # - runner-specific runners.yaml arguments
207    # - user-provided command line arguments
208    final_argv = runners_yaml['args'][runner_name] + runner_args
209
210    # 'user_args' contains parsed arguments which are:
211    #
212    # 1. provided on the command line, and
213    # 2. handled by add_parser_common(), and
214    # 3. *not* runner-specific
215    #
216    # 'final_argv' contains unparsed arguments from either:
217    #
218    # 1. runners.yaml, or
219    # 2. the command line
220    #
221    # We next have to:
222    #
223    # - parse 'final_argv' now that we have all the command line
224    #   arguments
225    # - create a RunnerConfig using 'user_args' and the result
226    #   of parsing 'final_argv'
227    parser = argparse.ArgumentParser(prog=runner_name, allow_abbrev=False)
228    add_parser_common(command, parser=parser)
229    runner_cls.add_parser(parser)
230    args, unknown = parser.parse_known_args(args=final_argv)
231    if unknown:
232        log.die(f'runner {runner_name} received unknown arguments: {unknown}')
233
234    # Override args with any user_args. The latter must take
235    # precedence, or e.g. --hex-file on the command line would be
236    # ignored in favor of a board.cmake setting.
237    for a, v in vars(user_args).items():
238        if v is not None:
239            setattr(args, a, v)
240
241    # Create the RunnerConfig from runners.yaml and any command line
242    # overrides.
243    runner_config = get_runner_config(build_dir, yaml_path, runners_yaml, args)
244    log.dbg(f'runner_config: {runner_config}', level=log.VERBOSE_VERY)
245
246    # Use that RunnerConfig to create the ZephyrBinaryRunner instance
247    # and call its run().
248    try:
249        runner = runner_cls.create(runner_config, args)
250        runner.run(command_name)
251    except ValueError as ve:
252        log.err(str(ve), fatal=True)
253        dump_traceback()
254        raise CommandError(1)
255    except MissingProgram as e:
256        log.die('required program', e.filename,
257                'not found; install it or add its location to PATH')
258    except RuntimeError as re:
259        if not user_args.verbose:
260            log.die(re)
261        else:
262            log.err('verbose mode enabled, dumping stack:', fatal=True)
263            raise
264
265def get_build_dir(args, die_if_none=True):
266    # Get the build directory for the given argument list and environment.
267    if args.build_dir:
268        return args.build_dir
269
270    guess = config.get('build', 'guess-dir', fallback='never')
271    guess = guess == 'runners'
272    dir = find_build_dir(None, guess)
273
274    if dir and is_zephyr_build(dir):
275        return dir
276    elif die_if_none:
277        msg = '--build-dir was not given, '
278        if dir:
279            msg = msg + 'and neither {} nor {} are zephyr build directories.'
280        else:
281            msg = msg + ('{} is not a build directory and the default build '
282                         'directory cannot be determined. Check your '
283                         'build.dir-fmt configuration option')
284        log.die(msg.format(getcwd(), dir))
285    else:
286        return None
287
288def load_cmake_cache(build_dir, args):
289    cache_file = path.join(build_dir, args.cmake_cache or zcmake.DEFAULT_CACHE)
290    try:
291        return zcmake.CMakeCache(cache_file)
292    except FileNotFoundError:
293        log.die(f'no CMake cache found (expected one at {cache_file})')
294
295def rebuild(command, build_dir, args):
296    _banner(f'west {command.name}: rebuilding')
297    try:
298        zcmake.run_build(build_dir)
299    except CalledProcessError:
300        if args.build_dir:
301            log.die(f're-build in {args.build_dir} failed')
302        else:
303            log.die(f're-build in {build_dir} failed (no --build-dir given)')
304
305def runners_yaml_path(build_dir, board):
306    ret = Path(build_dir) / 'zephyr' / 'runners.yaml'
307    if not ret.is_file():
308        log.die(f'either a pristine build is needed, or board {board} '
309                "doesn't support west flash/debug "
310                '(no ZEPHYR_RUNNERS_YAML in CMake cache)')
311    return ret
312
313def load_runners_yaml(path):
314    # Load runners.yaml and convert to Python object.
315
316    try:
317        with open(path, 'r') as f:
318            content = yaml.safe_load(f.read())
319    except FileNotFoundError:
320        log.die(f'runners.yaml file not found: {path}')
321
322    if not content.get('runners'):
323        log.wrn(f'no pre-configured runners in {path}; '
324                "this probably won't work")
325
326    return content
327
328def use_runner_cls(command, board, args, runners_yaml, cache):
329    # Get the ZephyrBinaryRunner class from its name, and make sure it
330    # supports the command. Print a message about the choice, and
331    # return the class.
332
333    runner = args.runner or runners_yaml.get(command.runner_key)
334    if runner is None:
335        log.die(f'no {command.name} runner available for board {board}. '
336                "Check the board's documentation for instructions.")
337
338    _banner(f'west {command.name}: using runner {runner}')
339
340    available = runners_yaml.get('runners', [])
341    if runner not in available:
342        if 'BOARD_DIR' in cache:
343            board_cmake = Path(cache['BOARD_DIR']) / 'board.cmake'
344        else:
345            board_cmake = 'board.cmake'
346        log.err(f'board {board} does not support runner {runner}',
347                fatal=True)
348        log.inf(f'To fix, configure this runner in {board_cmake} and rebuild.')
349        sys.exit(1)
350    try:
351        runner_cls = get_runner_cls(runner)
352    except ValueError as e:
353        log.die(e)
354    if command.name not in runner_cls.capabilities().commands:
355        log.die(f'runner {runner} does not support command {command.name}')
356
357    return runner_cls
358
359def get_runner_config(build_dir, yaml_path, runners_yaml, args=None):
360    # Get a RunnerConfig object for the current run. yaml_config is
361    # runners.yaml's config: map, and args are the command line arguments.
362    yaml_config = runners_yaml['config']
363    yaml_dir = yaml_path.parent
364    if args is None:
365        args = argparse.Namespace()
366
367    def output_file(filetype):
368
369        from_args = getattr(args, f'{filetype}_file', None)
370        if from_args is not None:
371            return from_args
372
373        from_yaml = yaml_config.get(f'{filetype}_file')
374        if from_yaml is not None:
375            # Output paths in runners.yaml are relative to the
376            # directory containing the runners.yaml file.
377            return fspath(yaml_dir / from_yaml)
378
379        return None
380
381    def config(attr, default=None):
382        return getattr(args, attr, None) or yaml_config.get(attr, default)
383
384    def filetype(attr):
385        ftype = str(getattr(args, attr, None)).lower()
386        if ftype == "hex":
387            return FileType.HEX
388        elif ftype == "bin":
389            return FileType.BIN
390        elif ftype == "elf":
391            return FileType.ELF
392        elif getattr(args, attr, None) is not None:
393            err = 'unknown --file-type ({}). Please use hex, bin or elf'
394            raise ValueError(err.format(ftype))
395
396        # file-type not provided, try to get from filename
397        file = getattr(args, "file", None)
398        if file is not None:
399            ext = Path(file).suffix
400            if ext == ".hex":
401                return FileType.HEX
402            if ext == ".bin":
403                return FileType.BIN
404            if ext == ".elf":
405                return FileType.ELF
406
407        # we couldn't get the file-type, set to
408        # OTHER and let the runner deal with it
409        return FileType.OTHER
410
411    return RunnerConfig(build_dir,
412                        yaml_config['board_dir'],
413                        output_file('elf'),
414                        output_file('exe'),
415                        output_file('hex'),
416                        output_file('bin'),
417                        output_file('uf2'),
418                        config('file'),
419                        filetype('file_type'),
420                        config('gdb'),
421                        config('openocd'),
422                        config('openocd_search', []))
423
424def dump_traceback():
425    # Save the current exception to a file and return its path.
426    fd, name = tempfile.mkstemp(prefix='west-exc-', suffix='.txt')
427    close(fd)        # traceback has no use for the fd
428    with open(name, 'w') as f:
429        traceback.print_exc(file=f)
430    log.inf("An exception trace has been saved in", name)
431
432#
433# west {command} --context
434#
435
436def dump_context(command, args, unknown_args):
437    build_dir = get_build_dir(args, die_if_none=False)
438    if build_dir is None:
439        log.wrn('no --build-dir given or found; output will be limited')
440        runners_yaml = None
441    else:
442        cache = load_cmake_cache(build_dir, args)
443        board = cache['CACHED_BOARD']
444        yaml_path = runners_yaml_path(build_dir, board)
445        runners_yaml = load_runners_yaml(yaml_path)
446
447    # Re-build unless asked not to, to make sure the output is up to date.
448    if build_dir and not args.skip_rebuild:
449        rebuild(command, build_dir, args)
450
451    if args.runner:
452        try:
453            cls = get_runner_cls(args.runner)
454        except ValueError:
455            log.die(f'invalid runner name {args.runner}; choices: ' +
456                    ', '.join(cls.name() for cls in
457                              ZephyrBinaryRunner.get_runners()))
458    else:
459        cls = None
460
461    if runners_yaml is None:
462        dump_context_no_config(command, cls)
463    else:
464        log.inf(f'build configuration:', colorize=True)
465        log.inf(f'{INDENT}build directory: {build_dir}')
466        log.inf(f'{INDENT}board: {board}')
467        log.inf(f'{INDENT}runners.yaml: {yaml_path}')
468        if cls:
469            dump_runner_context(command, cls, runners_yaml)
470        else:
471            dump_all_runner_context(command, runners_yaml, board, build_dir)
472
473def dump_context_no_config(command, cls):
474    if not cls:
475        all_cls = {cls.name(): cls for cls in ZephyrBinaryRunner.get_runners()
476                   if command.name in cls.capabilities().commands}
477        log.inf('all Zephyr runners which support {}:'.format(command.name),
478                colorize=True)
479        dump_wrapped_lines(', '.join(all_cls.keys()), INDENT)
480        log.inf()
481        log.inf('Note: use -r RUNNER to limit information to one runner.')
482    else:
483        # This does the right thing with a None argument.
484        dump_runner_context(command, cls, None)
485
486def dump_runner_context(command, cls, runners_yaml, indent=''):
487    dump_runner_caps(cls, indent)
488    dump_runner_option_help(cls, indent)
489
490    if runners_yaml is None:
491        return
492
493    if cls.name() in runners_yaml['runners']:
494        dump_runner_args(cls.name(), runners_yaml, indent)
495    else:
496        log.wrn(f'support for runner {cls.name()} is not configured '
497                f'in this build directory')
498
499def dump_runner_caps(cls, indent=''):
500    # Print RunnerCaps for the given runner class.
501
502    log.inf(f'{indent}{cls.name()} capabilities:', colorize=True)
503    log.inf(f'{indent}{INDENT}{cls.capabilities()}')
504
505def dump_runner_option_help(cls, indent=''):
506    # Print help text for class-specific command line options for the
507    # given runner class.
508
509    dummy_parser = argparse.ArgumentParser(prog='', add_help=False, allow_abbrev=False)
510    cls.add_parser(dummy_parser)
511    formatter = dummy_parser._get_formatter()
512    for group in dummy_parser._action_groups:
513        # Break the abstraction to filter out the 'flash', 'debug', etc.
514        # TODO: come up with something cleaner (may require changes
515        # in the runner core).
516        actions = group._group_actions
517        if len(actions) == 1 and actions[0].dest == 'command':
518            # This is the lone positional argument. Skip it.
519            continue
520        formatter.start_section('REMOVE ME')
521        formatter.add_text(group.description)
522        formatter.add_arguments(actions)
523        formatter.end_section()
524    # Get the runner help, with the "REMOVE ME" string gone
525    runner_help = f'\n{indent}'.join(formatter.format_help().splitlines()[1:])
526
527    log.inf(f'{indent}{cls.name()} options:', colorize=True)
528    log.inf(indent + runner_help)
529
530def dump_runner_args(group, runners_yaml, indent=''):
531    msg = f'{indent}{group} arguments from runners.yaml:'
532    args = runners_yaml['args'][group]
533    if args:
534        log.inf(msg, colorize=True)
535        for arg in args:
536            log.inf(f'{indent}{INDENT}{arg}')
537    else:
538        log.inf(f'{msg} (none)', colorize=True)
539
540def dump_all_runner_context(command, runners_yaml, board, build_dir):
541    all_cls = {cls.name(): cls for cls in ZephyrBinaryRunner.get_runners() if
542               command.name in cls.capabilities().commands}
543    available = runners_yaml['runners']
544    available_cls = {r: all_cls[r] for r in available if r in all_cls}
545    default_runner = runners_yaml[command.runner_key]
546    yaml_path = runners_yaml_path(build_dir, board)
547    runners_yaml = load_runners_yaml(yaml_path)
548
549    log.inf(f'zephyr runners which support "west {command.name}":',
550            colorize=True)
551    dump_wrapped_lines(', '.join(all_cls.keys()), INDENT)
552    log.inf()
553    dump_wrapped_lines('Note: not all may work with this board and build '
554                       'directory. Available runners are listed below.',
555                       INDENT)
556
557    log.inf(f'available runners in runners.yaml:',
558            colorize=True)
559    dump_wrapped_lines(', '.join(available), INDENT)
560    log.inf(f'default runner in runners.yaml:', colorize=True)
561    log.inf(INDENT + default_runner)
562    log.inf('common runner configuration:', colorize=True)
563    runner_config = get_runner_config(build_dir, yaml_path, runners_yaml)
564    for field, value in zip(runner_config._fields, runner_config):
565        log.inf(f'{INDENT}- {field}: {value}')
566    log.inf('runner-specific context:', colorize=True)
567    for cls in available_cls.values():
568        dump_runner_context(command, cls, runners_yaml, INDENT)
569
570    if len(available) > 1:
571        log.inf()
572        log.inf('Note: use -r RUNNER to limit information to one runner.')
573
574def dump_wrapped_lines(text, indent):
575    for line in textwrap.wrap(text, initial_indent=indent,
576                              subsequent_indent=indent,
577                              break_on_hyphens=False,
578                              break_long_words=False):
579        log.inf(line)
580