1#!/usr/bin/env python
2# coding=utf-8
3#
4# ESP-IDF helper script to enumerate the builds of multiple configurations of multiple apps.
5# Produces the list of builds. The list can be consumed by build_apps.py, which performs the actual builds.
6
7import argparse
8import glob
9import json
10import logging
11import os
12import re
13import sys
14import typing
15
16from find_build_apps import (BUILD_SYSTEM_CMAKE, BUILD_SYSTEMS, DEFAULT_TARGET, BuildItem, BuildSystem, ConfigRule,
17                             config_rules_from_str, setup_logging)
18
19
20# Helper functions
21def dict_from_sdkconfig(path):
22    """
23    Parse the sdkconfig file at 'path', return name:value pairs as a dict
24    """
25    regex = re.compile(r'^([^#=]+)=(.+)$')
26    result = {}
27    with open(path) as f:
28        for line in f:
29            m = regex.match(line)
30            if m:
31                val = m.group(2)
32                if val.startswith('"') and val.endswith('"'):
33                    val = val[1:-1]
34                result[m.group(1)] = val
35    return result
36
37
38# Main logic: enumerating apps and builds
39
40
41def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg,
42                        build_system, config_rules, preserve_artifacts=True):
43    # type: (str, str, str, str, str, str, typing.List[ConfigRule], bool) -> typing.List[BuildItem]
44    """
45    Find configurations (sdkconfig file fragments) for the given app, return them as BuildItem objects
46    :param app_path: app directory (can be / usually will be a relative path)
47    :param work_dir: directory where the app should be copied before building.
48                     May contain env. variables and placeholders.
49    :param build_dir: directory where the build will be done, relative to the work_dir. May contain placeholders.
50    :param build_log: path of the build log. May contain placeholders. May be None, in which case the log should go
51                      into stdout/stderr.
52    :param target_arg: the value of IDF_TARGET passed to the script. Used to filter out configurations with
53                       a different CONFIG_IDF_TARGET value.
54    :param build_system: name of the build system, index into BUILD_SYSTEMS dictionary
55    :param config_rules: mapping of sdkconfig file name patterns to configuration names
56    :param preserve_artifacts: determine if the built binary will be uploaded as artifacts.
57    :return: list of BuildItems representing build configuration of the app
58    """
59    build_items = []  # type: typing.List[BuildItem]
60    default_config_name = ''
61
62    for rule in config_rules:
63        if not rule.file_name:
64            default_config_name = rule.config_name
65            continue
66
67        sdkconfig_paths = glob.glob(os.path.join(app_path, rule.file_name))
68        sdkconfig_paths = sorted(sdkconfig_paths)
69        for sdkconfig_path in sdkconfig_paths:
70
71            # Check if the sdkconfig file specifies IDF_TARGET, and if it is matches the --target argument.
72            sdkconfig_dict = dict_from_sdkconfig(sdkconfig_path)
73            target_from_config = sdkconfig_dict.get('CONFIG_IDF_TARGET')
74            if target_from_config is not None and target_from_config != target_arg:
75                logging.debug('Skipping sdkconfig {} which requires target {}'.format(
76                    sdkconfig_path, target_from_config))
77                continue
78
79            # Figure out the config name
80            config_name = rule.config_name or ''
81            if '*' in rule.file_name:
82                # convert glob pattern into a regex
83                regex_str = r'.*' + rule.file_name.replace('.', r'\.').replace('*', r'(.*)')
84                groups = re.match(regex_str, sdkconfig_path)
85                assert groups
86                config_name = groups.group(1)
87
88            sdkconfig_path = os.path.relpath(sdkconfig_path, app_path)
89            logging.debug('Adding build: app {}, sdkconfig {}, config name "{}"'.format(
90                app_path, sdkconfig_path, config_name))
91            build_items.append(
92                BuildItem(
93                    app_path,
94                    work_dir,
95                    build_dir,
96                    build_log,
97                    target_arg,
98                    sdkconfig_path,
99                    config_name,
100                    build_system,
101                    preserve_artifacts,
102                ))
103
104    if not build_items:
105        logging.debug('Adding build: app {}, default sdkconfig, config name "{}"'.format(app_path, default_config_name))
106        return [
107            BuildItem(
108                app_path,
109                work_dir,
110                build_dir,
111                build_log,
112                target_arg,
113                None,
114                default_config_name,
115                build_system,
116                preserve_artifacts,
117            )
118        ]
119
120    return build_items
121
122
123def find_apps(build_system_class, path, recursive, exclude_list, target):
124    # type: (typing.Type[BuildSystem], str, bool, typing.List[str], str) -> typing.List[str]
125    """
126    Find app directories in path (possibly recursively), which contain apps for the given build system, compatible
127    with the given target.
128    :param build_system_class: class derived from BuildSystem, representing the build system in use
129    :param path: path where to look for apps
130    :param recursive: whether to recursively descend into nested directories if no app is found
131    :param exclude_list: list of paths to be excluded from the recursive search
132    :param target: desired value of IDF_TARGET; apps incompatible with the given target are skipped.
133    :return: list of paths of the apps found
134    """
135    build_system_name = build_system_class.NAME
136    logging.debug('Looking for {} apps in {}{}'.format(build_system_name, path, ' recursively' if recursive else ''))
137    if not recursive:
138        if exclude_list:
139            logging.warning('--exclude option is ignored when used without --recursive')
140        if not build_system_class.is_app(path):
141            logging.warning('Path {} specified without --recursive flag, but no {} app found there'.format(
142                path, build_system_name))
143            return []
144        return [path]
145
146    # The remaining part is for recursive == True
147    apps_found = []  # type: typing.List[str]
148    for root, dirs, _ in os.walk(path, topdown=True):
149        logging.debug('Entering {}'.format(root))
150        if root in exclude_list:
151            logging.debug('Skipping {} (excluded)'.format(root))
152            del dirs[:]
153            continue
154
155        if build_system_class.is_app(root):
156            logging.debug('Found {} app in {}'.format(build_system_name, root))
157            # Don't recurse into app subdirectories
158            del dirs[:]
159
160            supported_targets = build_system_class.supported_targets(root)
161            if supported_targets and (target in supported_targets):
162                apps_found.append(root)
163            else:
164                if supported_targets:
165                    logging.debug('Skipping, app only supports targets: ' + ', '.join(supported_targets))
166                else:
167                    logging.debug('Skipping, app has no supported targets')
168                continue
169
170    return apps_found
171
172
173def main():
174    parser = argparse.ArgumentParser(description='Tool to generate build steps for IDF apps')
175    parser.add_argument(
176        '-v',
177        '--verbose',
178        action='count',
179        help='Increase the logging level of the script. Can be specified multiple times.',
180    )
181    parser.add_argument(
182        '--log-file',
183        type=argparse.FileType('w'),
184        help='Write the script log to the specified file, instead of stderr',
185    )
186    parser.add_argument(
187        '--recursive',
188        action='store_true',
189        help='Look for apps in the specified directories recursively.',
190    )
191    parser.add_argument(
192        '--build-system',
193        choices=BUILD_SYSTEMS.keys()
194    )
195    parser.add_argument(
196        '--work-dir',
197        help='If set, the app is first copied into the specified directory, and then built.' +
198             'If not set, the work directory is the directory of the app.',
199    )
200    parser.add_argument(
201        '--config',
202        action='append',
203        help='Adds configurations (sdkconfig file names) to build. This can either be ' +
204             'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, ' +
205             'relative to the project directory, to be used. Optional NAME can be specified, ' +
206             'which can be used as a name of this configuration. FILEPATTERN is the name of ' +
207             'the sdkconfig file, relative to the project directory, with at most one wildcard. ' +
208             'The part captured by the wildcard is used as the name of the configuration.',
209    )
210    parser.add_argument(
211        '--build-dir',
212        help='If set, specifies the build directory name. Can expand placeholders. Can be either a ' +
213             'name relative to the work directory, or an absolute path.',
214    )
215    parser.add_argument(
216        '--build-log',
217        help='If specified, the build log will be written to this file. Can expand placeholders.',
218    )
219    parser.add_argument('--target', help='Build apps for given target.')
220    parser.add_argument(
221        '--format',
222        default='json',
223        choices=['json'],
224        help='Format to write the list of builds as',
225    )
226    parser.add_argument(
227        '--exclude',
228        action='append',
229        help='Ignore specified directory (if --recursive is given). Can be used multiple times.',
230    )
231    parser.add_argument(
232        '-o',
233        '--output',
234        type=argparse.FileType('w'),
235        help='Output the list of builds to the specified file',
236    )
237    parser.add_argument(
238        '--app-list',
239        default=None,
240        help='Scan tests results. Restrict the build/artifacts preservation behavior to apps need to be built. '
241             'If the file does not exist, will build all apps and upload all artifacts.'
242    )
243    parser.add_argument(
244        '-p', '--paths',
245        nargs='+',
246        help='One or more app paths.'
247    )
248    args = parser.parse_args()
249    setup_logging(args)
250
251    # Arguments Validation
252    if args.app_list:
253        conflict_args = [args.recursive, args.build_system, args.target, args.exclude, args.paths]
254        if any(conflict_args):
255            raise ValueError('Conflict settings. "recursive", "build_system", "target", "exclude", "paths" should not '
256                             'be specified with "app_list"')
257        if not os.path.exists(args.app_list):
258            raise OSError('File not found {}'.format(args.app_list))
259    else:
260        # If the build target is not set explicitly, get it from the environment or use the default one (esp32)
261        if not args.target:
262            env_target = os.environ.get('IDF_TARGET')
263            if env_target:
264                logging.info('--target argument not set, using IDF_TARGET={} from the environment'.format(env_target))
265                args.target = env_target
266            else:
267                logging.info('--target argument not set, using IDF_TARGET={} as the default'.format(DEFAULT_TARGET))
268                args.target = DEFAULT_TARGET
269        if not args.build_system:
270            logging.info('--build-system argument not set, using {} as the default'.format(BUILD_SYSTEM_CMAKE))
271            args.build_system = BUILD_SYSTEM_CMAKE
272        required_args = [args.build_system, args.target, args.paths]
273        if not all(required_args):
274            raise ValueError('If app_list not set, arguments "build_system", "target", "paths" are required.')
275
276    # Prepare the list of app paths, try to read from the scan_tests result.
277    # If the file exists, then follow the file's app_dir and build/artifacts behavior, won't do find_apps() again.
278    # If the file not exists, will do find_apps() first, then build all apps and upload all artifacts.
279    if args.app_list:
280        apps = [json.loads(line) for line in open(args.app_list)]
281    else:
282        app_dirs = []
283        build_system_class = BUILD_SYSTEMS[args.build_system]
284        for path in args.paths:
285            app_dirs += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target)
286        apps = [{'app_dir': app_dir, 'build': True, 'preserve': True} for app_dir in app_dirs]
287
288    if not apps:
289        logging.warning('No apps found')
290        SystemExit(0)
291
292    logging.info('Found {} apps'.format(len(apps)))
293    apps.sort(key=lambda x: x['app_dir'])
294
295    # Find compatible configurations of each app, collect them as BuildItems
296    build_items = []  # type: typing.List[BuildItem]
297    config_rules = config_rules_from_str(args.config or [])
298    for app in apps:
299        build_items += find_builds_for_app(
300            app['app_dir'],
301            args.work_dir,
302            args.build_dir,
303            args.build_log,
304            args.target or app['target'],
305            args.build_system or app['build_system'],
306            config_rules,
307            app['preserve'],
308        )
309    logging.info('Found {} builds'.format(len(build_items)))
310
311    # Write out the BuildItems. Only JSON supported now (will add YAML later).
312    if args.format != 'json':
313        raise NotImplementedError()
314
315    out = args.output or sys.stdout
316    out.writelines([item.to_json() + '\n' for item in build_items])
317
318
319if __name__ == '__main__':
320    main()
321