1#!/usr/bin/env python
2
3# coding=utf-8
4#
5# ESP-IDF helper script to build multiple applications. Consumes the input of find_apps.py.
6#
7
8import argparse
9import logging
10import os.path
11import re
12import sys
13
14from find_build_apps import BUILD_SYSTEMS, BuildError, BuildItem, setup_logging
15from find_build_apps.common import SIZE_JSON_FN, rmdir
16
17# This RE will match GCC errors and many other fatal build errors and warnings as well
18LOG_ERROR_WARNING = re.compile(r'(error|warning):', re.IGNORECASE)
19
20# Log this many trailing lines from a failed build log, also
21LOG_DEBUG_LINES = 25
22
23
24def main():  # type: () -> None
25    parser = argparse.ArgumentParser(description='ESP-IDF app builder')
26    parser.add_argument(
27        '-v',
28        '--verbose',
29        action='count',
30        help='Increase the logging level of the script. Can be specified multiple times.',
31    )
32    parser.add_argument(
33        '--build-verbose',
34        action='store_true',
35        help='Enable verbose output from build system.',
36    )
37    parser.add_argument(
38        '--log-file',
39        type=argparse.FileType('w'),
40        help='Write the script log to the specified file, instead of stderr',
41    )
42    parser.add_argument(
43        '--parallel-count',
44        default=1,
45        type=int,
46        help="Number of parallel build jobs. Note that this script doesn't start the jobs, " +
47             'it needs to be executed multiple times with same value of --parallel-count and ' +
48             'different values of --parallel-index.',
49    )
50    parser.add_argument(
51        '--parallel-index',
52        default=1,
53        type=int,
54        help='Index (1-based) of the job, out of the number specified by --parallel-count.',
55    )
56    parser.add_argument(
57        '--format',
58        default='json',
59        choices=['json'],
60        help='Format to read the list of builds',
61    )
62    parser.add_argument(
63        '--dry-run',
64        action='store_true',
65        help="Don't actually build, only print the build commands",
66    )
67    parser.add_argument(
68        '--keep-going',
69        action='store_true',
70        help="Don't exit immediately when a build fails.",
71    )
72    parser.add_argument(
73        '--output-build-list',
74        type=argparse.FileType('w'),
75        help='If specified, the list of builds (with all the placeholders expanded) will be written to this file.',
76    )
77    parser.add_argument(
78        '--size-info',
79        type=argparse.FileType('a'),
80        help='If specified, the test case name and size info json will be written to this file'
81    )
82    parser.add_argument(
83        'build_list',
84        type=argparse.FileType('r'),
85        nargs='?',
86        default=sys.stdin,
87        help='Name of the file to read the list of builds from. If not specified, read from stdin.',
88    )
89    args = parser.parse_args()
90
91    setup_logging(args)
92
93    build_items = [BuildItem.from_json(line) for line in args.build_list]
94    if not build_items:
95        logging.warning('Empty build list')
96        SystemExit(0)
97
98    num_builds = len(build_items)
99    num_jobs = args.parallel_count
100    job_index = args.parallel_index - 1  # convert to 0-based index
101    num_builds_per_job = (num_builds + num_jobs - 1) // num_jobs
102    min_job_index = num_builds_per_job * job_index
103    if min_job_index >= num_builds:
104        logging.warn('Nothing to do for job {} (build total: {}, per job: {})'.format(
105            job_index + 1, num_builds, num_builds_per_job))
106        raise SystemExit(0)
107
108    max_job_index = min(num_builds_per_job * (job_index + 1) - 1, num_builds - 1)
109    logging.info('Total {} builds, max. {} builds per job, running builds {}-{}'.format(
110        num_builds, num_builds_per_job, min_job_index + 1, max_job_index + 1))
111
112    builds_for_current_job = build_items[min_job_index:max_job_index + 1]
113    for i, build_info in enumerate(builds_for_current_job):
114        index = i + min_job_index + 1
115        build_info.index = index
116        build_info.dry_run = args.dry_run
117        build_info.verbose = args.build_verbose
118        build_info.keep_going = args.keep_going
119        logging.debug('    Build {}: {}'.format(index, repr(build_info)))
120        if args.output_build_list:
121            args.output_build_list.write(build_info.to_json_expanded() + '\n')
122
123    failed_builds = []
124    for build_info in builds_for_current_job:
125        logging.info('Running build {}: {}'.format(build_info.index, repr(build_info)))
126        build_system_class = BUILD_SYSTEMS[build_info.build_system]
127        try:
128            build_system_class.build(build_info)
129        except BuildError as e:
130            logging.error(str(e))
131            if build_info.build_log_path:
132                log_filename = os.path.basename(build_info.build_log_path)
133                with open(build_info.build_log_path, 'r') as f:
134                    lines = [line.rstrip() for line in f.readlines() if line.rstrip()]  # non-empty lines
135                    logging.debug('Error and warning lines from {}:'.format(log_filename))
136                    for line in lines:
137                        if LOG_ERROR_WARNING.search(line):
138                            logging.warning('>>> {}'.format(line))
139                    logging.debug('Last {} lines of {}:'.format(LOG_DEBUG_LINES, log_filename))
140                    for line in lines[-LOG_DEBUG_LINES:]:
141                        logging.debug('>>> {}'.format(line))
142            if args.keep_going:
143                failed_builds.append(build_info)
144            else:
145                raise SystemExit(1)
146        else:
147            if args.size_info:
148                build_info.write_size_info(args.size_info)
149            if not build_info.preserve:
150                logging.info('Removing build directory {}'.format(build_info.build_path))
151                # we only remove binaries here, log files are still needed by check_build_warnings.py
152                rmdir(build_info.build_path, exclude_file_pattern=SIZE_JSON_FN)
153
154    if failed_builds:
155        logging.error('The following build have failed:')
156        for build in failed_builds:
157            logging.error('    {}'.format(build))
158        raise SystemExit(1)
159
160
161if __name__ == '__main__':
162    main()
163