1# coding=utf-8
2import fnmatch
3import json
4import logging
5import os
6import re
7import shutil
8import subprocess
9import sys
10import typing
11from abc import abstractmethod
12from collections import namedtuple
13from io import open
14
15DEFAULT_TARGET = 'esp32'
16
17TARGET_PLACEHOLDER = '@t'
18WILDCARD_PLACEHOLDER = '@w'
19NAME_PLACEHOLDER = '@n'
20FULL_NAME_PLACEHOLDER = '@f'
21INDEX_PLACEHOLDER = '@i'
22
23IDF_SIZE_PY = os.path.join(os.environ['IDF_PATH'], 'tools', 'idf_size.py')
24SIZE_JSON_FN = 'size.json'
25
26SDKCONFIG_LINE_REGEX = re.compile(r"^([^=]+)=\"?([^\"\n]*)\"?\n*$")
27
28# If these keys are present in sdkconfig.defaults, they will be extracted and passed to CMake
29SDKCONFIG_TEST_OPTS = [
30    'EXCLUDE_COMPONENTS',
31    'TEST_EXCLUDE_COMPONENTS',
32    'TEST_COMPONENTS',
33]
34
35# These keys in sdkconfig.defaults are not propagated to the final sdkconfig file:
36SDKCONFIG_IGNORE_OPTS = [
37    'TEST_GROUPS'
38]
39
40# ConfigRule represents one --config argument of find_apps.py.
41# file_name is the name of the sdkconfig file fragment, optionally with a single wildcard ('*' character).
42# file_name can also be empty to indicate that the default configuration of the app should be used.
43# config_name is the name of the corresponding build configuration, or None if the value of wildcard is to be used.
44# For example:
45#   filename='', config_name='default' — represents the default app configuration, and gives it a name 'default'
46#   filename='sdkconfig.*', config_name=None - represents the set of configurations, names match the wildcard value
47ConfigRule = namedtuple('ConfigRule', ['file_name', 'config_name'])
48
49
50def config_rules_from_str(rule_strings):  # type: (typing.List[str]) -> typing.List[ConfigRule]
51    """
52    Helper function to convert strings like 'file_name=config_name' into ConfigRule objects
53    :param rule_strings: list of rules as strings
54    :return: list of ConfigRules
55    """
56    rules = []  # type: typing.List[ConfigRule]
57    for rule_str in rule_strings:
58        items = rule_str.split('=', 2)
59        rules.append(ConfigRule(items[0], items[1] if len(items) == 2 else None))
60    return rules
61
62
63def find_first_match(pattern, path):
64    for root, _, files in os.walk(path):
65        res = fnmatch.filter(files, pattern)
66        if res:
67            return os.path.join(root, res[0])
68    return None
69
70
71def rmdir(path, exclude_file_pattern=None):
72    if not exclude_file_pattern:
73        shutil.rmtree(path, ignore_errors=True)
74        return
75
76    for root, dirs, files in os.walk(path, topdown=False):
77        for f in files:
78            if not fnmatch.fnmatch(f, exclude_file_pattern):
79                os.remove(os.path.join(root, f))
80        for d in dirs:
81            try:
82                os.rmdir(os.path.join(root, d))
83            except OSError:
84                pass
85
86
87class BuildItem(object):
88    """
89    Instance of this class represents one build of an application.
90    The parameters which distinguish the build are passed to the constructor.
91    """
92
93    def __init__(
94            self,
95            app_path,
96            work_dir,
97            build_path,
98            build_log_path,
99            target,
100            sdkconfig_path,
101            config_name,
102            build_system,
103            preserve_artifacts,
104    ):
105        # These internal variables store the paths with environment variables and placeholders;
106        # Public properties with similar names use the _expand method to get the actual paths.
107        self._app_dir = app_path
108        self._work_dir = work_dir
109        self._build_dir = build_path
110        self._build_log_path = build_log_path
111
112        self.sdkconfig_path = sdkconfig_path
113        self.config_name = config_name
114        self.target = target
115        self.build_system = build_system
116
117        self.preserve = preserve_artifacts
118
119        self._app_name = os.path.basename(os.path.normpath(app_path))
120        self.size_json_fp = None
121
122        # Some miscellaneous build properties which are set later, at the build stage
123        self.index = None
124        self.verbose = False
125        self.dry_run = False
126        self.keep_going = False
127
128        self.work_path = self.work_dir or self.app_dir
129        if not self.build_dir:
130            self.build_path = os.path.join(self.work_path, 'build')
131        elif os.path.isabs(self.build_dir):
132            self.build_path = self.build_dir
133        else:
134            self.build_path = os.path.normpath(os.path.join(self.work_path, self.build_dir))
135
136    @property
137    def app_dir(self):
138        """
139        :return: directory of the app
140        """
141        return self._expand(self._app_dir)
142
143    @property
144    def work_dir(self):
145        """
146        :return: directory where the app should be copied to, prior to the build. Can be None, which means that the app
147                 directory should be used.
148        """
149        return self._expand(self._work_dir)
150
151    @property
152    def build_dir(self):
153        """
154        :return: build directory, either relative to the work directory (if relative path is used) or absolute path.
155        """
156        return self._expand(self._build_dir)
157
158    @property
159    def build_log_path(self):
160        """
161        :return: path of the build log file
162        """
163        return self._expand(self._build_log_path)
164
165    def __repr__(self):
166        return '({}) Build app {} for target {}, sdkconfig {} in {}'.format(
167            self.build_system,
168            self.app_dir,
169            self.target,
170            self.sdkconfig_path or '(default)',
171            self.build_dir,
172        )
173
174    def to_json(self):  # type: () -> str
175        """
176        :return: JSON string representing this object
177        """
178        return self._to_json(self._app_dir, self._work_dir, self._build_dir, self._build_log_path)
179
180    def to_json_expanded(self):  # type: () -> str
181        """
182        :return: JSON string representing this object, with all placeholders in paths expanded
183        """
184        return self._to_json(self.app_dir, self.work_dir, self.build_dir, self.build_log_path)
185
186    def _to_json(self, app_dir, work_dir, build_dir, build_log_path):  # type: (str, str, str, str) -> str
187        """
188        Internal function, called by to_json and to_json_expanded
189        """
190        return json.dumps({
191            'build_system': self.build_system,
192            'app_dir': app_dir,
193            'work_dir': work_dir,
194            'build_dir': build_dir,
195            'build_log_path': build_log_path,
196            'sdkconfig': self.sdkconfig_path,
197            'config': self.config_name,
198            'target': self.target,
199            'verbose': self.verbose,
200            'preserve': self.preserve,
201        })
202
203    @staticmethod
204    def from_json(json_str):  # type: (typing.Text) -> BuildItem
205        """
206        :return: Get the BuildItem from a JSON string
207        """
208        d = json.loads(str(json_str))
209        result = BuildItem(
210            app_path=d['app_dir'],
211            work_dir=d['work_dir'],
212            build_path=d['build_dir'],
213            build_log_path=d['build_log_path'],
214            sdkconfig_path=d['sdkconfig'],
215            config_name=d['config'],
216            target=d['target'],
217            build_system=d['build_system'],
218            preserve_artifacts=d['preserve']
219        )
220        result.verbose = d['verbose']
221        return result
222
223    def _expand(self, path):  # type: (str) -> str
224        """
225        Internal method, expands any of the placeholders in {app,work,build} paths.
226        """
227        if not path:
228            return path
229
230        if self.index is not None:
231            path = path.replace(INDEX_PLACEHOLDER, str(self.index))
232        path = path.replace(TARGET_PLACEHOLDER, self.target)
233        path = path.replace(NAME_PLACEHOLDER, self._app_name)
234        if (FULL_NAME_PLACEHOLDER in path):  # to avoid recursion to the call to app_dir in the next line:
235            path = path.replace(FULL_NAME_PLACEHOLDER, self.app_dir.replace(os.path.sep, '_'))
236        wildcard_pos = path.find(WILDCARD_PLACEHOLDER)
237        if wildcard_pos != -1:
238            if self.config_name:
239                # if config name is defined, put it in place of the placeholder
240                path = path.replace(WILDCARD_PLACEHOLDER, self.config_name)
241            else:
242                # otherwise, remove the placeholder and one character on the left
243                # (which is usually an underscore, dash, or other delimiter)
244                left_of_wildcard = max(0, wildcard_pos - 1)
245                right_of_wildcard = wildcard_pos + len(WILDCARD_PLACEHOLDER)
246                path = path[0:left_of_wildcard] + path[right_of_wildcard:]
247        path = os.path.expandvars(path)
248        return path
249
250    def get_size_json_fp(self):
251        if self.size_json_fp and os.path.exists(self.size_json_fp):
252            return self.size_json_fp
253
254        assert os.path.exists(self.build_path)
255        assert os.path.exists(self.work_path)
256
257        map_file = find_first_match('*.map', self.build_path)
258        if not map_file:
259            raise ValueError('.map file not found under "{}"'.format(self.build_path))
260
261        size_json_fp = os.path.join(self.build_path, SIZE_JSON_FN)
262        idf_size_args = [
263            sys.executable,
264            IDF_SIZE_PY,
265            '--json',
266            '-o', size_json_fp,
267            map_file
268        ]
269        subprocess.check_call(idf_size_args)
270        return size_json_fp
271
272    def write_size_info(self, size_info_fs):
273        if not self.size_json_fp or (not os.path.exists(self.size_json_fp)):
274            raise OSError('Run get_size_json_fp() for app {} after built binary'.format(self.app_dir))
275        size_info_dict = {
276            'app_name': self._app_name,
277            'config_name': self.config_name,
278            'target': self.target,
279            'path': self.size_json_fp,
280        }
281        size_info_fs.write(json.dumps(size_info_dict) + '\n')
282
283
284class BuildSystem:
285    """
286    Class representing a build system.
287    Derived classes implement the methods below.
288    Objects of these classes aren't instantiated, instead the class (type object) is used.
289    """
290    NAME = 'undefined'
291    SUPPORTED_TARGETS_REGEX = re.compile(r'Supported [Tt]argets((?:[ |]+(?:[0-9a-zA-Z\-]+))+)')
292
293    FORMAL_TO_USUAL = {
294        'ESP32': 'esp32',
295        'ESP32-S2': 'esp32s2',
296        'ESP32-S3': 'esp32s3',
297        'ESP32-C3': 'esp32c3',
298        'ESP32-H2': 'esp32h2',
299        'Linux': 'linux',
300    }
301
302    @classmethod
303    def build_prepare(cls, build_item):
304        app_path = build_item.app_dir
305        work_path = build_item.work_path
306        build_path = build_item.build_path
307
308        if work_path != app_path:
309            if os.path.exists(work_path):
310                logging.debug('Work directory {} exists, removing'.format(work_path))
311                if not build_item.dry_run:
312                    shutil.rmtree(work_path)
313            logging.debug('Copying app from {} to {}'.format(app_path, work_path))
314            if not build_item.dry_run:
315                shutil.copytree(app_path, work_path)
316
317        if os.path.exists(build_path):
318            logging.debug('Build directory {} exists, removing'.format(build_path))
319            if not build_item.dry_run:
320                shutil.rmtree(build_path)
321
322        if not build_item.dry_run:
323            os.makedirs(build_path)
324
325        # Prepare the sdkconfig file, from the contents of sdkconfig.defaults (if exists) and the contents of
326        # build_info.sdkconfig_path, i.e. the config-specific sdkconfig file.
327        #
328        # Note: the build system supports taking multiple sdkconfig.defaults files via SDKCONFIG_DEFAULTS
329        # CMake variable. However here we do this manually to perform environment variable expansion in the
330        # sdkconfig files.
331        sdkconfig_defaults_list = ['sdkconfig.defaults', 'sdkconfig.defaults.' + build_item.target]
332        if build_item.sdkconfig_path:
333            sdkconfig_defaults_list.append(build_item.sdkconfig_path)
334
335        sdkconfig_file = os.path.join(work_path, 'sdkconfig')
336        if os.path.exists(sdkconfig_file):
337            logging.debug('Removing sdkconfig file: {}'.format(sdkconfig_file))
338            if not build_item.dry_run:
339                os.unlink(sdkconfig_file)
340
341        logging.debug('Creating sdkconfig file: {}'.format(sdkconfig_file))
342        extra_cmakecache_items = {}
343        if not build_item.dry_run:
344            with open(sdkconfig_file, 'w') as f_out:
345                for sdkconfig_name in sdkconfig_defaults_list:
346                    sdkconfig_path = os.path.join(work_path, sdkconfig_name)
347                    if not sdkconfig_path or not os.path.exists(sdkconfig_path):
348                        continue
349                    logging.debug('Appending {} to sdkconfig'.format(sdkconfig_name))
350                    with open(sdkconfig_path, 'r') as f_in:
351                        for line in f_in:
352                            if not line.endswith('\n'):
353                                line += '\n'
354                            if cls.NAME == 'cmake':
355                                m = SDKCONFIG_LINE_REGEX.match(line)
356                                key = m.group(1) if m else None
357                                if key in SDKCONFIG_TEST_OPTS:
358                                    extra_cmakecache_items[key] = m.group(2)
359                                    continue
360                                if key in SDKCONFIG_IGNORE_OPTS:
361                                    continue
362                            f_out.write(os.path.expandvars(line))
363        else:
364            for sdkconfig_name in sdkconfig_defaults_list:
365                sdkconfig_path = os.path.join(app_path, sdkconfig_name)
366                if not sdkconfig_path:
367                    continue
368                logging.debug('Considering sdkconfig {}'.format(sdkconfig_path))
369                if not os.path.exists(sdkconfig_path):
370                    continue
371                logging.debug('Appending {} to sdkconfig'.format(sdkconfig_name))
372
373        # The preparation of build is finished. Implement the build part in sub classes.
374        if cls.NAME == 'cmake':
375            return build_path, work_path, extra_cmakecache_items
376        else:
377            return build_path, work_path
378
379    @staticmethod
380    @abstractmethod
381    def build(build_item):
382        pass
383
384    @staticmethod
385    @abstractmethod
386    def is_app(path):
387        pass
388
389    @staticmethod
390    def _read_readme(app_path):
391        # Markdown supported targets should be:
392        # e.g. | Supported Targets | ESP32 |
393        #      | ----------------- | ----- |
394        # reStructuredText supported targets should be:
395        # e.g. ================= =====
396        #      Supported Targets ESP32
397        #      ================= =====
398        def get_md_or_rst(app_path):
399            readme_path = os.path.join(app_path, 'README.md')
400            if not os.path.exists(readme_path):
401                readme_path = os.path.join(app_path, 'README.rst')
402                if not os.path.exists(readme_path):
403                    return None
404            return readme_path
405
406        readme_path = get_md_or_rst(app_path)
407        # Handle sub apps situation, e.g. master-slave
408        if not readme_path:
409            readme_path = get_md_or_rst(os.path.dirname(app_path))
410        if not readme_path:
411            return None
412        with open(readme_path, 'r', encoding='utf8') as readme_file:
413            return readme_file.read()
414
415    @classmethod
416    def _supported_targets(cls, app_path):
417        readme_file_content = BuildSystem._read_readme(app_path)
418        if not readme_file_content:
419            return cls.FORMAL_TO_USUAL.values()  # supports all targets if no readme found
420        match = re.findall(BuildSystem.SUPPORTED_TARGETS_REGEX, readme_file_content)
421        if not match:
422            return cls.FORMAL_TO_USUAL.values()  # supports all targets if no such header in readme
423        if len(match) > 1:
424            raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path))
425        support_str = match[0].strip()
426
427        targets = []
428        for part in support_str.split('|'):
429            for inner in part.split(' '):
430                inner = inner.strip()
431                if not inner:
432                    continue
433                elif inner in cls.FORMAL_TO_USUAL:
434                    targets.append(cls.FORMAL_TO_USUAL[inner])
435                else:
436                    raise NotImplementedError("Can't recognize value of target {} in {}, now we only support '{}'"
437                                              .format(inner, app_path, ', '.join(cls.FORMAL_TO_USUAL.keys())))
438        return targets
439
440    @classmethod
441    @abstractmethod
442    def supported_targets(cls, app_path):
443        pass
444
445
446class BuildError(RuntimeError):
447    pass
448
449
450def setup_logging(args):
451    """
452    Configure logging module according to the number of '--verbose'/'-v' arguments and the --log-file argument.
453    :param args: namespace obtained from argparse
454    """
455    if not args.verbose:
456        log_level = logging.WARNING
457    elif args.verbose == 1:
458        log_level = logging.INFO
459    else:
460        log_level = logging.DEBUG
461
462    logging.basicConfig(
463        format='%(levelname)s: %(message)s',
464        stream=args.log_file or sys.stderr,
465        level=log_level,
466    )
467