1#!/usr/bin/env python3
2#
3# Copyright (c) 2022, Nordic Semiconductor ASA
4#
5# SPDX-License-Identifier: Apache-2.0
6
7'''Internal snippets tool.
8This is part of the build system's support for snippets.
9It is not meant for use outside of the build system.
10
11Output CMake variables:
12
13- SNIPPET_NAMES: CMake list of discovered snippet names
14- SNIPPET_FOUND_{snippet}: one per discovered snippet
15'''
16
17from collections import defaultdict, UserDict
18from dataclasses import dataclass, field
19from pathlib import Path, PurePosixPath
20from typing import Dict, Iterable, List, Set
21from jsonschema.exceptions import best_match
22import argparse
23import logging
24import os
25import re
26import sys
27import yaml
28import platform
29import jsonschema
30
31# Marker type for an 'append:' configuration. Maps variables
32# to the list of values to append to them.
33Appends = Dict[str, List[str]]
34
35def _new_append():
36    return defaultdict(list)
37
38def _new_board2appends():
39    return defaultdict(_new_append)
40
41@dataclass
42class Snippet:
43    '''Class for keeping track of all the settings discovered for an
44    individual snippet.'''
45
46    name: str
47    appends: Appends = field(default_factory=_new_append)
48    board2appends: Dict[str, Appends] = field(default_factory=_new_board2appends)
49
50    def process_data(self, pathobj: Path, snippet_data: dict, sysbuild: bool):
51        '''Process the data in a snippet.yml file, after it is loaded into a
52        python object and validated by jsonschema.'''
53        def append_value(variable, value):
54            if variable in ('SB_EXTRA_CONF_FILE', 'EXTRA_DTC_OVERLAY_FILE', 'EXTRA_CONF_FILE'):
55                path = pathobj.parent / value
56                if not path.is_file():
57                    _err(f'snippet file {pathobj}: {variable}: file not found: {path}')
58                return f'"{path.as_posix()}"'
59            if variable in ('DTS_EXTRA_CPPFLAGS'):
60                return f'"{value}"'
61            _err(f'unknown append variable: {variable}')
62
63        for variable, value in snippet_data.get('append', {}).items():
64            if (sysbuild is True and variable[0:3] == 'SB_') or \
65            (sysbuild is False and variable[0:3] != 'SB_'):
66                self.appends[variable].append(append_value(variable, value))
67        for board, settings in snippet_data.get('boards', {}).items():
68            if board.startswith('/') and not board.endswith('/'):
69                _err(f"snippet file {pathobj}: board {board} starts with '/', so "
70                     "it must end with '/' to use a regular expression")
71            for variable, value in settings.get('append', {}).items():
72                if (sysbuild is True and variable[0:3] == 'SB_') or \
73                (sysbuild is False and variable[0:3] != 'SB_'):
74                    self.board2appends[board][variable].append(
75                        append_value(variable, value))
76
77class Snippets(UserDict):
78    '''Type for all the information we have discovered about all snippets.
79    As a dict, this maps a snippet's name onto the Snippet object.
80    Any additional global attributes about all snippets go here as
81    instance attributes.'''
82
83    def __init__(self, requested: Iterable[str] = None):
84        super().__init__()
85        self.paths: Set[Path] = set()
86        self.requested: List[str] = list(requested or [])
87
88class SnippetsError(Exception):
89    '''Class for signalling expected errors'''
90
91    def __init__(self, msg):
92        self.msg = msg
93
94class SnippetToCMakeOutput:
95    '''Helper class for outputting a Snippets's semantics to a .cmake
96    include file for use by snippets.cmake.'''
97
98    def __init__(self, snippets: Snippets):
99        self.snippets = snippets
100        self.section = '#' * 79
101
102    def output_cmake(self):
103        '''Output to the file provided to the constructor.'''
104        # TODO: add source file info
105        snippets = self.snippets
106        snippet_names = sorted(snippets.keys())
107        output = ''
108
109        if platform.system() == "Windows":
110            # Change to linux-style paths for windows to avoid cmake escape character code issues
111            snippets.paths = set(map(lambda x: str(PurePosixPath(x)), snippets.paths))
112
113            for this_snippet in snippets:
114                for snippet_append in (snippets[this_snippet].appends):
115                    snippets[this_snippet].appends[snippet_append] = \
116                        set(map(lambda x: str(x.replace("\\", "/")), \
117                            snippets[this_snippet].appends[snippet_append]))
118
119        snippet_path_list = " ".join(
120            sorted(f'"{path}"' for path in snippets.paths))
121
122        output += '''\
123# WARNING. THIS FILE IS AUTO-GENERATED. DO NOT MODIFY!
124#
125# This file contains build system settings derived from your snippets.
126# Its contents are an implementation detail that should not be used outside
127# of Zephyr's snippets CMake module.
128#
129# See the Snippets guide in the Zephyr documentation for more information.
130'''
131
132        output += f'''\
133{self.section}
134# Global information about all snippets.
135
136# The name of every snippet that was discovered.
137set(SNIPPET_NAMES {' '.join(f'"{name}"' for name in snippet_names)})
138# The paths to all the snippet.yml files. One snippet
139# can have multiple snippet.yml files.
140set(SNIPPET_PATHS {snippet_path_list})
141
142# Create variable scope for snippets build variables
143zephyr_create_scope(snippets)
144'''
145
146        for snippet_name in snippets.requested:
147            output += self.output_cmake_for(snippets[snippet_name])
148
149        return output
150
151    def output_cmake_for(self, snippet: Snippet):
152        output = f'''\
153{self.section}
154# Snippet '{snippet.name}'
155
156# Common variable appends.
157'''
158        output += self.output_appends(snippet.appends, 0)
159        for board, appends in snippet.board2appends.items():
160            output += self.output_appends_for_board(board, appends)
161        return output
162
163    def output_appends_for_board(self, board: str, appends: Appends):
164        output = ''
165        if board.startswith('/'):
166            board_re = board[1:-1]
167            output += f'''\
168# Appends for board regular expression '{board_re}'
169if("${{BOARD}}${{BOARD_QUALIFIERS}}" MATCHES "^{board_re}$")
170'''
171        else:
172            output += f'''\
173# Appends for board '{board}'
174if("${{BOARD}}${{BOARD_QUALIFIERS}}" STREQUAL "{board}")
175'''
176        output += self.output_appends(appends, 1)
177        output += 'endif()\n'
178        return output
179
180    def output_appends(self, appends: Appends, indent: int):
181        space = '  ' * indent
182        output = ''
183        for name, values in appends.items():
184            for value in values:
185                output += f'{space}zephyr_set({name} {value} SCOPE snippets APPEND)\n'
186        return output
187
188# Name of the file containing the jsonschema schema for snippet.yml
189# files.
190SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'snippet-schema.yaml')
191with open(SCHEMA_PATH, 'r') as f:
192    SNIPPET_SCHEMA = yaml.safe_load(f.read())
193
194# The name of the file which contains metadata about the snippets
195# being defined in a directory.
196SNIPPET_YML = 'snippet.yml'
197
198# Regular expression for validating snippet names. Snippet names must
199# begin with an alphanumeric character, and may contain alphanumeric
200# characters or underscores. This is intentionally very restrictive to
201# keep things consistent and easy to type and remember. We can relax
202# this a bit later if needed.
203SNIPPET_NAME_RE = re.compile('[A-Za-z0-9][A-Za-z0-9_-]*')
204
205# Logger for this module.
206LOG = logging.getLogger('snippets')
207
208def _err(msg):
209    raise SnippetsError(f'error: {msg}')
210
211def parse_args():
212    parser = argparse.ArgumentParser(description='snippets helper',
213                                     allow_abbrev=False)
214    parser.add_argument('--snippet-root', default=[], action='append', type=Path,
215                        help='''a SNIPPET_ROOT element; may be given
216                        multiple times''')
217    parser.add_argument('--snippet', dest='snippets', default=[], action='append',
218                        help='''a SNIPPET element; may be given
219                        multiple times''')
220    parser.add_argument('--cmake-out', type=Path,
221                        help='''file to write cmake output to; include()
222                        this file after calling this script''')
223    parser.add_argument('--sysbuild', action="store_true",
224                        help='''set if this is running as sysbuild''')
225    return parser.parse_args()
226
227def setup_logging():
228    logging.basicConfig(level=logging.INFO,
229                        format='  %(name)s: %(message)s')
230
231def process_snippets(args: argparse.Namespace) -> Snippets:
232    '''Process snippet.yml files under each *snippet_root*
233    by recursive search. Return a Snippets object describing
234    the results of the search.
235    '''
236    # This will contain information about all the snippets
237    # we discover in each snippet_root element.
238    snippets = Snippets(requested=args.snippets)
239
240    # Process each path in snippet_root in order, adjusting
241    # snippets as needed for each one.
242    for root in args.snippet_root:
243        process_snippets_in(root, snippets, args.sysbuild)
244
245    return snippets
246
247def find_snippets_in_roots(requested_snippets, snippet_roots) -> Snippets:
248    '''Process snippet.yml files under each *snippet_root*
249    by recursive search. Return a Snippets object describing
250    the results of the search.
251    '''
252    # This will contain information about all the snippets
253    # we discover in each snippet_root element.
254    snippets = Snippets(requested=requested_snippets)
255
256    # Process each path in snippet_root in order, adjusting
257    # snippets as needed for each one.
258    for root in snippet_roots:
259        process_snippets_in(root, snippets, False)
260
261    return snippets
262
263def process_snippets_in(root_dir: Path, snippets: Snippets, sysbuild: bool) -> None:
264    '''Process snippet.yml files in *root_dir*,
265    updating *snippets* as needed.'''
266
267    if not root_dir.is_dir():
268        LOG.warning(f'SNIPPET_ROOT {root_dir} '
269                    'is not a directory; ignoring it')
270        return
271
272    snippets_dir = root_dir / 'snippets'
273    if not snippets_dir.is_dir():
274        return
275
276    for dirpath, _, filenames in os.walk(snippets_dir):
277        if SNIPPET_YML not in filenames:
278            continue
279
280        snippet_yml = Path(dirpath) / SNIPPET_YML
281        snippet_data = load_snippet_yml(snippet_yml)
282        name = snippet_data['name']
283        if name not in snippets:
284            snippets[name] = Snippet(name=name)
285        snippets[name].process_data(snippet_yml, snippet_data, sysbuild)
286        snippets.paths.add(snippet_yml)
287
288def load_snippet_yml(snippet_yml: Path) -> dict:
289    '''Load a snippet.yml file *snippet_yml*, validate the contents
290    against the schema, and do other basic checks. Return the dict
291    of the resulting YAML data.'''
292
293    with open(snippet_yml, 'r') as f:
294        try:
295            snippet_data = yaml.safe_load(f.read())
296        except yaml.scanner.ScannerError:
297            _err(f'snippets file {snippet_yml} is invalid YAML')
298
299    validator_class = jsonschema.validators.validator_for(SNIPPET_SCHEMA)
300    validator_class.check_schema(SNIPPET_SCHEMA)
301    snippet_validator = validator_class(SNIPPET_SCHEMA)
302    errors = list(snippet_validator.iter_errors(snippet_data))
303
304    if errors:
305        sys.exit('ERROR: Malformed snippet YAML file: '
306                 f'{snippet_yml.as_posix()}\n'
307                 f'{best_match(errors).message} in {best_match(errors).json_path}')
308
309    name = snippet_data['name']
310    if not SNIPPET_NAME_RE.fullmatch(name):
311        _err(f"snippet file {snippet_yml}: invalid snippet name '{name}'; "
312             'snippet names must begin with a letter '
313             'or number, and may only contain letters, numbers, '
314             'dashes (-), and underscores (_)')
315
316    return snippet_data
317
318def check_for_errors(snippets: Snippets) -> None:
319    unknown_snippets = sorted(snippet for snippet in snippets.requested
320                              if snippet not in snippets)
321    if unknown_snippets:
322        all_snippets = '\n    '.join(sorted(snippets))
323        _err(f'''\
324snippets not found: {', '.join(unknown_snippets)}
325  Please choose from among the following snippets:
326    {all_snippets}''')
327
328def write_cmake_out(snippets: Snippets, cmake_out: Path) -> None:
329    '''Write a cmake include file to *cmake_out* which
330    reflects the information in *snippets*.
331
332    The contents of this file should be considered an implementation
333    detail and are not meant to be used outside of snippets.cmake.'''
334    if not cmake_out.parent.exists():
335        cmake_out.parent.mkdir()
336
337    snippet_data = SnippetToCMakeOutput(snippets).output_cmake()
338
339    if Path(cmake_out).is_file():
340        with open(cmake_out, encoding="utf-8") as fp:
341            if fp.read() == snippet_data:
342                return
343
344    with open(cmake_out, 'w', encoding="utf-8") as f:
345        f.write(snippet_data)
346
347def main():
348    args = parse_args()
349    setup_logging()
350    try:
351        snippets = process_snippets(args)
352        check_for_errors(snippets)
353    except SnippetsError as e:
354        LOG.critical(e.msg)
355        sys.exit(1)
356    write_cmake_out(snippets, args.cmake_out)
357
358if __name__ == "__main__":
359    main()
360