1#!/usr/bin/env python3
2#
3# Copyright (c) 2022, Nordic Semiconductor ASA
4#
5# SPDX-License-Identifier: Apache-2.0
6
7'''Internal snippets tool.
8This is part of the build system's support for snippets.
9It is not meant for use outside of the build system.
10
11Output CMake variables:
12
13- SNIPPET_NAMES: CMake list of discovered snippet names
14- SNIPPET_FOUND_{snippet}: one per discovered snippet
15'''
16
17from collections import defaultdict, UserDict
18from dataclasses import dataclass, field
19from pathlib import Path, PurePosixPath
20from typing import Dict, Iterable, List, Set
21import argparse
22import logging
23import os
24import pykwalify.core
25import pykwalify.errors
26import re
27import sys
28import textwrap
29import yaml
30import platform
31
32# Marker type for an 'append:' configuration. Maps variables
33# to the list of values to append to them.
34Appends = Dict[str, List[str]]
35
36def _new_append():
37    return defaultdict(list)
38
39def _new_board2appends():
40    return defaultdict(_new_append)
41
42@dataclass
43class Snippet:
44    '''Class for keeping track of all the settings discovered for an
45    individual snippet.'''
46
47    name: str
48    appends: Appends = field(default_factory=_new_append)
49    board2appends: Dict[str, Appends] = field(default_factory=_new_board2appends)
50
51    def process_data(self, pathobj: Path, snippet_data: dict, sysbuild: bool):
52        '''Process the data in a snippet.yml file, after it is loaded into a
53        python object and validated by pykwalify.'''
54        def append_value(variable, value):
55            if variable in ('SB_EXTRA_CONF_FILE', 'EXTRA_DTC_OVERLAY_FILE', 'EXTRA_CONF_FILE'):
56                path = pathobj.parent / value
57                if not path.is_file():
58                    _err(f'snippet file {pathobj}: {variable}: file not found: {path}')
59                return f'"{path.as_posix()}"'
60            if variable in ('DTS_EXTRA_CPPFLAGS'):
61                return f'"{value}"'
62            _err(f'unknown append variable: {variable}')
63
64        for variable, value in snippet_data.get('append', {}).items():
65            if (sysbuild is True and variable[0:3] == 'SB_') or \
66            (sysbuild is False and variable[0:3] != 'SB_'):
67                self.appends[variable].append(append_value(variable, value))
68        for board, settings in snippet_data.get('boards', {}).items():
69            if board.startswith('/') and not board.endswith('/'):
70                _err(f"snippet file {pathobj}: board {board} starts with '/', so "
71                     "it must end with '/' to use a regular expression")
72            for variable, value in settings.get('append', {}).items():
73                if (sysbuild is True and variable[0:3] == 'SB_') or \
74                (sysbuild is False and variable[0:3] != 'SB_'):
75                    self.board2appends[board][variable].append(
76                        append_value(variable, value))
77
78class Snippets(UserDict):
79    '''Type for all the information we have discovered about all snippets.
80    As a dict, this maps a snippet's name onto the Snippet object.
81    Any additional global attributes about all snippets go here as
82    instance attributes.'''
83
84    def __init__(self, requested: Iterable[str] = None):
85        super().__init__()
86        self.paths: Set[Path] = set()
87        self.requested: List[str] = list(requested or [])
88
89class SnippetsError(Exception):
90    '''Class for signalling expected errors'''
91
92    def __init__(self, msg):
93        self.msg = msg
94
95class SnippetToCMakePrinter:
96    '''Helper class for printing a Snippets's semantics to a .cmake
97    include file for use by snippets.cmake.'''
98
99    def __init__(self, snippets: Snippets, out_file):
100        self.snippets = snippets
101        self.out_file = out_file
102        self.section = '#' * 79
103
104    def print_cmake(self):
105        '''Print to the output file provided to the constructor.'''
106        # TODO: add source file info
107        snippets = self.snippets
108        snippet_names = sorted(snippets.keys())
109
110        if platform.system() == "Windows":
111            # Change to linux-style paths for windows to avoid cmake escape character code issues
112            snippets.paths = set(map(lambda x: str(PurePosixPath(x)), snippets.paths))
113
114            for this_snippet in snippets:
115                for snippet_append in (snippets[this_snippet].appends):
116                    snippets[this_snippet].appends[snippet_append] = \
117                        set(map(lambda x: str(x.replace("\\", "/")), \
118                            snippets[this_snippet].appends[snippet_append]))
119
120        snippet_path_list = " ".join(
121            sorted(f'"{path}"' for path in snippets.paths))
122
123        self.print('''\
124# WARNING. THIS FILE IS AUTO-GENERATED. DO NOT MODIFY!
125#
126# This file contains build system settings derived from your snippets.
127# Its contents are an implementation detail that should not be used outside
128# of Zephyr's snippets CMake module.
129#
130# See the Snippets guide in the Zephyr documentation for more information.
131''')
132
133        self.print(f'''\
134{self.section}
135# Global information about all snippets.
136
137# The name of every snippet that was discovered.
138set(SNIPPET_NAMES {' '.join(f'"{name}"' for name in snippet_names)})
139# The paths to all the snippet.yml files. One snippet
140# can have multiple snippet.yml files.
141set(SNIPPET_PATHS {snippet_path_list})
142
143# Create variable scope for snippets build variables
144zephyr_create_scope(snippets)
145''')
146
147        for snippet_name in snippets.requested:
148            self.print_cmake_for(snippets[snippet_name])
149            self.print()
150
151    def print_cmake_for(self, snippet: Snippet):
152        self.print(f'''\
153{self.section}
154# Snippet '{snippet.name}'
155
156# Common variable appends.''')
157        self.print_appends(snippet.appends, 0)
158        for board, appends in snippet.board2appends.items():
159            self.print_appends_for_board(board, appends)
160
161    def print_appends_for_board(self, board: str, appends: Appends):
162        if board.startswith('/'):
163            board_re = board[1:-1]
164            self.print(f'''\
165# Appends for board regular expression '{board_re}'
166if("${{BOARD}}${{BOARD_QUALIFIERS}}" MATCHES "^{board_re}$")''')
167        else:
168            self.print(f'''\
169# Appends for board '{board}'
170if("${{BOARD}}${{BOARD_QUALIFIERS}}" STREQUAL "{board}")''')
171        self.print_appends(appends, 1)
172        self.print('endif()')
173
174    def print_appends(self, appends: Appends, indent: int):
175        space = '  ' * indent
176        for name, values in appends.items():
177            for value in values:
178                self.print(f'{space}zephyr_set({name} {value} SCOPE snippets APPEND)')
179
180    def print(self, *args, **kwargs):
181        kwargs['file'] = self.out_file
182        print(*args, **kwargs)
183
184# Name of the file containing the pykwalify schema for snippet.yml
185# files.
186SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'snippet-schema.yml')
187with open(SCHEMA_PATH, 'r') as f:
188    SNIPPET_SCHEMA = yaml.safe_load(f.read())
189
190# The name of the file which contains metadata about the snippets
191# being defined in a directory.
192SNIPPET_YML = 'snippet.yml'
193
194# Regular expression for validating snippet names. Snippet names must
195# begin with an alphanumeric character, and may contain alphanumeric
196# characters or underscores. This is intentionally very restrictive to
197# keep things consistent and easy to type and remember. We can relax
198# this a bit later if needed.
199SNIPPET_NAME_RE = re.compile('[A-Za-z0-9][A-Za-z0-9_-]*')
200
201# Logger for this module.
202LOG = logging.getLogger('snippets')
203
204def _err(msg):
205    raise SnippetsError(f'error: {msg}')
206
207def parse_args():
208    parser = argparse.ArgumentParser(description='snippets helper',
209                                     allow_abbrev=False)
210    parser.add_argument('--snippet-root', default=[], action='append', type=Path,
211                        help='''a SNIPPET_ROOT element; may be given
212                        multiple times''')
213    parser.add_argument('--snippet', dest='snippets', default=[], action='append',
214                        help='''a SNIPPET element; may be given
215                        multiple times''')
216    parser.add_argument('--cmake-out', type=Path,
217                        help='''file to write cmake output to; include()
218                        this file after calling this script''')
219    parser.add_argument('--sysbuild', action="store_true",
220                        help='''set if this is running as sysbuild''')
221    return parser.parse_args()
222
223def setup_logging():
224    # Silence validation errors from pykwalify, which are logged at
225    # logging.ERROR level. We want to handle those ourselves as
226    # needed.
227    logging.getLogger('pykwalify').setLevel(logging.CRITICAL)
228    logging.basicConfig(level=logging.INFO,
229                        format='  %(name)s: %(message)s')
230
231def process_snippets(args: argparse.Namespace) -> Snippets:
232    '''Process snippet.yml files under each *snippet_root*
233    by recursive search. Return a Snippets object describing
234    the results of the search.
235    '''
236    # This will contain information about all the snippets
237    # we discover in each snippet_root element.
238    snippets = Snippets(requested=args.snippets)
239
240    # Process each path in snippet_root in order, adjusting
241    # snippets as needed for each one.
242    for root in args.snippet_root:
243        process_snippets_in(root, snippets, args.sysbuild)
244
245    return snippets
246
247def find_snippets_in_roots(requested_snippets, snippet_roots) -> Snippets:
248    '''Process snippet.yml files under each *snippet_root*
249    by recursive search. Return a Snippets object describing
250    the results of the search.
251    '''
252    # This will contain information about all the snippets
253    # we discover in each snippet_root element.
254    snippets = Snippets(requested=requested_snippets)
255
256    # Process each path in snippet_root in order, adjusting
257    # snippets as needed for each one.
258    for root in snippet_roots:
259        process_snippets_in(root, snippets, False)
260
261    return snippets
262
263def process_snippets_in(root_dir: Path, snippets: Snippets, sysbuild: bool) -> None:
264    '''Process snippet.yml files in *root_dir*,
265    updating *snippets* as needed.'''
266
267    if not root_dir.is_dir():
268        LOG.warning(f'SNIPPET_ROOT {root_dir} '
269                    'is not a directory; ignoring it')
270        return
271
272    snippets_dir = root_dir / 'snippets'
273    if not snippets_dir.is_dir():
274        return
275
276    for dirpath, _, filenames in os.walk(snippets_dir):
277        if SNIPPET_YML not in filenames:
278            continue
279
280        snippet_yml = Path(dirpath) / SNIPPET_YML
281        snippet_data = load_snippet_yml(snippet_yml)
282        name = snippet_data['name']
283        if name not in snippets:
284            snippets[name] = Snippet(name=name)
285        snippets[name].process_data(snippet_yml, snippet_data, sysbuild)
286        snippets.paths.add(snippet_yml)
287
288def load_snippet_yml(snippet_yml: Path) -> dict:
289    '''Load a snippet.yml file *snippet_yml*, validate the contents
290    against the schema, and do other basic checks. Return the dict
291    of the resulting YAML data.'''
292
293    with open(snippet_yml, 'r') as f:
294        try:
295            snippet_data = yaml.safe_load(f.read())
296        except yaml.scanner.ScannerError:
297            _err(f'snippets file {snippet_yml} is invalid YAML')
298
299    def pykwalify_err(e):
300        return f'''\
301invalid {SNIPPET_YML} file: {snippet_yml}
302{textwrap.indent(e.msg, '  ')}
303'''
304
305    try:
306        pykwalify.core.Core(source_data=snippet_data,
307                            schema_data=SNIPPET_SCHEMA).validate()
308    except pykwalify.errors.PyKwalifyException as e:
309        _err(pykwalify_err(e))
310
311    name = snippet_data['name']
312    if not SNIPPET_NAME_RE.fullmatch(name):
313        _err(f"snippet file {snippet_yml}: invalid snippet name '{name}'; "
314             'snippet names must begin with a letter '
315             'or number, and may only contain letters, numbers, '
316             'dashes (-), and underscores (_)')
317
318    return snippet_data
319
320def check_for_errors(snippets: Snippets) -> None:
321    unknown_snippets = sorted(snippet for snippet in snippets.requested
322                              if snippet not in snippets)
323    if unknown_snippets:
324        all_snippets = '\n    '.join(sorted(snippets))
325        _err(f'''\
326snippets not found: {', '.join(unknown_snippets)}
327  Please choose from among the following snippets:
328    {all_snippets}''')
329
330def write_cmake_out(snippets: Snippets, cmake_out: Path) -> None:
331    '''Write a cmake include file to *cmake_out* which
332    reflects the information in *snippets*.
333
334    The contents of this file should be considered an implementation
335    detail and are not meant to be used outside of snippets.cmake.'''
336    if not cmake_out.parent.exists():
337        cmake_out.parent.mkdir()
338    with open(cmake_out, 'w', encoding="utf-8") as f:
339        SnippetToCMakePrinter(snippets, f).print_cmake()
340
341def main():
342    args = parse_args()
343    setup_logging()
344    try:
345        snippets = process_snippets(args)
346        check_for_errors(snippets)
347    except SnippetsError as e:
348        LOG.critical(e.msg)
349        sys.exit(1)
350    write_cmake_out(snippets, args.cmake_out)
351
352if __name__ == "__main__":
353    main()
354