1#!/usr/bin/env python3
2#
3# Copyright (c) 2022, Nordic Semiconductor ASA
4#
5# SPDX-License-Identifier: Apache-2.0
6
7'''Internal snippets tool.
8This is part of the build system's support for snippets.
9It is not meant for use outside of the build system.
10
11Output CMake variables:
12
13- SNIPPET_NAMES: CMake list of discovered snippet names
14- SNIPPET_FOUND_{snippet}: one per discovered snippet
15'''
16
17from collections import defaultdict, UserDict
18from dataclasses import dataclass, field
19from pathlib import Path, PurePosixPath
20from typing import Dict, Iterable, List, Set
21import argparse
22import logging
23import os
24import pykwalify.core
25import pykwalify.errors
26import re
27import sys
28import textwrap
29import yaml
30import platform
31
32# Marker type for an 'append:' configuration. Maps variables
33# to the list of values to append to them.
34Appends = Dict[str, List[str]]
35
36def _new_append():
37    return defaultdict(list)
38
39def _new_board2appends():
40    return defaultdict(_new_append)
41
42@dataclass
43class Snippet:
44    '''Class for keeping track of all the settings discovered for an
45    individual snippet.'''
46
47    name: str
48    appends: Appends = field(default_factory=_new_append)
49    board2appends: Dict[str, Appends] = field(default_factory=_new_board2appends)
50
51    def process_data(self, pathobj: Path, snippet_data: dict):
52        '''Process the data in a snippet.yml file, after it is loaded into a
53        python object and validated by pykwalify.'''
54        def append_value(variable, value):
55            if variable in ('EXTRA_DTC_OVERLAY_FILE', 'EXTRA_CONF_FILE'):
56                path = pathobj.parent / value
57                if not path.is_file():
58                    _err(f'snippet file {pathobj}: {variable}: file not found: {path}')
59                return f'"{path.as_posix()}"'
60            if variable in ('DTS_EXTRA_CPPFLAGS'):
61                return f'"{value}"'
62            _err(f'unknown append variable: {variable}')
63
64        for variable, value in snippet_data.get('append', {}).items():
65            self.appends[variable].append(append_value(variable, value))
66        for board, settings in snippet_data.get('boards', {}).items():
67            if board.startswith('/') and not board.endswith('/'):
68                _err(f"snippet file {pathobj}: board {board} starts with '/', so "
69                     "it must end with '/' to use a regular expression")
70            for variable, value in settings.get('append', {}).items():
71                self.board2appends[board][variable].append(
72                    append_value(variable, value))
73
74class Snippets(UserDict):
75    '''Type for all the information we have discovered about all snippets.
76    As a dict, this maps a snippet's name onto the Snippet object.
77    Any additional global attributes about all snippets go here as
78    instance attributes.'''
79
80    def __init__(self, requested: Iterable[str] = None):
81        super().__init__()
82        self.paths: Set[Path] = set()
83        self.requested: List[str] = list(requested or [])
84
85class SnippetsError(Exception):
86    '''Class for signalling expected errors'''
87
88    def __init__(self, msg):
89        self.msg = msg
90
91class SnippetToCMakePrinter:
92    '''Helper class for printing a Snippets's semantics to a .cmake
93    include file for use by snippets.cmake.'''
94
95    def __init__(self, snippets: Snippets, out_file):
96        self.snippets = snippets
97        self.out_file = out_file
98        self.section = '#' * 79
99
100    def print_cmake(self):
101        '''Print to the output file provided to the constructor.'''
102        # TODO: add source file info
103        snippets = self.snippets
104        snippet_names = sorted(snippets.keys())
105
106        if platform.system() == "Windows":
107            # Change to linux-style paths for windows to avoid cmake escape character code issues
108            snippets.paths = set(map(lambda x: str(PurePosixPath(x)), snippets.paths))
109
110            for this_snippet in snippets:
111                for snippet_append in (snippets[this_snippet].appends):
112                    snippets[this_snippet].appends[snippet_append] = \
113                        set(map(lambda x: str(x.replace("\\", "/")), \
114                            snippets[this_snippet].appends[snippet_append]))
115
116        snippet_path_list = " ".join(
117            sorted(f'"{path}"' for path in snippets.paths))
118
119        self.print('''\
120# WARNING. THIS FILE IS AUTO-GENERATED. DO NOT MODIFY!
121#
122# This file contains build system settings derived from your snippets.
123# Its contents are an implementation detail that should not be used outside
124# of Zephyr's snippets CMake module.
125#
126# See the Snippets guide in the Zephyr documentation for more information.
127''')
128
129        self.print(f'''\
130{self.section}
131# Global information about all snippets.
132
133# The name of every snippet that was discovered.
134set(SNIPPET_NAMES {' '.join(f'"{name}"' for name in snippet_names)})
135# The paths to all the snippet.yml files. One snippet
136# can have multiple snippet.yml files.
137set(SNIPPET_PATHS {snippet_path_list})
138
139# Create variable scope for snippets build variables
140zephyr_create_scope(snippets)
141''')
142
143        for snippet_name in snippets.requested:
144            self.print_cmake_for(snippets[snippet_name])
145            self.print()
146
147    def print_cmake_for(self, snippet: Snippet):
148        self.print(f'''\
149{self.section}
150# Snippet '{snippet.name}'
151
152# Common variable appends.''')
153        self.print_appends(snippet.appends, 0)
154        for board, appends in snippet.board2appends.items():
155            self.print_appends_for_board(board, appends)
156
157    def print_appends_for_board(self, board: str, appends: Appends):
158        if board.startswith('/'):
159            board_re = board[1:-1]
160            self.print(f'''\
161# Appends for board regular expression '{board_re}'
162if("${{BOARD}}${{BOARD_QUALIFIERS}}" MATCHES "^{board_re}$")''')
163        else:
164            self.print(f'''\
165# Appends for board '{board}'
166if("${{BOARD}}${{BOARD_QUALIFIERS}}" STREQUAL "{board}")''')
167        self.print_appends(appends, 1)
168        self.print('endif()')
169
170    def print_appends(self, appends: Appends, indent: int):
171        space = '  ' * indent
172        for name, values in appends.items():
173            for value in values:
174                self.print(f'{space}zephyr_set({name} {value} SCOPE snippets APPEND)')
175
176    def print(self, *args, **kwargs):
177        kwargs['file'] = self.out_file
178        print(*args, **kwargs)
179
180# Name of the file containing the pykwalify schema for snippet.yml
181# files.
182SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'snippet-schema.yml')
183with open(SCHEMA_PATH, 'r') as f:
184    SNIPPET_SCHEMA = yaml.safe_load(f.read())
185
186# The name of the file which contains metadata about the snippets
187# being defined in a directory.
188SNIPPET_YML = 'snippet.yml'
189
190# Regular expression for validating snippet names. Snippet names must
191# begin with an alphanumeric character, and may contain alphanumeric
192# characters or underscores. This is intentionally very restrictive to
193# keep things consistent and easy to type and remember. We can relax
194# this a bit later if needed.
195SNIPPET_NAME_RE = re.compile('[A-Za-z0-9][A-Za-z0-9_-]*')
196
197# Logger for this module.
198LOG = logging.getLogger('snippets')
199
200def _err(msg):
201    raise SnippetsError(f'error: {msg}')
202
203def parse_args():
204    parser = argparse.ArgumentParser(description='snippets helper',
205                                     allow_abbrev=False)
206    parser.add_argument('--snippet-root', default=[], action='append', type=Path,
207                        help='''a SNIPPET_ROOT element; may be given
208                        multiple times''')
209    parser.add_argument('--snippet', dest='snippets', default=[], action='append',
210                        help='''a SNIPPET element; may be given
211                        multiple times''')
212    parser.add_argument('--cmake-out', type=Path,
213                        help='''file to write cmake output to; include()
214                        this file after calling this script''')
215    return parser.parse_args()
216
217def setup_logging():
218    # Silence validation errors from pykwalify, which are logged at
219    # logging.ERROR level. We want to handle those ourselves as
220    # needed.
221    logging.getLogger('pykwalify').setLevel(logging.CRITICAL)
222    logging.basicConfig(level=logging.INFO,
223                        format='  %(name)s: %(message)s')
224
225def process_snippets(args: argparse.Namespace) -> Snippets:
226    '''Process snippet.yml files under each *snippet_root*
227    by recursive search. Return a Snippets object describing
228    the results of the search.
229    '''
230    # This will contain information about all the snippets
231    # we discover in each snippet_root element.
232    snippets = Snippets(requested=args.snippets)
233
234    # Process each path in snippet_root in order, adjusting
235    # snippets as needed for each one.
236    for root in args.snippet_root:
237        process_snippets_in(root, snippets)
238
239    return snippets
240
241def find_snippets_in_roots(requested_snippets, snippet_roots) -> Snippets:
242    '''Process snippet.yml files under each *snippet_root*
243    by recursive search. Return a Snippets object describing
244    the results of the search.
245    '''
246    # This will contain information about all the snippets
247    # we discover in each snippet_root element.
248    snippets = Snippets(requested=requested_snippets)
249
250    # Process each path in snippet_root in order, adjusting
251    # snippets as needed for each one.
252    for root in snippet_roots:
253        process_snippets_in(root, snippets)
254
255    return snippets
256
257def process_snippets_in(root_dir: Path, snippets: Snippets) -> None:
258    '''Process snippet.yml files in *root_dir*,
259    updating *snippets* as needed.'''
260
261    if not root_dir.is_dir():
262        LOG.warning(f'SNIPPET_ROOT {root_dir} '
263                    'is not a directory; ignoring it')
264        return
265
266    snippets_dir = root_dir / 'snippets'
267    if not snippets_dir.is_dir():
268        return
269
270    for dirpath, _, filenames in os.walk(snippets_dir):
271        if SNIPPET_YML not in filenames:
272            continue
273
274        snippet_yml = Path(dirpath) / SNIPPET_YML
275        snippet_data = load_snippet_yml(snippet_yml)
276        name = snippet_data['name']
277        if name not in snippets:
278            snippets[name] = Snippet(name=name)
279        snippets[name].process_data(snippet_yml, snippet_data)
280        snippets.paths.add(snippet_yml)
281
282def load_snippet_yml(snippet_yml: Path) -> dict:
283    '''Load a snippet.yml file *snippet_yml*, validate the contents
284    against the schema, and do other basic checks. Return the dict
285    of the resulting YAML data.'''
286
287    with open(snippet_yml, 'r') as f:
288        try:
289            snippet_data = yaml.safe_load(f.read())
290        except yaml.scanner.ScannerError:
291            _err(f'snippets file {snippet_yml} is invalid YAML')
292
293    def pykwalify_err(e):
294        return f'''\
295invalid {SNIPPET_YML} file: {snippet_yml}
296{textwrap.indent(e.msg, '  ')}
297'''
298
299    try:
300        pykwalify.core.Core(source_data=snippet_data,
301                            schema_data=SNIPPET_SCHEMA).validate()
302    except pykwalify.errors.PyKwalifyException as e:
303        _err(pykwalify_err(e))
304
305    name = snippet_data['name']
306    if not SNIPPET_NAME_RE.fullmatch(name):
307        _err(f"snippet file {snippet_yml}: invalid snippet name '{name}'; "
308             'snippet names must begin with a letter '
309             'or number, and may only contain letters, numbers, '
310             'dashes (-), and underscores (_)')
311
312    return snippet_data
313
314def check_for_errors(snippets: Snippets) -> None:
315    unknown_snippets = sorted(snippet for snippet in snippets.requested
316                              if snippet not in snippets)
317    if unknown_snippets:
318        all_snippets = '\n    '.join(sorted(snippets))
319        _err(f'''\
320snippets not found: {', '.join(unknown_snippets)}
321  Please choose from among the following snippets:
322    {all_snippets}''')
323
324def write_cmake_out(snippets: Snippets, cmake_out: Path) -> None:
325    '''Write a cmake include file to *cmake_out* which
326    reflects the information in *snippets*.
327
328    The contents of this file should be considered an implementation
329    detail and are not meant to be used outside of snippets.cmake.'''
330    if not cmake_out.parent.exists():
331        cmake_out.parent.mkdir()
332    with open(cmake_out, 'w', encoding="utf-8") as f:
333        SnippetToCMakePrinter(snippets, f).print_cmake()
334
335def main():
336    args = parse_args()
337    setup_logging()
338    try:
339        snippets = process_snippets(args)
340        check_for_errors(snippets)
341    except SnippetsError as e:
342        LOG.critical(e.msg)
343        sys.exit(1)
344    write_cmake_out(snippets, args.cmake_out)
345
346if __name__ == "__main__":
347    main()
348