1#!/usr/bin/env python3
2#
3# Copyright (c) 2022, Nordic Semiconductor ASA
4#
5# SPDX-License-Identifier: Apache-2.0
6
7'''Internal snippets tool.
8This is part of the build system's support for snippets.
9It is not meant for use outside of the build system.
10
11Output CMake variables:
12
13- SNIPPET_NAMES: CMake list of discovered snippet names
14- SNIPPET_FOUND_{snippet}: one per discovered snippet
15'''
16
17from collections import defaultdict, UserDict
18from dataclasses import dataclass, field
19from pathlib import Path, PurePosixPath
20from typing import Dict, Iterable, List, Set
21import argparse
22import logging
23import os
24import pykwalify.core
25import pykwalify.errors
26import re
27import sys
28import textwrap
29import yaml
30import platform
31
32# Marker type for an 'append:' configuration. Maps variables
33# to the list of values to append to them.
34Appends = Dict[str, List[str]]
35
36def _new_append():
37    return defaultdict(list)
38
39def _new_board2appends():
40    return defaultdict(_new_append)
41
42@dataclass
43class Snippet:
44    '''Class for keeping track of all the settings discovered for an
45    individual snippet.'''
46
47    name: str
48    appends: Appends = field(default_factory=_new_append)
49    board2appends: Dict[str, Appends] = field(default_factory=_new_board2appends)
50
51    def process_data(self, pathobj: Path, snippet_data: dict):
52        '''Process the data in a snippet.yml file, after it is loaded into a
53        python object and validated by pykwalify.'''
54        def append_value(variable, value):
55            if variable in ('EXTRA_DTC_OVERLAY_FILE', 'EXTRA_CONF_FILE'):
56                path = pathobj.parent / value
57                if not path.is_file():
58                    _err(f'snippet file {pathobj}: {variable}: file not found: {path}')
59                return f'"{path}"'
60            _err(f'unknown append variable: {variable}')
61
62        for variable, value in snippet_data.get('append', {}).items():
63            self.appends[variable].append(append_value(variable, value))
64        for board, settings in snippet_data.get('boards', {}).items():
65            if board.startswith('/') and not board.endswith('/'):
66                _err(f"snippet file {pathobj}: board {board} starts with '/', so "
67                     "it must end with '/' to use a regular expression")
68            for variable, value in settings.get('append', {}).items():
69                self.board2appends[board][variable].append(
70                    append_value(variable, value))
71
72class Snippets(UserDict):
73    '''Type for all the information we have discovered about all snippets.
74    As a dict, this maps a snippet's name onto the Snippet object.
75    Any additional global attributes about all snippets go here as
76    instance attributes.'''
77
78    def __init__(self, requested: Iterable[str] = None):
79        super().__init__()
80        self.paths: Set[Path] = set()
81        self.requested: List[str] = list(requested or [])
82
83class SnippetsError(Exception):
84    '''Class for signalling expected errors'''
85
86    def __init__(self, msg):
87        self.msg = msg
88
89class SnippetToCMakePrinter:
90    '''Helper class for printing a Snippets's semantics to a .cmake
91    include file for use by snippets.cmake.'''
92
93    def __init__(self, snippets: Snippets, out_file):
94        self.snippets = snippets
95        self.out_file = out_file
96        self.section = '#' * 79
97
98    def print_cmake(self):
99        '''Print to the output file provided to the constructor.'''
100        # TODO: add source file info
101        snippets = self.snippets
102        snippet_names = sorted(snippets.keys())
103
104        if platform.system() == "Windows":
105            # Change to linux-style paths for windows to avoid cmake escape character code issues
106            snippets.paths = set(map(lambda x: str(PurePosixPath(x)), snippets.paths))
107
108            for this_snippet in snippets:
109                for snippet_append in (snippets[this_snippet].appends):
110                    snippets[this_snippet].appends[snippet_append] = \
111                        set(map(lambda x: str(x.replace("\\", "/")), \
112                            snippets[this_snippet].appends[snippet_append]))
113
114        snippet_path_list = " ".join(
115            sorted(f'"{path}"' for path in snippets.paths))
116
117        self.print('''\
118# WARNING. THIS FILE IS AUTO-GENERATED. DO NOT MODIFY!
119#
120# This file contains build system settings derived from your snippets.
121# Its contents are an implementation detail that should not be used outside
122# of Zephyr's snippets CMake module.
123#
124# See the Snippets guide in the Zephyr documentation for more information.
125''')
126
127        self.print(f'''\
128{self.section}
129# Global information about all snippets.
130
131# The name of every snippet that was discovered.
132set(SNIPPET_NAMES {' '.join(f'"{name}"' for name in snippet_names)})
133# The paths to all the snippet.yml files. One snippet
134# can have multiple snippet.yml files.
135set(SNIPPET_PATHS {snippet_path_list})
136
137# Create variable scope for snippets build variables
138zephyr_create_scope(snippets)
139''')
140
141        for snippet_name in snippets.requested:
142            self.print_cmake_for(snippets[snippet_name])
143            self.print()
144
145    def print_cmake_for(self, snippet: Snippet):
146        self.print(f'''\
147{self.section}
148# Snippet '{snippet.name}'
149
150# Common variable appends.''')
151        self.print_appends(snippet.appends, 0)
152        for board, appends in snippet.board2appends.items():
153            self.print_appends_for_board(board, appends)
154
155    def print_appends_for_board(self, board: str, appends: Appends):
156        if board.startswith('/'):
157            board_re = board[1:-1]
158            self.print(f'''\
159# Appends for board regular expression '{board_re}'
160if("${{BOARD}}" MATCHES "^{board_re}$")''')
161        else:
162            self.print(f'''\
163# Appends for board '{board}'
164if("${{BOARD}}" STREQUAL "{board}")''')
165        self.print_appends(appends, 1)
166        self.print('endif()')
167
168    def print_appends(self, appends: Appends, indent: int):
169        space = '  ' * indent
170        for name, values in appends.items():
171            for value in values:
172                self.print(f'{space}zephyr_set({name} {value} SCOPE snippets APPEND)')
173
174    def print(self, *args, **kwargs):
175        kwargs['file'] = self.out_file
176        print(*args, **kwargs)
177
178# Name of the file containing the pykwalify schema for snippet.yml
179# files.
180SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'snippet-schema.yml')
181with open(SCHEMA_PATH, 'r') as f:
182    SNIPPET_SCHEMA = yaml.safe_load(f.read())
183
184# The name of the file which contains metadata about the snippets
185# being defined in a directory.
186SNIPPET_YML = 'snippet.yml'
187
188# Regular expression for validating snippet names. Snippet names must
189# begin with an alphanumeric character, and may contain alphanumeric
190# characters or underscores. This is intentionally very restrictive to
191# keep things consistent and easy to type and remember. We can relax
192# this a bit later if needed.
193SNIPPET_NAME_RE = re.compile('[A-Za-z0-9][A-Za-z0-9_-]*')
194
195# Logger for this module.
196LOG = logging.getLogger('snippets')
197
198def _err(msg):
199    raise SnippetsError(f'error: {msg}')
200
201def parse_args():
202    parser = argparse.ArgumentParser(description='snippets helper',
203                                     allow_abbrev=False)
204    parser.add_argument('--snippet-root', default=[], action='append', type=Path,
205                        help='''a SNIPPET_ROOT element; may be given
206                        multiple times''')
207    parser.add_argument('--snippet', dest='snippets', default=[], action='append',
208                        help='''a SNIPPET element; may be given
209                        multiple times''')
210    parser.add_argument('--cmake-out', type=Path,
211                        help='''file to write cmake output to; include()
212                        this file after calling this script''')
213    return parser.parse_args()
214
215def setup_logging():
216    # Silence validation errors from pykwalify, which are logged at
217    # logging.ERROR level. We want to handle those ourselves as
218    # needed.
219    logging.getLogger('pykwalify').setLevel(logging.CRITICAL)
220    logging.basicConfig(level=logging.INFO,
221                        format='  %(name)s: %(message)s')
222
223def process_snippets(args: argparse.Namespace) -> Snippets:
224    '''Process snippet.yml files under each *snippet_root*
225    by recursive search. Return a Snippets object describing
226    the results of the search.
227    '''
228    # This will contain information about all the snippets
229    # we discover in each snippet_root element.
230    snippets = Snippets(requested=args.snippets)
231
232    # Process each path in snippet_root in order, adjusting
233    # snippets as needed for each one.
234    for root in args.snippet_root:
235        process_snippets_in(root, snippets)
236
237    return snippets
238
239def process_snippets_in(root_dir: Path, snippets: Snippets) -> None:
240    '''Process snippet.yml files in *root_dir*,
241    updating *snippets* as needed.'''
242
243    if not root_dir.is_dir():
244        LOG.warning(f'SNIPPET_ROOT {root_dir} '
245                    'is not a directory; ignoring it')
246        return
247
248    snippets_dir = root_dir / 'snippets'
249    if not snippets_dir.is_dir():
250        return
251
252    for dirpath, _, filenames in os.walk(snippets_dir):
253        if SNIPPET_YML not in filenames:
254            continue
255
256        snippet_yml = Path(dirpath) / SNIPPET_YML
257        snippet_data = load_snippet_yml(snippet_yml)
258        name = snippet_data['name']
259        if name not in snippets:
260            snippets[name] = Snippet(name=name)
261        snippets[name].process_data(snippet_yml, snippet_data)
262        snippets.paths.add(snippet_yml)
263
264def load_snippet_yml(snippet_yml: Path) -> dict:
265    '''Load a snippet.yml file *snippet_yml*, validate the contents
266    against the schema, and do other basic checks. Return the dict
267    of the resulting YAML data.'''
268
269    with open(snippet_yml, 'r') as f:
270        try:
271            snippet_data = yaml.safe_load(f.read())
272        except yaml.scanner.ScannerError:
273            _err(f'snippets file {snippet_yml} is invalid YAML')
274
275    def pykwalify_err(e):
276        return f'''\
277invalid {SNIPPET_YML} file: {snippet_yml}
278{textwrap.indent(e.msg, '  ')}
279'''
280
281    try:
282        pykwalify.core.Core(source_data=snippet_data,
283                            schema_data=SNIPPET_SCHEMA).validate()
284    except pykwalify.errors.PyKwalifyException as e:
285        _err(pykwalify_err(e))
286
287    name = snippet_data['name']
288    if not SNIPPET_NAME_RE.fullmatch(name):
289        _err(f"snippet file {snippet_yml}: invalid snippet name '{name}'; "
290             'snippet names must begin with a letter '
291             'or number, and may only contain letters, numbers, '
292             'dashes (-), and underscores (_)')
293
294    return snippet_data
295
296def check_for_errors(snippets: Snippets) -> None:
297    unknown_snippets = sorted(snippet for snippet in snippets.requested
298                              if snippet not in snippets)
299    if unknown_snippets:
300        all_snippets = '\n    '.join(sorted(snippets))
301        _err(f'''\
302snippets not found: {', '.join(unknown_snippets)}
303  Please choose from among the following snippets:
304    {all_snippets}''')
305
306def write_cmake_out(snippets: Snippets, cmake_out: Path) -> None:
307    '''Write a cmake include file to *cmake_out* which
308    reflects the information in *snippets*.
309
310    The contents of this file should be considered an implementation
311    detail and are not meant to be used outside of snippets.cmake.'''
312    if not cmake_out.parent.exists():
313        cmake_out.parent.mkdir()
314    with open(cmake_out, 'w', encoding="utf-8") as f:
315        SnippetToCMakePrinter(snippets, f).print_cmake()
316
317def main():
318    args = parse_args()
319    setup_logging()
320    try:
321        snippets = process_snippets(args)
322        check_for_errors(snippets)
323    except SnippetsError as e:
324        LOG.critical(e.msg)
325        sys.exit(1)
326    write_cmake_out(snippets, args.cmake_out)
327
328if __name__ == "__main__":
329    main()
330