1#!/usr/bin/env python3
2#
3# Copyright (c) 2019, Nordic Semiconductor ASA
4#
5# SPDX-License-Identifier: Apache-2.0
6
7'''Tool for parsing a list of projects to determine if they are Zephyr
8projects. If no projects are given then the output from `west list` will be
9used as project list.
10
11Include file is generated for Kconfig using --kconfig-out.
12A <name>:<path> text file is generated for use with CMake using --cmake-out.
13
14Using --twister-out <filename> an argument file for twister script will
15be generated which would point to test and sample roots available in modules
16that can be included during a twister run. This allows testing code
17maintained in modules in addition to what is available in the main Zephyr tree.
18'''
19
20import argparse
21import os
22import re
23import sys
24import yaml
25import pykwalify.core
26from pathlib import Path, PurePath
27from collections import namedtuple
28
29METADATA_SCHEMA = '''
30## A pykwalify schema for basic validation of the structure of a
31## metadata YAML file.
32##
33# The zephyr/module.yml file is a simple list of key value pairs to be used by
34# the build system.
35type: map
36mapping:
37  name:
38    required: false
39    type: str
40  build:
41    required: false
42    type: map
43    mapping:
44      cmake:
45        required: false
46        type: str
47      kconfig:
48        required: false
49        type: str
50      cmake-ext:
51        required: false
52        type: bool
53        default: false
54      kconfig-ext:
55        required: false
56        type: bool
57        default: false
58      depends:
59        required: false
60        type: seq
61        sequence:
62          - type: str
63      settings:
64        required: false
65        type: map
66        mapping:
67          board_root:
68            required: false
69            type: str
70          dts_root:
71            required: false
72            type: str
73          soc_root:
74            required: false
75            type: str
76          arch_root:
77            required: false
78            type: str
79          module_ext_root:
80            required: false
81            type: str
82  tests:
83    required: false
84    type: seq
85    sequence:
86      - type: str
87  samples:
88    required: false
89    type: seq
90    sequence:
91      - type: str
92  boards:
93    required: false
94    type: seq
95    sequence:
96      - type: str
97'''
98
99schema = yaml.safe_load(METADATA_SCHEMA)
100
101
102def validate_setting(setting, module_path, filename=None):
103    if setting is not None:
104        if filename is not None:
105            checkfile = os.path.join(module_path, setting, filename)
106        else:
107            checkfile = os.path.join(module_path, setting)
108        if not os.path.isfile(checkfile):
109            return False
110    return True
111
112
113def process_module(module):
114    module_path = PurePath(module)
115    module_yml = module_path.joinpath('zephyr/module.yml')
116
117    # The input is a module if zephyr/module.yml is a valid yaml file
118    # or if both zephyr/CMakeLists.txt and zephyr/Kconfig are present.
119
120    if Path(module_yml).is_file():
121        with Path(module_yml).open('r') as f:
122            meta = yaml.safe_load(f.read())
123
124        try:
125            pykwalify.core.Core(source_data=meta, schema_data=schema)\
126                .validate()
127        except pykwalify.errors.SchemaError as e:
128            sys.exit('ERROR: Malformed "build" section in file: {}\n{}'
129                     .format(module_yml.as_posix(), e))
130
131        meta['name'] = meta.get('name', module_path.name)
132        meta['name-sanitized'] = re.sub('[^a-zA-Z0-9]', '_', meta['name'])
133        return meta
134
135    if Path(module_path.joinpath('zephyr/CMakeLists.txt')).is_file() and \
136       Path(module_path.joinpath('zephyr/Kconfig')).is_file():
137        return {'name': module_path.name,
138                'name-sanitized': re.sub('[^a-zA-Z0-9]', '_', module_path.name),
139                'build': {'cmake': 'zephyr', 'kconfig': 'zephyr/Kconfig'}}
140
141    return None
142
143
144def process_cmake(module, meta):
145    section = meta.get('build', dict())
146    module_path = PurePath(module)
147    module_yml = module_path.joinpath('zephyr/module.yml')
148
149    cmake_extern = section.get('cmake-ext', False)
150    if cmake_extern:
151        return('\"{}\":\"{}\":\"{}\"\n'
152               .format(meta['name'],
153                       module_path.as_posix(),
154                       "${ZEPHYR_" + meta['name-sanitized'].upper() + "_CMAKE_DIR}"))
155
156    cmake_setting = section.get('cmake', None)
157    if not validate_setting(cmake_setting, module, 'CMakeLists.txt'):
158        sys.exit('ERROR: "cmake" key in {} has folder value "{}" which '
159                 'does not contain a CMakeLists.txt file.'
160                 .format(module_yml.as_posix(), cmake_setting))
161
162    cmake_path = os.path.join(module, cmake_setting or 'zephyr')
163    cmake_file = os.path.join(cmake_path, 'CMakeLists.txt')
164    if os.path.isfile(cmake_file):
165        return('\"{}\":\"{}\":\"{}\"\n'
166               .format(meta['name'],
167                       module_path.as_posix(),
168                       Path(cmake_path).resolve().as_posix()))
169    else:
170        return('\"{}\":\"{}\":\"\"\n'
171               .format(meta['name'],
172                       module_path.as_posix()))
173
174
175def process_settings(module, meta):
176    section = meta.get('build', dict())
177    build_settings = section.get('settings', None)
178    out_text = ""
179
180    if build_settings is not None:
181        for root in ['board', 'dts', 'soc', 'arch', 'module_ext']:
182            setting = build_settings.get(root+'_root', None)
183            if setting is not None:
184                root_path = PurePath(module) / setting
185                out_text += f'"{root.upper()}_ROOT":'
186                out_text += f'"{root_path.as_posix()}"\n'
187
188    return out_text
189
190
191def kconfig_snippet(meta, path, kconfig_file=None):
192    name = meta['name']
193    name_sanitized = meta['name-sanitized']
194
195    snippet = (f'menu "{name} ({path})"',
196               f'osource "{kconfig_file.resolve().as_posix()}"' if kconfig_file
197               else f'osource "$(ZEPHYR_{name_sanitized.upper()}_KCONFIG)"',
198               f'config ZEPHYR_{name_sanitized.upper()}_MODULE',
199               '	bool',
200               '	default y',
201               'endmenu\n')
202    return '\n'.join(snippet)
203
204
205def process_kconfig(module, meta):
206    section = meta.get('build', dict())
207    module_path = PurePath(module)
208    module_yml = module_path.joinpath('zephyr/module.yml')
209    kconfig_extern = section.get('kconfig-ext', False)
210    if kconfig_extern:
211        return kconfig_snippet(meta, module_path)
212
213    kconfig_setting = section.get('kconfig', None)
214    if not validate_setting(kconfig_setting, module):
215        sys.exit('ERROR: "kconfig" key in {} has value "{}" which does '
216                 'not point to a valid Kconfig file.'
217                 .format(module_yml, kconfig_setting))
218
219    kconfig_file = os.path.join(module, kconfig_setting or 'zephyr/Kconfig')
220    if os.path.isfile(kconfig_file):
221        return kconfig_snippet(meta, module_path, Path(kconfig_file))
222    else:
223        return ""
224
225
226def process_twister(module, meta):
227
228    out = ""
229    tests = meta.get('tests', [])
230    samples = meta.get('samples', [])
231    boards = meta.get('boards', [])
232
233    for pth in tests + samples:
234        if pth:
235            dir = os.path.join(module, pth)
236            out += '-T\n{}\n'.format(PurePath(os.path.abspath(dir))
237                                     .as_posix())
238
239    for pth in boards:
240        if pth:
241            dir = os.path.join(module, pth)
242            out += '--board-root\n{}\n'.format(PurePath(os.path.abspath(dir))
243                                               .as_posix())
244
245    return out
246
247
248def parse_modules(zephyr_base, modules=None, extra_modules=None):
249    if modules is None:
250        # West is imported here, as it is optional
251        # (and thus maybe not installed)
252        # if user is providing a specific modules list.
253        from west.manifest import Manifest
254        from west.util import WestNotFound
255        from west.version import __version__ as WestVersion
256        from packaging import version
257        try:
258            manifest = Manifest.from_file()
259            if version.parse(WestVersion) >= version.parse('0.9.0'):
260                projects = [p.posixpath for p in manifest.get_projects([])
261                            if manifest.is_active(p)]
262            else:
263                projects = [p.posixpath for p in manifest.get_projects([])]
264        except WestNotFound:
265            # Only accept WestNotFound, meaning we are not in a west
266            # workspace. Such setup is allowed, as west may be installed
267            # but the project is not required to use west.
268            projects = []
269    else:
270        projects = modules.copy()
271
272    if extra_modules is None:
273        extra_modules = []
274
275    projects += extra_modules
276
277    Module = namedtuple('Module', ['project', 'meta', 'depends'])
278    # dep_modules is a list of all modules that has an unresolved dependency
279    dep_modules = []
280    # start_modules is a list modules with no depends left (no incoming edge)
281    start_modules = []
282    # sorted_modules is a topological sorted list of the modules
283    sorted_modules = []
284
285    for project in projects:
286        # Avoid including Zephyr base project as module.
287        if project == zephyr_base:
288            continue
289
290        meta = process_module(project)
291        if meta:
292            section = meta.get('build', dict())
293            deps = section.get('depends', [])
294            if not deps:
295                start_modules.append(Module(project, meta, []))
296            else:
297                dep_modules.append(Module(project, meta, deps))
298        elif project in extra_modules:
299            sys.exit(f'{project}, given in ZEPHYR_EXTRA_MODULES, '
300                     'is not a valid zephyr module')
301
302    # This will do a topological sort to ensure the modules are ordered
303    # according to dependency settings.
304    while start_modules:
305        node = start_modules.pop(0)
306        sorted_modules.append(node)
307        node_name = node.meta['name']
308        to_remove = []
309        for module in dep_modules:
310            if node_name in module.depends:
311                module.depends.remove(node_name)
312                if not module.depends:
313                    start_modules.append(module)
314                    to_remove.append(module)
315        for module in to_remove:
316            dep_modules.remove(module)
317
318    if dep_modules:
319        # If there are any modules with unresolved dependencies, then the
320        # modules contains unmet or cyclic dependencies. Error out.
321        error = 'Unmet or cyclic dependencies in modules:\n'
322        for module in dep_modules:
323            error += f'{module.project} depends on: {module.depends}\n'
324        sys.exit(error)
325
326    return sorted_modules
327
328
329def main():
330    parser = argparse.ArgumentParser(description='''
331    Process a list of projects and create Kconfig / CMake include files for
332    projects which are also a Zephyr module''')
333
334    parser.add_argument('--kconfig-out',
335                        help="""File to write with resulting KConfig import
336                             statements.""")
337    parser.add_argument('--twister-out',
338                        help="""File to write with resulting twister
339                             parameters.""")
340    parser.add_argument('--cmake-out',
341                        help="""File to write with resulting <name>:<path>
342                             values to use for including in CMake""")
343    parser.add_argument('--settings-out',
344                        help="""File to write with resulting <name>:<value>
345                             values to use for including in CMake""")
346    parser.add_argument('-m', '--modules', nargs='+',
347                        help="""List of modules to parse instead of using `west
348                             list`""")
349    parser.add_argument('-x', '--extra-modules', nargs='+',
350                        help='List of extra modules to parse')
351    parser.add_argument('-z', '--zephyr-base',
352                        help='Path to zephyr repository')
353    args = parser.parse_args()
354
355    kconfig = ""
356    cmake = ""
357    settings = ""
358    twister = ""
359
360    modules = parse_modules(args.zephyr_base, args.modules, args.extra_modules)
361
362    for module in modules:
363        kconfig += process_kconfig(module.project, module.meta)
364        cmake += process_cmake(module.project, module.meta)
365        settings += process_settings(module.project, module.meta)
366        twister += process_twister(module.project, module.meta)
367
368    if args.kconfig_out:
369        with open(args.kconfig_out, 'w', encoding="utf-8") as fp:
370            fp.write(kconfig)
371
372    if args.cmake_out:
373        with open(args.cmake_out, 'w', encoding="utf-8") as fp:
374            fp.write(cmake)
375
376    if args.settings_out:
377        with open(args.settings_out, 'w', encoding="utf-8") as fp:
378            fp.write('''\
379# WARNING. THIS FILE IS AUTO-GENERATED. DO NOT MODIFY!
380#
381# This file contains build system settings derived from your modules.
382#
383# Modules may be set via ZEPHYR_MODULES, ZEPHYR_EXTRA_MODULES,
384# and/or the west manifest file.
385#
386# See the Modules guide for more information.
387''')
388            fp.write(settings)
389
390    if args.twister_out:
391        with open(args.twister_out, 'w', encoding="utf-8") as fp:
392            fp.write(twister)
393
394
395if __name__ == "__main__":
396    main()
397