1#!/usr/bin/env python3
2#
3# Copyright (c) 2019, Nordic Semiconductor ASA
4#
5# SPDX-License-Identifier: Apache-2.0
6
7'''Tool for parsing a list of projects to determine if they are Zephyr
8projects. If no projects are given then the output from `west list` will be
9used as project list.
10
11Include file is generated for Kconfig using --kconfig-out.
12A <name>:<path> text file is generated for use with CMake using --cmake-out.
13
14Using --twister-out <filename> an argument file for twister script will
15be generated which would point to test and sample roots available in modules
16that can be included during a twister run. This allows testing code
17maintained in modules in addition to what is available in the main Zephyr tree.
18'''
19
20import argparse
21import hashlib
22import os
23import re
24import subprocess
25import sys
26import yaml
27import pykwalify.core
28from pathlib import Path, PurePath
29from collections import namedtuple
30
31METADATA_SCHEMA = '''
32## A pykwalify schema for basic validation of the structure of a
33## metadata YAML file.
34##
35# The zephyr/module.yml file is a simple list of key value pairs to be used by
36# the build system.
37type: map
38mapping:
39  name:
40    required: false
41    type: str
42  build:
43    required: false
44    type: map
45    mapping:
46      cmake:
47        required: false
48        type: str
49      kconfig:
50        required: false
51        type: str
52      cmake-ext:
53        required: false
54        type: bool
55        default: false
56      kconfig-ext:
57        required: false
58        type: bool
59        default: false
60      sysbuild-cmake:
61        required: false
62        type: str
63      sysbuild-kconfig:
64        required: false
65        type: str
66      sysbuild-cmake-ext:
67        required: false
68        type: bool
69        default: false
70      sysbuild-kconfig-ext:
71        required: false
72        type: bool
73        default: false
74      depends:
75        required: false
76        type: seq
77        sequence:
78          - type: str
79      settings:
80        required: false
81        type: map
82        mapping:
83          board_root:
84            required: false
85            type: str
86          dts_root:
87            required: false
88            type: str
89          snippet_root:
90            required: false
91            type: str
92          soc_root:
93            required: false
94            type: str
95          arch_root:
96            required: false
97            type: str
98          module_ext_root:
99            required: false
100            type: str
101          sca_root:
102            required: false
103            type: str
104  tests:
105    required: false
106    type: seq
107    sequence:
108      - type: str
109  samples:
110    required: false
111    type: seq
112    sequence:
113      - type: str
114  boards:
115    required: false
116    type: seq
117    sequence:
118      - type: str
119  blobs:
120    required: false
121    type: seq
122    sequence:
123      - type: map
124        mapping:
125          path:
126            required: true
127            type: str
128          sha256:
129            required: true
130            type: str
131          type:
132            required: true
133            type: str
134            enum: ['img', 'lib']
135          version:
136            required: true
137            type: str
138          license-path:
139            required: true
140            type: str
141          url:
142            required: true
143            type: str
144          description:
145            required: true
146            type: str
147          doc-url:
148            required: false
149            type: str
150'''
151
152MODULE_YML_PATH = PurePath('zephyr/module.yml')
153# Path to the blobs folder
154MODULE_BLOBS_PATH = PurePath('zephyr/blobs')
155BLOB_PRESENT = 'A'
156BLOB_NOT_PRESENT = 'D'
157BLOB_OUTDATED = 'M'
158
159schema = yaml.safe_load(METADATA_SCHEMA)
160
161
162def validate_setting(setting, module_path, filename=None):
163    if setting is not None:
164        if filename is not None:
165            checkfile = Path(module_path) / setting / filename
166        else:
167            checkfile = Path(module_path) / setting
168        if not checkfile.resolve().is_file():
169            return False
170    return True
171
172
173def process_module(module):
174    module_path = PurePath(module)
175
176    # The input is a module if zephyr/module.{yml,yaml} is a valid yaml file
177    # or if both zephyr/CMakeLists.txt and zephyr/Kconfig are present.
178
179    for module_yml in [module_path / MODULE_YML_PATH,
180                       module_path / MODULE_YML_PATH.with_suffix('.yaml')]:
181        if Path(module_yml).is_file():
182            with Path(module_yml).open('r') as f:
183                meta = yaml.safe_load(f.read())
184
185            try:
186                pykwalify.core.Core(source_data=meta, schema_data=schema)\
187                    .validate()
188            except pykwalify.errors.SchemaError as e:
189                sys.exit('ERROR: Malformed "build" section in file: {}\n{}'
190                        .format(module_yml.as_posix(), e))
191
192            meta['name'] = meta.get('name', module_path.name)
193            meta['name-sanitized'] = re.sub('[^a-zA-Z0-9]', '_', meta['name'])
194            return meta
195
196    if Path(module_path.joinpath('zephyr/CMakeLists.txt')).is_file() and \
197       Path(module_path.joinpath('zephyr/Kconfig')).is_file():
198        return {'name': module_path.name,
199                'name-sanitized': re.sub('[^a-zA-Z0-9]', '_', module_path.name),
200                'build': {'cmake': 'zephyr', 'kconfig': 'zephyr/Kconfig'}}
201
202    return None
203
204
205def process_cmake(module, meta):
206    section = meta.get('build', dict())
207    module_path = PurePath(module)
208    module_yml = module_path.joinpath('zephyr/module.yml')
209
210    cmake_extern = section.get('cmake-ext', False)
211    if cmake_extern:
212        return('\"{}\":\"{}\":\"{}\"\n'
213               .format(meta['name'],
214                       module_path.as_posix(),
215                       "${ZEPHYR_" + meta['name-sanitized'].upper() + "_CMAKE_DIR}"))
216
217    cmake_setting = section.get('cmake', None)
218    if not validate_setting(cmake_setting, module, 'CMakeLists.txt'):
219        sys.exit('ERROR: "cmake" key in {} has folder value "{}" which '
220                 'does not contain a CMakeLists.txt file.'
221                 .format(module_yml.as_posix(), cmake_setting))
222
223    cmake_path = os.path.join(module, cmake_setting or 'zephyr')
224    cmake_file = os.path.join(cmake_path, 'CMakeLists.txt')
225    if os.path.isfile(cmake_file):
226        return('\"{}\":\"{}\":\"{}\"\n'
227               .format(meta['name'],
228                       module_path.as_posix(),
229                       Path(cmake_path).resolve().as_posix()))
230    else:
231        return('\"{}\":\"{}\":\"\"\n'
232               .format(meta['name'],
233                       module_path.as_posix()))
234
235
236def process_sysbuildcmake(module, meta):
237    section = meta.get('build', dict())
238    module_path = PurePath(module)
239    module_yml = module_path.joinpath('zephyr/module.yml')
240
241    cmake_extern = section.get('sysbuild-cmake-ext', False)
242    if cmake_extern:
243        return('\"{}\":\"{}\":\"{}\"\n'
244               .format(meta['name'],
245                       module_path.as_posix(),
246                       "${SYSBUILD_" + meta['name-sanitized'].upper() + "_CMAKE_DIR}"))
247
248    cmake_setting = section.get('sysbuild-cmake', None)
249    if not validate_setting(cmake_setting, module, 'CMakeLists.txt'):
250        sys.exit('ERROR: "cmake" key in {} has folder value "{}" which '
251                 'does not contain a CMakeLists.txt file.'
252                 .format(module_yml.as_posix(), cmake_setting))
253
254    if cmake_setting is None:
255        return ""
256
257    cmake_path = os.path.join(module, cmake_setting or 'zephyr')
258    cmake_file = os.path.join(cmake_path, 'CMakeLists.txt')
259    if os.path.isfile(cmake_file):
260        return('\"{}\":\"{}\":\"{}\"\n'
261               .format(meta['name'],
262                       module_path.as_posix(),
263                       Path(cmake_path).resolve().as_posix()))
264    else:
265        return('\"{}\":\"{}\":\"\"\n'
266               .format(meta['name'],
267                       module_path.as_posix()))
268
269
270def process_settings(module, meta):
271    section = meta.get('build', dict())
272    build_settings = section.get('settings', None)
273    out_text = ""
274
275    if build_settings is not None:
276        for root in ['board', 'dts', 'snippet', 'soc', 'arch', 'module_ext', 'sca']:
277            setting = build_settings.get(root+'_root', None)
278            if setting is not None:
279                root_path = PurePath(module) / setting
280                out_text += f'"{root.upper()}_ROOT":'
281                out_text += f'"{root_path.as_posix()}"\n'
282
283    return out_text
284
285
286def get_blob_status(path, sha256):
287    if not path.is_file():
288        return BLOB_NOT_PRESENT
289    with path.open('rb') as f:
290        m = hashlib.sha256()
291        m.update(f.read())
292        if sha256.lower() == m.hexdigest():
293            return BLOB_PRESENT
294        else:
295            return BLOB_OUTDATED
296
297
298def process_blobs(module, meta):
299    blobs = []
300    mblobs = meta.get('blobs', None)
301    if not mblobs:
302        return blobs
303
304    blobs_path = Path(module) / MODULE_BLOBS_PATH
305    for blob in mblobs:
306        blob['module'] = meta.get('name', None)
307        blob['abspath'] = blobs_path / Path(blob['path'])
308        blob['status'] = get_blob_status(blob['abspath'], blob['sha256'])
309        blobs.append(blob)
310
311    return blobs
312
313
314def kconfig_snippet(meta, path, kconfig_file=None, blobs=False, sysbuild=False):
315    name = meta['name']
316    name_sanitized = meta['name-sanitized']
317
318    snippet = [f'menu "{name} ({path.as_posix()})"',
319               f'osource "{kconfig_file.resolve().as_posix()}"' if kconfig_file
320               else f'osource "$(SYSBUILD_{name_sanitized.upper()}_KCONFIG)"' if sysbuild is True
321	       else f'osource "$(ZEPHYR_{name_sanitized.upper()}_KCONFIG)"',
322               f'config ZEPHYR_{name_sanitized.upper()}_MODULE',
323               '	bool',
324               '	default y',
325               'endmenu\n']
326
327    if blobs:
328        snippet.insert(-1, '	select TAINT_BLOBS')
329    return '\n'.join(snippet)
330
331
332def process_kconfig(module, meta):
333    blobs = process_blobs(module, meta)
334    taint_blobs = len(tuple(filter(lambda b: b['status'] != 'D', blobs))) != 0
335    section = meta.get('build', dict())
336    module_path = PurePath(module)
337    module_yml = module_path.joinpath('zephyr/module.yml')
338    kconfig_extern = section.get('kconfig-ext', False)
339    if kconfig_extern:
340        return kconfig_snippet(meta, module_path, blobs=taint_blobs)
341
342    kconfig_setting = section.get('kconfig', None)
343    if not validate_setting(kconfig_setting, module):
344        sys.exit('ERROR: "kconfig" key in {} has value "{}" which does '
345                 'not point to a valid Kconfig file.'
346                 .format(module_yml, kconfig_setting))
347
348    kconfig_file = os.path.join(module, kconfig_setting or 'zephyr/Kconfig')
349    if os.path.isfile(kconfig_file):
350        return kconfig_snippet(meta, module_path, Path(kconfig_file),
351                               blobs=taint_blobs)
352    else:
353        name_sanitized = meta['name-sanitized']
354        return (f'config ZEPHYR_{name_sanitized.upper()}_MODULE\n'
355                f'   bool\n'
356                f'   default y\n')
357
358
359def process_sysbuildkconfig(module, meta):
360    section = meta.get('build', dict())
361    module_path = PurePath(module)
362    module_yml = module_path.joinpath('zephyr/module.yml')
363    kconfig_extern = section.get('sysbuild-kconfig-ext', False)
364    if kconfig_extern:
365        return kconfig_snippet(meta, module_path, sysbuild=True)
366
367    kconfig_setting = section.get('sysbuild-kconfig', None)
368    if not validate_setting(kconfig_setting, module):
369        sys.exit('ERROR: "kconfig" key in {} has value "{}" which does '
370                 'not point to a valid Kconfig file.'
371                 .format(module_yml, kconfig_setting))
372
373    if kconfig_setting is None:
374        return ""
375
376    kconfig_file = os.path.join(module, kconfig_setting)
377    if os.path.isfile(kconfig_file):
378        return kconfig_snippet(meta, module_path, Path(kconfig_file))
379    else:
380        return ""
381
382
383def process_twister(module, meta):
384
385    out = ""
386    tests = meta.get('tests', [])
387    samples = meta.get('samples', [])
388    boards = meta.get('boards', [])
389
390    for pth in tests + samples:
391        if pth:
392            dir = os.path.join(module, pth)
393            out += '-T\n{}\n'.format(PurePath(os.path.abspath(dir))
394                                     .as_posix())
395
396    for pth in boards:
397        if pth:
398            dir = os.path.join(module, pth)
399            out += '--board-root\n{}\n'.format(PurePath(os.path.abspath(dir))
400                                               .as_posix())
401
402    return out
403
404
405def process_meta(zephyr_base, west_projs, modules, extra_modules=None,
406                 propagate_state=False):
407    # Process zephyr_base, projects, and modules and create a dictionary
408    # with meta information for each input.
409    #
410    # The dictionary will contain meta info in the following lists:
411    # - zephyr:        path and revision
412    # - modules:       name, path, and revision
413    # - west-projects: path and revision
414    #
415    # returns the dictionary with said lists
416
417    meta = {'zephyr': None, 'modules': None, 'workspace': None}
418
419    workspace_dirty = False
420    workspace_extra = extra_modules is not None
421    workspace_off = False
422
423    def git_revision(path):
424        rc = subprocess.Popen(['git', 'rev-parse', '--is-inside-work-tree'],
425                              stdout=subprocess.PIPE,
426                              stderr=subprocess.PIPE,
427                              cwd=path).wait()
428        if rc == 0:
429            # A git repo.
430            popen = subprocess.Popen(['git', 'rev-parse', 'HEAD'],
431                                     stdout=subprocess.PIPE,
432                                     stderr=subprocess.PIPE,
433                                     cwd=path)
434            stdout, stderr = popen.communicate()
435            stdout = stdout.decode('utf-8')
436
437            if not (popen.returncode or stderr):
438                revision = stdout.rstrip()
439
440                rc = subprocess.Popen(['git', 'diff-index', '--quiet', 'HEAD',
441                                       '--'],
442                                      stdout=None,
443                                      stderr=None,
444                                      cwd=path).wait()
445                if rc:
446                    return revision + '-dirty', True
447                return revision, False
448        return None, False
449
450    zephyr_revision, zephyr_dirty = git_revision(zephyr_base)
451    zephyr_project = {'path': zephyr_base,
452                      'revision': zephyr_revision}
453    meta['zephyr'] = zephyr_project
454    meta['workspace'] = {}
455    workspace_dirty |= zephyr_dirty
456
457    if west_projs is not None:
458        from west.manifest import MANIFEST_REV_BRANCH
459        projects = west_projs['projects']
460        meta_projects = []
461
462        # Special treatment of manifest project.
463        manifest_proj_path = PurePath(projects[0].posixpath).as_posix()
464        manifest_revision, manifest_dirty = git_revision(manifest_proj_path)
465        workspace_dirty |= manifest_dirty
466        manifest_project = {'path': manifest_proj_path,
467                            'revision': manifest_revision}
468        meta_projects.append(manifest_project)
469
470        for project in projects[1:]:
471            project_path = PurePath(project.posixpath).as_posix()
472            revision, dirty = git_revision(project_path)
473            workspace_dirty |= dirty
474            if project.sha(MANIFEST_REV_BRANCH) != revision:
475                revision += '-off'
476                workspace_off = True
477            meta_project = {'path': project_path,
478                            'revision': revision}
479            meta_projects.append(meta_project)
480
481        meta.update({'west': {'manifest': west_projs['manifest_path'],
482                              'projects': meta_projects}})
483        meta['workspace'].update({'off': workspace_off})
484
485    meta_projects = []
486    for module in modules:
487        module_path = PurePath(module.project).as_posix()
488        revision, dirty = git_revision(module_path)
489        workspace_dirty |= dirty
490        meta_project = {'name': module.meta['name'],
491                        'path': module_path,
492                        'revision': revision}
493        meta_projects.append(meta_project)
494    meta['modules'] = meta_projects
495
496    meta['workspace'].update({'dirty': workspace_dirty,
497                              'extra': workspace_extra})
498
499    if propagate_state:
500        if workspace_dirty and not zephyr_dirty:
501            zephyr_revision += '-dirty'
502        if workspace_extra:
503            zephyr_revision += '-extra'
504        if workspace_off:
505            zephyr_revision += '-off'
506        zephyr_project.update({'revision': zephyr_revision})
507
508        if west_projs is not None:
509            if workspace_dirty and not manifest_dirty:
510                manifest_revision += '-dirty'
511            if workspace_extra:
512                manifest_revision += '-extra'
513            if workspace_off:
514                manifest_revision += '-off'
515            manifest_project.update({'revision': manifest_revision})
516
517    return meta
518
519
520def west_projects(manifest = None):
521    manifest_path = None
522    projects = []
523    # West is imported here, as it is optional
524    # (and thus maybe not installed)
525    # if user is providing a specific modules list.
526    try:
527        from west.manifest import Manifest
528    except ImportError:
529        # West is not installed, so don't return any projects.
530        return None
531
532    # If west *is* installed, we need all of the following imports to
533    # work. West versions that are excessively old may fail here:
534    # west.configuration.MalformedConfig was
535    # west.manifest.MalformedConfig until west v0.14.0, for example.
536    # These should be hard errors.
537    from west.manifest import \
538        ManifestImportFailed, MalformedManifest, ManifestVersionError
539    from west.configuration import MalformedConfig
540    from west.util import WestNotFound
541    from west.version import __version__ as WestVersion
542
543    from packaging import version
544    try:
545        if not manifest:
546            manifest = Manifest.from_file()
547        if version.parse(WestVersion) >= version.parse('0.9.0'):
548            projects = [p for p in manifest.get_projects([])
549                        if manifest.is_active(p)]
550        else:
551            projects = manifest.get_projects([])
552        manifest_path = manifest.path
553        return {'manifest_path': manifest_path, 'projects': projects}
554    except (ManifestImportFailed, MalformedManifest,
555            ManifestVersionError, MalformedConfig) as e:
556        sys.exit(f'ERROR: {e}')
557    except WestNotFound:
558        # Only accept WestNotFound, meaning we are not in a west
559        # workspace. Such setup is allowed, as west may be installed
560        # but the project is not required to use west.
561        pass
562    return None
563
564
565def parse_modules(zephyr_base, manifest=None, west_projs=None, modules=None,
566                  extra_modules=None):
567
568    if modules is None:
569        west_projs = west_projs or west_projects(manifest)
570        modules = ([p.posixpath for p in west_projs['projects']]
571                   if west_projs else [])
572
573    if extra_modules is None:
574        extra_modules = []
575
576    Module = namedtuple('Module', ['project', 'meta', 'depends'])
577
578    all_modules_by_name = {}
579    # dep_modules is a list of all modules that has an unresolved dependency
580    dep_modules = []
581    # start_modules is a list modules with no depends left (no incoming edge)
582    start_modules = []
583    # sorted_modules is a topological sorted list of the modules
584    sorted_modules = []
585
586    for project in modules + extra_modules:
587        # Avoid including Zephyr base project as module.
588        if project == zephyr_base:
589            continue
590
591        meta = process_module(project)
592        if meta:
593            depends = meta.get('build', {}).get('depends', [])
594            all_modules_by_name[meta['name']] = Module(project, meta, depends)
595
596        elif project in extra_modules:
597            sys.exit(f'{project}, given in ZEPHYR_EXTRA_MODULES, '
598                     'is not a valid zephyr module')
599
600    for module in all_modules_by_name.values():
601        if not module.depends:
602            start_modules.append(module)
603        else:
604            dep_modules.append(module)
605
606    # This will do a topological sort to ensure the modules are ordered
607    # according to dependency settings.
608    while start_modules:
609        node = start_modules.pop(0)
610        sorted_modules.append(node)
611        node_name = node.meta['name']
612        to_remove = []
613        for module in dep_modules:
614            if node_name in module.depends:
615                module.depends.remove(node_name)
616                if not module.depends:
617                    start_modules.append(module)
618                    to_remove.append(module)
619        for module in to_remove:
620            dep_modules.remove(module)
621
622    if dep_modules:
623        # If there are any modules with unresolved dependencies, then the
624        # modules contains unmet or cyclic dependencies. Error out.
625        error = 'Unmet or cyclic dependencies in modules:\n'
626        for module in dep_modules:
627            error += f'{module.project} depends on: {module.depends}\n'
628        sys.exit(error)
629
630    return sorted_modules
631
632
633def main():
634    parser = argparse.ArgumentParser(description='''
635    Process a list of projects and create Kconfig / CMake include files for
636    projects which are also a Zephyr module''', allow_abbrev=False)
637
638    parser.add_argument('--kconfig-out',
639                        help="""File to write with resulting KConfig import
640                             statements.""")
641    parser.add_argument('--twister-out',
642                        help="""File to write with resulting twister
643                             parameters.""")
644    parser.add_argument('--cmake-out',
645                        help="""File to write with resulting <name>:<path>
646                             values to use for including in CMake""")
647    parser.add_argument('--sysbuild-kconfig-out',
648                        help="""File to write with resulting KConfig import
649                             statements.""")
650    parser.add_argument('--sysbuild-cmake-out',
651                        help="""File to write with resulting <name>:<path>
652                             values to use for including in CMake""")
653    parser.add_argument('--meta-out',
654                        help="""Write a build meta YaML file containing a list
655                             of Zephyr modules and west projects.
656                             If a module or project is also a git repository
657                             the current SHA revision will also be written.""")
658    parser.add_argument('--meta-state-propagate', action='store_true',
659                        help="""Propagate state of modules and west projects
660                             to the suffix of the Zephyr SHA and if west is
661                             used, to the suffix of the manifest SHA""")
662    parser.add_argument('--settings-out',
663                        help="""File to write with resulting <name>:<value>
664                             values to use for including in CMake""")
665    parser.add_argument('-m', '--modules', nargs='+',
666                        help="""List of modules to parse instead of using `west
667                             list`""")
668    parser.add_argument('-x', '--extra-modules', nargs='+',
669                        help='List of extra modules to parse')
670    parser.add_argument('-z', '--zephyr-base',
671                        help='Path to zephyr repository')
672    args = parser.parse_args()
673
674    kconfig = ""
675    cmake = ""
676    sysbuild_kconfig = ""
677    sysbuild_cmake = ""
678    settings = ""
679    twister = ""
680
681    west_projs = west_projects()
682    modules = parse_modules(args.zephyr_base, None, west_projs,
683                            args.modules, args.extra_modules)
684
685    for module in modules:
686        kconfig += process_kconfig(module.project, module.meta)
687        cmake += process_cmake(module.project, module.meta)
688        sysbuild_kconfig += process_sysbuildkconfig(module.project, module.meta)
689        sysbuild_cmake += process_sysbuildcmake(module.project, module.meta)
690        settings += process_settings(module.project, module.meta)
691        twister += process_twister(module.project, module.meta)
692
693    if args.kconfig_out:
694        with open(args.kconfig_out, 'w', encoding="utf-8") as fp:
695            fp.write(kconfig)
696
697    if args.cmake_out:
698        with open(args.cmake_out, 'w', encoding="utf-8") as fp:
699            fp.write(cmake)
700
701    if args.sysbuild_kconfig_out:
702        with open(args.sysbuild_kconfig_out, 'w', encoding="utf-8") as fp:
703            fp.write(sysbuild_kconfig)
704
705    if args.sysbuild_cmake_out:
706        with open(args.sysbuild_cmake_out, 'w', encoding="utf-8") as fp:
707            fp.write(sysbuild_cmake)
708
709    if args.settings_out:
710        with open(args.settings_out, 'w', encoding="utf-8") as fp:
711            fp.write('''\
712# WARNING. THIS FILE IS AUTO-GENERATED. DO NOT MODIFY!
713#
714# This file contains build system settings derived from your modules.
715#
716# Modules may be set via ZEPHYR_MODULES, ZEPHYR_EXTRA_MODULES,
717# and/or the west manifest file.
718#
719# See the Modules guide for more information.
720''')
721            fp.write(settings)
722
723    if args.twister_out:
724        with open(args.twister_out, 'w', encoding="utf-8") as fp:
725            fp.write(twister)
726
727    if args.meta_out:
728        meta = process_meta(args.zephyr_base, west_projs, modules,
729                            args.extra_modules, args.meta_state_propagate)
730
731        with open(args.meta_out, 'w', encoding="utf-8") as fp:
732            fp.write(yaml.dump(meta))
733
734
735if __name__ == "__main__":
736    main()
737