1#!/usr/bin/env python3
2#
3# Copyright (c) 2019, Nordic Semiconductor ASA
4#
5# SPDX-License-Identifier: Apache-2.0
6
7'''Tool for parsing a list of projects to determine if they are Zephyr
8projects. If no projects are given then the output from `west list` will be
9used as project list.
10
11Include file is generated for Kconfig using --kconfig-out.
12A <name>:<path> text file is generated for use with CMake using --cmake-out.
13
14Using --twister-out <filename> an argument file for twister script will
15be generated which would point to test and sample roots available in modules
16that can be included during a twister run. This allows testing code
17maintained in modules in addition to what is available in the main Zephyr tree.
18'''
19
20import argparse
21import hashlib
22import os
23import re
24import subprocess
25import sys
26import yaml
27import pykwalify.core
28from pathlib import Path, PurePath
29from collections import namedtuple
30
31METADATA_SCHEMA = '''
32## A pykwalify schema for basic validation of the structure of a
33## metadata YAML file.
34##
35# The zephyr/module.yml file is a simple list of key value pairs to be used by
36# the build system.
37type: map
38mapping:
39  name:
40    required: false
41    type: str
42  build:
43    required: false
44    type: map
45    mapping:
46      cmake:
47        required: false
48        type: str
49      kconfig:
50        required: false
51        type: str
52      cmake-ext:
53        required: false
54        type: bool
55        default: false
56      kconfig-ext:
57        required: false
58        type: bool
59        default: false
60      sysbuild-cmake:
61        required: false
62        type: str
63      sysbuild-kconfig:
64        required: false
65        type: str
66      sysbuild-cmake-ext:
67        required: false
68        type: bool
69        default: false
70      sysbuild-kconfig-ext:
71        required: false
72        type: bool
73        default: false
74      depends:
75        required: false
76        type: seq
77        sequence:
78          - type: str
79      settings:
80        required: false
81        type: map
82        mapping:
83          board_root:
84            required: false
85            type: str
86          dts_root:
87            required: false
88            type: str
89          snippet_root:
90            required: false
91            type: str
92          soc_root:
93            required: false
94            type: str
95          arch_root:
96            required: false
97            type: str
98          module_ext_root:
99            required: false
100            type: str
101          sca_root:
102            required: false
103            type: str
104  tests:
105    required: false
106    type: seq
107    sequence:
108      - type: str
109  samples:
110    required: false
111    type: seq
112    sequence:
113      - type: str
114  boards:
115    required: false
116    type: seq
117    sequence:
118      - type: str
119  blobs:
120    required: false
121    type: seq
122    sequence:
123      - type: map
124        mapping:
125          path:
126            required: true
127            type: str
128          sha256:
129            required: true
130            type: str
131          type:
132            required: true
133            type: str
134            enum: ['img', 'lib']
135          version:
136            required: true
137            type: str
138          license-path:
139            required: true
140            type: str
141          url:
142            required: true
143            type: str
144          description:
145            required: true
146            type: str
147          doc-url:
148            required: false
149            type: str
150'''
151
152MODULE_YML_PATH = PurePath('zephyr/module.yml')
153# Path to the blobs folder
154MODULE_BLOBS_PATH = PurePath('zephyr/blobs')
155BLOB_PRESENT = 'A'
156BLOB_NOT_PRESENT = 'D'
157BLOB_OUTDATED = 'M'
158
159schema = yaml.safe_load(METADATA_SCHEMA)
160
161
162def validate_setting(setting, module_path, filename=None):
163    if setting is not None:
164        if filename is not None:
165            checkfile = Path(module_path) / setting / filename
166        else:
167            checkfile = Path(module_path) / setting
168        if not checkfile.resolve().is_file():
169            return False
170    return True
171
172
173def process_module(module):
174    module_path = PurePath(module)
175
176    # The input is a module if zephyr/module.{yml,yaml} is a valid yaml file
177    # or if both zephyr/CMakeLists.txt and zephyr/Kconfig are present.
178
179    for module_yml in [module_path / MODULE_YML_PATH,
180                       module_path / MODULE_YML_PATH.with_suffix('.yaml')]:
181        if Path(module_yml).is_file():
182            with Path(module_yml).open('r') as f:
183                meta = yaml.safe_load(f.read())
184
185            try:
186                pykwalify.core.Core(source_data=meta, schema_data=schema)\
187                    .validate()
188            except pykwalify.errors.SchemaError as e:
189                sys.exit('ERROR: Malformed "build" section in file: {}\n{}'
190                        .format(module_yml.as_posix(), e))
191
192            meta['name'] = meta.get('name', module_path.name)
193            meta['name-sanitized'] = re.sub('[^a-zA-Z0-9]', '_', meta['name'])
194            return meta
195
196    if Path(module_path.joinpath('zephyr/CMakeLists.txt')).is_file() and \
197       Path(module_path.joinpath('zephyr/Kconfig')).is_file():
198        return {'name': module_path.name,
199                'name-sanitized': re.sub('[^a-zA-Z0-9]', '_', module_path.name),
200                'build': {'cmake': 'zephyr', 'kconfig': 'zephyr/Kconfig'}}
201
202    return None
203
204
205def process_cmake(module, meta):
206    section = meta.get('build', dict())
207    module_path = PurePath(module)
208    module_yml = module_path.joinpath('zephyr/module.yml')
209
210    cmake_extern = section.get('cmake-ext', False)
211    if cmake_extern:
212        return('\"{}\":\"{}\":\"{}\"\n'
213               .format(meta['name'],
214                       module_path.as_posix(),
215                       "${ZEPHYR_" + meta['name-sanitized'].upper() + "_CMAKE_DIR}"))
216
217    cmake_setting = section.get('cmake', None)
218    if not validate_setting(cmake_setting, module, 'CMakeLists.txt'):
219        sys.exit('ERROR: "cmake" key in {} has folder value "{}" which '
220                 'does not contain a CMakeLists.txt file.'
221                 .format(module_yml.as_posix(), cmake_setting))
222
223    cmake_path = os.path.join(module, cmake_setting or 'zephyr')
224    cmake_file = os.path.join(cmake_path, 'CMakeLists.txt')
225    if os.path.isfile(cmake_file):
226        return('\"{}\":\"{}\":\"{}\"\n'
227               .format(meta['name'],
228                       module_path.as_posix(),
229                       Path(cmake_path).resolve().as_posix()))
230    else:
231        return('\"{}\":\"{}\":\"\"\n'
232               .format(meta['name'],
233                       module_path.as_posix()))
234
235
236def process_sysbuildcmake(module, meta):
237    section = meta.get('build', dict())
238    module_path = PurePath(module)
239    module_yml = module_path.joinpath('zephyr/module.yml')
240
241    cmake_extern = section.get('sysbuild-cmake-ext', False)
242    if cmake_extern:
243        return('\"{}\":\"{}\":\"{}\"\n'
244               .format(meta['name'],
245                       module_path.as_posix(),
246                       "${SYSBUILD_" + meta['name-sanitized'].upper() + "_CMAKE_DIR}"))
247
248    cmake_setting = section.get('sysbuild-cmake', None)
249    if not validate_setting(cmake_setting, module, 'CMakeLists.txt'):
250        sys.exit('ERROR: "cmake" key in {} has folder value "{}" which '
251                 'does not contain a CMakeLists.txt file.'
252                 .format(module_yml.as_posix(), cmake_setting))
253
254    if cmake_setting is None:
255        return ""
256
257    cmake_path = os.path.join(module, cmake_setting or 'zephyr')
258    cmake_file = os.path.join(cmake_path, 'CMakeLists.txt')
259    if os.path.isfile(cmake_file):
260        return('\"{}\":\"{}\":\"{}\"\n'
261               .format(meta['name'],
262                       module_path.as_posix(),
263                       Path(cmake_path).resolve().as_posix()))
264    else:
265        return('\"{}\":\"{}\":\"\"\n'
266               .format(meta['name'],
267                       module_path.as_posix()))
268
269
270def process_settings(module, meta):
271    section = meta.get('build', dict())
272    build_settings = section.get('settings', None)
273    out_text = ""
274
275    if build_settings is not None:
276        for root in ['board', 'dts', 'snippet', 'soc', 'arch', 'module_ext', 'sca']:
277            setting = build_settings.get(root+'_root', None)
278            if setting is not None:
279                root_path = PurePath(module) / setting
280                out_text += f'"{root.upper()}_ROOT":'
281                out_text += f'"{root_path.as_posix()}"\n'
282
283    return out_text
284
285
286def get_blob_status(path, sha256):
287    if not path.is_file():
288        return BLOB_NOT_PRESENT
289    with path.open('rb') as f:
290        m = hashlib.sha256()
291        m.update(f.read())
292        if sha256.lower() == m.hexdigest():
293            return BLOB_PRESENT
294        else:
295            return BLOB_OUTDATED
296
297
298def process_blobs(module, meta):
299    blobs = []
300    mblobs = meta.get('blobs', None)
301    if not mblobs:
302        return blobs
303
304    blobs_path = Path(module) / MODULE_BLOBS_PATH
305    for blob in mblobs:
306        blob['module'] = meta.get('name', None)
307        blob['abspath'] = blobs_path / Path(blob['path'])
308        blob['status'] = get_blob_status(blob['abspath'], blob['sha256'])
309        blobs.append(blob)
310
311    return blobs
312
313
314def kconfig_snippet(meta, path, kconfig_file=None, blobs=False, sysbuild=False):
315    name = meta['name']
316    name_sanitized = meta['name-sanitized']
317
318    snippet = [f'menu "{name} ({path.as_posix()})"',
319               f'osource "{kconfig_file.resolve().as_posix()}"' if kconfig_file
320               else f'osource "$(SYSBUILD_{name_sanitized.upper()}_KCONFIG)"' if sysbuild is True
321	       else f'osource "$(ZEPHYR_{name_sanitized.upper()}_KCONFIG)"',
322               f'config ZEPHYR_{name_sanitized.upper()}_MODULE',
323               '	bool',
324               '	default y',
325               'endmenu\n']
326
327    if blobs:
328        snippet.insert(-1, '	select TAINT_BLOBS')
329    return '\n'.join(snippet)
330
331
332def process_kconfig(module, meta):
333    blobs = process_blobs(module, meta)
334    taint_blobs = len(tuple(filter(lambda b: b['status'] != 'D', blobs))) != 0
335    section = meta.get('build', dict())
336    module_path = PurePath(module)
337    module_yml = module_path.joinpath('zephyr/module.yml')
338    kconfig_extern = section.get('kconfig-ext', False)
339    if kconfig_extern:
340        return kconfig_snippet(meta, module_path, blobs=taint_blobs)
341
342    kconfig_setting = section.get('kconfig', None)
343    if not validate_setting(kconfig_setting, module):
344        sys.exit('ERROR: "kconfig" key in {} has value "{}" which does '
345                 'not point to a valid Kconfig file.'
346                 .format(module_yml, kconfig_setting))
347
348    kconfig_file = os.path.join(module, kconfig_setting or 'zephyr/Kconfig')
349    if os.path.isfile(kconfig_file):
350        return kconfig_snippet(meta, module_path, Path(kconfig_file),
351                               blobs=taint_blobs)
352    else:
353        return ""
354
355
356def process_sysbuildkconfig(module, meta):
357    section = meta.get('build', dict())
358    module_path = PurePath(module)
359    module_yml = module_path.joinpath('zephyr/module.yml')
360    kconfig_extern = section.get('sysbuild-kconfig-ext', False)
361    if kconfig_extern:
362        return kconfig_snippet(meta, module_path, sysbuild=True)
363
364    kconfig_setting = section.get('sysbuild-kconfig', None)
365    if not validate_setting(kconfig_setting, module):
366        sys.exit('ERROR: "kconfig" key in {} has value "{}" which does '
367                 'not point to a valid Kconfig file.'
368                 .format(module_yml, kconfig_setting))
369
370    if kconfig_setting is None:
371        return ""
372
373    kconfig_file = os.path.join(module, kconfig_setting)
374    if os.path.isfile(kconfig_file):
375        return kconfig_snippet(meta, module_path, Path(kconfig_file))
376    else:
377        return ""
378
379
380def process_twister(module, meta):
381
382    out = ""
383    tests = meta.get('tests', [])
384    samples = meta.get('samples', [])
385    boards = meta.get('boards', [])
386
387    for pth in tests + samples:
388        if pth:
389            dir = os.path.join(module, pth)
390            out += '-T\n{}\n'.format(PurePath(os.path.abspath(dir))
391                                     .as_posix())
392
393    for pth in boards:
394        if pth:
395            dir = os.path.join(module, pth)
396            out += '--board-root\n{}\n'.format(PurePath(os.path.abspath(dir))
397                                               .as_posix())
398
399    return out
400
401
402def process_meta(zephyr_base, west_projs, modules, extra_modules=None,
403                 propagate_state=False):
404    # Process zephyr_base, projects, and modules and create a dictionary
405    # with meta information for each input.
406    #
407    # The dictionary will contain meta info in the following lists:
408    # - zephyr:        path and revision
409    # - modules:       name, path, and revision
410    # - west-projects: path and revision
411    #
412    # returns the dictionary with said lists
413
414    meta = {'zephyr': None, 'modules': None, 'workspace': None}
415
416    workspace_dirty = False
417    workspace_extra = extra_modules is not None
418    workspace_off = False
419
420    def git_revision(path):
421        rc = subprocess.Popen(['git', 'rev-parse', '--is-inside-work-tree'],
422                              stdout=subprocess.PIPE,
423                              stderr=subprocess.PIPE,
424                              cwd=path).wait()
425        if rc == 0:
426            # A git repo.
427            popen = subprocess.Popen(['git', 'rev-parse', 'HEAD'],
428                                     stdout=subprocess.PIPE,
429                                     stderr=subprocess.PIPE,
430                                     cwd=path)
431            stdout, stderr = popen.communicate()
432            stdout = stdout.decode('utf-8')
433
434            if not (popen.returncode or stderr):
435                revision = stdout.rstrip()
436
437                rc = subprocess.Popen(['git', 'diff-index', '--quiet', 'HEAD',
438                                       '--'],
439                                      stdout=None,
440                                      stderr=None,
441                                      cwd=path).wait()
442                if rc:
443                    return revision + '-dirty', True
444                return revision, False
445        return None, False
446
447    zephyr_revision, zephyr_dirty = git_revision(zephyr_base)
448    zephyr_project = {'path': zephyr_base,
449                      'revision': zephyr_revision}
450    meta['zephyr'] = zephyr_project
451    meta['workspace'] = {}
452    workspace_dirty |= zephyr_dirty
453
454    if west_projs is not None:
455        from west.manifest import MANIFEST_REV_BRANCH
456        projects = west_projs['projects']
457        meta_projects = []
458
459        # Special treatment of manifest project.
460        manifest_proj_path = PurePath(projects[0].posixpath).as_posix()
461        manifest_revision, manifest_dirty = git_revision(manifest_proj_path)
462        workspace_dirty |= manifest_dirty
463        manifest_project = {'path': manifest_proj_path,
464                            'revision': manifest_revision}
465        meta_projects.append(manifest_project)
466
467        for project in projects[1:]:
468            project_path = PurePath(project.posixpath).as_posix()
469            revision, dirty = git_revision(project_path)
470            workspace_dirty |= dirty
471            if project.sha(MANIFEST_REV_BRANCH) != revision:
472                revision += '-off'
473                workspace_off = True
474            meta_project = {'path': project_path,
475                            'revision': revision}
476            meta_projects.append(meta_project)
477
478        meta.update({'west': {'manifest': west_projs['manifest_path'],
479                              'projects': meta_projects}})
480        meta['workspace'].update({'off': workspace_off})
481
482    meta_projects = []
483    for module in modules:
484        module_path = PurePath(module.project).as_posix()
485        revision, dirty = git_revision(module_path)
486        workspace_dirty |= dirty
487        meta_project = {'name': module.meta['name'],
488                        'path': module_path,
489                        'revision': revision}
490        meta_projects.append(meta_project)
491    meta['modules'] = meta_projects
492
493    meta['workspace'].update({'dirty': workspace_dirty,
494                              'extra': workspace_extra})
495
496    if propagate_state:
497        if workspace_dirty and not zephyr_dirty:
498            zephyr_revision += '-dirty'
499        if workspace_extra:
500            zephyr_revision += '-extra'
501        if workspace_off:
502            zephyr_revision += '-off'
503        zephyr_project.update({'revision': zephyr_revision})
504
505        if west_projs is not None:
506            if workspace_dirty and not manifest_dirty:
507                manifest_revision += '-dirty'
508            if workspace_extra:
509                manifest_revision += '-extra'
510            if workspace_off:
511                manifest_revision += '-off'
512            manifest_project.update({'revision': manifest_revision})
513
514    return meta
515
516
517def west_projects(manifest = None):
518    manifest_path = None
519    projects = []
520    # West is imported here, as it is optional
521    # (and thus maybe not installed)
522    # if user is providing a specific modules list.
523    try:
524        from west.manifest import Manifest
525    except ImportError:
526        # West is not installed, so don't return any projects.
527        return None
528
529    # If west *is* installed, we need all of the following imports to
530    # work. West versions that are excessively old may fail here:
531    # west.configuration.MalformedConfig was
532    # west.manifest.MalformedConfig until west v0.14.0, for example.
533    # These should be hard errors.
534    from west.manifest import \
535        ManifestImportFailed, MalformedManifest, ManifestVersionError
536    from west.configuration import MalformedConfig
537    from west.util import WestNotFound
538    from west.version import __version__ as WestVersion
539
540    from packaging import version
541    try:
542        if not manifest:
543            manifest = Manifest.from_file()
544        if version.parse(WestVersion) >= version.parse('0.9.0'):
545            projects = [p for p in manifest.get_projects([])
546                        if manifest.is_active(p)]
547        else:
548            projects = manifest.get_projects([])
549        manifest_path = manifest.path
550        return {'manifest_path': manifest_path, 'projects': projects}
551    except (ManifestImportFailed, MalformedManifest,
552            ManifestVersionError, MalformedConfig) as e:
553        sys.exit(f'ERROR: {e}')
554    except WestNotFound:
555        # Only accept WestNotFound, meaning we are not in a west
556        # workspace. Such setup is allowed, as west may be installed
557        # but the project is not required to use west.
558        pass
559    return None
560
561
562def parse_modules(zephyr_base, manifest=None, west_projs=None, modules=None,
563                  extra_modules=None):
564
565    if modules is None:
566        west_projs = west_projs or west_projects(manifest)
567        modules = ([p.posixpath for p in west_projs['projects']]
568                   if west_projs else [])
569
570    if extra_modules is None:
571        extra_modules = []
572
573    Module = namedtuple('Module', ['project', 'meta', 'depends'])
574
575    all_modules_by_name = {}
576    # dep_modules is a list of all modules that has an unresolved dependency
577    dep_modules = []
578    # start_modules is a list modules with no depends left (no incoming edge)
579    start_modules = []
580    # sorted_modules is a topological sorted list of the modules
581    sorted_modules = []
582
583    for project in modules + extra_modules:
584        # Avoid including Zephyr base project as module.
585        if project == zephyr_base:
586            continue
587
588        meta = process_module(project)
589        if meta:
590            depends = meta.get('build', {}).get('depends', [])
591            all_modules_by_name[meta['name']] = Module(project, meta, depends)
592
593        elif project in extra_modules:
594            sys.exit(f'{project}, given in ZEPHYR_EXTRA_MODULES, '
595                     'is not a valid zephyr module')
596
597    for module in all_modules_by_name.values():
598        if not module.depends:
599            start_modules.append(module)
600        else:
601            dep_modules.append(module)
602
603    # This will do a topological sort to ensure the modules are ordered
604    # according to dependency settings.
605    while start_modules:
606        node = start_modules.pop(0)
607        sorted_modules.append(node)
608        node_name = node.meta['name']
609        to_remove = []
610        for module in dep_modules:
611            if node_name in module.depends:
612                module.depends.remove(node_name)
613                if not module.depends:
614                    start_modules.append(module)
615                    to_remove.append(module)
616        for module in to_remove:
617            dep_modules.remove(module)
618
619    if dep_modules:
620        # If there are any modules with unresolved dependencies, then the
621        # modules contains unmet or cyclic dependencies. Error out.
622        error = 'Unmet or cyclic dependencies in modules:\n'
623        for module in dep_modules:
624            error += f'{module.project} depends on: {module.depends}\n'
625        sys.exit(error)
626
627    return sorted_modules
628
629
630def main():
631    parser = argparse.ArgumentParser(description='''
632    Process a list of projects and create Kconfig / CMake include files for
633    projects which are also a Zephyr module''', allow_abbrev=False)
634
635    parser.add_argument('--kconfig-out',
636                        help="""File to write with resulting KConfig import
637                             statements.""")
638    parser.add_argument('--twister-out',
639                        help="""File to write with resulting twister
640                             parameters.""")
641    parser.add_argument('--cmake-out',
642                        help="""File to write with resulting <name>:<path>
643                             values to use for including in CMake""")
644    parser.add_argument('--sysbuild-kconfig-out',
645                        help="""File to write with resulting KConfig import
646                             statements.""")
647    parser.add_argument('--sysbuild-cmake-out',
648                        help="""File to write with resulting <name>:<path>
649                             values to use for including in CMake""")
650    parser.add_argument('--meta-out',
651                        help="""Write a build meta YaML file containing a list
652                             of Zephyr modules and west projects.
653                             If a module or project is also a git repository
654                             the current SHA revision will also be written.""")
655    parser.add_argument('--meta-state-propagate', action='store_true',
656                        help="""Propagate state of modules and west projects
657                             to the suffix of the Zephyr SHA and if west is
658                             used, to the suffix of the manifest SHA""")
659    parser.add_argument('--settings-out',
660                        help="""File to write with resulting <name>:<value>
661                             values to use for including in CMake""")
662    parser.add_argument('-m', '--modules', nargs='+',
663                        help="""List of modules to parse instead of using `west
664                             list`""")
665    parser.add_argument('-x', '--extra-modules', nargs='+',
666                        help='List of extra modules to parse')
667    parser.add_argument('-z', '--zephyr-base',
668                        help='Path to zephyr repository')
669    args = parser.parse_args()
670
671    kconfig = ""
672    cmake = ""
673    sysbuild_kconfig = ""
674    sysbuild_cmake = ""
675    settings = ""
676    twister = ""
677
678    west_projs = west_projects()
679    modules = parse_modules(args.zephyr_base, None, west_projs,
680                            args.modules, args.extra_modules)
681
682    for module in modules:
683        kconfig += process_kconfig(module.project, module.meta)
684        cmake += process_cmake(module.project, module.meta)
685        sysbuild_kconfig += process_sysbuildkconfig(module.project, module.meta)
686        sysbuild_cmake += process_sysbuildcmake(module.project, module.meta)
687        settings += process_settings(module.project, module.meta)
688        twister += process_twister(module.project, module.meta)
689
690    if args.kconfig_out:
691        with open(args.kconfig_out, 'w', encoding="utf-8") as fp:
692            fp.write(kconfig)
693
694    if args.cmake_out:
695        with open(args.cmake_out, 'w', encoding="utf-8") as fp:
696            fp.write(cmake)
697
698    if args.sysbuild_kconfig_out:
699        with open(args.sysbuild_kconfig_out, 'w', encoding="utf-8") as fp:
700            fp.write(sysbuild_kconfig)
701
702    if args.sysbuild_cmake_out:
703        with open(args.sysbuild_cmake_out, 'w', encoding="utf-8") as fp:
704            fp.write(sysbuild_cmake)
705
706    if args.settings_out:
707        with open(args.settings_out, 'w', encoding="utf-8") as fp:
708            fp.write('''\
709# WARNING. THIS FILE IS AUTO-GENERATED. DO NOT MODIFY!
710#
711# This file contains build system settings derived from your modules.
712#
713# Modules may be set via ZEPHYR_MODULES, ZEPHYR_EXTRA_MODULES,
714# and/or the west manifest file.
715#
716# See the Modules guide for more information.
717''')
718            fp.write(settings)
719
720    if args.twister_out:
721        with open(args.twister_out, 'w', encoding="utf-8") as fp:
722            fp.write(twister)
723
724    if args.meta_out:
725        meta = process_meta(args.zephyr_base, west_projs, modules,
726                            args.extra_modules, args.meta_state_propagate)
727
728        with open(args.meta_out, 'w', encoding="utf-8") as fp:
729            fp.write(yaml.dump(meta))
730
731
732if __name__ == "__main__":
733    main()
734