1#!/usr/bin/env python3 2# 3# Copyright (c) 2019, Nordic Semiconductor ASA 4# 5# SPDX-License-Identifier: Apache-2.0 6 7'''Tool for parsing a list of projects to determine if they are Zephyr 8projects. If no projects are given then the output from `west list` will be 9used as project list. 10 11Include file is generated for Kconfig using --kconfig-out. 12A <name>:<path> text file is generated for use with CMake using --cmake-out. 13 14Using --twister-out <filename> an argument file for twister script will 15be generated which would point to test and sample roots available in modules 16that can be included during a twister run. This allows testing code 17maintained in modules in addition to what is available in the main Zephyr tree. 18''' 19 20import argparse 21import hashlib 22import os 23import re 24import subprocess 25import sys 26import yaml 27import pykwalify.core 28from pathlib import Path, PurePath, PurePosixPath 29from collections import namedtuple 30 31try: 32 from yaml import CSafeLoader as SafeLoader 33except ImportError: 34 from yaml import SafeLoader 35 36METADATA_SCHEMA = ''' 37## A pykwalify schema for basic validation of the structure of a 38## metadata YAML file. 39## 40# The zephyr/module.yml file is a simple list of key value pairs to be used by 41# the build system. 42type: map 43mapping: 44 name: 45 required: false 46 type: str 47 build: 48 required: false 49 type: map 50 mapping: 51 cmake: 52 required: false 53 type: str 54 kconfig: 55 required: false 56 type: str 57 cmake-ext: 58 required: false 59 type: bool 60 default: false 61 kconfig-ext: 62 required: false 63 type: bool 64 default: false 65 sysbuild-cmake: 66 required: false 67 type: str 68 sysbuild-kconfig: 69 required: false 70 type: str 71 sysbuild-cmake-ext: 72 required: false 73 type: bool 74 default: false 75 sysbuild-kconfig-ext: 76 required: false 77 type: bool 78 default: false 79 depends: 80 required: false 81 type: seq 82 sequence: 83 - type: str 84 settings: 85 required: false 86 type: map 87 mapping: 88 board_root: 89 required: false 90 type: str 91 dts_root: 92 required: false 93 type: str 94 snippet_root: 95 required: false 96 type: str 97 soc_root: 98 required: false 99 type: str 100 arch_root: 101 required: false 102 type: str 103 module_ext_root: 104 required: false 105 type: str 106 sca_root: 107 required: false 108 type: str 109 tests: 110 required: false 111 type: seq 112 sequence: 113 - type: str 114 samples: 115 required: false 116 type: seq 117 sequence: 118 - type: str 119 boards: 120 required: false 121 type: seq 122 sequence: 123 - type: str 124 blobs: 125 required: false 126 type: seq 127 sequence: 128 - type: map 129 mapping: 130 path: 131 required: true 132 type: str 133 sha256: 134 required: true 135 type: str 136 type: 137 required: true 138 type: str 139 enum: ['img', 'lib'] 140 version: 141 required: true 142 type: str 143 license-path: 144 required: true 145 type: str 146 click-through: 147 required: false 148 type: bool 149 default: false 150 url: 151 required: true 152 type: str 153 description: 154 required: true 155 type: str 156 doc-url: 157 required: false 158 type: str 159 security: 160 required: false 161 type: map 162 mapping: 163 external-references: 164 required: false 165 type: seq 166 sequence: 167 - type: str 168 package-managers: 169 required: false 170 type: map 171 mapping: 172 pip: 173 required: false 174 type: map 175 mapping: 176 requirement-files: 177 required: false 178 type: seq 179 sequence: 180 - type: str 181 runners: 182 required: false 183 type: seq 184 sequence: 185 - type: map 186 mapping: 187 file: 188 required: true 189 type: str 190''' 191 192MODULE_YML_PATH = PurePath('zephyr/module.yml') 193# Path to the blobs folder 194MODULE_BLOBS_PATH = PurePath('zephyr/blobs') 195BLOB_PRESENT = 'A' 196BLOB_NOT_PRESENT = 'D' 197BLOB_OUTDATED = 'M' 198 199schema = yaml.load(METADATA_SCHEMA, Loader=SafeLoader) 200 201 202def validate_setting(setting, module_path, filename=None): 203 if setting is not None: 204 if filename is not None: 205 checkfile = Path(module_path) / setting / filename 206 else: 207 checkfile = Path(module_path) / setting 208 if not checkfile.resolve().is_file(): 209 return False 210 return True 211 212 213def process_module(module): 214 module_path = PurePath(module) 215 216 # The input is a module if zephyr/module.{yml,yaml} is a valid yaml file 217 # or if both zephyr/CMakeLists.txt and zephyr/Kconfig are present. 218 219 for module_yml in [module_path / MODULE_YML_PATH, 220 module_path / MODULE_YML_PATH.with_suffix('.yaml')]: 221 if Path(module_yml).is_file(): 222 with Path(module_yml).open('r', encoding='utf-8') as f: 223 meta = yaml.load(f.read(), Loader=SafeLoader) 224 225 try: 226 pykwalify.core.Core(source_data=meta, schema_data=schema)\ 227 .validate() 228 except pykwalify.errors.SchemaError as e: 229 sys.exit('ERROR: Malformed "build" section in file: {}\n{}' 230 .format(module_yml.as_posix(), e)) 231 232 meta['name'] = meta.get('name', module_path.name) 233 meta['name-sanitized'] = re.sub('[^a-zA-Z0-9]', '_', meta['name']) 234 return meta 235 236 if Path(module_path.joinpath('zephyr/CMakeLists.txt')).is_file() and \ 237 Path(module_path.joinpath('zephyr/Kconfig')).is_file(): 238 return {'name': module_path.name, 239 'name-sanitized': re.sub('[^a-zA-Z0-9]', '_', module_path.name), 240 'build': {'cmake': 'zephyr', 'kconfig': 'zephyr/Kconfig'}} 241 242 return None 243 244 245def process_cmake(module, meta): 246 section = meta.get('build', dict()) 247 module_path = PurePath(module) 248 module_yml = module_path.joinpath('zephyr/module.yml') 249 250 cmake_extern = section.get('cmake-ext', False) 251 if cmake_extern: 252 return('\"{}\":\"{}\":\"{}\"\n' 253 .format(meta['name'], 254 module_path.as_posix(), 255 "${ZEPHYR_" + meta['name-sanitized'].upper() + "_CMAKE_DIR}")) 256 257 cmake_setting = section.get('cmake', None) 258 if not validate_setting(cmake_setting, module, 'CMakeLists.txt'): 259 sys.exit('ERROR: "cmake" key in {} has folder value "{}" which ' 260 'does not contain a CMakeLists.txt file.' 261 .format(module_yml.as_posix(), cmake_setting)) 262 263 cmake_path = os.path.join(module, cmake_setting or 'zephyr') 264 cmake_file = os.path.join(cmake_path, 'CMakeLists.txt') 265 if os.path.isfile(cmake_file): 266 return('\"{}\":\"{}\":\"{}\"\n' 267 .format(meta['name'], 268 module_path.as_posix(), 269 Path(cmake_path).resolve().as_posix())) 270 else: 271 return('\"{}\":\"{}\":\"\"\n' 272 .format(meta['name'], 273 module_path.as_posix())) 274 275 276def process_sysbuildcmake(module, meta): 277 section = meta.get('build', dict()) 278 module_path = PurePath(module) 279 module_yml = module_path.joinpath('zephyr/module.yml') 280 281 cmake_extern = section.get('sysbuild-cmake-ext', False) 282 if cmake_extern: 283 return('\"{}\":\"{}\":\"{}\"\n' 284 .format(meta['name'], 285 module_path.as_posix(), 286 "${SYSBUILD_" + meta['name-sanitized'].upper() + "_CMAKE_DIR}")) 287 288 cmake_setting = section.get('sysbuild-cmake', None) 289 if not validate_setting(cmake_setting, module, 'CMakeLists.txt'): 290 sys.exit('ERROR: "cmake" key in {} has folder value "{}" which ' 291 'does not contain a CMakeLists.txt file.' 292 .format(module_yml.as_posix(), cmake_setting)) 293 294 if cmake_setting is None: 295 return "" 296 297 cmake_path = os.path.join(module, cmake_setting or 'zephyr') 298 cmake_file = os.path.join(cmake_path, 'CMakeLists.txt') 299 if os.path.isfile(cmake_file): 300 return('\"{}\":\"{}\":\"{}\"\n' 301 .format(meta['name'], 302 module_path.as_posix(), 303 Path(cmake_path).resolve().as_posix())) 304 else: 305 return('\"{}\":\"{}\":\"\"\n' 306 .format(meta['name'], 307 module_path.as_posix())) 308 309 310def process_settings(module, meta): 311 section = meta.get('build', dict()) 312 build_settings = section.get('settings', None) 313 out_text = "" 314 315 if build_settings is not None: 316 for root in ['board', 'dts', 'snippet', 'soc', 'arch', 'module_ext', 'sca']: 317 setting = build_settings.get(root+'_root', None) 318 if setting is not None: 319 root_path = PurePath(module) / setting 320 out_text += f'"{root.upper()}_ROOT":' 321 out_text += f'"{root_path.as_posix()}"\n' 322 323 return out_text 324 325 326def get_blob_status(path, sha256): 327 if not path.is_file(): 328 return BLOB_NOT_PRESENT 329 with path.open('rb') as f: 330 m = hashlib.sha256() 331 m.update(f.read()) 332 if sha256.lower() == m.hexdigest(): 333 return BLOB_PRESENT 334 else: 335 return BLOB_OUTDATED 336 337 338def process_blobs(module, meta): 339 blobs = [] 340 mblobs = meta.get('blobs', None) 341 if not mblobs: 342 return blobs 343 344 blobs_path = Path(module) / MODULE_BLOBS_PATH 345 for blob in mblobs: 346 blob['module'] = meta.get('name', None) 347 blob['abspath'] = blobs_path / Path(blob['path']) 348 blob['license-abspath'] = Path(module) / Path(blob['license-path']) 349 blob['status'] = get_blob_status(blob['abspath'], blob['sha256']) 350 blobs.append(blob) 351 352 return blobs 353 354 355def kconfig_module_opts(name_sanitized, blobs, taint_blobs): 356 snippet = [f'config ZEPHYR_{name_sanitized.upper()}_MODULE', 357 ' bool', 358 ' default y'] 359 360 if taint_blobs: 361 snippet += [' select TAINT_BLOBS'] 362 363 if blobs: 364 snippet += [f'\nconfig ZEPHYR_{name_sanitized.upper()}_MODULE_BLOBS', 365 ' bool'] 366 if taint_blobs: 367 snippet += [' default y'] 368 369 return snippet 370 371 372def kconfig_snippet(meta, path, kconfig_file=None, blobs=False, taint_blobs=False, sysbuild=False): 373 name = meta['name'] 374 name_sanitized = meta['name-sanitized'] 375 376 snippet = [f'menu "{name} ({path.as_posix()})"'] 377 378 snippet += [f'osource "{kconfig_file.resolve().as_posix()}"' if kconfig_file 379 else f'osource "$(SYSBUILD_{name_sanitized.upper()}_KCONFIG)"' if sysbuild is True 380 else f'osource "$(ZEPHYR_{name_sanitized.upper()}_KCONFIG)"'] 381 382 snippet += kconfig_module_opts(name_sanitized, blobs, taint_blobs) 383 384 snippet += ['endmenu\n'] 385 386 return '\n'.join(snippet) 387 388 389def process_kconfig_module_dir(module, meta, cmake_output): 390 module_path = PurePath(module) 391 name_sanitized = meta['name-sanitized'] 392 393 if cmake_output is False: 394 return f'ZEPHYR_{name_sanitized.upper()}_MODULE_DIR={module_path.as_posix()}\n' 395 return f'list(APPEND kconfig_env_dirs ZEPHYR_{name_sanitized.upper()}_MODULE_DIR={module_path.as_posix()})\n' 396 397 398def process_kconfig(module, meta): 399 blobs = process_blobs(module, meta) 400 taint_blobs = any(b['status'] != BLOB_NOT_PRESENT for b in blobs) 401 section = meta.get('build', dict()) 402 module_path = PurePath(module) 403 module_yml = module_path.joinpath('zephyr/module.yml') 404 kconfig_extern = section.get('kconfig-ext', False) 405 406 if kconfig_extern: 407 return kconfig_snippet(meta, module_path, blobs=blobs, taint_blobs=taint_blobs) 408 409 kconfig_setting = section.get('kconfig', None) 410 if not validate_setting(kconfig_setting, module): 411 sys.exit('ERROR: "kconfig" key in {} has value "{}" which does ' 412 'not point to a valid Kconfig file.' 413 .format(module_yml, kconfig_setting)) 414 415 kconfig_file = os.path.join(module, kconfig_setting or 'zephyr/Kconfig') 416 if os.path.isfile(kconfig_file): 417 return kconfig_snippet(meta, module_path, Path(kconfig_file), 418 blobs=blobs, taint_blobs=taint_blobs) 419 else: 420 name_sanitized = meta['name-sanitized'] 421 return '\n'.join(kconfig_module_opts(name_sanitized, blobs, taint_blobs)) + '\n' 422 423 424def process_sysbuildkconfig(module, meta): 425 section = meta.get('build', dict()) 426 module_path = PurePath(module) 427 module_yml = module_path.joinpath('zephyr/module.yml') 428 kconfig_extern = section.get('sysbuild-kconfig-ext', False) 429 name_sanitized = meta['name-sanitized'] 430 431 if kconfig_extern: 432 return kconfig_snippet(meta, module_path, sysbuild=True) 433 434 kconfig_setting = section.get('sysbuild-kconfig', None) 435 if not validate_setting(kconfig_setting, module): 436 sys.exit('ERROR: "kconfig" key in {} has value "{}" which does ' 437 'not point to a valid Kconfig file.' 438 .format(module_yml, kconfig_setting)) 439 440 if kconfig_setting is not None: 441 kconfig_file = os.path.join(module, kconfig_setting) 442 if os.path.isfile(kconfig_file): 443 return kconfig_snippet(meta, module_path, Path(kconfig_file)) 444 445 return (f'config ZEPHYR_{name_sanitized.upper()}_MODULE\n' 446 f' bool\n' 447 f' default y\n') 448 449 450def process_twister(module, meta): 451 452 out = "" 453 tests = meta.get('tests', []) 454 samples = meta.get('samples', []) 455 boards = meta.get('boards', []) 456 457 for pth in tests + samples: 458 if pth: 459 dir = os.path.join(module, pth) 460 out += '-T\n{}\n'.format(PurePath(os.path.abspath(dir)) 461 .as_posix()) 462 463 for pth in boards: 464 if pth: 465 dir = os.path.join(module, pth) 466 out += '--board-root\n{}\n'.format(PurePath(os.path.abspath(dir)) 467 .as_posix()) 468 469 return out 470 471def is_valid_git_revision(revision): 472 """ 473 Returns True if the given string is a valid git revision hash (40 hex digits). 474 """ 475 if not isinstance(revision, str): 476 return False 477 return bool(re.fullmatch(r'[0-9a-fA-F]{40}', revision)) 478 479def _create_meta_project(project_path): 480 def git_revision(path): 481 rc = subprocess.Popen(['git', 'rev-parse', '--is-inside-work-tree'], 482 stdout=subprocess.PIPE, 483 stderr=subprocess.PIPE, 484 cwd=path).wait() 485 if rc == 0: 486 # A git repo. 487 popen = subprocess.Popen(['git', 'rev-parse', 'HEAD'], 488 stdout=subprocess.PIPE, 489 stderr=subprocess.PIPE, 490 cwd=path) 491 stdout, stderr = popen.communicate() 492 stdout = stdout.decode('utf-8') 493 494 if not (popen.returncode or stderr): 495 revision = stdout.rstrip() 496 497 rc = subprocess.Popen(['git', 'diff-index', '--quiet', 'HEAD', 498 '--'], 499 stdout=None, 500 stderr=None, 501 cwd=path).wait() 502 if rc: 503 return revision + '-dirty', True 504 return revision, False 505 return "unknown", False 506 507 def git_remote(path): 508 popen = subprocess.Popen(['git', 'remote'], 509 stdout=subprocess.PIPE, 510 stderr=subprocess.PIPE, 511 cwd=path) 512 stdout, stderr = popen.communicate() 513 stdout = stdout.decode('utf-8') 514 515 remotes_name = [] 516 if not (popen.returncode or stderr): 517 remotes_name = stdout.rstrip().split('\n') 518 519 remote_url = None 520 521 # If more than one remote, do not return any remote 522 if len(remotes_name) == 1: 523 remote = remotes_name[0] 524 popen = subprocess.Popen(['git', 'remote', 'get-url', remote], 525 stdout=subprocess.PIPE, 526 stderr=subprocess.PIPE, 527 cwd=path) 528 stdout, stderr = popen.communicate() 529 stdout = stdout.decode('utf-8') 530 531 if not (popen.returncode or stderr): 532 remote_url = stdout.rstrip() 533 534 return remote_url 535 536 def git_tags(path, revision): 537 if not revision or len(revision) == 0: 538 return None 539 540 popen = subprocess.Popen(['git', '-P', 'tag', '--points-at', revision], 541 stdout=subprocess.PIPE, 542 stderr=subprocess.PIPE, 543 cwd=path) 544 stdout, stderr = popen.communicate() 545 stdout = stdout.decode('utf-8') 546 547 tags = None 548 if not (popen.returncode or stderr): 549 tags = stdout.rstrip().splitlines() 550 551 return tags 552 553 workspace_dirty = False 554 path = PurePath(project_path).as_posix() 555 556 revision, dirty = git_revision(path) 557 workspace_dirty |= dirty 558 remote = git_remote(path) 559 tags = git_tags(path, revision) 560 561 meta_project = {'path': path, 562 'revision': revision} 563 564 if remote: 565 meta_project['remote'] = remote 566 567 if tags: 568 meta_project['tags'] = tags 569 570 return meta_project, workspace_dirty 571 572 573def _get_meta_project(meta_projects_list, project_path): 574 projects = [ prj for prj in meta_projects_list[1:] if prj["path"] == project_path ] 575 576 return projects[0] if len(projects) == 1 else None 577 578 579def process_meta(zephyr_base, west_projs, modules, extra_modules=None, 580 propagate_state=False): 581 # Process zephyr_base, projects, and modules and create a dictionary 582 # with meta information for each input. 583 # 584 # The dictionary will contain meta info in the following lists: 585 # - zephyr: path and revision 586 # - modules: name, path, and revision 587 # - west-projects: path and revision 588 # 589 # returns the dictionary with said lists 590 591 meta = {'zephyr': None, 'modules': None, 'workspace': None} 592 593 zephyr_project, zephyr_dirty = _create_meta_project(zephyr_base) 594 zephyr_off = zephyr_project.get("remote") is None 595 596 workspace_dirty = zephyr_dirty 597 workspace_extra = extra_modules is not None 598 workspace_off = zephyr_off 599 600 if zephyr_off and is_valid_git_revision(zephyr_project['revision']): 601 zephyr_project['revision'] += '-off' 602 603 meta['zephyr'] = zephyr_project 604 meta['workspace'] = {} 605 606 if west_projs is not None: 607 from west.manifest import MANIFEST_REV_BRANCH 608 projects = west_projs['projects'] 609 meta_projects = [] 610 611 manifest_path = projects[0].posixpath 612 613 # Special treatment of manifest project 614 # Git information (remote/revision) are not provided by west for the Manifest (west.yml) 615 # To mitigate this, we check if we don't use the manifest from the zephyr repository or an other project. 616 # If it's from zephyr, reuse zephyr information 617 # If it's from an other project, ignore it, it will be added later 618 # If it's not found, we extract data manually (remote/revision) from the directory 619 620 manifest_project = None 621 manifest_dirty = False 622 manifest_off = False 623 624 if zephyr_base == manifest_path: 625 manifest_project = zephyr_project 626 manifest_dirty = zephyr_dirty 627 manifest_off = zephyr_off 628 elif not [ prj for prj in projects[1:] if prj.posixpath == manifest_path ]: 629 manifest_project, manifest_dirty = _create_meta_project( 630 projects[0].posixpath) 631 manifest_off = manifest_project.get("remote") is None 632 if manifest_off and is_valid_git_revision(manifest_project['revision']): 633 manifest_project["revision"] += "-off" 634 635 if manifest_project: 636 workspace_off |= manifest_off 637 workspace_dirty |= manifest_dirty 638 meta_projects.append(manifest_project) 639 640 # Iterates on all projects except the first one (manifest) 641 for project in projects[1:]: 642 meta_project, dirty = _create_meta_project(project.posixpath) 643 workspace_dirty |= dirty 644 meta_projects.append(meta_project) 645 646 off = False 647 if not meta_project.get("remote") or project.sha(MANIFEST_REV_BRANCH) != meta_project['revision'].removesuffix("-dirty"): 648 off = True 649 if not meta_project.get('remote') or project.url != meta_project['remote']: 650 # Force manifest URL and set commit as 'off' 651 meta_project['url'] = project.url 652 off = True 653 654 if off: 655 if is_valid_git_revision(meta_project['revision']): 656 meta_project['revision'] += '-off' 657 workspace_off |= off 658 659 # If manifest is in project, updates related variables 660 if project.posixpath == manifest_path: 661 manifest_dirty |= dirty 662 manifest_off |= off 663 manifest_project = meta_project 664 665 meta.update({'west': {'manifest': west_projs['manifest_path'], 666 'projects': meta_projects}}) 667 meta['workspace'].update({'off': workspace_off}) 668 669 # Iterates on all modules 670 meta_modules = [] 671 for module in modules: 672 # Check if modules is not in projects 673 # It allows to have the "-off" flag since `modules` variable` does not provide URL/remote 674 meta_module = _get_meta_project(meta_projects, module.project) 675 676 if not meta_module: 677 meta_module, dirty = _create_meta_project(module.project) 678 workspace_dirty |= dirty 679 680 meta_module['name'] = module.meta.get('name') 681 682 if module.meta.get('security'): 683 meta_module['security'] = module.meta.get('security') 684 meta_modules.append(meta_module) 685 686 meta['modules'] = meta_modules 687 688 meta['workspace'].update({'dirty': workspace_dirty, 689 'extra': workspace_extra}) 690 691 if propagate_state: 692 zephyr_revision = zephyr_project['revision'] 693 if is_valid_git_revision(zephyr_revision): 694 if workspace_dirty and not zephyr_dirty: 695 zephyr_revision += '-dirty' 696 if workspace_extra: 697 zephyr_revision += '-extra' 698 if workspace_off and not zephyr_off: 699 zephyr_revision += '-off' 700 zephyr_project.update({'revision': zephyr_revision}) 701 702 if west_projs is not None: 703 manifest_revision = manifest_project['revision'] 704 if is_valid_git_revision(manifest_revision): 705 if workspace_dirty and not manifest_dirty: 706 manifest_revision += '-dirty' 707 if workspace_extra: 708 manifest_revision += '-extra' 709 if workspace_off and not manifest_off: 710 manifest_revision += '-off' 711 manifest_project.update({'revision': manifest_revision}) 712 713 return meta 714 715 716def west_projects(manifest=None): 717 manifest_path = None 718 projects = [] 719 # West is imported here, as it is optional 720 # (and thus maybe not installed) 721 # if user is providing a specific modules list. 722 try: 723 from west.manifest import Manifest 724 except ImportError: 725 # West is not installed, so don't return any projects. 726 return None 727 728 # If west *is* installed, we need all of the following imports to 729 # work. West versions that are excessively old may fail here: 730 # west.configuration.MalformedConfig was 731 # west.manifest.MalformedConfig until west v0.14.0, for example. 732 # These should be hard errors. 733 from west.manifest import \ 734 ManifestImportFailed, MalformedManifest, ManifestVersionError 735 from west.configuration import MalformedConfig 736 from west.util import WestNotFound 737 from west.version import __version__ as WestVersion 738 739 from packaging import version 740 try: 741 if not manifest: 742 manifest = Manifest.from_file() 743 if version.parse(WestVersion) >= version.parse('0.9.0'): 744 projects = [p for p in manifest.get_projects([]) 745 if manifest.is_active(p)] 746 else: 747 projects = manifest.get_projects([]) 748 manifest_path = manifest.path 749 return {'manifest_path': manifest_path, 'projects': projects} 750 except (ManifestImportFailed, MalformedManifest, 751 ManifestVersionError, MalformedConfig) as e: 752 sys.exit(f'ERROR: {e}') 753 except WestNotFound: 754 # Only accept WestNotFound, meaning we are not in a west 755 # workspace. Such setup is allowed, as west may be installed 756 # but the project is not required to use west. 757 pass 758 return None 759 760 761def parse_modules(zephyr_base, manifest=None, west_projs=None, modules=None, 762 extra_modules=None): 763 764 if modules is None: 765 west_projs = west_projs or west_projects(manifest) 766 modules = ([p.posixpath for p in west_projs['projects']] 767 if west_projs else []) 768 769 if extra_modules is None: 770 extra_modules = [] 771 for var in ['EXTRA_ZEPHYR_MODULES', 'ZEPHYR_EXTRA_MODULES']: 772 extra_module = os.environ.get(var, None) 773 if not extra_module: 774 continue 775 extra_modules.extend(PurePosixPath(p) for p in extra_module.split(';') if p) 776 777 Module = namedtuple('Module', ['project', 'meta', 'depends']) 778 779 all_modules_by_name = {} 780 # dep_modules is a list of all modules that has an unresolved dependency 781 dep_modules = [] 782 # start_modules is a list modules with no depends left (no incoming edge) 783 start_modules = [] 784 # sorted_modules is a topological sorted list of the modules 785 sorted_modules = [] 786 787 for project in modules + extra_modules: 788 # Avoid including Zephyr base project as module. 789 if project == zephyr_base: 790 continue 791 792 meta = process_module(project) 793 if meta: 794 depends = meta.get('build', {}).get('depends', []) 795 all_modules_by_name[meta['name']] = Module(project, meta, depends) 796 797 elif project in extra_modules: 798 sys.exit(f'{project}, given in ZEPHYR_EXTRA_MODULES, ' 799 'is not a valid zephyr module') 800 801 for module in all_modules_by_name.values(): 802 if not module.depends: 803 start_modules.append(module) 804 else: 805 dep_modules.append(module) 806 807 # This will do a topological sort to ensure the modules are ordered 808 # according to dependency settings. 809 while start_modules: 810 node = start_modules.pop(0) 811 sorted_modules.append(node) 812 node_name = node.meta['name'] 813 to_remove = [] 814 for module in dep_modules: 815 if node_name in module.depends: 816 module.depends.remove(node_name) 817 if not module.depends: 818 start_modules.append(module) 819 to_remove.append(module) 820 for module in to_remove: 821 dep_modules.remove(module) 822 823 if dep_modules: 824 # If there are any modules with unresolved dependencies, then the 825 # modules contains unmet or cyclic dependencies. Error out. 826 error = 'Unmet or cyclic dependencies in modules:\n' 827 for module in dep_modules: 828 error += f'{module.project} depends on: {module.depends}\n' 829 sys.exit(error) 830 831 return sorted_modules 832 833def write_if_different(file, data): 834 if Path(file).is_file(): 835 with open(file, encoding="utf-8") as fp: 836 if fp.read() == data: 837 return 838 839 with open(file, 'w', encoding="utf-8") as fp: 840 fp.write(data) 841 842def main(): 843 parser = argparse.ArgumentParser(description=''' 844 Process a list of projects and create Kconfig / CMake include files for 845 projects which are also a Zephyr module''', allow_abbrev=False) 846 847 parser.add_argument('--kconfig-out', 848 help="""File to write with resulting KConfig import 849 statements.""") 850 parser.add_argument('--twister-out', 851 help="""File to write with resulting twister 852 parameters.""") 853 parser.add_argument('--cmake-out', 854 help="""File to write with resulting <name>:<path> 855 values to use for including in CMake""") 856 parser.add_argument('--sysbuild-kconfig-out', 857 help="""File to write with resulting KConfig import 858 statements.""") 859 parser.add_argument('--sysbuild-cmake-out', 860 help="""File to write with resulting <name>:<path> 861 values to use for including in CMake""") 862 parser.add_argument('--meta-out', 863 help="""Write a build meta YaML file containing a list 864 of Zephyr modules and west projects. 865 If a module or project is also a git repository 866 the current SHA revision will also be written.""") 867 parser.add_argument('--meta-state-propagate', action='store_true', 868 help="""Propagate state of modules and west projects 869 to the suffix of the Zephyr SHA and if west is 870 used, to the suffix of the manifest SHA""") 871 parser.add_argument('--settings-out', 872 help="""File to write with resulting <name>:<value> 873 values to use for including in CMake""") 874 parser.add_argument('-m', '--modules', nargs='+', 875 help="""List of modules to parse instead of using `west 876 list`""") 877 parser.add_argument('-x', '--extra-modules', nargs='+', 878 help='List of extra modules to parse') 879 parser.add_argument('-z', '--zephyr-base', 880 help='Path to zephyr repository') 881 args = parser.parse_args() 882 883 kconfig_module_dirs = "" 884 kconfig_module_dirs_cmake = "set(kconfig_env_dirs)\n" 885 kconfig = "" 886 cmake = "" 887 sysbuild_kconfig = "" 888 sysbuild_cmake = "" 889 twister = "" 890 settings = '''\ 891# WARNING. THIS FILE IS AUTO-GENERATED. DO NOT MODIFY! 892# 893# This file contains build system settings derived from your modules. 894# 895# Modules may be set via ZEPHYR_MODULES, ZEPHYR_EXTRA_MODULES, 896# and/or the west manifest file. 897# 898# See the Modules guide for more information. 899''' 900 901 west_projs = west_projects() 902 modules = parse_modules(args.zephyr_base, None, west_projs, 903 args.modules, args.extra_modules) 904 905 for module in modules: 906 kconfig_module_dirs += process_kconfig_module_dir(module.project, module.meta, False) 907 kconfig_module_dirs_cmake += process_kconfig_module_dir(module.project, module.meta, True) 908 kconfig += process_kconfig(module.project, module.meta) 909 cmake += process_cmake(module.project, module.meta) 910 sysbuild_kconfig += process_sysbuildkconfig( 911 module.project, module.meta) 912 sysbuild_cmake += process_sysbuildcmake(module.project, module.meta) 913 settings += process_settings(module.project, module.meta) 914 twister += process_twister(module.project, module.meta) 915 916 if args.kconfig_out or args.sysbuild_kconfig_out: 917 if args.kconfig_out: 918 kconfig_module_dirs_out = PurePath(args.kconfig_out).parent / 'kconfig_module_dirs.env' 919 kconfig_module_dirs_cmake_out = PurePath(args.kconfig_out).parent / \ 920 'kconfig_module_dirs.cmake' 921 elif args.sysbuild_kconfig_out: 922 kconfig_module_dirs_out = PurePath(args.sysbuild_kconfig_out).parent / \ 923 'kconfig_module_dirs.env' 924 kconfig_module_dirs_cmake_out = PurePath(args.sysbuild_kconfig_out).parent / \ 925 'kconfig_module_dirs.cmake' 926 927 write_if_different(kconfig_module_dirs_out, kconfig_module_dirs) 928 write_if_different(kconfig_module_dirs_cmake_out, kconfig_module_dirs_cmake) 929 930 if args.kconfig_out: 931 write_if_different(args.kconfig_out, kconfig) 932 933 if args.cmake_out: 934 write_if_different(args.cmake_out, cmake) 935 936 if args.sysbuild_kconfig_out: 937 write_if_different(args.sysbuild_kconfig_out, sysbuild_kconfig) 938 939 if args.sysbuild_cmake_out: 940 write_if_different(args.sysbuild_cmake_out, sysbuild_cmake) 941 942 if args.settings_out: 943 write_if_different(args.settings_out, settings) 944 945 if args.twister_out: 946 write_if_different(args.twister_out, twister) 947 948 if args.meta_out: 949 meta = process_meta(args.zephyr_base, west_projs, modules, 950 args.extra_modules, args.meta_state_propagate) 951 952 # Ignore references and insert data instead 953 yaml.Dumper.ignore_aliases = lambda self, data: True 954 write_if_different(args.meta_out, yaml.dump(meta)) 955 956 957if __name__ == "__main__": 958 main() 959
