1#!/usr/bin/env python 2# 3# Command line tool to take in ESP-IDF sdkconfig files with project 4# settings and output data in multiple formats (update config, generate 5# header file, generate .cmake include file, documentation, etc). 6# 7# Used internally by the ESP-IDF build system. But designed to be 8# non-IDF-specific. 9# 10# Copyright 2018-2020 Espressif Systems (Shanghai) PTE LTD 11# 12# Licensed under the Apache License, Version 2.0 (the "License"); 13# you may not use this file except in compliance with the License. 14# You may obtain a copy of the License at 15# 16# http:#www.apache.org/licenses/LICENSE-2.0 17# 18# Unless required by applicable law or agreed to in writing, software 19# distributed under the License is distributed on an "AS IS" BASIS, 20# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 21# See the License for the specific language governing permissions and 22# limitations under the License. 23from __future__ import print_function 24 25import argparse 26import json 27import os 28import os.path 29import re 30import sys 31import tempfile 32from pathlib import Path 33 34# This relies on this file being in hal_espressif/tools/kconfig_new/confgen.py 35# If you move this file, you'll break it, so be careful. 36THIS_ZEPHYR = Path(__file__).parents[5] / 'zephyr' 37ZEPHYR_BASE = Path(os.environ.get('ZEPHYR_BASE', THIS_ZEPHYR)) 38 39sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts", "west_commands")) 40 41# use zephyr kconfiglib 42sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts", "kconfig")) 43 44import gen_kconfig_doc # noqa: E402 45import kconfiglib # noqa: E402 46 47__version__ = '0.1' 48 49if 'IDF_CMAKE' not in os.environ: 50 os.environ['IDF_CMAKE'] = '' 51 52 53class DeprecatedOptions(object): 54 _REN_FILE = 'sdkconfig.rename' 55 _DEP_OP_BEGIN = '# Deprecated options for backward compatibility' 56 _DEP_OP_END = '# End of deprecated options' 57 _RE_DEP_OP_BEGIN = re.compile(_DEP_OP_BEGIN) 58 _RE_DEP_OP_END = re.compile(_DEP_OP_END) 59 60 def __init__(self, config_prefix, path_rename_files=[]): 61 self.config_prefix = config_prefix 62 # r_dic maps deprecated options to new options; rev_r_dic maps in the opposite direction 63 self.r_dic, self.rev_r_dic = self._parse_replacements(path_rename_files) 64 65 # note the '=' at the end of regex for not getting partial match of configs 66 self._RE_CONFIG = re.compile(r'{}(\w+)='.format(self.config_prefix)) 67 68 def _parse_replacements(self, repl_paths): 69 rep_dic = {} 70 rev_rep_dic = {} 71 72 def remove_config_prefix(string): 73 if string.startswith(self.config_prefix): 74 return string[len(self.config_prefix):] 75 raise RuntimeError('Error in {} (line {}): Config {} is not prefixed with {}' 76 ''.format(rep_path, line_number, string, self.config_prefix)) 77 78 for rep_path in repl_paths: 79 with open(rep_path) as f_rep: 80 for line_number, line in enumerate(f_rep, start=1): 81 sp_line = line.split() 82 if len(sp_line) == 0 or sp_line[0].startswith('#'): 83 # empty line or comment 84 continue 85 if len(sp_line) != 2 or not all(x.startswith(self.config_prefix) for x in sp_line): 86 raise RuntimeError('Syntax error in {} (line {})'.format(rep_path, line_number)) 87 if sp_line[0] in rep_dic: 88 raise RuntimeError('Error in {} (line {}): Replacement {} exist for {} and new ' 89 'replacement {} is defined'.format(rep_path, line_number, 90 rep_dic[sp_line[0]], sp_line[0], 91 sp_line[1])) 92 93 (dep_opt, new_opt) = (remove_config_prefix(x) for x in sp_line) 94 rep_dic[dep_opt] = new_opt 95 rev_rep_dic[new_opt] = dep_opt 96 return rep_dic, rev_rep_dic 97 98 def get_deprecated_option(self, new_option): 99 return self.rev_r_dic.get(new_option, None) 100 101 def get_new_option(self, deprecated_option): 102 return self.r_dic.get(deprecated_option, None) 103 104 def replace(self, sdkconfig_in, sdkconfig_out): 105 replace_enabled = True 106 with open(sdkconfig_in, 'r') as f_in, open(sdkconfig_out, 'w') as f_out: 107 for line_num, line in enumerate(f_in, start=1): 108 if self._RE_DEP_OP_BEGIN.search(line): 109 replace_enabled = False 110 elif self._RE_DEP_OP_END.search(line): 111 replace_enabled = True 112 elif replace_enabled: 113 m = self._RE_CONFIG.search(line) 114 if m and m.group(1) in self.r_dic: 115 depr_opt = self.config_prefix + m.group(1) 116 new_opt = self.config_prefix + self.r_dic[m.group(1)] 117 line = line.replace(depr_opt, new_opt) 118 print('{}:{} {} was replaced with {}'.format(sdkconfig_in, line_num, depr_opt, new_opt)) 119 f_out.write(line) 120 121 def append_doc(self, config, visibility, path_output): 122 123 def option_was_written(opt): 124 # named choices were written if any of the symbols in the choice were visible 125 if new_opt in config.named_choices: 126 syms = config.named_choices[new_opt].syms 127 for s in syms: 128 if any(visibility.visible(node) for node in s.nodes): 129 return True 130 return False 131 else: 132 try: 133 # otherwise if any of the nodes associated with the option was visible 134 return any(visibility.visible(node) for node in config.syms[opt].nodes) 135 except KeyError: 136 return False 137 138 if len(self.r_dic) > 0: 139 with open(path_output, 'a') as f_o: 140 header = 'Deprecated options and their replacements' 141 f_o.write('.. _configuration-deprecated-options:\n\n{}\n{}\n\n'.format(header, '-' * len(header))) 142 for dep_opt in sorted(self.r_dic): 143 new_opt = self.r_dic[dep_opt] 144 if option_was_written(new_opt) and (new_opt not in config.syms or config.syms[new_opt].choice is None): 145 # everything except config for a choice (no link reference for those in the docs) 146 f_o.write('- {}{} (:ref:`{}{}`)\n'.format(config.config_prefix, dep_opt, 147 config.config_prefix, new_opt)) 148 149 if new_opt in config.named_choices: 150 # here are printed config options which were filtered out 151 syms = config.named_choices[new_opt].syms 152 for sym in syms: 153 if sym.name in self.rev_r_dic: 154 # only if the symbol has been renamed 155 dep_name = self.rev_r_dic[sym.name] 156 157 # config options doesn't have references 158 f_o.write(' - {}{}\n'.format(config.config_prefix, dep_name)) 159 160 def append_config(self, config, path_output): 161 tmp_list = [] 162 163 def append_config_node_process(node): 164 item = node.item 165 if isinstance(item, kconfiglib.Symbol) and item.env_var is None: 166 if item.name in self.rev_r_dic: 167 c_string = item.config_string 168 if c_string: 169 tmp_list.append(c_string.replace(self.config_prefix + item.name, 170 self.config_prefix + self.rev_r_dic[item.name])) 171 172 for n in config.node_iter(): 173 append_config_node_process(n) 174 175 if len(tmp_list) > 0: 176 with open(path_output, 'a') as f_o: 177 f_o.write('\n{}\n'.format(self._DEP_OP_BEGIN)) 178 f_o.writelines(tmp_list) 179 f_o.write('{}\n'.format(self._DEP_OP_END)) 180 181 def append_header(self, config, path_output): 182 def _opt_defined(opt): 183 if not opt.visibility: 184 return False 185 return not (opt.orig_type in (kconfiglib.BOOL, kconfiglib.TRISTATE) and opt.str_value == 'n') 186 187 if len(self.r_dic) > 0: 188 with open(path_output, 'a') as f_o: 189 f_o.write('\n/* List of deprecated options */\n') 190 for dep_opt in sorted(self.r_dic): 191 new_opt = self.r_dic[dep_opt] 192 if new_opt in config.syms and _opt_defined(config.syms[new_opt]): 193 f_o.write('#define {}{} {}{}\n'.format(self.config_prefix, dep_opt, self.config_prefix, new_opt)) 194 195 196def main(): 197 parser = argparse.ArgumentParser(description='confgen.py v%s - Config Generation Tool' % __version__, prog=os.path.basename(sys.argv[0])) 198 199 parser.add_argument('--config', 200 help='Project configuration settings', 201 nargs='?', 202 default=None) 203 204 parser.add_argument('--defaults', 205 help='Optional project defaults file, used if --config file doesn\'t exist. ' 206 'Multiple files can be specified using multiple --defaults arguments.', 207 nargs='?', 208 default=[], 209 action='append') 210 211 parser.add_argument('--kconfig', 212 help='KConfig file with config item definitions', 213 required=True) 214 215 parser.add_argument('--sdkconfig-rename', 216 help='File with deprecated Kconfig options', 217 required=False) 218 219 parser.add_argument('--dont-write-deprecated', 220 help='Do not write compatibility statements for deprecated values', 221 action='store_true') 222 223 parser.add_argument('--output', nargs=2, action='append', 224 help='Write output file (format and output filename)', 225 metavar=('FORMAT', 'FILENAME'), 226 default=[]) 227 228 parser.add_argument('--env', action='append', default=[], 229 help='Environment to set when evaluating the config file', metavar='NAME=VAL') 230 231 parser.add_argument('--env-file', type=argparse.FileType('r'), 232 help='Optional file to load environment variables from. Contents ' 233 'should be a JSON object where each key/value pair is a variable.') 234 235 args = parser.parse_args() 236 237 for fmt, filename in args.output: 238 if fmt not in OUTPUT_FORMATS.keys(): 239 print("Format '%s' not recognised. Known formats: %s" % (fmt, OUTPUT_FORMATS.keys())) 240 sys.exit(1) 241 242 try: 243 args.env = [(name,value) for (name,value) in (e.split('=',1) for e in args.env)] 244 except ValueError: 245 print("--env arguments must each contain =. To unset an environment variable, use 'ENV='") 246 sys.exit(1) 247 248 for name, value in args.env: 249 os.environ[name] = value 250 251 if args.env_file is not None: 252 os.environ.update(json.load(args.env_file)) 253 254 config = kconfiglib.Kconfig(args.kconfig) 255 config.warn_assign_redun = False 256 config.warn_assign_override = False 257 258 sdkconfig_renames = [args.sdkconfig_rename] if args.sdkconfig_rename else [] 259 sdkconfig_renames += os.environ.get('COMPONENT_SDKCONFIG_RENAMES', '').split() 260 deprecated_options = DeprecatedOptions(config.config_prefix, path_rename_files=sdkconfig_renames) 261 262 if len(args.defaults) > 0: 263 def _replace_empty_assignments(path_in, path_out): 264 with open(path_in, 'r') as f_in, open(path_out, 'w') as f_out: 265 for line_num, line in enumerate(f_in, start=1): 266 line = line.strip() 267 if line.endswith('='): 268 line += 'n' 269 print('{}:{} line was updated to {}'.format(path_out, line_num, line)) 270 f_out.write(line) 271 f_out.write('\n') 272 273 # always load defaults first, so any items which are not defined in that config 274 # will have the default defined in the defaults file 275 for name in args.defaults: 276 print('Loading defaults file %s...' % name) 277 if not os.path.exists(name): 278 raise RuntimeError('Defaults file not found: %s' % name) 279 try: 280 with tempfile.NamedTemporaryFile(prefix='confgen_tmp', delete=False) as f: 281 temp_file1 = f.name 282 with tempfile.NamedTemporaryFile(prefix='confgen_tmp', delete=False) as f: 283 temp_file2 = f.name 284 deprecated_options.replace(sdkconfig_in=name, sdkconfig_out=temp_file1) 285 _replace_empty_assignments(temp_file1, temp_file2) 286 config.load_config(temp_file2, replace=False) 287 finally: 288 try: 289 os.remove(temp_file1) 290 os.remove(temp_file2) 291 except OSError: 292 pass 293 294 # If config file previously exists, load it 295 if args.config and os.path.exists(args.config): 296 # ... but replace deprecated options before that 297 with tempfile.NamedTemporaryFile(prefix='confgen_tmp', delete=False) as f: 298 temp_file = f.name 299 try: 300 deprecated_options.replace(sdkconfig_in=args.config, sdkconfig_out=temp_file) 301 config.load_config(temp_file, replace=False) 302 update_if_changed(temp_file, args.config) 303 finally: 304 try: 305 os.remove(temp_file) 306 except OSError: 307 pass 308 309 if args.dont_write_deprecated: 310 # The deprecated object was useful until now for replacements. Now it will be redefined with no configurations 311 # and as the consequence, it won't generate output with deprecated statements. 312 deprecated_options = DeprecatedOptions('', path_rename_files=[]) 313 314 # Output the files specified in the arguments 315 for output_type, filename in args.output: 316 with tempfile.NamedTemporaryFile(prefix='confgen_tmp', delete=False) as f: 317 temp_file = f.name 318 try: 319 output_function = OUTPUT_FORMATS[output_type] 320 output_function(deprecated_options, config, temp_file) 321 update_if_changed(temp_file, filename) 322 finally: 323 try: 324 os.remove(temp_file) 325 except OSError: 326 pass 327 328 329def write_config(deprecated_options, config, filename): 330 CONFIG_HEADING = """# 331# Automatically generated file. DO NOT EDIT. 332# Espressif IoT Development Framework (ESP-IDF) Project Configuration 333# 334""" 335 config.write_config(filename, header=CONFIG_HEADING) 336 deprecated_options.append_config(config, filename) 337 338 339def write_makefile(deprecated_options, config, filename): 340 CONFIG_HEADING = """# 341# Automatically generated file. DO NOT EDIT. 342# Espressif IoT Development Framework (ESP-IDF) Project Makefile Configuration 343# 344""" 345 with open(filename, 'w') as f: 346 tmp_dep_lines = [] 347 f.write(CONFIG_HEADING) 348 349 def get_makefile_config_string(name, value, orig_type): 350 if orig_type in (kconfiglib.BOOL, kconfiglib.TRISTATE): 351 value = '' if value == 'n' else value 352 elif orig_type == kconfiglib.INT: 353 try: 354 value = int(value) 355 except ValueError: 356 value = '' 357 elif orig_type == kconfiglib.HEX: 358 try: 359 value = hex(int(value, 16)) # ensure 0x prefix 360 except ValueError: 361 value = '' 362 elif orig_type == kconfiglib.STRING: 363 value = '"{}"'.format(kconfiglib.escape(value)) 364 else: 365 raise RuntimeError('{}{}: unknown type {}'.format(config.config_prefix, name, orig_type)) 366 367 return '{}{}={}\n'.format(config.config_prefix, name, value) 368 369 def write_makefile_node(node): 370 item = node.item 371 if isinstance(item, kconfiglib.Symbol) and item.env_var is None: 372 # item.config_string cannot be used because it ignores hidden config items 373 val = item.str_value 374 f.write(get_makefile_config_string(item.name, val, item.orig_type)) 375 376 dep_opt = deprecated_options.get_deprecated_option(item.name) 377 if dep_opt: 378 # the same string but with the deprecated name 379 tmp_dep_lines.append(get_makefile_config_string(dep_opt, val, item.orig_type)) 380 381 for n in config.node_iter(True): 382 write_makefile_node(n) 383 384 if len(tmp_dep_lines) > 0: 385 f.write('\n# List of deprecated options\n') 386 f.writelines(tmp_dep_lines) 387 388 389def write_header(deprecated_options, config, filename): 390 CONFIG_HEADING = """/* 391 * Automatically generated file. DO NOT EDIT. 392 * Espressif IoT Development Framework (ESP-IDF) Configuration Header 393 */ 394#pragma once 395""" 396 config.write_autoconf(filename, header=CONFIG_HEADING) 397 deprecated_options.append_header(config, filename) 398 399 400def write_cmake(deprecated_options, config, filename): 401 with open(filename, 'w') as f: 402 tmp_dep_list = [] 403 write = f.write 404 prefix = config.config_prefix 405 406 write("""# 407# Automatically generated file. DO NOT EDIT. 408# Espressif IoT Development Framework (ESP-IDF) Configuration cmake include file 409# 410""") 411 412 configs_list = list() 413 414 def write_node(node): 415 sym = node.item 416 if not isinstance(sym, kconfiglib.Symbol): 417 return 418 419 if sym.config_string: 420 val = sym.str_value 421 if sym.orig_type in (kconfiglib.BOOL, kconfiglib.TRISTATE) and val == 'n': 422 val = '' # write unset values as empty variables 423 elif sym.orig_type == kconfiglib.STRING: 424 val = kconfiglib.escape(val) 425 elif sym.orig_type == kconfiglib.HEX: 426 val = hex(int(val, 16)) # ensure 0x prefix 427 write('set({}{} "{}")\n'.format(prefix, sym.name, val)) 428 429 configs_list.append(prefix + sym.name) 430 dep_opt = deprecated_options.get_deprecated_option(sym.name) 431 if dep_opt: 432 tmp_dep_list.append('set({}{} "{}")\n'.format(prefix, dep_opt, val)) 433 configs_list.append(prefix + dep_opt) 434 435 for n in config.node_iter(): 436 write_node(n) 437 write('set(CONFIGS_LIST {})'.format(';'.join(configs_list))) 438 439 if len(tmp_dep_list) > 0: 440 write('\n# List of deprecated options for backward compatibility\n') 441 f.writelines(tmp_dep_list) 442 443 444def get_json_values(config): 445 config_dict = {} 446 447 def write_node(node): 448 sym = node.item 449 if not isinstance(sym, kconfiglib.Symbol): 450 return 451 452 if sym.config_string: 453 val = sym.str_value 454 if sym.type in [kconfiglib.BOOL, kconfiglib.TRISTATE]: 455 val = (val != 'n') 456 elif sym.type == kconfiglib.HEX: 457 val = int(val, 16) 458 elif sym.type == kconfiglib.INT: 459 val = int(val) 460 config_dict[sym.name] = val 461 for n in config.node_iter(False): 462 write_node(n) 463 return config_dict 464 465 466def write_json(deprecated_options, config, filename): 467 config_dict = get_json_values(config) 468 with open(filename, 'w') as f: 469 json.dump(config_dict, f, indent=4, sort_keys=True) 470 471 472def get_menu_node_id(node): 473 """ Given a menu node, return a unique id 474 which can be used to identify it in the menu structure 475 476 Will either be the config symbol name, or a menu identifier 477 'slug' 478 479 """ 480 try: 481 if not isinstance(node.item, kconfiglib.Choice): 482 return node.item.name 483 except AttributeError: 484 pass 485 486 result = [] 487 while node.parent is not None: 488 slug = re.sub(r'\W+', '-', node.prompt[0]).lower() 489 result.append(slug) 490 node = node.parent 491 492 result = '-'.join(reversed(result)) 493 return result 494 495 496def write_json_menus(deprecated_options, config, filename): 497 existing_ids = set() 498 result = [] # root level items 499 node_lookup = {} # lookup from MenuNode to an item in result 500 501 def write_node(node): 502 try: 503 json_parent = node_lookup[node.parent]['children'] 504 except KeyError: 505 assert node.parent not in node_lookup # if fails, we have a parent node with no "children" entity (ie a bug) 506 json_parent = result # root level node 507 508 # node.kconfig.y means node has no dependency, 509 if node.dep is node.kconfig.y: 510 depends = None 511 else: 512 depends = kconfiglib.expr_str(node.dep) 513 514 try: 515 # node.is_menuconfig is True in newer kconfiglibs for menus and choices as well 516 is_menuconfig = node.is_menuconfig and isinstance(node.item, kconfiglib.Symbol) 517 except AttributeError: 518 is_menuconfig = False 519 520 new_json = None 521 if node.item == kconfiglib.MENU or is_menuconfig: 522 new_json = {'type': 'menu', 523 'title': node.prompt[0], 524 'depends_on': depends, 525 'children': [], 526 } 527 if is_menuconfig: 528 sym = node.item 529 new_json['name'] = sym.name 530 new_json['help'] = node.help 531 new_json['is_menuconfig'] = is_menuconfig 532 greatest_range = None 533 if len(sym.ranges) > 0: 534 # Note: Evaluating the condition using kconfiglib's expr_value 535 # should have one condition which is true 536 for min_range, max_range, cond_expr in sym.ranges: 537 if kconfiglib.expr_value(cond_expr): 538 greatest_range = [min_range, max_range] 539 new_json['range'] = greatest_range 540 541 elif isinstance(node.item, kconfiglib.Symbol): 542 sym = node.item 543 greatest_range = None 544 if len(sym.ranges) > 0: 545 # Note: Evaluating the condition using kconfiglib's expr_value 546 # should have one condition which is true 547 for min_range, max_range, cond_expr in sym.ranges: 548 if kconfiglib.expr_value(cond_expr): 549 base = 16 if sym.type == kconfiglib.HEX else 10 550 greatest_range = [int(min_range.str_value, base), int(max_range.str_value, base)] 551 break 552 553 new_json = { 554 'type': kconfiglib.TYPE_TO_STR[sym.type], 555 'name': sym.name, 556 'title': node.prompt[0] if node.prompt else None, 557 'depends_on': depends, 558 'help': node.help, 559 'range': greatest_range, 560 'children': [], 561 } 562 elif isinstance(node.item, kconfiglib.Choice): 563 choice = node.item 564 new_json = { 565 'type': 'choice', 566 'title': node.prompt[0], 567 'name': choice.name, 568 'depends_on': depends, 569 'help': node.help, 570 'children': [] 571 } 572 573 if new_json: 574 node_id = get_menu_node_id(node) 575 if node_id in existing_ids: 576 raise RuntimeError('Config file contains two items with the same id: %s (%s). ' + 577 'Please rename one of these items to avoid ambiguity.' % (node_id, node.prompt[0])) 578 new_json['id'] = node_id 579 580 json_parent.append(new_json) 581 node_lookup[node] = new_json 582 583 for n in config.node_iter(): 584 write_node(n) 585 with open(filename, 'w') as f: 586 f.write(json.dumps(result, sort_keys=True, indent=4)) 587 588 589def write_docs(deprecated_options, config, filename): 590 try: 591 target = os.environ['IDF_TARGET'] 592 except KeyError: 593 print('IDF_TARGET environment variable must be defined!') 594 sys.exit(1) 595 596 visibility = gen_kconfig_doc.ConfigTargetVisibility(config, target) 597 gen_kconfig_doc.write_docs(config, visibility, filename) 598 deprecated_options.append_doc(config, visibility, filename) 599 600 601def update_if_changed(source, destination): 602 with open(source, 'r') as f: 603 source_contents = f.read() 604 605 if os.path.exists(destination): 606 with open(destination, 'r') as f: 607 dest_contents = f.read() 608 if source_contents == dest_contents: 609 return # nothing to update 610 611 with open(destination, 'w') as f: 612 f.write(source_contents) 613 614 615OUTPUT_FORMATS = {'config': write_config, 616 'makefile': write_makefile, # only used with make in order to generate auto.conf 617 'header': write_header, 618 'cmake': write_cmake, 619 'docs': write_docs, 620 'json': write_json, 621 'json_menus': write_json_menus, 622 } 623 624 625class FatalError(RuntimeError): 626 """ 627 Class for runtime errors (not caused by bugs but by user input). 628 """ 629 pass 630 631 632if __name__ == '__main__': 633 try: 634 main() 635 except FatalError as e: 636 print('A fatal error occurred: %s' % e) 637 sys.exit(2) 638
