1#!/usr/bin/env python 2# 3# Command line tool to take in ESP-IDF sdkconfig files with project 4# settings and output data in multiple formats (update config, generate 5# header file, generate .cmake include file, documentation, etc). 6# 7# Used internally by the ESP-IDF build system. But designed to be 8# non-IDF-specific. 9# 10# Copyright 2018-2020 Espressif Systems (Shanghai) PTE LTD 11# 12# Licensed under the Apache License, Version 2.0 (the "License"); 13# you may not use this file except in compliance with the License. 14# You may obtain a copy of the License at 15# 16# http:#www.apache.org/licenses/LICENSE-2.0 17# 18# Unless required by applicable law or agreed to in writing, software 19# distributed under the License is distributed on an "AS IS" BASIS, 20# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 21# See the License for the specific language governing permissions and 22# limitations under the License. 23from __future__ import print_function 24 25import argparse 26import json 27import os 28import os.path 29import re 30import sys 31import tempfile 32 33 34if 'ZEPHYR_BASE' not in os.environ: 35 raise RuntimeError('Could not find ZEPHYR_BASE environment path') 36 37# use zephyr kconfiglib 38sys.path.insert(0, os.path.join(os.environ['ZEPHYR_BASE'], "scripts", "kconfig")) 39 40import gen_kconfig_doc # noqa: E402 41import kconfiglib # noqa: E402 42 43__version__ = '0.1' 44 45if 'IDF_CMAKE' not in os.environ: 46 os.environ['IDF_CMAKE'] = '' 47 48 49class DeprecatedOptions(object): 50 _REN_FILE = 'sdkconfig.rename' 51 _DEP_OP_BEGIN = '# Deprecated options for backward compatibility' 52 _DEP_OP_END = '# End of deprecated options' 53 _RE_DEP_OP_BEGIN = re.compile(_DEP_OP_BEGIN) 54 _RE_DEP_OP_END = re.compile(_DEP_OP_END) 55 56 def __init__(self, config_prefix, path_rename_files=[]): 57 self.config_prefix = config_prefix 58 # r_dic maps deprecated options to new options; rev_r_dic maps in the opposite direction 59 self.r_dic, self.rev_r_dic = self._parse_replacements(path_rename_files) 60 61 # note the '=' at the end of regex for not getting partial match of configs 62 self._RE_CONFIG = re.compile(r'{}(\w+)='.format(self.config_prefix)) 63 64 def _parse_replacements(self, repl_paths): 65 rep_dic = {} 66 rev_rep_dic = {} 67 68 def remove_config_prefix(string): 69 if string.startswith(self.config_prefix): 70 return string[len(self.config_prefix):] 71 raise RuntimeError('Error in {} (line {}): Config {} is not prefixed with {}' 72 ''.format(rep_path, line_number, string, self.config_prefix)) 73 74 for rep_path in repl_paths: 75 with open(rep_path) as f_rep: 76 for line_number, line in enumerate(f_rep, start=1): 77 sp_line = line.split() 78 if len(sp_line) == 0 or sp_line[0].startswith('#'): 79 # empty line or comment 80 continue 81 if len(sp_line) != 2 or not all(x.startswith(self.config_prefix) for x in sp_line): 82 raise RuntimeError('Syntax error in {} (line {})'.format(rep_path, line_number)) 83 if sp_line[0] in rep_dic: 84 raise RuntimeError('Error in {} (line {}): Replacement {} exist for {} and new ' 85 'replacement {} is defined'.format(rep_path, line_number, 86 rep_dic[sp_line[0]], sp_line[0], 87 sp_line[1])) 88 89 (dep_opt, new_opt) = (remove_config_prefix(x) for x in sp_line) 90 rep_dic[dep_opt] = new_opt 91 rev_rep_dic[new_opt] = dep_opt 92 return rep_dic, rev_rep_dic 93 94 def get_deprecated_option(self, new_option): 95 return self.rev_r_dic.get(new_option, None) 96 97 def get_new_option(self, deprecated_option): 98 return self.r_dic.get(deprecated_option, None) 99 100 def replace(self, sdkconfig_in, sdkconfig_out): 101 replace_enabled = True 102 with open(sdkconfig_in, 'r') as f_in, open(sdkconfig_out, 'w') as f_out: 103 for line_num, line in enumerate(f_in, start=1): 104 if self._RE_DEP_OP_BEGIN.search(line): 105 replace_enabled = False 106 elif self._RE_DEP_OP_END.search(line): 107 replace_enabled = True 108 elif replace_enabled: 109 m = self._RE_CONFIG.search(line) 110 if m and m.group(1) in self.r_dic: 111 depr_opt = self.config_prefix + m.group(1) 112 new_opt = self.config_prefix + self.r_dic[m.group(1)] 113 line = line.replace(depr_opt, new_opt) 114 print('{}:{} {} was replaced with {}'.format(sdkconfig_in, line_num, depr_opt, new_opt)) 115 f_out.write(line) 116 117 def append_doc(self, config, visibility, path_output): 118 119 def option_was_written(opt): 120 # named choices were written if any of the symbols in the choice were visible 121 if new_opt in config.named_choices: 122 syms = config.named_choices[new_opt].syms 123 for s in syms: 124 if any(visibility.visible(node) for node in s.nodes): 125 return True 126 return False 127 else: 128 # otherwise if any of the nodes associated with the option was visible 129 return any(visibility.visible(node) for node in config.syms[opt].nodes) 130 131 if len(self.r_dic) > 0: 132 with open(path_output, 'a') as f_o: 133 header = 'Deprecated options and their replacements' 134 f_o.write('.. _configuration-deprecated-options:\n\n{}\n{}\n\n'.format(header, '-' * len(header))) 135 for dep_opt in sorted(self.r_dic): 136 new_opt = self.r_dic[dep_opt] 137 if option_was_written(new_opt) and (new_opt not in config.syms or config.syms[new_opt].choice is None): 138 # everything except config for a choice (no link reference for those in the docs) 139 f_o.write('- {}{} (:ref:`{}{}`)\n'.format(config.config_prefix, dep_opt, 140 config.config_prefix, new_opt)) 141 142 if new_opt in config.named_choices: 143 # here are printed config options which were filtered out 144 syms = config.named_choices[new_opt].syms 145 for sym in syms: 146 if sym.name in self.rev_r_dic: 147 # only if the symbol has been renamed 148 dep_name = self.rev_r_dic[sym.name] 149 150 # config options doesn't have references 151 f_o.write(' - {}{}\n'.format(config.config_prefix, dep_name)) 152 153 def append_config(self, config, path_output): 154 tmp_list = [] 155 156 def append_config_node_process(node): 157 item = node.item 158 if isinstance(item, kconfiglib.Symbol) and item.env_var is None: 159 if item.name in self.rev_r_dic: 160 c_string = item.config_string 161 if c_string: 162 tmp_list.append(c_string.replace(self.config_prefix + item.name, 163 self.config_prefix + self.rev_r_dic[item.name])) 164 165 for n in config.node_iter(): 166 append_config_node_process(n) 167 168 if len(tmp_list) > 0: 169 with open(path_output, 'a') as f_o: 170 f_o.write('\n{}\n'.format(self._DEP_OP_BEGIN)) 171 f_o.writelines(tmp_list) 172 f_o.write('{}\n'.format(self._DEP_OP_END)) 173 174 def append_header(self, config, path_output): 175 def _opt_defined(opt): 176 if not opt.visibility: 177 return False 178 return not (opt.orig_type in (kconfiglib.BOOL, kconfiglib.TRISTATE) and opt.str_value == 'n') 179 180 if len(self.r_dic) > 0: 181 with open(path_output, 'a') as f_o: 182 f_o.write('\n/* List of deprecated options */\n') 183 for dep_opt in sorted(self.r_dic): 184 new_opt = self.r_dic[dep_opt] 185 if new_opt in config.syms and _opt_defined(config.syms[new_opt]): 186 f_o.write('#define {}{} {}{}\n'.format(self.config_prefix, dep_opt, self.config_prefix, new_opt)) 187 188 189def main(): 190 parser = argparse.ArgumentParser(description='confgen.py v%s - Config Generation Tool' % __version__, prog=os.path.basename(sys.argv[0])) 191 192 parser.add_argument('--config', 193 help='Project configuration settings', 194 nargs='?', 195 default=None) 196 197 parser.add_argument('--defaults', 198 help='Optional project defaults file, used if --config file doesn\'t exist. ' 199 'Multiple files can be specified using multiple --defaults arguments.', 200 nargs='?', 201 default=[], 202 action='append') 203 204 parser.add_argument('--kconfig', 205 help='KConfig file with config item definitions', 206 required=True) 207 208 parser.add_argument('--sdkconfig-rename', 209 help='File with deprecated Kconfig options', 210 required=False) 211 212 parser.add_argument('--dont-write-deprecated', 213 help='Do not write compatibility statements for deprecated values', 214 action='store_true') 215 216 parser.add_argument('--output', nargs=2, action='append', 217 help='Write output file (format and output filename)', 218 metavar=('FORMAT', 'FILENAME'), 219 default=[]) 220 221 parser.add_argument('--env', action='append', default=[], 222 help='Environment to set when evaluating the config file', metavar='NAME=VAL') 223 224 parser.add_argument('--env-file', type=argparse.FileType('r'), 225 help='Optional file to load environment variables from. Contents ' 226 'should be a JSON object where each key/value pair is a variable.') 227 228 args = parser.parse_args() 229 230 for fmt, filename in args.output: 231 if fmt not in OUTPUT_FORMATS.keys(): 232 print("Format '%s' not recognised. Known formats: %s" % (fmt, OUTPUT_FORMATS.keys())) 233 sys.exit(1) 234 235 try: 236 args.env = [(name,value) for (name,value) in (e.split('=',1) for e in args.env)] 237 except ValueError: 238 print("--env arguments must each contain =. To unset an environment variable, use 'ENV='") 239 sys.exit(1) 240 241 for name, value in args.env: 242 os.environ[name] = value 243 244 if args.env_file is not None: 245 os.environ.update(json.load(args.env_file)) 246 247 config = kconfiglib.Kconfig(args.kconfig) 248 config.warn_assign_redun = False 249 config.warn_assign_override = False 250 251 sdkconfig_renames = [args.sdkconfig_rename] if args.sdkconfig_rename else [] 252 sdkconfig_renames += os.environ.get('COMPONENT_SDKCONFIG_RENAMES', '').split() 253 deprecated_options = DeprecatedOptions(config.config_prefix, path_rename_files=sdkconfig_renames) 254 255 if len(args.defaults) > 0: 256 def _replace_empty_assignments(path_in, path_out): 257 with open(path_in, 'r') as f_in, open(path_out, 'w') as f_out: 258 for line_num, line in enumerate(f_in, start=1): 259 line = line.strip() 260 if line.endswith('='): 261 line += 'n' 262 print('{}:{} line was updated to {}'.format(path_out, line_num, line)) 263 f_out.write(line) 264 f_out.write('\n') 265 266 # always load defaults first, so any items which are not defined in that config 267 # will have the default defined in the defaults file 268 for name in args.defaults: 269 print('Loading defaults file %s...' % name) 270 if not os.path.exists(name): 271 raise RuntimeError('Defaults file not found: %s' % name) 272 try: 273 with tempfile.NamedTemporaryFile(prefix='confgen_tmp', delete=False) as f: 274 temp_file1 = f.name 275 with tempfile.NamedTemporaryFile(prefix='confgen_tmp', delete=False) as f: 276 temp_file2 = f.name 277 deprecated_options.replace(sdkconfig_in=name, sdkconfig_out=temp_file1) 278 _replace_empty_assignments(temp_file1, temp_file2) 279 config.load_config(temp_file2, replace=False) 280 finally: 281 try: 282 os.remove(temp_file1) 283 os.remove(temp_file2) 284 except OSError: 285 pass 286 287 # If config file previously exists, load it 288 if args.config and os.path.exists(args.config): 289 # ... but replace deprecated options before that 290 with tempfile.NamedTemporaryFile(prefix='confgen_tmp', delete=False) as f: 291 temp_file = f.name 292 try: 293 deprecated_options.replace(sdkconfig_in=args.config, sdkconfig_out=temp_file) 294 config.load_config(temp_file, replace=False) 295 update_if_changed(temp_file, args.config) 296 finally: 297 try: 298 os.remove(temp_file) 299 except OSError: 300 pass 301 302 if args.dont_write_deprecated: 303 # The deprecated object was useful until now for replacements. Now it will be redefined with no configurations 304 # and as the consequence, it won't generate output with deprecated statements. 305 deprecated_options = DeprecatedOptions('', path_rename_files=[]) 306 307 # Output the files specified in the arguments 308 for output_type, filename in args.output: 309 with tempfile.NamedTemporaryFile(prefix='confgen_tmp', delete=False) as f: 310 temp_file = f.name 311 try: 312 output_function = OUTPUT_FORMATS[output_type] 313 output_function(deprecated_options, config, temp_file) 314 update_if_changed(temp_file, filename) 315 finally: 316 try: 317 os.remove(temp_file) 318 except OSError: 319 pass 320 321 322def write_config(deprecated_options, config, filename): 323 CONFIG_HEADING = """# 324# Automatically generated file. DO NOT EDIT. 325# Espressif IoT Development Framework (ESP-IDF) Project Configuration 326# 327""" 328 config.write_config(filename, header=CONFIG_HEADING) 329 deprecated_options.append_config(config, filename) 330 331 332def write_makefile(deprecated_options, config, filename): 333 CONFIG_HEADING = """# 334# Automatically generated file. DO NOT EDIT. 335# Espressif IoT Development Framework (ESP-IDF) Project Makefile Configuration 336# 337""" 338 with open(filename, 'w') as f: 339 tmp_dep_lines = [] 340 f.write(CONFIG_HEADING) 341 342 def get_makefile_config_string(name, value, orig_type): 343 if orig_type in (kconfiglib.BOOL, kconfiglib.TRISTATE): 344 value = '' if value == 'n' else value 345 elif orig_type == kconfiglib.INT: 346 try: 347 value = int(value) 348 except ValueError: 349 value = '' 350 elif orig_type == kconfiglib.HEX: 351 try: 352 value = hex(int(value, 16)) # ensure 0x prefix 353 except ValueError: 354 value = '' 355 elif orig_type == kconfiglib.STRING: 356 value = '"{}"'.format(kconfiglib.escape(value)) 357 else: 358 raise RuntimeError('{}{}: unknown type {}'.format(config.config_prefix, name, orig_type)) 359 360 return '{}{}={}\n'.format(config.config_prefix, name, value) 361 362 def write_makefile_node(node): 363 item = node.item 364 if isinstance(item, kconfiglib.Symbol) and item.env_var is None: 365 # item.config_string cannot be used because it ignores hidden config items 366 val = item.str_value 367 f.write(get_makefile_config_string(item.name, val, item.orig_type)) 368 369 dep_opt = deprecated_options.get_deprecated_option(item.name) 370 if dep_opt: 371 # the same string but with the deprecated name 372 tmp_dep_lines.append(get_makefile_config_string(dep_opt, val, item.orig_type)) 373 374 for n in config.node_iter(True): 375 write_makefile_node(n) 376 377 if len(tmp_dep_lines) > 0: 378 f.write('\n# List of deprecated options\n') 379 f.writelines(tmp_dep_lines) 380 381 382def write_header(deprecated_options, config, filename): 383 CONFIG_HEADING = """/* 384 * Automatically generated file. DO NOT EDIT. 385 * Espressif IoT Development Framework (ESP-IDF) Configuration Header 386 */ 387#pragma once 388""" 389 config.write_autoconf(filename, header=CONFIG_HEADING) 390 deprecated_options.append_header(config, filename) 391 392 393def write_cmake(deprecated_options, config, filename): 394 with open(filename, 'w') as f: 395 tmp_dep_list = [] 396 write = f.write 397 prefix = config.config_prefix 398 399 write("""# 400# Automatically generated file. DO NOT EDIT. 401# Espressif IoT Development Framework (ESP-IDF) Configuration cmake include file 402# 403""") 404 405 configs_list = list() 406 407 def write_node(node): 408 sym = node.item 409 if not isinstance(sym, kconfiglib.Symbol): 410 return 411 412 if sym.config_string: 413 val = sym.str_value 414 if sym.orig_type in (kconfiglib.BOOL, kconfiglib.TRISTATE) and val == 'n': 415 val = '' # write unset values as empty variables 416 elif sym.orig_type == kconfiglib.STRING: 417 val = kconfiglib.escape(val) 418 elif sym.orig_type == kconfiglib.HEX: 419 val = hex(int(val, 16)) # ensure 0x prefix 420 write('set({}{} "{}")\n'.format(prefix, sym.name, val)) 421 422 configs_list.append(prefix + sym.name) 423 dep_opt = deprecated_options.get_deprecated_option(sym.name) 424 if dep_opt: 425 tmp_dep_list.append('set({}{} "{}")\n'.format(prefix, dep_opt, val)) 426 configs_list.append(prefix + dep_opt) 427 428 for n in config.node_iter(): 429 write_node(n) 430 write('set(CONFIGS_LIST {})'.format(';'.join(configs_list))) 431 432 if len(tmp_dep_list) > 0: 433 write('\n# List of deprecated options for backward compatibility\n') 434 f.writelines(tmp_dep_list) 435 436 437def get_json_values(config): 438 config_dict = {} 439 440 def write_node(node): 441 sym = node.item 442 if not isinstance(sym, kconfiglib.Symbol): 443 return 444 445 if sym.config_string: 446 val = sym.str_value 447 if sym.type in [kconfiglib.BOOL, kconfiglib.TRISTATE]: 448 val = (val != 'n') 449 elif sym.type == kconfiglib.HEX: 450 val = int(val, 16) 451 elif sym.type == kconfiglib.INT: 452 val = int(val) 453 config_dict[sym.name] = val 454 for n in config.node_iter(False): 455 write_node(n) 456 return config_dict 457 458 459def write_json(deprecated_options, config, filename): 460 config_dict = get_json_values(config) 461 with open(filename, 'w') as f: 462 json.dump(config_dict, f, indent=4, sort_keys=True) 463 464 465def get_menu_node_id(node): 466 """ Given a menu node, return a unique id 467 which can be used to identify it in the menu structure 468 469 Will either be the config symbol name, or a menu identifier 470 'slug' 471 472 """ 473 try: 474 if not isinstance(node.item, kconfiglib.Choice): 475 return node.item.name 476 except AttributeError: 477 pass 478 479 result = [] 480 while node.parent is not None: 481 slug = re.sub(r'\W+', '-', node.prompt[0]).lower() 482 result.append(slug) 483 node = node.parent 484 485 result = '-'.join(reversed(result)) 486 return result 487 488 489def write_json_menus(deprecated_options, config, filename): 490 existing_ids = set() 491 result = [] # root level items 492 node_lookup = {} # lookup from MenuNode to an item in result 493 494 def write_node(node): 495 try: 496 json_parent = node_lookup[node.parent]['children'] 497 except KeyError: 498 assert node.parent not in node_lookup # if fails, we have a parent node with no "children" entity (ie a bug) 499 json_parent = result # root level node 500 501 # node.kconfig.y means node has no dependency, 502 if node.dep is node.kconfig.y: 503 depends = None 504 else: 505 depends = kconfiglib.expr_str(node.dep) 506 507 try: 508 # node.is_menuconfig is True in newer kconfiglibs for menus and choices as well 509 is_menuconfig = node.is_menuconfig and isinstance(node.item, kconfiglib.Symbol) 510 except AttributeError: 511 is_menuconfig = False 512 513 new_json = None 514 if node.item == kconfiglib.MENU or is_menuconfig: 515 new_json = {'type': 'menu', 516 'title': node.prompt[0], 517 'depends_on': depends, 518 'children': [], 519 } 520 if is_menuconfig: 521 sym = node.item 522 new_json['name'] = sym.name 523 new_json['help'] = node.help 524 new_json['is_menuconfig'] = is_menuconfig 525 greatest_range = None 526 if len(sym.ranges) > 0: 527 # Note: Evaluating the condition using kconfiglib's expr_value 528 # should have one condition which is true 529 for min_range, max_range, cond_expr in sym.ranges: 530 if kconfiglib.expr_value(cond_expr): 531 greatest_range = [min_range, max_range] 532 new_json['range'] = greatest_range 533 534 elif isinstance(node.item, kconfiglib.Symbol): 535 sym = node.item 536 greatest_range = None 537 if len(sym.ranges) > 0: 538 # Note: Evaluating the condition using kconfiglib's expr_value 539 # should have one condition which is true 540 for min_range, max_range, cond_expr in sym.ranges: 541 if kconfiglib.expr_value(cond_expr): 542 base = 16 if sym.type == kconfiglib.HEX else 10 543 greatest_range = [int(min_range.str_value, base), int(max_range.str_value, base)] 544 break 545 546 new_json = { 547 'type': kconfiglib.TYPE_TO_STR[sym.type], 548 'name': sym.name, 549 'title': node.prompt[0] if node.prompt else None, 550 'depends_on': depends, 551 'help': node.help, 552 'range': greatest_range, 553 'children': [], 554 } 555 elif isinstance(node.item, kconfiglib.Choice): 556 choice = node.item 557 new_json = { 558 'type': 'choice', 559 'title': node.prompt[0], 560 'name': choice.name, 561 'depends_on': depends, 562 'help': node.help, 563 'children': [] 564 } 565 566 if new_json: 567 node_id = get_menu_node_id(node) 568 if node_id in existing_ids: 569 raise RuntimeError('Config file contains two items with the same id: %s (%s). ' + 570 'Please rename one of these items to avoid ambiguity.' % (node_id, node.prompt[0])) 571 new_json['id'] = node_id 572 573 json_parent.append(new_json) 574 node_lookup[node] = new_json 575 576 for n in config.node_iter(): 577 write_node(n) 578 with open(filename, 'w') as f: 579 f.write(json.dumps(result, sort_keys=True, indent=4)) 580 581 582def write_docs(deprecated_options, config, filename): 583 try: 584 target = os.environ['IDF_TARGET'] 585 except KeyError: 586 print('IDF_TARGET environment variable must be defined!') 587 sys.exit(1) 588 589 visibility = gen_kconfig_doc.ConfigTargetVisibility(config, target) 590 gen_kconfig_doc.write_docs(config, visibility, filename) 591 deprecated_options.append_doc(config, visibility, filename) 592 593 594def update_if_changed(source, destination): 595 with open(source, 'r') as f: 596 source_contents = f.read() 597 598 if os.path.exists(destination): 599 with open(destination, 'r') as f: 600 dest_contents = f.read() 601 if source_contents == dest_contents: 602 return # nothing to update 603 604 with open(destination, 'w') as f: 605 f.write(source_contents) 606 607 608OUTPUT_FORMATS = {'config': write_config, 609 'makefile': write_makefile, # only used with make in order to generate auto.conf 610 'header': write_header, 611 'cmake': write_cmake, 612 'docs': write_docs, 613 'json': write_json, 614 'json_menus': write_json_menus, 615 } 616 617 618class FatalError(RuntimeError): 619 """ 620 Class for runtime errors (not caused by bugs but by user input). 621 """ 622 pass 623 624 625if __name__ == '__main__': 626 try: 627 main() 628 except FatalError as e: 629 print('A fatal error occurred: %s' % e) 630 sys.exit(2) 631
