1#!/usr/bin/env python 2# coding=utf-8 3# 4# This script helps installing tools required to use the ESP-IDF, and updating PATH 5# to use the installed tools. It can also create a Python virtual environment, 6# and install Python requirements into it. 7# It does not install OS dependencies. It does install tools such as the Xtensa 8# GCC toolchain and ESP32 ULP coprocessor toolchain. 9# 10# By default, downloaded tools will be installed under $HOME/.espressif directory 11# (%USERPROFILE%/.espressif on Windows). This path can be modified by setting 12# IDF_TOOLS_PATH variable prior to running this tool. 13# 14# Users do not need to interact with this script directly. In IDF root directory, 15# install.sh (.bat) and export.sh (.bat) scripts are provided to invoke this script. 16# 17# Usage: 18# 19# * To install the tools, run `idf_tools.py install`. 20# 21# * To install the Python environment, run `idf_tools.py install-python-env`. 22# 23# * To start using the tools, run `eval "$(idf_tools.py export)"` — this will update 24# the PATH to point to the installed tools and set up other environment variables 25# needed by the tools. 26# 27### 28# 29# Copyright 2019 Espressif Systems (Shanghai) PTE LTD 30# 31# Licensed under the Apache License, Version 2.0 (the "License"); 32# you may not use this file except in compliance with the License. 33# You may obtain a copy of the License at 34# 35# http://www.apache.org/licenses/LICENSE-2.0 36# 37# Unless required by applicable law or agreed to in writing, software 38# distributed under the License is distributed on an "AS IS" BASIS, 39# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 40# See the License for the specific language governing permissions and 41# limitations under the License. 42 43import argparse 44import contextlib 45import copy 46import errno 47import functools 48import hashlib 49import json 50import os 51import platform 52import re 53import shutil 54import ssl 55import subprocess 56import sys 57import tarfile 58import zipfile 59from collections import OrderedDict, namedtuple 60 61try: 62 import typing # noqa: F401 63except ImportError: 64 pass 65 66try: 67 from urllib.error import ContentTooShortError 68 from urllib.request import urlopen 69except ImportError: 70 from urllib import ContentTooShortError, urlopen 71 72try: 73 from exceptions import WindowsError 74except ImportError: 75 class WindowsError(OSError): 76 pass 77 78 79TOOLS_FILE = 'tools/tools.json' 80TOOLS_SCHEMA_FILE = 'tools/tools_schema.json' 81TOOLS_FILE_NEW = 'tools/tools.new.json' 82TOOLS_FILE_VERSION = 1 83IDF_TOOLS_PATH_DEFAULT = os.path.join('~', '.espressif') 84UNKNOWN_VERSION = 'unknown' 85SUBST_TOOL_PATH_REGEX = re.compile(r'\${TOOL_PATH}') 86VERSION_REGEX_REPLACE_DEFAULT = r'\1' 87IDF_MAINTAINER = os.environ.get('IDF_MAINTAINER') or False 88TODO_MESSAGE = 'TODO' 89DOWNLOAD_RETRY_COUNT = 3 90URL_PREFIX_MAP_SEPARATOR = ',' 91IDF_TOOLS_INSTALL_CMD = os.environ.get('IDF_TOOLS_INSTALL_CMD') 92IDF_TOOLS_EXPORT_CMD = os.environ.get('IDF_TOOLS_INSTALL_CMD') 93 94PYTHON_PLATFORM = platform.system() + '-' + platform.machine() 95 96# Identifiers used in tools.json for different platforms. 97PLATFORM_WIN32 = 'win32' 98PLATFORM_WIN64 = 'win64' 99PLATFORM_MACOS = 'macos' 100PLATFORM_LINUX32 = 'linux-i686' 101PLATFORM_LINUX64 = 'linux-amd64' 102PLATFORM_LINUX_ARM32 = 'linux-armel' 103PLATFORM_LINUX_ARM64 = 'linux-arm64' 104 105 106# Mappings from various other names these platforms are known as, to the identifiers above. 107# This includes strings produced from "platform.system() + '-' + platform.machine()", see PYTHON_PLATFORM 108# definition above. 109# This list also includes various strings used in release archives of xtensa-esp32-elf-gcc, OpenOCD, etc. 110PLATFORM_FROM_NAME = { 111 # Windows 112 PLATFORM_WIN32: PLATFORM_WIN32, 113 'Windows-i686': PLATFORM_WIN32, 114 'Windows-x86': PLATFORM_WIN32, 115 PLATFORM_WIN64: PLATFORM_WIN64, 116 'Windows-x86_64': PLATFORM_WIN64, 117 'Windows-AMD64': PLATFORM_WIN64, 118 # macOS 119 PLATFORM_MACOS: PLATFORM_MACOS, 120 'osx': PLATFORM_MACOS, 121 'darwin': PLATFORM_MACOS, 122 'Darwin-x86_64': PLATFORM_MACOS, 123 # pretend it is x86_64 until Darwin-arm64 tool builds are available: 124 'Darwin-arm64': PLATFORM_MACOS, 125 # Linux 126 PLATFORM_LINUX64: PLATFORM_LINUX64, 127 'linux64': PLATFORM_LINUX64, 128 'Linux-x86_64': PLATFORM_LINUX64, 129 'FreeBSD-amd64': PLATFORM_LINUX64, 130 PLATFORM_LINUX32: PLATFORM_LINUX32, 131 'linux32': PLATFORM_LINUX32, 132 'Linux-i686': PLATFORM_LINUX32, 133 'FreeBSD-i386': PLATFORM_LINUX32, 134 PLATFORM_LINUX_ARM32: PLATFORM_LINUX_ARM32, 135 'Linux-arm': PLATFORM_LINUX_ARM32, 136 'Linux-armv7l': PLATFORM_LINUX_ARM32, 137 PLATFORM_LINUX_ARM64: PLATFORM_LINUX_ARM64, 138 'Linux-arm64': PLATFORM_LINUX_ARM64, 139 'Linux-aarch64': PLATFORM_LINUX_ARM64, 140 'Linux-armv8l': PLATFORM_LINUX_ARM64, 141} 142 143UNKNOWN_PLATFORM = 'unknown' 144CURRENT_PLATFORM = PLATFORM_FROM_NAME.get(PYTHON_PLATFORM, UNKNOWN_PLATFORM) 145 146EXPORT_SHELL = 'shell' 147EXPORT_KEY_VALUE = 'key-value' 148 149ISRG_X1_ROOT_CERT = u""" 150-----BEGIN CERTIFICATE----- 151MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw 152TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh 153cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4 154WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu 155ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY 156MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc 157h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+ 1580TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U 159A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW 160T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH 161B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC 162B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv 163KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn 164OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn 165jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw 166qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI 167rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV 168HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq 169hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL 170ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ 1713BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK 172NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5 173ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur 174TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC 175jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc 176oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq 1774RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA 178mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d 179emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc= 180-----END CERTIFICATE----- 181""" 182 183 184global_quiet = False 185global_non_interactive = False 186global_idf_path = None # type: typing.Optional[str] 187global_idf_tools_path = None # type: typing.Optional[str] 188global_tools_json = None # type: typing.Optional[str] 189 190 191def fatal(text, *args): 192 if not global_quiet: 193 sys.stderr.write('ERROR: ' + text + '\n', *args) 194 195 196def warn(text, *args): 197 if not global_quiet: 198 sys.stderr.write('WARNING: ' + text + '\n', *args) 199 200 201def info(text, f=None, *args): 202 if not global_quiet: 203 if f is None: 204 f = sys.stdout 205 f.write(text + '\n', *args) 206 207 208def run_cmd_check_output(cmd, input_text=None, extra_paths=None): 209 # If extra_paths is given, locate the executable in one of these directories. 210 # Note: it would seem logical to add extra_paths to env[PATH], instead, and let OS do the job of finding the 211 # executable for us. However this does not work on Windows: https://bugs.python.org/issue8557. 212 if extra_paths: 213 found = False 214 extensions = [''] 215 if sys.platform == 'win32': 216 extensions.append('.exe') 217 for path in extra_paths: 218 for ext in extensions: 219 fullpath = os.path.join(path, cmd[0] + ext) 220 if os.path.exists(fullpath): 221 cmd[0] = fullpath 222 found = True 223 break 224 if found: 225 break 226 227 try: 228 if input_text: 229 input_text = input_text.encode() 230 result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, input=input_text) 231 return result.stdout + result.stderr 232 except (AttributeError, TypeError): 233 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) 234 stdout, stderr = p.communicate(input_text) 235 if p.returncode != 0: 236 try: 237 raise subprocess.CalledProcessError(p.returncode, cmd, stdout, stderr) 238 except TypeError: 239 raise subprocess.CalledProcessError(p.returncode, cmd, stdout) 240 return stdout + stderr 241 242 243def to_shell_specific_paths(paths_list): 244 if sys.platform == 'win32': 245 paths_list = [p.replace('/', os.path.sep) if os.path.sep in p else p for p in paths_list] 246 247 if 'MSYSTEM' in os.environ: 248 paths_msys = run_cmd_check_output(['cygpath', '-u', '-f', '-'], 249 input_text='\n'.join(paths_list)) 250 paths_list = paths_msys.decode().strip().split('\n') 251 252 return paths_list 253 254 255def get_env_for_extra_paths(extra_paths): 256 """ 257 Return a copy of environment variables dict, prepending paths listed in extra_paths 258 to the PATH environment variable. 259 """ 260 env_arg = os.environ.copy() 261 new_path = os.pathsep.join(extra_paths) + os.pathsep + env_arg['PATH'] 262 if sys.version_info.major == 2: 263 env_arg['PATH'] = new_path.encode('utf8') 264 else: 265 env_arg['PATH'] = new_path 266 return env_arg 267 268 269def get_file_size_sha256(filename, block_size=65536): 270 sha256 = hashlib.sha256() 271 size = 0 272 with open(filename, 'rb') as f: 273 for block in iter(lambda: f.read(block_size), b''): 274 sha256.update(block) 275 size += len(block) 276 return size, sha256.hexdigest() 277 278 279def report_progress(count, block_size, total_size): 280 percent = int(count * block_size * 100 / total_size) 281 percent = min(100, percent) 282 sys.stdout.write('\r%d%%' % percent) 283 sys.stdout.flush() 284 285 286def mkdir_p(path): 287 try: 288 os.makedirs(path) 289 except OSError as exc: 290 if exc.errno != errno.EEXIST or not os.path.isdir(path): 291 raise 292 293 294def unpack(filename, destination): 295 info('Extracting {0} to {1}'.format(filename, destination)) 296 if filename.endswith(('.tar.gz', '.tgz')): 297 archive_obj = tarfile.open(filename, 'r:gz') 298 elif filename.endswith('zip'): 299 archive_obj = zipfile.ZipFile(filename) 300 else: 301 raise NotImplementedError('Unsupported archive type') 302 if sys.version_info.major == 2: 303 # This is a workaround for the issue that unicode destination is not handled: 304 # https://bugs.python.org/issue17153 305 destination = str(destination) 306 archive_obj.extractall(destination) 307 308 309def splittype(url): 310 match = re.match('([^/:]+):(.*)', url, re.DOTALL) 311 if match: 312 scheme, data = match.groups() 313 return scheme.lower(), data 314 return None, url 315 316 317# An alternative version of urlretrieve which takes SSL context as an argument 318def urlretrieve_ctx(url, filename, reporthook=None, data=None, context=None): 319 url_type, path = splittype(url) 320 321 # urlopen doesn't have context argument in Python <=2.7.9 322 extra_urlopen_args = {} 323 if context: 324 extra_urlopen_args['context'] = context 325 with contextlib.closing(urlopen(url, data, **extra_urlopen_args)) as fp: 326 headers = fp.info() 327 328 # Just return the local path and the "headers" for file:// 329 # URLs. No sense in performing a copy unless requested. 330 if url_type == 'file' and not filename: 331 return os.path.normpath(path), headers 332 333 # Handle temporary file setup. 334 tfp = open(filename, 'wb') 335 336 with tfp: 337 result = filename, headers 338 bs = 1024 * 8 339 size = int(headers.get('content-length', -1)) 340 read = 0 341 blocknum = 0 342 343 if reporthook: 344 reporthook(blocknum, bs, size) 345 346 while True: 347 block = fp.read(bs) 348 if not block: 349 break 350 read += len(block) 351 tfp.write(block) 352 blocknum += 1 353 if reporthook: 354 reporthook(blocknum, bs, size) 355 356 if size >= 0 and read < size: 357 raise ContentTooShortError( 358 'retrieval incomplete: got only %i out of %i bytes' 359 % (read, size), result) 360 361 return result 362 363 364# Sometimes renaming a directory on Windows (randomly?) causes a PermissionError. 365# This is confirmed to be a workaround: 366# https://github.com/espressif/esp-idf/issues/3819#issuecomment-515167118 367# https://github.com/espressif/esp-idf/issues/4063#issuecomment-531490140 368# https://stackoverflow.com/a/43046729 369def rename_with_retry(path_from, path_to): 370 if sys.platform.startswith('win'): 371 retry_count = 100 372 else: 373 retry_count = 1 374 375 for retry in range(retry_count): 376 try: 377 os.rename(path_from, path_to) 378 return 379 except (OSError, WindowsError): # WindowsError until Python 3.3, then OSError 380 if retry == retry_count - 1: 381 raise 382 warn('Rename {} to {} failed, retrying...'.format(path_from, path_to)) 383 384 385def strip_container_dirs(path, levels): 386 assert levels > 0 387 # move the original directory out of the way (add a .tmp suffix) 388 tmp_path = path + '.tmp' 389 if os.path.exists(tmp_path): 390 shutil.rmtree(tmp_path) 391 rename_with_retry(path, tmp_path) 392 os.mkdir(path) 393 base_path = tmp_path 394 # walk given number of levels down 395 for level in range(levels): 396 contents = os.listdir(base_path) 397 if len(contents) > 1: 398 raise RuntimeError('at level {}, expected 1 entry, got {}'.format(level, contents)) 399 base_path = os.path.join(base_path, contents[0]) 400 if not os.path.isdir(base_path): 401 raise RuntimeError('at level {}, {} is not a directory'.format(level, contents[0])) 402 # get the list of directories/files to move 403 contents = os.listdir(base_path) 404 for name in contents: 405 move_from = os.path.join(base_path, name) 406 move_to = os.path.join(path, name) 407 rename_with_retry(move_from, move_to) 408 shutil.rmtree(tmp_path) 409 410 411class ToolNotFound(RuntimeError): 412 pass 413 414 415class ToolExecError(RuntimeError): 416 pass 417 418 419class DownloadError(RuntimeError): 420 pass 421 422 423class IDFToolDownload(object): 424 def __init__(self, platform_name, url, size, sha256): 425 self.platform_name = platform_name 426 self.url = url 427 self.size = size 428 self.sha256 = sha256 429 self.platform_name = platform_name 430 431 432@functools.total_ordering 433class IDFToolVersion(object): 434 STATUS_RECOMMENDED = 'recommended' 435 STATUS_SUPPORTED = 'supported' 436 STATUS_DEPRECATED = 'deprecated' 437 438 STATUS_VALUES = [STATUS_RECOMMENDED, STATUS_SUPPORTED, STATUS_DEPRECATED] 439 440 def __init__(self, version, status): 441 self.version = version 442 self.status = status 443 self.downloads = OrderedDict() 444 self.latest = False 445 446 def __lt__(self, other): 447 if self.status != other.status: 448 return self.status > other.status 449 else: 450 assert not (self.status == IDFToolVersion.STATUS_RECOMMENDED 451 and other.status == IDFToolVersion.STATUS_RECOMMENDED) 452 return self.version < other.version 453 454 def __eq__(self, other): 455 return self.status == other.status and self.version == other.version 456 457 def add_download(self, platform_name, url, size, sha256): 458 self.downloads[platform_name] = IDFToolDownload(platform_name, url, size, sha256) 459 460 def get_download_for_platform(self, platform_name): # type: (str) -> IDFToolDownload 461 if platform_name in PLATFORM_FROM_NAME.keys(): 462 platform_name = PLATFORM_FROM_NAME[platform_name] 463 if platform_name in self.downloads.keys(): 464 return self.downloads[platform_name] 465 if 'any' in self.downloads.keys(): 466 return self.downloads['any'] 467 return None 468 469 def compatible_with_platform(self, platform_name=PYTHON_PLATFORM): 470 return self.get_download_for_platform(platform_name) is not None 471 472 def get_supported_platforms(self): # type: () -> typing.Set[str] 473 return set(self.downloads.keys()) 474 475 476OPTIONS_LIST = ['version_cmd', 477 'version_regex', 478 'version_regex_replace', 479 'export_paths', 480 'export_vars', 481 'install', 482 'info_url', 483 'license', 484 'strip_container_dirs'] 485 486IDFToolOptions = namedtuple('IDFToolOptions', OPTIONS_LIST) 487 488 489class IDFTool(object): 490 # possible values of 'install' field 491 INSTALL_ALWAYS = 'always' 492 INSTALL_ON_REQUEST = 'on_request' 493 INSTALL_NEVER = 'never' 494 495 def __init__(self, name, description, install, info_url, license, version_cmd, version_regex, version_regex_replace=None, 496 strip_container_dirs=0): 497 self.name = name 498 self.description = description 499 self.versions = OrderedDict() # type: typing.Dict[str, IDFToolVersion] 500 self.version_in_path = None 501 self.versions_installed = [] 502 if version_regex_replace is None: 503 version_regex_replace = VERSION_REGEX_REPLACE_DEFAULT 504 self.options = IDFToolOptions(version_cmd, version_regex, version_regex_replace, 505 [], OrderedDict(), install, info_url, license, strip_container_dirs) 506 self.platform_overrides = [] 507 self._platform = CURRENT_PLATFORM 508 self._update_current_options() 509 510 def copy_for_platform(self, platform): # type: (str) -> IDFTool 511 result = copy.deepcopy(self) 512 result._platform = platform 513 result._update_current_options() 514 return result 515 516 def _update_current_options(self): 517 self._current_options = IDFToolOptions(*self.options) 518 for override in self.platform_overrides: 519 if self._platform not in override['platforms']: 520 continue 521 override_dict = override.copy() 522 del override_dict['platforms'] 523 self._current_options = self._current_options._replace(**override_dict) 524 525 def add_version(self, version): 526 assert(type(version) is IDFToolVersion) 527 self.versions[version.version] = version 528 529 def get_path(self): # type: () -> str 530 return os.path.join(global_idf_tools_path, 'tools', self.name) 531 532 def get_path_for_version(self, version): # type: (str) -> str 533 assert(version in self.versions) 534 return os.path.join(self.get_path(), version) 535 536 def get_export_paths(self, version): # type: (str) -> typing.List[str] 537 tool_path = self.get_path_for_version(version) 538 return [os.path.join(tool_path, *p) for p in self._current_options.export_paths] 539 540 def get_export_vars(self, version): # type: (str) -> typing.Dict[str] 541 """ 542 Get the dictionary of environment variables to be exported, for the given version. 543 Expands: 544 - ${TOOL_PATH} => the actual path where the version is installed 545 """ 546 result = {} 547 for k, v in self._current_options.export_vars.items(): 548 replace_path = self.get_path_for_version(version).replace('\\', '\\\\') 549 v_repl = re.sub(SUBST_TOOL_PATH_REGEX, replace_path, v) 550 if v_repl != v: 551 v_repl = to_shell_specific_paths([v_repl])[0] 552 result[k] = v_repl 553 return result 554 555 def check_version(self, extra_paths=None): # type: (typing.Optional[typing.List[str]]) -> str 556 """ 557 Execute the tool, optionally prepending extra_paths to PATH, 558 extract the version string and return it as a result. 559 Raises ToolNotFound if the tool is not found (not present in the paths). 560 Raises ToolExecError if the tool returns with a non-zero exit code. 561 Returns 'unknown' if tool returns something from which version string 562 can not be extracted. 563 """ 564 # this function can not be called for a different platform 565 assert self._platform == CURRENT_PLATFORM 566 cmd = self._current_options.version_cmd 567 try: 568 version_cmd_result = run_cmd_check_output(cmd, None, extra_paths) 569 except OSError: 570 # tool is not on the path 571 raise ToolNotFound('Tool {} not found'.format(self.name)) 572 except subprocess.CalledProcessError as e: 573 raise ToolExecError('Command {} has returned non-zero exit code ({})\n'.format( 574 ' '.join(self._current_options.version_cmd), e.returncode)) 575 576 in_str = version_cmd_result.decode('utf-8') 577 match = re.search(self._current_options.version_regex, in_str) 578 if not match: 579 return UNKNOWN_VERSION 580 return re.sub(self._current_options.version_regex, self._current_options.version_regex_replace, match.group(0)) 581 582 def get_install_type(self): 583 return self._current_options.install 584 585 def compatible_with_platform(self): 586 return any([v.compatible_with_platform() for v in self.versions.values()]) 587 588 def get_supported_platforms(self): # type: () -> typing.Set[str] 589 result = set() 590 for v in self.versions.values(): 591 result.update(v.get_supported_platforms()) 592 return result 593 594 def get_recommended_version(self): 595 recommended_versions = [k for k, v in self.versions.items() 596 if v.status == IDFToolVersion.STATUS_RECOMMENDED 597 and v.compatible_with_platform(self._platform)] 598 assert len(recommended_versions) <= 1 599 if recommended_versions: 600 return recommended_versions[0] 601 return None 602 603 def get_preferred_installed_version(self): 604 recommended_versions = [k for k in self.versions_installed 605 if self.versions[k].status == IDFToolVersion.STATUS_RECOMMENDED 606 and self.versions[k].compatible_with_platform(self._platform)] 607 assert len(recommended_versions) <= 1 608 if recommended_versions: 609 return recommended_versions[0] 610 return None 611 612 def find_installed_versions(self): 613 """ 614 Checks whether the tool can be found in PATH and in global_idf_tools_path. 615 Writes results to self.version_in_path and self.versions_installed. 616 """ 617 # this function can not be called for a different platform 618 assert self._platform == CURRENT_PLATFORM 619 # First check if the tool is in system PATH 620 try: 621 ver_str = self.check_version() 622 except ToolNotFound: 623 # not in PATH 624 pass 625 except ToolExecError: 626 warn('tool {} found in path, but failed to run'.format(self.name)) 627 else: 628 self.version_in_path = ver_str 629 630 # Now check all the versions installed in global_idf_tools_path 631 self.versions_installed = [] 632 for version, version_obj in self.versions.items(): 633 if not version_obj.compatible_with_platform(): 634 continue 635 tool_path = self.get_path_for_version(version) 636 if not os.path.exists(tool_path): 637 # version not installed 638 continue 639 try: 640 ver_str = self.check_version(self.get_export_paths(version)) 641 except ToolNotFound: 642 warn('directory for tool {} version {} is present, but tool was not found'.format( 643 self.name, version)) 644 except ToolExecError: 645 warn('tool {} version {} is installed, but the tool failed to run'.format( 646 self.name, version)) 647 else: 648 if ver_str != version: 649 warn('tool {} version {} is installed, but has reported version {}'.format( 650 self.name, version, ver_str)) 651 else: 652 self.versions_installed.append(version) 653 654 def download(self, version): 655 assert(version in self.versions) 656 download_obj = self.versions[version].get_download_for_platform(self._platform) 657 if not download_obj: 658 fatal('No packages for tool {} platform {}!'.format(self.name, self._platform)) 659 raise DownloadError() 660 661 url = download_obj.url 662 archive_name = os.path.basename(url) 663 local_path = os.path.join(global_idf_tools_path, 'dist', archive_name) 664 mkdir_p(os.path.dirname(local_path)) 665 666 if os.path.isfile(local_path): 667 if not self.check_download_file(download_obj, local_path): 668 warn('removing downloaded file {0} and downloading again'.format(archive_name)) 669 os.unlink(local_path) 670 else: 671 info('file {0} is already downloaded'.format(archive_name)) 672 return 673 674 downloaded = False 675 for retry in range(DOWNLOAD_RETRY_COUNT): 676 local_temp_path = local_path + '.tmp' 677 info('Downloading {} to {}'.format(archive_name, local_temp_path)) 678 try: 679 ctx = None 680 # For dl.espressif.com, add the ISRG x1 root certificate. 681 # This works around the issue with outdated certificate stores in some installations. 682 if 'dl.espressif.com' in url: 683 try: 684 ctx = ssl.create_default_context() 685 ctx.load_verify_locations(cadata=ISRG_X1_ROOT_CERT) 686 except AttributeError: 687 # no ssl.create_default_context or load_verify_locations cadata argument 688 # in Python <=2.7.8 689 pass 690 691 urlretrieve_ctx(url, local_temp_path, report_progress if not global_non_interactive else None, context=ctx) 692 sys.stdout.write('\rDone\n') 693 except Exception as e: 694 # urlretrieve could throw different exceptions, e.g. IOError when the server is down 695 # Errors are ignored because the downloaded file is checked a couple of lines later. 696 warn('Download failure {}'.format(e)) 697 sys.stdout.flush() 698 if not os.path.isfile(local_temp_path) or not self.check_download_file(download_obj, local_temp_path): 699 warn('Failed to download {} to {}'.format(url, local_temp_path)) 700 continue 701 rename_with_retry(local_temp_path, local_path) 702 downloaded = True 703 break 704 if not downloaded: 705 fatal('Failed to download, and retry count has expired') 706 raise DownloadError() 707 708 def install(self, version): 709 # Currently this is called after calling 'download' method, so here are a few asserts 710 # for the conditions which should be true once that method is done. 711 assert (version in self.versions) 712 download_obj = self.versions[version].get_download_for_platform(self._platform) 713 assert (download_obj is not None) 714 archive_name = os.path.basename(download_obj.url) 715 archive_path = os.path.join(global_idf_tools_path, 'dist', archive_name) 716 assert (os.path.isfile(archive_path)) 717 dest_dir = self.get_path_for_version(version) 718 if os.path.exists(dest_dir): 719 warn('destination path already exists, removing') 720 shutil.rmtree(dest_dir) 721 mkdir_p(dest_dir) 722 unpack(archive_path, dest_dir) 723 if self._current_options.strip_container_dirs: 724 strip_container_dirs(dest_dir, self._current_options.strip_container_dirs) 725 726 @staticmethod 727 def check_download_file(download_obj, local_path): 728 expected_sha256 = download_obj.sha256 729 expected_size = download_obj.size 730 file_size, file_sha256 = get_file_size_sha256(local_path) 731 if file_size != expected_size: 732 warn('file size mismatch for {}, expected {}, got {}'.format(local_path, expected_size, file_size)) 733 return False 734 if file_sha256 != expected_sha256: 735 warn('hash mismatch for {}, expected {}, got {}'.format(local_path, expected_sha256, file_sha256)) 736 return False 737 return True 738 739 @classmethod 740 def from_json(cls, tool_dict): 741 # json.load will return 'str' types in Python 3 and 'unicode' in Python 2 742 expected_str_type = type(u'') 743 744 # Validate json fields 745 tool_name = tool_dict.get('name') 746 if type(tool_name) is not expected_str_type: 747 raise RuntimeError('tool_name is not a string') 748 749 description = tool_dict.get('description') 750 if type(description) is not expected_str_type: 751 raise RuntimeError('description is not a string') 752 753 version_cmd = tool_dict.get('version_cmd') 754 if type(version_cmd) is not list: 755 raise RuntimeError('version_cmd for tool %s is not a list of strings' % tool_name) 756 757 version_regex = tool_dict.get('version_regex') 758 if type(version_regex) is not expected_str_type or not version_regex: 759 raise RuntimeError('version_regex for tool %s is not a non-empty string' % tool_name) 760 761 version_regex_replace = tool_dict.get('version_regex_replace') 762 if version_regex_replace and type(version_regex_replace) is not expected_str_type: 763 raise RuntimeError('version_regex_replace for tool %s is not a string' % tool_name) 764 765 export_paths = tool_dict.get('export_paths') 766 if type(export_paths) is not list: 767 raise RuntimeError('export_paths for tool %s is not a list' % tool_name) 768 769 export_vars = tool_dict.get('export_vars', {}) 770 if type(export_vars) is not dict: 771 raise RuntimeError('export_vars for tool %s is not a mapping' % tool_name) 772 773 versions = tool_dict.get('versions') 774 if type(versions) is not list: 775 raise RuntimeError('versions for tool %s is not an array' % tool_name) 776 777 install = tool_dict.get('install', False) 778 if type(install) is not expected_str_type: 779 raise RuntimeError('install for tool %s is not a string' % tool_name) 780 781 info_url = tool_dict.get('info_url', False) 782 if type(info_url) is not expected_str_type: 783 raise RuntimeError('info_url for tool %s is not a string' % tool_name) 784 785 license = tool_dict.get('license', False) 786 if type(license) is not expected_str_type: 787 raise RuntimeError('license for tool %s is not a string' % tool_name) 788 789 strip_container_dirs = tool_dict.get('strip_container_dirs', 0) 790 if strip_container_dirs and type(strip_container_dirs) is not int: 791 raise RuntimeError('strip_container_dirs for tool %s is not an int' % tool_name) 792 793 overrides_list = tool_dict.get('platform_overrides', []) 794 if type(overrides_list) is not list: 795 raise RuntimeError('platform_overrides for tool %s is not a list' % tool_name) 796 797 # Create the object 798 tool_obj = cls(tool_name, description, install, info_url, license, 799 version_cmd, version_regex, version_regex_replace, 800 strip_container_dirs) 801 802 for path in export_paths: 803 tool_obj.options.export_paths.append(path) 804 805 for name, value in export_vars.items(): 806 tool_obj.options.export_vars[name] = value 807 808 for index, override in enumerate(overrides_list): 809 platforms_list = override.get('platforms') 810 if type(platforms_list) is not list: 811 raise RuntimeError('platforms for override %d of tool %s is not a list' % (index, tool_name)) 812 813 install = override.get('install') 814 if install is not None and type(install) is not expected_str_type: 815 raise RuntimeError('install for override %d of tool %s is not a string' % (index, tool_name)) 816 817 version_cmd = override.get('version_cmd') 818 if version_cmd is not None and type(version_cmd) is not list: 819 raise RuntimeError('version_cmd for override %d of tool %s is not a list of strings' % 820 (index, tool_name)) 821 822 version_regex = override.get('version_regex') 823 if version_regex is not None and (type(version_regex) is not expected_str_type or not version_regex): 824 raise RuntimeError('version_regex for override %d of tool %s is not a non-empty string' % 825 (index, tool_name)) 826 827 version_regex_replace = override.get('version_regex_replace') 828 if version_regex_replace is not None and type(version_regex_replace) is not expected_str_type: 829 raise RuntimeError('version_regex_replace for override %d of tool %s is not a string' % 830 (index, tool_name)) 831 832 export_paths = override.get('export_paths') 833 if export_paths is not None and type(export_paths) is not list: 834 raise RuntimeError('export_paths for override %d of tool %s is not a list' % (index, tool_name)) 835 836 export_vars = override.get('export_vars') 837 if export_vars is not None and type(export_vars) is not dict: 838 raise RuntimeError('export_vars for override %d of tool %s is not a mapping' % (index, tool_name)) 839 tool_obj.platform_overrides.append(override) 840 841 recommended_versions = {} 842 for version_dict in versions: 843 version = version_dict.get('name') 844 if type(version) is not expected_str_type: 845 raise RuntimeError('version name for tool {} is not a string'.format(tool_name)) 846 847 version_status = version_dict.get('status') 848 if type(version_status) is not expected_str_type and version_status not in IDFToolVersion.STATUS_VALUES: 849 raise RuntimeError('tool {} version {} status is not one of {}', tool_name, version, 850 IDFToolVersion.STATUS_VALUES) 851 852 version_obj = IDFToolVersion(version, version_status) 853 for platform_id, platform_dict in version_dict.items(): 854 if platform_id in ['name', 'status']: 855 continue 856 if platform_id not in PLATFORM_FROM_NAME.keys(): 857 raise RuntimeError('invalid platform %s for tool %s version %s' % 858 (platform_id, tool_name, version)) 859 860 version_obj.add_download(platform_id, 861 platform_dict['url'], platform_dict['size'], platform_dict['sha256']) 862 863 if version_status == IDFToolVersion.STATUS_RECOMMENDED: 864 if platform_id not in recommended_versions: 865 recommended_versions[platform_id] = [] 866 recommended_versions[platform_id].append(version) 867 868 tool_obj.add_version(version_obj) 869 for platform_id, version_list in recommended_versions.items(): 870 if len(version_list) > 1: 871 raise RuntimeError('tool {} for platform {} has {} recommended versions'.format( 872 tool_name, platform_id, len(recommended_versions))) 873 if install != IDFTool.INSTALL_NEVER and len(recommended_versions) == 0: 874 raise RuntimeError('required/optional tool {} for platform {} has no recommended versions'.format( 875 tool_name, platform_id)) 876 877 tool_obj._update_current_options() 878 return tool_obj 879 880 def to_json(self): 881 versions_array = [] 882 for version, version_obj in self.versions.items(): 883 version_json = { 884 'name': version, 885 'status': version_obj.status 886 } 887 for platform_id, download in version_obj.downloads.items(): 888 version_json[platform_id] = { 889 'url': download.url, 890 'size': download.size, 891 'sha256': download.sha256 892 } 893 versions_array.append(version_json) 894 overrides_array = self.platform_overrides 895 896 tool_json = { 897 'name': self.name, 898 'description': self.description, 899 'export_paths': self.options.export_paths, 900 'export_vars': self.options.export_vars, 901 'install': self.options.install, 902 'info_url': self.options.info_url, 903 'license': self.options.license, 904 'version_cmd': self.options.version_cmd, 905 'version_regex': self.options.version_regex, 906 'versions': versions_array, 907 } 908 if self.options.version_regex_replace != VERSION_REGEX_REPLACE_DEFAULT: 909 tool_json['version_regex_replace'] = self.options.version_regex_replace 910 if overrides_array: 911 tool_json['platform_overrides'] = overrides_array 912 if self.options.strip_container_dirs: 913 tool_json['strip_container_dirs'] = self.options.strip_container_dirs 914 return tool_json 915 916 917def load_tools_info(): # type: () -> typing.Dict[str, IDFTool] 918 """ 919 Load tools metadata from tools.json, return a dictionary: tool name - tool info 920 """ 921 tool_versions_file_name = global_tools_json 922 923 with open(tool_versions_file_name, 'r') as f: 924 tools_info = json.load(f) 925 926 return parse_tools_info_json(tools_info) 927 928 929def parse_tools_info_json(tools_info): 930 """ 931 Parse and validate the dictionary obtained by loading the tools.json file. 932 Returns a dictionary of tools (key: tool name, value: IDFTool object). 933 """ 934 if tools_info['version'] != TOOLS_FILE_VERSION: 935 raise RuntimeError('Invalid version') 936 937 tools_dict = OrderedDict() 938 939 tools_array = tools_info.get('tools') 940 if type(tools_array) is not list: 941 raise RuntimeError('tools property is missing or not an array') 942 943 for tool_dict in tools_array: 944 tool = IDFTool.from_json(tool_dict) 945 tools_dict[tool.name] = tool 946 947 return tools_dict 948 949 950def dump_tools_json(tools_info): 951 tools_array = [] 952 for tool_name, tool_obj in tools_info.items(): 953 tool_json = tool_obj.to_json() 954 tools_array.append(tool_json) 955 file_json = {'version': TOOLS_FILE_VERSION, 'tools': tools_array} 956 return json.dumps(file_json, indent=2, separators=(',', ': '), sort_keys=True) 957 958 959def get_python_env_path(): 960 python_ver_major_minor = '{}.{}'.format(sys.version_info.major, sys.version_info.minor) 961 962 version_file_path = os.path.join(global_idf_path, 'version.txt') 963 if os.path.exists(version_file_path): 964 with open(version_file_path, 'r') as version_file: 965 idf_version_str = version_file.read() 966 else: 967 try: 968 idf_version_str = subprocess.check_output(['git', 'describe'], 969 cwd=global_idf_path, env=os.environ).decode() 970 except subprocess.CalledProcessError as e: 971 warn('Git describe was unsuccessul: {}'.format(e)) 972 idf_version_str = '' 973 match = re.match(r'^v([0-9]+\.[0-9]+).*', idf_version_str) 974 if match: 975 idf_version = match.group(1) 976 else: 977 idf_version = None 978 # fallback when IDF is a shallow clone 979 try: 980 with open(os.path.join(global_idf_path, 'components', 'esp_common', 'include', 'esp_idf_version.h')) as f: 981 m = re.search(r'^#define\s+ESP_IDF_VERSION_MAJOR\s+(\d+).+?^#define\s+ESP_IDF_VERSION_MINOR\s+(\d+)', 982 f.read(), re.DOTALL | re.MULTILINE) 983 if m: 984 idf_version = '.'.join((m.group(1), m.group(2))) 985 else: 986 warn('Reading IDF version from C header file failed!') 987 except Exception as e: 988 warn('Is it not possible to determine the IDF version: {}'.format(e)) 989 990 if idf_version is None: 991 fatal('IDF version cannot be determined') 992 raise SystemExit(1) 993 994 idf_python_env_path = os.path.join(global_idf_tools_path, 'python_env', 995 'idf{}_py{}_env'.format(idf_version, python_ver_major_minor)) 996 997 if sys.platform == 'win32': 998 subdir = 'Scripts' 999 python_exe = 'python.exe' 1000 else: 1001 subdir = 'bin' 1002 python_exe = 'python' 1003 1004 idf_python_export_path = os.path.join(idf_python_env_path, subdir) 1005 virtualenv_python = os.path.join(idf_python_export_path, python_exe) 1006 1007 return idf_python_env_path, idf_python_export_path, virtualenv_python 1008 1009 1010def action_list(args): 1011 tools_info = load_tools_info() 1012 for name, tool in tools_info.items(): 1013 if tool.get_install_type() == IDFTool.INSTALL_NEVER: 1014 continue 1015 optional_str = ' (optional)' if tool.get_install_type() == IDFTool.INSTALL_ON_REQUEST else '' 1016 info('* {}: {}{}'.format(name, tool.description, optional_str)) 1017 tool.find_installed_versions() 1018 versions_for_platform = {k: v for k, v in tool.versions.items() if v.compatible_with_platform()} 1019 if not versions_for_platform: 1020 info(' (no versions compatible with platform {})'.format(PYTHON_PLATFORM)) 1021 continue 1022 versions_sorted = sorted(versions_for_platform.keys(), key=tool.versions.get, reverse=True) 1023 for version in versions_sorted: 1024 version_obj = tool.versions[version] 1025 info(' - {} ({}{})'.format(version, version_obj.status, 1026 ', installed' if version in tool.versions_installed else '')) 1027 1028 1029def action_check(args): 1030 tools_info = load_tools_info() 1031 not_found_list = [] 1032 info('Checking for installed tools...') 1033 for name, tool in tools_info.items(): 1034 if tool.get_install_type() == IDFTool.INSTALL_NEVER: 1035 continue 1036 tool_found_somewhere = False 1037 info('Checking tool %s' % name) 1038 tool.find_installed_versions() 1039 if tool.version_in_path: 1040 info(' version found in PATH: %s' % tool.version_in_path) 1041 tool_found_somewhere = True 1042 else: 1043 info(' no version found in PATH') 1044 1045 for version in tool.versions_installed: 1046 info(' version installed in tools directory: %s' % version) 1047 tool_found_somewhere = True 1048 if not tool_found_somewhere and tool.get_install_type() == IDFTool.INSTALL_ALWAYS: 1049 not_found_list.append(name) 1050 if not_found_list: 1051 fatal('The following required tools were not found: ' + ' '.join(not_found_list)) 1052 raise SystemExit(1) 1053 1054 1055def action_export(args): 1056 tools_info = load_tools_info() 1057 all_tools_found = True 1058 export_vars = {} 1059 paths_to_export = [] 1060 for name, tool in tools_info.items(): 1061 if tool.get_install_type() == IDFTool.INSTALL_NEVER: 1062 continue 1063 tool.find_installed_versions() 1064 1065 if tool.version_in_path: 1066 if tool.version_in_path not in tool.versions: 1067 # unsupported version 1068 if args.prefer_system: 1069 warn('using an unsupported version of tool {} found in PATH: {}'.format( 1070 tool.name, tool.version_in_path)) 1071 continue 1072 else: 1073 # unsupported version in path 1074 pass 1075 else: 1076 # supported/deprecated version in PATH, use it 1077 version_obj = tool.versions[tool.version_in_path] 1078 if version_obj.status == IDFToolVersion.STATUS_SUPPORTED: 1079 info('Using a supported version of tool {} found in PATH: {}.'.format(name, tool.version_in_path), 1080 f=sys.stderr) 1081 info('However the recommended version is {}.'.format(tool.get_recommended_version()), 1082 f=sys.stderr) 1083 elif version_obj.status == IDFToolVersion.STATUS_DEPRECATED: 1084 warn('using a deprecated version of tool {} found in PATH: {}'.format(name, tool.version_in_path)) 1085 continue 1086 1087 self_restart_cmd = '{} {}{}'.format(sys.executable, __file__, 1088 (' --tools-json ' + args.tools_json) if args.tools_json else '') 1089 self_restart_cmd = to_shell_specific_paths([self_restart_cmd])[0] 1090 1091 if IDF_TOOLS_EXPORT_CMD: 1092 prefer_system_hint = '' 1093 else: 1094 prefer_system_hint = ' To use it, run \'{} export --prefer-system\''.format(self_restart_cmd) 1095 1096 if IDF_TOOLS_INSTALL_CMD: 1097 install_cmd = to_shell_specific_paths([IDF_TOOLS_INSTALL_CMD])[0] 1098 else: 1099 install_cmd = self_restart_cmd + ' install' 1100 1101 if not tool.versions_installed: 1102 if tool.get_install_type() == IDFTool.INSTALL_ALWAYS: 1103 all_tools_found = False 1104 fatal('tool {} has no installed versions. Please run \'{}\' to install it.'.format( 1105 tool.name, install_cmd)) 1106 if tool.version_in_path and tool.version_in_path not in tool.versions: 1107 info('An unsupported version of tool {} was found in PATH: {}. '.format(name, tool.version_in_path) + 1108 prefer_system_hint, f=sys.stderr) 1109 continue 1110 else: 1111 # tool is optional, and does not have versions installed 1112 # use whatever is available in PATH 1113 continue 1114 1115 if tool.version_in_path and tool.version_in_path not in tool.versions: 1116 info('Not using an unsupported version of tool {} found in PATH: {}.'.format( 1117 tool.name, tool.version_in_path) + prefer_system_hint, f=sys.stderr) 1118 1119 version_to_use = tool.get_preferred_installed_version() 1120 export_paths = tool.get_export_paths(version_to_use) 1121 if export_paths: 1122 paths_to_export += export_paths 1123 tool_export_vars = tool.get_export_vars(version_to_use) 1124 for k, v in tool_export_vars.items(): 1125 old_v = os.environ.get(k) 1126 if old_v is None or old_v != v: 1127 export_vars[k] = v 1128 1129 current_path = os.getenv('PATH') 1130 idf_python_env_path, idf_python_export_path, virtualenv_python = get_python_env_path() 1131 if os.path.exists(virtualenv_python): 1132 idf_python_env_path = to_shell_specific_paths([idf_python_env_path])[0] 1133 if os.getenv('IDF_PYTHON_ENV_PATH') != idf_python_env_path: 1134 export_vars['IDF_PYTHON_ENV_PATH'] = to_shell_specific_paths([idf_python_env_path])[0] 1135 if idf_python_export_path not in current_path: 1136 paths_to_export.append(idf_python_export_path) 1137 1138 idf_tools_dir = os.path.join(global_idf_path, 'tools') 1139 idf_tools_dir = to_shell_specific_paths([idf_tools_dir])[0] 1140 if idf_tools_dir not in current_path: 1141 paths_to_export.append(idf_tools_dir) 1142 1143 if sys.platform == 'win32' and 'MSYSTEM' not in os.environ: 1144 old_path = '%PATH%' 1145 path_sep = ';' 1146 else: 1147 old_path = '$PATH' 1148 # can't trust os.pathsep here, since for Windows Python started from MSYS shell, 1149 # os.pathsep will be ';' 1150 path_sep = ':' 1151 1152 if args.format == EXPORT_SHELL: 1153 if sys.platform == 'win32' and 'MSYSTEM' not in os.environ: 1154 export_format = 'SET "{}={}"' 1155 export_sep = '\n' 1156 else: 1157 export_format = 'export {}="{}"' 1158 export_sep = ';' 1159 elif args.format == EXPORT_KEY_VALUE: 1160 export_format = '{}={}' 1161 export_sep = '\n' 1162 else: 1163 raise NotImplementedError('unsupported export format {}'.format(args.format)) 1164 1165 if paths_to_export: 1166 export_vars['PATH'] = path_sep.join(to_shell_specific_paths(paths_to_export) + [old_path]) 1167 1168 export_statements = export_sep.join([export_format.format(k, v) for k, v in export_vars.items()]) 1169 1170 if export_statements: 1171 print(export_statements) 1172 1173 if not all_tools_found: 1174 raise SystemExit(1) 1175 1176 1177def apply_url_mirrors(args, tool_download_obj): 1178 apply_mirror_prefix_map(args, tool_download_obj) 1179 apply_github_assets_option(tool_download_obj) 1180 1181 1182def apply_mirror_prefix_map(args, tool_download_obj): 1183 """Rewrite URL for given tool_obj, given tool_version, and current platform, 1184 if --mirror-prefix-map flag or IDF_MIRROR_PREFIX_MAP environment variable is given. 1185 """ 1186 mirror_prefix_map = None 1187 mirror_prefix_map_env = os.getenv('IDF_MIRROR_PREFIX_MAP') 1188 if mirror_prefix_map_env: 1189 mirror_prefix_map = mirror_prefix_map_env.split(';') 1190 if IDF_MAINTAINER and args.mirror_prefix_map: 1191 if mirror_prefix_map: 1192 warn('Both IDF_MIRROR_PREFIX_MAP environment variable and --mirror-prefix-map flag are specified, ' + 1193 'will use the value from the command line.') 1194 mirror_prefix_map = args.mirror_prefix_map 1195 if mirror_prefix_map and tool_download_obj: 1196 for item in mirror_prefix_map: 1197 if URL_PREFIX_MAP_SEPARATOR not in item: 1198 warn('invalid mirror-prefix-map item (missing \'{}\') {}'.format(URL_PREFIX_MAP_SEPARATOR, item)) 1199 continue 1200 search, replace = item.split(URL_PREFIX_MAP_SEPARATOR, 1) 1201 old_url = tool_download_obj.url 1202 new_url = re.sub(search, replace, old_url) 1203 if new_url != old_url: 1204 info('Changed download URL: {} => {}'.format(old_url, new_url)) 1205 tool_download_obj.url = new_url 1206 break 1207 1208 1209def apply_github_assets_option(tool_download_obj): 1210 """ Rewrite URL for given tool_obj if the download URL is an https://github.com/ URL and the variable 1211 IDF_GITHUB_ASSETS is set. The github.com part of the URL will be replaced. 1212 """ 1213 try: 1214 github_assets = os.environ['IDF_GITHUB_ASSETS'].strip() 1215 except KeyError: 1216 return # no IDF_GITHUB_ASSETS 1217 if not github_assets: # variable exists but is empty 1218 return 1219 1220 # check no URL qualifier in the mirror URL 1221 if '://' in github_assets: 1222 fatal("IDF_GITHUB_ASSETS shouldn't include any URL qualifier, https:// is assumed") 1223 raise SystemExit(1) 1224 1225 # Strip any trailing / from the mirror URL 1226 github_assets = github_assets.rstrip('/') 1227 1228 old_url = tool_download_obj.url 1229 new_url = re.sub(r'^https://github.com/', 'https://{}/'.format(github_assets), old_url) 1230 if new_url != old_url: 1231 info('Using GitHub assets mirror for URL: {} => {}'.format(old_url, new_url)) 1232 tool_download_obj.url = new_url 1233 1234 1235def action_download(args): 1236 tools_info = load_tools_info() 1237 tools_spec = args.tools 1238 1239 if args.platform not in PLATFORM_FROM_NAME: 1240 fatal('unknown platform: {}' % args.platform) 1241 raise SystemExit(1) 1242 platform = PLATFORM_FROM_NAME[args.platform] 1243 1244 tools_info_for_platform = OrderedDict() 1245 for name, tool_obj in tools_info.items(): 1246 tool_for_platform = tool_obj.copy_for_platform(platform) 1247 tools_info_for_platform[name] = tool_for_platform 1248 1249 if not tools_spec or 'required' in tools_spec: 1250 tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS] 1251 info('Downloading tools for {}: {}'.format(platform, ', '.join(tools_spec))) 1252 elif 'all' in tools_spec: 1253 tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() != IDFTool.INSTALL_NEVER] 1254 info('Downloading tools for {}: {}'.format(platform, ', '.join(tools_spec))) 1255 1256 for tool_spec in tools_spec: 1257 if '@' not in tool_spec: 1258 tool_name = tool_spec 1259 tool_version = None 1260 else: 1261 tool_name, tool_version = tool_spec.split('@', 1) 1262 if tool_name not in tools_info_for_platform: 1263 fatal('unknown tool name: {}'.format(tool_name)) 1264 raise SystemExit(1) 1265 tool_obj = tools_info_for_platform[tool_name] 1266 if tool_version is not None and tool_version not in tool_obj.versions: 1267 fatal('unknown version for tool {}: {}'.format(tool_name, tool_version)) 1268 raise SystemExit(1) 1269 if tool_version is None: 1270 tool_version = tool_obj.get_recommended_version() 1271 if tool_version is None: 1272 fatal('tool {} not found for {} platform'.format(tool_name, platform)) 1273 raise SystemExit(1) 1274 tool_spec = '{}@{}'.format(tool_name, tool_version) 1275 1276 info('Downloading {}'.format(tool_spec)) 1277 apply_url_mirrors(args, tool_obj.versions[tool_version].get_download_for_platform(platform)) 1278 1279 tool_obj.download(tool_version) 1280 1281 1282def action_install(args): 1283 tools_info = load_tools_info() 1284 tools_spec = args.tools 1285 if not tools_spec or 'required' in tools_spec: 1286 tools_spec = [k for k, v in tools_info.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS] 1287 info('Installing tools: {}'.format(', '.join(tools_spec))) 1288 elif 'all' in tools_spec: 1289 tools_spec = [k for k, v in tools_info.items() if v.get_install_type() != IDFTool.INSTALL_NEVER] 1290 info('Installing tools: {}'.format(', '.join(tools_spec))) 1291 1292 for tool_spec in tools_spec: 1293 if '@' not in tool_spec: 1294 tool_name = tool_spec 1295 tool_version = None 1296 else: 1297 tool_name, tool_version = tool_spec.split('@', 1) 1298 if tool_name not in tools_info: 1299 fatal('unknown tool name: {}'.format(tool_name)) 1300 raise SystemExit(1) 1301 tool_obj = tools_info[tool_name] 1302 if not tool_obj.compatible_with_platform(): 1303 fatal('tool {} does not have versions compatible with platform {}'.format(tool_name, CURRENT_PLATFORM)) 1304 raise SystemExit(1) 1305 if tool_version is not None and tool_version not in tool_obj.versions: 1306 fatal('unknown version for tool {}: {}'.format(tool_name, tool_version)) 1307 raise SystemExit(1) 1308 if tool_version is None: 1309 tool_version = tool_obj.get_recommended_version() 1310 assert tool_version is not None 1311 tool_obj.find_installed_versions() 1312 tool_spec = '{}@{}'.format(tool_name, tool_version) 1313 if tool_version in tool_obj.versions_installed: 1314 info('Skipping {} (already installed)'.format(tool_spec)) 1315 continue 1316 1317 info('Installing {}'.format(tool_spec)) 1318 apply_url_mirrors(args, tool_obj.versions[tool_version].get_download_for_platform(PYTHON_PLATFORM)) 1319 1320 tool_obj.download(tool_version) 1321 tool_obj.install(tool_version) 1322 1323 1324def get_wheels_dir(): 1325 tools_info = load_tools_info() 1326 wheels_package_name = 'idf-python-wheels' 1327 if wheels_package_name not in tools_info: 1328 return None 1329 wheels_package = tools_info[wheels_package_name] 1330 recommended_version = wheels_package.get_recommended_version() 1331 if recommended_version is None: 1332 return None 1333 wheels_dir = wheels_package.get_path_for_version(recommended_version) 1334 if not os.path.exists(wheels_dir): 1335 return None 1336 return wheels_dir 1337 1338 1339def action_install_python_env(args): # type: ignore 1340 reinstall = args.reinstall 1341 idf_python_env_path, _, virtualenv_python = get_python_env_path() 1342 1343 is_virtualenv = hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix) 1344 if is_virtualenv and (not os.path.exists(idf_python_env_path) or reinstall): 1345 fatal('This script was called from a virtual environment, can not create a virtual environment again') 1346 raise SystemExit(1) 1347 1348 if os.path.exists(virtualenv_python): 1349 try: 1350 subprocess.check_call([virtualenv_python, '--version'], stdout=sys.stdout, stderr=sys.stderr) 1351 except subprocess.CalledProcessError: 1352 # At this point we can reinstall the virtual environment if it is non-functional. This can happen at least 1353 # when the Python interpreter was removed which was used to create the virtual environment. 1354 reinstall = True 1355 1356 try: 1357 subprocess.check_call([virtualenv_python, '-m', 'pip', '--version'], stdout=sys.stdout, stderr=sys.stderr) 1358 except subprocess.CalledProcessError: 1359 warn('PIP is not available in the virtual environment.') 1360 # Reinstallation of the virtual environment could help if PIP was installed for the main Python 1361 reinstall = True 1362 1363 if reinstall and os.path.exists(idf_python_env_path): 1364 warn('Removing the existing Python environment in {}'.format(idf_python_env_path)) 1365 shutil.rmtree(idf_python_env_path) 1366 1367 if not os.path.exists(virtualenv_python): 1368 info('Creating a new Python environment in {}'.format(idf_python_env_path)) 1369 1370 try: 1371 import virtualenv # noqa: F401 1372 except ImportError: 1373 info('Installing virtualenv') 1374 subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--user', 'virtualenv'], 1375 stdout=sys.stdout, stderr=sys.stderr) 1376 1377 subprocess.check_call([sys.executable, '-m', 'virtualenv', idf_python_env_path], 1378 stdout=sys.stdout, stderr=sys.stderr) 1379 run_args = [virtualenv_python, '-m', 'pip', 'install', '--no-warn-script-location'] 1380 requirements_txt = os.path.join(global_idf_path, 'requirements.txt') 1381 run_args += ['-r', requirements_txt] 1382 if args.extra_wheels_dir: 1383 run_args += ['--find-links', args.extra_wheels_dir] 1384 if args.no_index: 1385 run_args += ['--no-index'] 1386 if args.extra_wheels_url: 1387 run_args += ['--extra-index-url', args.extra_wheels_url] 1388 1389 wheels_dir = get_wheels_dir() 1390 if wheels_dir is not None: 1391 run_args += ['--find-links', wheels_dir] 1392 1393 info('Installing Python packages from {}'.format(requirements_txt)) 1394 subprocess.check_call(run_args, stdout=sys.stdout, stderr=sys.stderr) 1395 1396 1397def action_add_version(args): 1398 tools_info = load_tools_info() 1399 tool_name = args.tool 1400 tool_obj = tools_info.get(tool_name) 1401 if not tool_obj: 1402 info('Creating new tool entry for {}'.format(tool_name)) 1403 tool_obj = IDFTool(tool_name, TODO_MESSAGE, IDFTool.INSTALL_ALWAYS, 1404 TODO_MESSAGE, TODO_MESSAGE, [TODO_MESSAGE], TODO_MESSAGE) 1405 tools_info[tool_name] = tool_obj 1406 version = args.version 1407 version_obj = tool_obj.versions.get(version) 1408 if version not in tool_obj.versions: 1409 info('Creating new version {}'.format(version)) 1410 version_obj = IDFToolVersion(version, IDFToolVersion.STATUS_SUPPORTED) 1411 tool_obj.versions[version] = version_obj 1412 url_prefix = args.url_prefix or 'https://%s/' % TODO_MESSAGE 1413 for file_path in args.files: 1414 file_name = os.path.basename(file_path) 1415 # Guess which platform this file is for 1416 found_platform = None 1417 for platform_alias, platform_id in PLATFORM_FROM_NAME.items(): 1418 if platform_alias in file_name: 1419 found_platform = platform_id 1420 break 1421 if found_platform is None: 1422 info('Could not guess platform for file {}'.format(file_name)) 1423 found_platform = TODO_MESSAGE 1424 # Get file size and calculate the SHA256 1425 file_size, file_sha256 = get_file_size_sha256(file_path) 1426 url = url_prefix + file_name 1427 info('Adding download for platform {}'.format(found_platform)) 1428 info(' size: {}'.format(file_size)) 1429 info(' SHA256: {}'.format(file_sha256)) 1430 info(' URL: {}'.format(url)) 1431 version_obj.add_download(found_platform, url, file_size, file_sha256) 1432 json_str = dump_tools_json(tools_info) 1433 if not args.output: 1434 args.output = os.path.join(global_idf_path, TOOLS_FILE_NEW) 1435 with open(args.output, 'w') as f: 1436 f.write(json_str) 1437 f.write('\n') 1438 info('Wrote output to {}'.format(args.output)) 1439 1440 1441def action_rewrite(args): 1442 tools_info = load_tools_info() 1443 json_str = dump_tools_json(tools_info) 1444 if not args.output: 1445 args.output = os.path.join(global_idf_path, TOOLS_FILE_NEW) 1446 with open(args.output, 'w') as f: 1447 f.write(json_str) 1448 f.write('\n') 1449 info('Wrote output to {}'.format(args.output)) 1450 1451 1452def action_validate(args): 1453 try: 1454 import jsonschema 1455 except ImportError: 1456 fatal('You need to install jsonschema package to use validate command') 1457 raise SystemExit(1) 1458 1459 with open(os.path.join(global_idf_path, TOOLS_FILE), 'r') as tools_file: 1460 tools_json = json.load(tools_file) 1461 1462 with open(os.path.join(global_idf_path, TOOLS_SCHEMA_FILE), 'r') as schema_file: 1463 schema_json = json.load(schema_file) 1464 jsonschema.validate(tools_json, schema_json) 1465 # on failure, this will raise an exception with a fairly verbose diagnostic message 1466 1467 1468def action_gen_doc(args): 1469 f = args.output 1470 tools_info = load_tools_info() 1471 1472 def print_out(text): 1473 f.write(text + '\n') 1474 1475 print_out('.. |zwsp| unicode:: U+200B') 1476 print_out(' :trim:') 1477 print_out('') 1478 1479 idf_gh_url = 'https://github.com/espressif/esp-idf' 1480 for tool_name, tool_obj in tools_info.items(): 1481 info_url = tool_obj.options.info_url 1482 if idf_gh_url + '/tree' in info_url: 1483 info_url = re.sub(idf_gh_url + r'/tree/\w+/(.*)', r':idf:`\1`', info_url) 1484 1485 license_url = 'https://spdx.org/licenses/' + tool_obj.options.license 1486 1487 print_out(""" 1488.. _tool-{name}: 1489 1490{name} 1491{underline} 1492 1493{description} 1494 1495.. include:: idf-tools-notes.inc 1496 :start-after: tool-{name}-notes 1497 :end-before: --- 1498 1499License: `{license} <{license_url}>`_ 1500 1501More info: {info_url} 1502 1503.. list-table:: 1504 :widths: 10 10 80 1505 :header-rows: 1 1506 1507 * - Platform 1508 - Required 1509 - Download 1510""".rstrip().format(name=tool_name, 1511 underline=args.heading_underline_char * len(tool_name), 1512 description=tool_obj.description, 1513 license=tool_obj.options.license, 1514 license_url=license_url, 1515 info_url=info_url)) 1516 1517 for platform_name in sorted(tool_obj.get_supported_platforms()): 1518 platform_tool = tool_obj.copy_for_platform(platform_name) 1519 install_type = platform_tool.get_install_type() 1520 if install_type == IDFTool.INSTALL_NEVER: 1521 continue 1522 elif install_type == IDFTool.INSTALL_ALWAYS: 1523 install_type_str = 'required' 1524 elif install_type == IDFTool.INSTALL_ON_REQUEST: 1525 install_type_str = 'optional' 1526 else: 1527 raise NotImplementedError() 1528 1529 version = platform_tool.get_recommended_version() 1530 version_obj = platform_tool.versions[version] 1531 download_obj = version_obj.get_download_for_platform(platform_name) 1532 1533 # Note: keep the list entries indented to the same number of columns 1534 # as the list header above. 1535 print_out(""" 1536 * - {} 1537 - {} 1538 - {} 1539 1540 .. rst-class:: tool-sha256 1541 1542 SHA256: {} 1543""".strip('\n').format(platform_name, install_type_str, download_obj.url, download_obj.sha256)) 1544 1545 print_out('') 1546 print_out('') 1547 1548 1549def main(argv): 1550 parser = argparse.ArgumentParser() 1551 1552 parser.add_argument('--quiet', help='Don\'t output diagnostic messages to stdout/stderr', action='store_true') 1553 parser.add_argument('--non-interactive', help='Don\'t output interactive messages and questions', action='store_true') 1554 parser.add_argument('--tools-json', help='Path to the tools.json file to use') 1555 parser.add_argument('--idf-path', help='ESP-IDF path to use') 1556 1557 subparsers = parser.add_subparsers(dest='action') 1558 subparsers.add_parser('list', help='List tools and versions available') 1559 subparsers.add_parser('check', help='Print summary of tools installed or found in PATH') 1560 export = subparsers.add_parser('export', help='Output command for setting tool paths, suitable for shell') 1561 export.add_argument('--format', choices=[EXPORT_SHELL, EXPORT_KEY_VALUE], default=EXPORT_SHELL, 1562 help='Format of the output: shell (suitable for printing into shell), ' + 1563 'or key-value (suitable for parsing by other tools') 1564 export.add_argument('--prefer-system', help='Normally, if the tool is already present in PATH, ' + 1565 'but has an unsupported version, a version from the tools directory ' + 1566 'will be used instead. If this flag is given, the version in PATH ' + 1567 'will be used.', action='store_true') 1568 install = subparsers.add_parser('install', help='Download and install tools into the tools directory') 1569 install.add_argument('tools', metavar='TOOL', nargs='*', default=['required'], 1570 help='Tools to install. ' + 1571 'To install a specific version use <tool_name>@<version> syntax. ' + 1572 'Use empty or \'required\' to install required tools, not optional ones. ' + 1573 'Use \'all\' to install all tools, including the optional ones.') 1574 1575 download = subparsers.add_parser('download', help='Download the tools into the dist directory') 1576 download.add_argument('--platform', help='Platform to download the tools for') 1577 download.add_argument('tools', metavar='TOOL', nargs='*', default=['required'], 1578 help='Tools to download. ' + 1579 'To download a specific version use <tool_name>@<version> syntax. ' + 1580 'Use empty or \'required\' to download required tools, not optional ones. ' + 1581 'Use \'all\' to download all tools, including the optional ones.') 1582 1583 if IDF_MAINTAINER: 1584 for subparser in [download, install]: 1585 subparser.add_argument('--mirror-prefix-map', nargs='*', 1586 help='Pattern to rewrite download URLs, with source and replacement separated by comma.' + 1587 ' E.g. http://foo.com,http://test.foo.com') 1588 1589 install_python_env = subparsers.add_parser('install-python-env', 1590 help='Create Python virtual environment and install the ' + 1591 'required Python packages') 1592 install_python_env.add_argument('--reinstall', help='Discard the previously installed environment', 1593 action='store_true') 1594 install_python_env.add_argument('--extra-wheels-dir', help='Additional directories with wheels ' + 1595 'to use during installation') 1596 install_python_env.add_argument('--extra-wheels-url', help='Additional URL with wheels', default='https://dl.espressif.com/pypi') 1597 install_python_env.add_argument('--no-index', help='Work offline without retrieving wheels index') 1598 1599 if IDF_MAINTAINER: 1600 add_version = subparsers.add_parser('add-version', help='Add or update download info for a version') 1601 add_version.add_argument('--output', help='Save new tools.json into this file') 1602 add_version.add_argument('--tool', help='Tool name to set add a version for', required=True) 1603 add_version.add_argument('--version', help='Version identifier', required=True) 1604 add_version.add_argument('--url-prefix', help='String to prepend to file names to obtain download URLs') 1605 add_version.add_argument('files', help='File names of the download artifacts', nargs='*') 1606 1607 rewrite = subparsers.add_parser('rewrite', help='Load tools.json, validate, and save the result back into JSON') 1608 rewrite.add_argument('--output', help='Save new tools.json into this file') 1609 1610 subparsers.add_parser('validate', help='Validate tools.json against schema file') 1611 1612 gen_doc = subparsers.add_parser('gen-doc', help='Write the list of tools as a documentation page') 1613 gen_doc.add_argument('--output', type=argparse.FileType('w'), default=sys.stdout, 1614 help='Output file name') 1615 gen_doc.add_argument('--heading-underline-char', help='Character to use when generating RST sections', default='~') 1616 1617 args = parser.parse_args(argv) 1618 1619 if args.action is None: 1620 parser.print_help() 1621 parser.exit(1) 1622 1623 if args.quiet: 1624 global global_quiet 1625 global_quiet = True 1626 1627 if args.non_interactive: 1628 global global_non_interactive 1629 global_non_interactive = True 1630 1631 global global_idf_path 1632 global_idf_path = os.environ.get('IDF_PATH') 1633 if args.idf_path: 1634 global_idf_path = args.idf_path 1635 if not global_idf_path: 1636 global_idf_path = os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) 1637 os.environ['IDF_PATH'] = global_idf_path 1638 1639 global global_idf_tools_path 1640 global_idf_tools_path = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(IDF_TOOLS_PATH_DEFAULT) 1641 1642 # On macOS, unset __PYVENV_LAUNCHER__ variable if it is set. 1643 # Otherwise sys.executable keeps pointing to the system Python, even when a python binary from a virtualenv is invoked. 1644 # See https://bugs.python.org/issue22490#msg283859. 1645 os.environ.pop('__PYVENV_LAUNCHER__', None) 1646 1647 if sys.version_info.major == 2: 1648 try: 1649 global_idf_tools_path.decode('ascii') 1650 except UnicodeDecodeError: 1651 fatal('IDF_TOOLS_PATH contains non-ASCII characters: {}'.format(global_idf_tools_path) + 1652 '\nThis is not supported yet with Python 2. ' + 1653 'Please set IDF_TOOLS_PATH to a directory with an ASCII name, or switch to Python 3.') 1654 raise SystemExit(1) 1655 1656 if CURRENT_PLATFORM == UNKNOWN_PLATFORM: 1657 fatal('Platform {} appears to be unsupported'.format(PYTHON_PLATFORM)) 1658 raise SystemExit(1) 1659 1660 global global_tools_json 1661 if args.tools_json: 1662 global_tools_json = args.tools_json 1663 else: 1664 global_tools_json = os.path.join(global_idf_path, TOOLS_FILE) 1665 1666 action_func_name = 'action_' + args.action.replace('-', '_') 1667 action_func = globals()[action_func_name] 1668 1669 action_func(args) 1670 1671 1672if __name__ == '__main__': 1673 main(sys.argv[1:]) 1674
