1#!/usr/bin/env python
2
3# internal use only for CI
4# download archive of one commit instead of cloning entire submodule repo
5
6import argparse
7import os
8import re
9import shutil
10import subprocess
11import time
12
13import gitlab_api
14
15SUBMODULE_PATTERN = re.compile(r"\[submodule \"([^\"]+)\"]")
16PATH_PATTERN = re.compile(r'path\s+=\s+(\S+)')
17URL_PATTERN = re.compile(r'url\s+=\s+(\S+)')
18
19SUBMODULE_ARCHIVE_TEMP_FOLDER = 'submodule_archive'
20
21
22class SubModule(object):
23    # We don't need to support recursive submodule clone now
24
25    GIT_LS_TREE_OUTPUT_PATTERN = re.compile(r'\d+\s+commit\s+([0-9a-f]+)\s+')
26
27    def __init__(self, gitlab_inst, path, url):
28        self.path = path
29        self.gitlab_inst = gitlab_inst
30        self.project_id = self._get_project_id(url)
31        self.commit_id = self._get_commit_id(path)
32
33    def _get_commit_id(self, path):
34        output = subprocess.check_output(['git', 'ls-tree', 'HEAD', path])
35        output = output.decode()
36        # example output: 160000 commit d88a262fbdf35e5abb372280eb08008749c3faa0	components/esp_wifi/lib
37        match = self.GIT_LS_TREE_OUTPUT_PATTERN.search(output)
38        return match.group(1)
39
40    def _get_project_id(self, url):
41        base_name = os.path.basename(url)
42        project_id = self.gitlab_inst.get_project_id(os.path.splitext(base_name)[0],  # remove .git
43                                                     namespace='espressif')
44        return project_id
45
46    def download_archive(self):
47        print('Update submodule: {}: {}'.format(self.path, self.commit_id))
48        path_name = self.gitlab_inst.download_archive(self.commit_id, SUBMODULE_ARCHIVE_TEMP_FOLDER,
49                                                      self.project_id)
50        renamed_path = os.path.join(os.path.dirname(path_name), os.path.basename(self.path))
51        os.rename(path_name, renamed_path)
52        shutil.rmtree(self.path, ignore_errors=True)
53        shutil.move(renamed_path, os.path.dirname(self.path))
54
55
56def update_submodule(git_module_file, submodules_to_update):
57    gitlab_inst = gitlab_api.Gitlab()
58    submodules = []
59    with open(git_module_file, 'r') as f:
60        data = f.read()
61    match = SUBMODULE_PATTERN.search(data)
62    while True:
63        next_match = SUBMODULE_PATTERN.search(data, pos=match.end())
64        if next_match:
65            end_pos = next_match.start()
66        else:
67            end_pos = len(data)
68        path_match = PATH_PATTERN.search(data, pos=match.end(), endpos=end_pos)
69        url_match = URL_PATTERN.search(data, pos=match.end(), endpos=end_pos)
70        path = path_match.group(1)
71        url = url_match.group(1)
72
73        filter_result = True
74        if submodules_to_update:
75            if path not in submodules_to_update:
76                filter_result = False
77        if filter_result:
78            submodules.append(SubModule(gitlab_inst, path, url))
79
80        match = next_match
81        if not match:
82            break
83
84    shutil.rmtree(SUBMODULE_ARCHIVE_TEMP_FOLDER, ignore_errors=True)
85
86    for submodule in submodules:
87        submodule.download_archive()
88
89
90if __name__ == '__main__':
91    start_time = time.time()
92    parser = argparse.ArgumentParser()
93    parser.add_argument('--repo_path', '-p', default='.', help='repo path')
94    parser.add_argument('--submodule', '-s', default='all',
95                        help='Submodules to update. By default update all submodules. '
96                             'For multiple submodules, separate them with `;`. '
97                             '`all` and `none` are special values that indicates we fetch all / none submodules')
98    args = parser.parse_args()
99    if args.submodule == 'none':
100        print("don't need to update submodules")
101        exit(0)
102    if args.submodule == 'all':
103        _submodules = []
104    else:
105        _submodules = args.submodule.split(';')
106    update_submodule(os.path.join(args.repo_path, '.gitmodules'), _submodules)
107    print('total time spent on update submodule: {:.02f}s'.format(time.time() - start_time))
108