1#!/usr/bin/env python3
2#
3# CI script to deploy docs to a webserver. Not useful outside of CI environment
4#
5#
6# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
7#
8# Licensed under the Apache License, Version 2.0 (the "License");
9# you may not use this file except in compliance with the License.
10# You may obtain a copy of the License at
11#
12#     http://www.apache.org/licenses/LICENSE-2.0
13#
14# Unless required by applicable law or agreed to in writing, software
15# distributed under the License is distributed on an "AS IS" BASIS,
16# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17# See the License for the specific language governing permissions and
18# limitations under the License.
19#
20import glob
21import os
22import os.path
23import re
24import stat
25import subprocess
26import sys
27import tarfile
28
29import packaging.version
30
31
32def env(variable, default=None):
33    """ Shortcut to return the expanded version of an environment variable """
34    return os.path.expandvars(os.environ.get(variable, default) if default else os.environ[variable])
35
36
37# import sanitize_version from the docs directory, shared with here
38sys.path.append(os.path.join(env('IDF_PATH'), 'docs'))
39from sanitize_version import sanitize_version  # noqa
40
41
42def main():
43    # if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI
44    git_ver = env('GIT_VER')  # output of git describe --always
45    ci_ver = env('CI_COMMIT_REF_NAME', git_ver)  # branch or tag we're building for (used for 'release' & URL)
46
47    version = sanitize_version(ci_ver)
48    print('Git version: {}'.format(git_ver))
49    print('CI Version: {}'.format(ci_ver))
50    print('Deployment version: {}'.format(version))
51
52    if not version:
53        raise RuntimeError('A version is needed to deploy')
54
55    build_dir = env('DOCS_BUILD_DIR')  # top-level local build dir, where docs have already been built
56
57    if not build_dir:
58        raise RuntimeError('Valid DOCS_BUILD_DIR is needed to deploy')
59
60    url_base = env('DOCS_DEPLOY_URL_BASE')  # base for HTTP URLs, used to print the URL to the log after deploying
61
62    docs_server = env('DOCS_DEPLOY_SERVER')  # ssh server to deploy to
63    docs_user = env('DOCS_DEPLOY_SERVER_USER')
64    docs_path = env('DOCS_DEPLOY_PATH')  # filesystem path on DOCS_SERVER
65
66    if not docs_server:
67        raise RuntimeError('Valid DOCS_DEPLOY_SERVER is needed to deploy')
68
69    if not docs_user:
70        raise RuntimeError('Valid DOCS_DEPLOY_SERVER_USER is needed to deploy')
71
72    docs_server = '{}@{}'.format(docs_user, docs_server)
73
74    if not docs_path:
75        raise RuntimeError('Valid DOCS_DEPLOY_PATH is needed to deploy')
76
77    print('DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}'.format(docs_server, docs_path))
78
79    tarball_path, version_urls = build_doc_tarball(version, git_ver, build_dir)
80
81    deploy(version, tarball_path, docs_path, docs_server)
82
83    print('Docs URLs:')
84    doc_deploy_type = os.getenv('TYPE')
85    for vurl in version_urls:
86        language, _, target = vurl.split('/')
87        tag = '{}_{}'.format(language, target)
88        url = '{}/{}/index.html'.format(url_base, vurl)  # (index.html needed for the preview server)
89        url = re.sub(r'([^:])//', r'\1/', url)  # get rid of any // that isn't in the https:// part
90        print('[document {}][{}] {}'.format(doc_deploy_type, tag, url))
91
92    # note: it would be neater to use symlinks for stable, but because of the directory order
93    # (language first) it's kind of a pain to do on a remote server, so we just repeat the
94    # process but call the version 'stable' this time
95    if is_stable_version(version):
96        print('Deploying again as stable version...')
97        tarball_path, version_urls = build_doc_tarball('stable', git_ver, build_dir)
98        deploy('stable', tarball_path, docs_path, docs_server)
99
100
101def deploy(version, tarball_path, docs_path, docs_server):
102    def run_ssh(commands):
103        """ Log into docs_server and run a sequence of commands using ssh """
104        print('Running ssh: {}'.format(commands))
105        subprocess.run(['ssh', '-o', 'BatchMode=yes', docs_server, '-x', ' && '.join(commands)], check=True)
106
107    # copy the version tarball to the server
108    run_ssh(['mkdir -p {}'.format(docs_path)])
109    print('Running scp {} to {}'.format(tarball_path, '{}:{}'.format(docs_server, docs_path)))
110    subprocess.run(['scp', '-B', tarball_path, '{}:{}'.format(docs_server, docs_path)], check=True)
111
112    tarball_name = os.path.basename(tarball_path)
113
114    run_ssh(['cd {}'.format(docs_path),
115             'rm -rf ./*/{}'.format(version),  # remove any pre-existing docs matching this version
116             'tar -zxvf {}'.format(tarball_name),  # untar the archive with the new docs
117             'rm {}'.format(tarball_name)])
118
119    # Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc
120    # as the version will be invalid for a window of time. Better to do it atomically, but this is
121    # another thing made much more complex by the directory structure putting language before version...
122
123
124def build_doc_tarball(version, git_ver, build_dir):
125    """ Make a tar.gz archive of the docs, in the directory structure used to deploy as
126        the given version """
127    version_paths = []
128    tarball_path = '{}/{}.tar.gz'.format(build_dir, version)
129
130    # find all the 'html/' directories under build_dir
131    html_dirs = glob.glob('{}/**/html/'.format(build_dir), recursive=True)
132    print('Found %d html directories' % len(html_dirs))
133
134    pdfs = glob.glob('{}/**/latex/build/*.pdf'.format(build_dir), recursive=True)
135    print('Found %d PDFs in latex directories' % len(pdfs))
136
137    # add symlink for stable and latest and adds them to PDF blob
138    symlinks = create_and_add_symlinks(version, git_ver, pdfs)
139
140    def not_sources_dir(ti):
141        """ Filter the _sources directories out of the tarballs """
142        if ti.name.endswith('/_sources'):
143            return None
144
145        ti.mode |= stat.S_IWGRP  # make everything group-writeable
146        return ti
147
148    try:
149        os.remove(tarball_path)
150    except OSError:
151        pass
152
153    with tarfile.open(tarball_path, 'w:gz') as tarball:
154        for html_dir in html_dirs:
155            # html_dir has the form '<ignored>/<language>/<target>/html/'
156            target_dirname = os.path.dirname(os.path.dirname(html_dir))
157            target = os.path.basename(target_dirname)
158            language = os.path.basename(os.path.dirname(target_dirname))
159
160            # when deploying, we want the top-level directory layout 'language/version/target'
161            archive_path = '{}/{}/{}'.format(language, version, target)
162            print("Archiving '{}' as '{}'...".format(html_dir, archive_path))
163            tarball.add(html_dir, archive_path, filter=not_sources_dir)
164            version_paths.append(archive_path)
165
166        for pdf_path in pdfs:
167            # pdf_path has the form '<ignored>/<language>/<target>/latex/build'
168            latex_dirname = os.path.dirname(pdf_path)
169            pdf_filename = os.path.basename(pdf_path)
170            target_dirname = os.path.dirname(os.path.dirname(latex_dirname))
171            target = os.path.basename(target_dirname)
172            language = os.path.basename(os.path.dirname(target_dirname))
173
174            # when deploying, we want the layout 'language/version/target/pdf'
175            archive_path = '{}/{}/{}/{}'.format(language, version, target, pdf_filename)
176            print("Archiving '{}' as '{}'...".format(pdf_path, archive_path))
177            tarball.add(pdf_path, archive_path)
178
179    for symlink in symlinks:
180        os.unlink(symlink)
181
182    return (os.path.abspath(tarball_path), version_paths)
183
184
185def create_and_add_symlinks(version, git_ver, pdfs):
186    """ Create symbolic links for PDFs for 'latest' and 'stable' releases """
187
188    symlinks = []
189    if 'stable' in version or 'latest' in version:
190        for pdf_path in pdfs:
191            symlink_path = pdf_path.replace(git_ver, version)
192            os.symlink(pdf_path, symlink_path)
193            symlinks.append(symlink_path)
194
195        pdfs.extend(symlinks)
196        print('Found %d PDFs in latex directories after adding symlink' % len(pdfs))
197
198    return symlinks
199
200
201def is_stable_version(version):
202    """ Heuristic for whether this is the latest stable release """
203    if not version.startswith('v'):
204        return False  # branch name
205    if '-' in version:
206        return False  # prerelease tag
207
208    git_out = subprocess.check_output(['git', 'tag', '-l']).decode('utf-8')
209
210    versions = [v.strip() for v in git_out.split('\n')]
211    versions = [v for v in versions if re.match(r'^v[\d\.]+$', v)]  # include vX.Y.Z only
212
213    versions = [packaging.version.parse(v) for v in versions]
214
215    max_version = max(versions)
216
217    if max_version.public != version[1:]:
218        print('Stable version is v{}. This version is {}.'.format(max_version.public, version))
219        return False
220    else:
221        print('This version {} is the stable version'.format(version))
222        return True
223
224
225if __name__ == '__main__':
226    main()
227